From nobody Tue Oct 7 05:19:45 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D43CF23BCED; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; cv=none; b=NRrB4vJNALSP6mxDOk3ZEeMGaR2OgcgG4WJwZXrSwpCE7MPKlQ9tp/VJ+eASGBtEnDkOQqH2/LZrPnqlzd62Pdr+crQnSnVQZnFYUx/qhZRuygTXPrrH48ZYjSGuZleEbybP25V2EUpN3/OZnAjOY03qWvYqtjaeeHf8PNjR6JQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; c=relaxed/simple; bh=pcCiP5FTA+Lm/bobq6YiOZWGbXIV/H7h3fpmxDOHmBo=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=E/sloOcH0S3EccYeX3chk710dua7s8SeQS4SmKWsMHte0Q4a8MQFzvvqicrhscx8ecpQqoI3pV/NofwWmXQ2m7m6PTXl2h42tl2ND7j3qNMnuSbpDzSF4MRpHnoxyfto9VH2nqQP39rZPrctYamfp0/tGcaQ6G8CiiyW2tR+tPI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=q+1HR/nS; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="q+1HR/nS" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=Af6nUOSfZkjof68kUCgJ+wjJzCjJKMFbRfwFCJ9l/cg=; b=q+1HR/nSjCuAMzUiwaWE8NHWZ2 hf12akAIGdg18oYqUIV4zB6+r46LTYo/6Hlkc5HnkUNcs/5yc0UzmFmUbrQAnysUEdVrRELdl1wxU SZJGaaMugzWBbIaB2a2Twi6ZuRAM456xqWXw9A3hKWfSOt5uxBq9Mk1ux4sS1qeEcZz1QPAjBf/zM +cA0WzGJ9GgCJ1tsJCDs83KETK6TYVxBfUlONoUOXkQUzRl5F05ejG5MB95+OAfaD3yl6G99ftUY1 jhd4w0cVplk+JZhnT7x/J0vxn0ks8915aRCIvZo7hcdkLwXPnQCRne+7K4LzoYBRQSzlKmVsz8Lx6 iZ97NlFg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGfz-00000006uK3-41hI; Mon, 14 Jul 2025 10:44:52 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id CD3CB30039A; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103439.637049932@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:12 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 01/16] x86/kvm/emulate: Implement test_cc() in C References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Current test_cc() uses the fastop infrastructure to test flags using SETcc instructions. However, int3_emulate_jcc() already fully implements the flags->CC mapping, use that. Removes a pile of gnarly asm. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/include/asm/text-patching.h | 20 +++++++++++++------- arch/x86/kvm/emulate.c | 34 ++----------------------------= ---- 2 files changed, 15 insertions(+), 39 deletions(-) --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -177,9 +177,9 @@ void int3_emulate_ret(struct pt_regs *re } =20 static __always_inline -void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsig= ned long disp) +bool __emulate_cc(unsigned long flags, u8 cc) { - static const unsigned long jcc_mask[6] =3D { + static const unsigned long cc_mask[6] =3D { [0] =3D X86_EFLAGS_OF, [1] =3D X86_EFLAGS_CF, [2] =3D X86_EFLAGS_ZF, @@ -192,15 +192,21 @@ void int3_emulate_jcc(struct pt_regs *re bool match; =20 if (cc < 0xc) { - match =3D regs->flags & jcc_mask[cc >> 1]; + match =3D flags & cc_mask[cc >> 1]; } else { - match =3D ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ - ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); + match =3D ((flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ + ((flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); if (cc >=3D 0xe) - match =3D match || (regs->flags & X86_EFLAGS_ZF); + match =3D match || (flags & X86_EFLAGS_ZF); } =20 - if ((match && !invert) || (!match && invert)) + return (match && !invert) || (!match && invert); +} + +static __always_inline +void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsig= ned long disp) +{ + if (__emulate_cc(regs->flags, cc)) ip +=3D disp; =20 int3_emulate_jmp(regs, ip); --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -26,6 +26,7 @@ #include #include #include +#include =20 #include "x86.h" #include "tss.h" @@ -416,31 +417,6 @@ static int fastop(struct x86_emulate_ctx ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END =20 -/* Special case for SETcc - 1 instruction per cc */ -#define FOP_SETCC(op) \ - FOP_FUNC(op) \ - #op " %al \n\t" \ - FOP_RET(op) - -FOP_START(setcc) -FOP_SETCC(seto) -FOP_SETCC(setno) -FOP_SETCC(setc) -FOP_SETCC(setnc) -FOP_SETCC(setz) -FOP_SETCC(setnz) -FOP_SETCC(setbe) -FOP_SETCC(setnbe) -FOP_SETCC(sets) -FOP_SETCC(setns) -FOP_SETCC(setp) -FOP_SETCC(setnp) -FOP_SETCC(setl) -FOP_SETCC(setnl) -FOP_SETCC(setle) -FOP_SETCC(setnle) -FOP_END; - FOP_START(salc) FOP_FUNC(salc) "pushf; sbb %al, %al; popf \n\t" @@ -1068,13 +1044,7 @@ static int em_bsr_c(struct x86_emulate_c =20 static __always_inline u8 test_cc(unsigned int condition, unsigned long fl= ags) { - u8 rc; - void (*fop)(void) =3D (void *)em_setcc + FASTOP_SIZE * (condition & 0xf); - - flags =3D (flags & EFLAGS_MASK) | X86_EFLAGS_IF; - asm("push %[flags]; popf; " CALL_NOSPEC - : "=3Da"(rc), ASM_CALL_CONSTRAINT : [thunk_target]"r"(fop), [flags]"r= "(flags)); - return rc; + return __emulate_cc(flags, condition & 0xf); } =20 static void fetch_register_operand(struct operand *op) From nobody Tue Oct 7 05:19:45 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 64F782475C2; Mon, 14 Jul 2025 10:45:08 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489910; cv=none; b=b8dCiB36PUj6bZndvGC2hkY5XUMhc5go6Z00nCqubwfr5oEn0H8dSxMY3HUxbywwQ7ZauvPFsR04RTkc4beFVAlifJ1WtFRjI+qaj4O+uENTIB2psGdz/v2qXSeAeTCiT7GGuPFUdFwWDWQOWtlipSP+BiJVIBnfKlc/GJfLuZ0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489910; c=relaxed/simple; bh=HTdqaT+oNcjQwtUabrIRrG0MBwoQaXbobzT0+Yi/Nqs=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=b1uwHc0OIo4Mufk4+5pEHRct54Fd2+svpKuDvm/ZnybhZOX+IXpBiUxixFjd1rg9sLduXC+rQ3sfbXZAhmkVjkwwpx4em3hGauaruT/+jKljnoSugC8C0Ls+yxaePF2zHiGRcFp6m/9yd+whulo4EJTun0ErGOBFGJS9iwotzBg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=uFuWY4AE; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="uFuWY4AE" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=8jYQtoRPpor4h8Zc2IcsTcKFBlClEZNWIyaWDZfA3xw=; b=uFuWY4AEv6rVu3xZqnPvflBX5U wJ+lT0ed3wOgfcWv+4kUXI426E2p3ud2ayqqunahvxTeEw5C+QrRRv+G6QasIy/NKqGEOPjSBVhaf xKKsrOxzS0c6UaNqt6qbEx+G2Z8O/G+yDPbXJBMM5VkeKQ252k3nC/nGWlV3y1YU1Q/dKDePr543N dqlCs6xL5jzIKUQlkgUaqTTTX8mUSTYJfyylUYwdmz+rnVLM4TtU45EGhmlQFiXdNa1evalr7fAmr IczUlhqZ2Ir8ahc2+QbnUcFg5H71sKSQpAhMnaQ/nliGiwcCq9PcWXIz8WBkxvm401QUqhQwoawF/ uXOSKrQw==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg0-00000006uK4-00bn; Mon, 14 Jul 2025 10:44:52 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id D20773007A0; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103439.773781574@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:13 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 02/16] x86/kvm/emulate: Introduce EM_ASM_1 References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace fastops with C based stubs. There are a bunch of problems with the current fastop infrastructure, most all related to their special calling convention, which bypasses the normal C-ABI. There are two immediate problems with this at present: - it relies on RET preserving EFLAGS; whereas C-ABI does not. - it circumvents compiler based control-flow-integrity checking because its all asm magic. The first is a problem for some mitigations where the x86_indirect_return_thunk needs to include non-trivial work that clobbers EFLAGS (eg. the Skylake call depth tracking thing). The second is a problem because it presents a 'naked' indirect call on kCFI builds, making it a prime target for control flow hijacking. Additionally, given that a large chunk of virtual machine performance relies on absolutely avoiding vmexit these days, this emulation stuff just isn't that critical for performance anymore. As such, replace the fastop calls with normal C functions using the 'execute' member. As noted by Paolo: this code was performance critical for pre-Westmere (2010) and only when running big real mode code. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 71 ++++++++++++++++++++++++++++++++++++++++----= ----- 1 file changed, 58 insertions(+), 13 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -267,11 +267,56 @@ static void invalidate_registers(struct X86_EFLAGS_PF|X86_EFLAGS_CF) =20 #ifdef CONFIG_X86_64 -#define ON64(x) x +#define ON64(x...) x #else -#define ON64(x) +#define ON64(x...) #endif =20 +#define EM_ASM_START(op) \ +static int em_##op(struct x86_emulate_ctxt *ctxt) \ +{ \ + unsigned long flags =3D (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; \ + int bytes =3D 1, ok =3D 1; \ + if (!(ctxt->d & ByteOp)) \ + bytes =3D ctxt->dst.bytes; \ + switch (bytes) { + +#define __EM_ASM(str) \ + asm("push %[flags]; popf \n\t" \ + "10: " str \ + "pushf; pop %[flags] \n\t" \ + "11: \n\t" \ + : "+a" (ctxt->dst.val), \ + "+d" (ctxt->src.val), \ + [flags] "+D" (flags), \ + "+S" (ok) \ + : "c" (ctxt->src2.val)) + +#define __EM_ASM_1(op, dst) \ + __EM_ASM(#op " %%" #dst " \n\t") + +#define __EM_ASM_1_EX(op, dst) \ + __EM_ASM(#op " %%" #dst " \n\t" \ + _ASM_EXTABLE_TYPE_REG(10b, 11f, EX_TYPE_ZERO_REG, %%esi)) + +#define __EM_ASM_2(op, dst, src) \ + __EM_ASM(#op " %%" #src ", %%" #dst " \n\t") + +#define EM_ASM_END \ + } \ + ctxt->eflags =3D (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \ + return !ok ? emulate_de(ctxt) : X86EMUL_CONTINUE; \ +} + +/* 1-operand, using "a" (dst) */ +#define EM_ASM_1(op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_1(op##b, al); break; \ + case 2: __EM_ASM_1(op##w, ax); break; \ + case 4: __EM_ASM_1(op##l, eax); break; \ + ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ + EM_ASM_END + /* * fastop functions have a special calling convention: * @@ -1002,10 +1047,10 @@ FASTOP3WCL(shrd); =20 FASTOP2W(imul); =20 -FASTOP1(not); -FASTOP1(neg); -FASTOP1(inc); -FASTOP1(dec); +EM_ASM_1(not); +EM_ASM_1(neg); +EM_ASM_1(inc); +EM_ASM_1(dec); =20 FASTOP2CL(rol); FASTOP2CL(ror); @@ -4021,8 +4066,8 @@ static const struct opcode group2[] =3D { static const struct opcode group3[] =3D { F(DstMem | SrcImm | NoWrite, em_test), F(DstMem | SrcImm | NoWrite, em_test), - F(DstMem | SrcNone | Lock, em_not), - F(DstMem | SrcNone | Lock, em_neg), + I(DstMem | SrcNone | Lock, em_not), + I(DstMem | SrcNone | Lock, em_neg), F(DstXacc | Src2Mem, em_mul_ex), F(DstXacc | Src2Mem, em_imul_ex), F(DstXacc | Src2Mem, em_div_ex), @@ -4030,14 +4075,14 @@ static const struct opcode group3[] =3D { }; =20 static const struct opcode group4[] =3D { - F(ByteOp | DstMem | SrcNone | Lock, em_inc), - F(ByteOp | DstMem | SrcNone | Lock, em_dec), + I(ByteOp | DstMem | SrcNone | Lock, em_inc), + I(ByteOp | DstMem | SrcNone | Lock, em_dec), N, N, N, N, N, N, }; =20 static const struct opcode group5[] =3D { - F(DstMem | SrcNone | Lock, em_inc), - F(DstMem | SrcNone | Lock, em_dec), + I(DstMem | SrcNone | Lock, em_inc), + I(DstMem | SrcNone | Lock, em_dec), I(SrcMem | NearBranch | IsBranch, em_call_near_abs), I(SrcMemFAddr | ImplicitOps | IsBranch, em_call_far), I(SrcMem | NearBranch | IsBranch, em_jmp_abs), @@ -4237,7 +4282,7 @@ static const struct opcode opcode_table[ /* 0x38 - 0x3F */ F6ALU(NoWrite, em_cmp), N, N, /* 0x40 - 0x4F */ - X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)), + X8(I(DstReg, em_inc)), X8(I(DstReg, em_dec)), /* 0x50 - 0x57 */ X8(I(SrcReg | Stack, em_push)), /* 0x58 - 0x5F */ From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 13ADF2472BD; Mon, 14 Jul 2025 10:45:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489910; cv=none; b=Yt7d5ed49T3pAEAlkbM5f0zuxevFmZSKtY0M7zv0VKMUwmSEjaToyVYjl/dspkAdxJU5OktRTfANHJc0iko0ieGFNsP9LzFyFwaFiHP6vng6Ww0YiDv3hIGpy5TZoVtTM0/WtyHxP3R0ojz9DMLJbgo/LtmABQCjMp0dvNlDSQk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489910; c=relaxed/simple; bh=fTgYF2PsGCwY0Pjg20fnmDCk2omHnQd0aXz+zTQIPzU=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=dsFTrctb+weXUWKL8sTldZDy88ACMXtaZ7sgClTZKjMxLEji0o8bUPWmB97YdfTddL5pNRN3fMeKUWwlrsDGupnhDtVDpIhv2xWVkLgu62OsCv+GED1hD56lZbC2/2Oyp5hKj+x9fQTYGsIElXxqVMHh2WC7tOtfLgI0rZDGYeY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=WmbNuKXm; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="WmbNuKXm" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=I75Per9iZErX40IGaiBRNDtT7hqKwKBYiyZ5s6JY19U=; b=WmbNuKXmKkjgJlREf8zjge3ONa /p38PPMwiE1NMuDJSHVKMuVz2y+l0qRh9YdzfavyRnS7Q/2fWjjYCxBQgHuRNs6LXytpDf4ZMvl+S pfTfENAlx0xazbtkZTKm+TrYnViW5Hb6HgGSYJXux4EjcaDJAxy69K2rWj6YfFOZi1id/durWwrdv Hyq+EDOBKTf1e0e+gHoVWnBwjxUC2tyKTxcky/lFqHMJ5ICxmADDLIdf1eRvgFuhQVPqT8PW+MTK5 8ruOW2sZgtUHMCTR+bQUT7vxQJ1pLlgTfUzaCQlEHo4Y9MhI4l2/itkWGgaIbXcTR2M0/ts/l/BHn xPgZRk+g==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGfz-00000009kca-3A1l; Mon, 14 Jul 2025 10:44:51 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id D613F300DD3; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103439.903697475@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:14 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 03/16] x86/kvm/emulate: Introduce EM_ASM_2 References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the FASTOP2 instructions. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 85 +++++++++++++++++++++++++++-----------------= ----- 1 file changed, 47 insertions(+), 38 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -317,6 +317,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END =20 +/* 2-operand, using "a" (dst), "d" (src) */ +#define EM_ASM_2(op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_2(op##b, al, dl); break; \ + case 2: __EM_ASM_2(op##w, ax, dx); break; \ + case 4: __EM_ASM_2(op##l, eax, edx); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ + EM_ASM_END + /* * fastop functions have a special calling convention: * @@ -1027,15 +1036,16 @@ static int read_descriptor(struct x86_em return rc; } =20 -FASTOP2(add); -FASTOP2(or); -FASTOP2(adc); -FASTOP2(sbb); -FASTOP2(and); -FASTOP2(sub); -FASTOP2(xor); -FASTOP2(cmp); -FASTOP2(test); +EM_ASM_2(add); +EM_ASM_2(or); +EM_ASM_2(adc); +EM_ASM_2(sbb); +EM_ASM_2(and); +EM_ASM_2(sub); +EM_ASM_2(xor); +EM_ASM_2(cmp); +EM_ASM_2(test); +EM_ASM_2(xadd); =20 FASTOP1SRC2(mul, mul_ex); FASTOP1SRC2(imul, imul_ex); @@ -1067,7 +1077,6 @@ FASTOP2W(bts); FASTOP2W(btr); FASTOP2W(btc); =20 -FASTOP2(xadd); =20 FASTOP2R(cmp, cmp_r); =20 @@ -2304,7 +2313,7 @@ static int em_cmpxchg(struct x86_emulate ctxt->dst.val =3D reg_read(ctxt, VCPU_REGS_RAX); ctxt->src.orig_val =3D ctxt->src.val; ctxt->src.val =3D ctxt->dst.orig_val; - fastop(ctxt, em_cmp); + em_cmp(ctxt); =20 if (ctxt->eflags & X86_EFLAGS_ZF) { /* Success: write back to memory; no update of EAX */ @@ -3069,7 +3078,7 @@ static int em_das(struct x86_emulate_ctx ctxt->src.type =3D OP_IMM; ctxt->src.val =3D 0; ctxt->src.bytes =3D 1; - fastop(ctxt, em_or); + em_or(ctxt); ctxt->eflags &=3D ~(X86_EFLAGS_AF | X86_EFLAGS_CF); if (cf) ctxt->eflags |=3D X86_EFLAGS_CF; @@ -3095,7 +3104,7 @@ static int em_aam(struct x86_emulate_ctx ctxt->src.type =3D OP_IMM; ctxt->src.val =3D 0; ctxt->src.bytes =3D 1; - fastop(ctxt, em_or); + em_or(ctxt); =20 return X86EMUL_CONTINUE; } @@ -3113,7 +3122,7 @@ static int em_aad(struct x86_emulate_ctx ctxt->src.type =3D OP_IMM; ctxt->src.val =3D 0; ctxt->src.bytes =3D 1; - fastop(ctxt, em_or); + em_or(ctxt); =20 return X86EMUL_CONTINUE; } @@ -3998,9 +4007,9 @@ static int check_perm_out(struct x86_emu #define I2bvIP(_f, _e, _i, _p) \ IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p) =20 -#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \ - F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ - F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) +#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \ + I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ + I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) =20 static const struct opcode group7_rm0[] =3D { N, @@ -4038,14 +4047,14 @@ static const struct opcode group7_rm7[] }; =20 static const struct opcode group1[] =3D { - F(Lock, em_add), - F(Lock | PageTable, em_or), - F(Lock, em_adc), - F(Lock, em_sbb), - F(Lock | PageTable, em_and), - F(Lock, em_sub), - F(Lock, em_xor), - F(NoWrite, em_cmp), + I(Lock, em_add), + I(Lock | PageTable, em_or), + I(Lock, em_adc), + I(Lock, em_sbb), + I(Lock | PageTable, em_and), + I(Lock, em_sub), + I(Lock, em_xor), + I(NoWrite, em_cmp), }; =20 static const struct opcode group1A[] =3D { @@ -4064,8 +4073,8 @@ static const struct opcode group2[] =3D { }; =20 static const struct opcode group3[] =3D { - F(DstMem | SrcImm | NoWrite, em_test), - F(DstMem | SrcImm | NoWrite, em_test), + I(DstMem | SrcImm | NoWrite, em_test), + I(DstMem | SrcImm | NoWrite, em_test), I(DstMem | SrcNone | Lock, em_not), I(DstMem | SrcNone | Lock, em_neg), F(DstXacc | Src2Mem, em_mul_ex), @@ -4258,29 +4267,29 @@ static const struct instr_dual instr_dua =20 static const struct opcode opcode_table[256] =3D { /* 0x00 - 0x07 */ - F6ALU(Lock, em_add), + I6ALU(Lock, em_add), I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg), I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg), /* 0x08 - 0x0F */ - F6ALU(Lock | PageTable, em_or), + I6ALU(Lock | PageTable, em_or), I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg), N, /* 0x10 - 0x17 */ - F6ALU(Lock, em_adc), + I6ALU(Lock, em_adc), I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg), I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg), /* 0x18 - 0x1F */ - F6ALU(Lock, em_sbb), + I6ALU(Lock, em_sbb), I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg), I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg), /* 0x20 - 0x27 */ - F6ALU(Lock | PageTable, em_and), N, N, + I6ALU(Lock | PageTable, em_and), N, N, /* 0x28 - 0x2F */ - F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das), + I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das), /* 0x30 - 0x37 */ - F6ALU(Lock, em_xor), N, N, + I6ALU(Lock, em_xor), N, N, /* 0x38 - 0x3F */ - F6ALU(NoWrite, em_cmp), N, N, + I6ALU(NoWrite, em_cmp), N, N, /* 0x40 - 0x4F */ X8(I(DstReg, em_inc)), X8(I(DstReg, em_dec)), /* 0x50 - 0x57 */ @@ -4306,7 +4315,7 @@ static const struct opcode opcode_table[ G(DstMem | SrcImm, group1), G(ByteOp | DstMem | SrcImm | No64, group1), G(DstMem | SrcImmByte, group1), - F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test), + I2bv(DstMem | SrcReg | ModRM | NoWrite, em_test), I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg), /* 0x88 - 0x8F */ I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov), @@ -4329,7 +4338,7 @@ static const struct opcode opcode_table[ I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov), F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r), /* 0xA8 - 0xAF */ - F2bv(DstAcc | SrcImm | NoWrite, em_test), + I2bv(DstAcc | SrcImm | NoWrite, em_test), I2bv(SrcAcc | DstDI | Mov | String, em_mov), I2bv(SrcSI | DstAcc | Mov | String, em_mov), F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r), @@ -4467,7 +4476,7 @@ static const struct opcode twobyte_table I(DstReg | SrcMem | ModRM, em_bsr_c), D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov), /* 0xC0 - 0xC7 */ - F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd), + I2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd), N, ID(0, &instr_dual_0f_c3), N, N, N, GD(0, &group9), /* 0xC8 - 0xCF */ From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D134024467E; Mon, 14 Jul 2025 10:45:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489907; cv=none; b=m7Jawxxl9FC6wuTaROkefdqoxYDdJ2SmB8YGajMtkrhXEgG4At5YuHt5SCCZ+xXxAW/XfwyqmGFIdxYrYhJtoWEBwXnIKakRnqXIxHOW4tRIGspYexc2+CD8VkWFc0XrNWF+nc3rg8qDkNW7yNXQFacLAeQ0Lb+MB4w4OHXAkrs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489907; c=relaxed/simple; bh=tuWqoc0GQbR2JTEQ+wVBO1amFPf+D71o88hvLaHFndU=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=Wc0V1F9Itp/dCeze0mDK0T49KM5pCaxky4bdtVYe5hffXI5Zu/ITiEoBFyp/Q8bajdSuCJ3P6oLdq3SPvWpBUJDbknw/dO4n0Ir9/8ULsyfgCFsCsYj8ZlU58vQ9/Ovwl/Porp8VSGjQ0RXPFGD3oT265VzcYwRDQrtJ2zQkPjw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=KSXuacWx; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="KSXuacWx" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=fy6j2Zut50EkLTl5aAIvm4GTA07w7XyXzsCJhihcPoI=; b=KSXuacWxkXt1D2/owpptiCy8Ys D3ujdk+OLSrRw8uTv5iuV6Q13KVK/FGuv2TMYi78KfcPea+tTzBtsf3h/WpQqsZNiYEHx7RQDip07 ysiEd+8g9PwgS6FwSvna85L5D8bF6SuxTMjZhkjq4Iv5EyrbE0pKlBbmZSJRhfas7Uk4J7mGpKb2N TvSFYisclONYICygkzXlatLbgk0MZQKDiIqszYi1OODE7OtYvSmFRz+SLwnuYErnG4pa3llhDxOHl +r5nZv2gi7Ytg0H97FL7p/BRxO47lEN8+BZdArzx99vLL0YcPob7kTOC5OJPFb/IE6wyD5wa7Scu9 mY8nkZhg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGfz-00000009kcb-38bL; Mon, 14 Jul 2025 10:44:51 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id DA222300F1A; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.024933524@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:15 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 04/16] x86/kvm/emulate: Introduce EM_ASM_2R References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the FASTOP2R instruction. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -326,6 +326,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ EM_ASM_END =20 +/* 2-operand, reversed */ +#define EM_ASM_2R(op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_2(op##b, dl, al); break; \ + case 2: __EM_ASM_2(op##w, dx, ax); break; \ + case 4: __EM_ASM_2(op##l, edx, eax); break; \ + ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \ + EM_ASM_END + /* * fastop functions have a special calling convention: * @@ -1077,8 +1086,7 @@ FASTOP2W(bts); FASTOP2W(btr); FASTOP2W(btc); =20 - -FASTOP2R(cmp, cmp_r); +EM_ASM_2R(cmp, cmp_r); =20 static int em_bsf_c(struct x86_emulate_ctxt *ctxt) { @@ -4336,12 +4344,12 @@ static const struct opcode opcode_table[ I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov), I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov), I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov), - F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r), + I2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r), /* 0xA8 - 0xAF */ I2bv(DstAcc | SrcImm | NoWrite, em_test), I2bv(SrcAcc | DstDI | Mov | String, em_mov), I2bv(SrcSI | DstAcc | Mov | String, em_mov), - F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r), + I2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r), /* 0xB0 - 0xB7 */ X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)), /* 0xB8 - 0xBF */ From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D13A1246348; Mon, 14 Jul 2025 10:45:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; cv=none; b=jDdMmFJJDxvHHUUhqGWuyKev8AXlfgcwaU9Wd2lY8DH4MBIZFdE5RdzE16OwLw8/sWab0fJME7RNJ0FYRDv1Z9cxkc47M1LGQ7CeR5KULnOOrtdBtuPp3ZkavdVYZubH0qLH5LCvnJmp+vDvQpgvDXzVlz9Mng18g/p/MMO8Vzs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; c=relaxed/simple; bh=PdAeXVf6Vqp6oyKtuCeP7lKnTP97pY4tt8hC+7YL5Y8=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=MP86hnMvQtAIFpOsKkiq/wG6UvIee38VER058bYlMgkBa3jUEKtBmrl0pRn4VTQMhQT4QjV7191eoWLLDYSZTVybX8admDPPN9DLWZmDBrcRaV2DKFq856hYODafnBKjcCWe17v8cxIufo1OsbpKJYb5sGZfTwz3zVLB39VzUEk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=q2oXozo+; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="q2oXozo+" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=uP3uw2B1ls5qkdz2Wl3msyHx5+mDer9y8QVhMyZ1gnc=; b=q2oXozo+F0MPpsnTwUJOz6e2Fc 5FyXmhE9A0S891VO0Jw8CZvzEz7Lp5OlCO01zEmRJ+ICU04kKQEGz9COkm++85lh6rEZ+XcnI8+VK AKc7nDnnHzUqgTHMKUpB6zhH2USYyi6gAXDfdwkVIr64Ro1tf/vlb3MOqYp6izn/9to/WeG9HCsX3 99qUmINyk6wf6uaVv+lSH11qvoANZHk358h21f0JUWa9Bl6U9wMUtaLmjgdSxfeFmsz9tNC2k/5uV e+R6yG97H5MrICXgCysnpe0/BYj32PtDAJ98ZLrUyZiuCfG9QjyjQYvLPhJk7psqRcuV1b6nbzlvp G8myFu+g==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg0-00000009kcg-268u; Mon, 14 Jul 2025 10:44:52 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id DE4E5300F1D; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.142923581@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:16 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 05/16] x86/kvm/emulate: Introduce EM_ASM_2W References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the FASTOP2W instructions. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 47 ++++++++++++++++++++++++++++----------------= --- 1 file changed, 28 insertions(+), 19 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -335,6 +335,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \ EM_ASM_END =20 +/* 2-operand, word only (no byte op) */ +#define EM_ASM_2W(op) \ + EM_ASM_START(op) \ + case 1: break; \ + case 2: __EM_ASM_2(op##w, ax, dx); break; \ + case 4: __EM_ASM_2(op##l, eax, edx); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ + EM_ASM_END + /* * fastop functions have a special calling convention: * @@ -1064,7 +1073,7 @@ FASTOP1SRC2EX(idiv, idiv_ex); FASTOP3WCL(shld); FASTOP3WCL(shrd); =20 -FASTOP2W(imul); +EM_ASM_2W(imul); =20 EM_ASM_1(not); EM_ASM_1(neg); @@ -1079,12 +1088,12 @@ FASTOP2CL(shl); FASTOP2CL(shr); FASTOP2CL(sar); =20 -FASTOP2W(bsf); -FASTOP2W(bsr); -FASTOP2W(bt); -FASTOP2W(bts); -FASTOP2W(btr); -FASTOP2W(btc); +EM_ASM_2W(bsf); +EM_ASM_2W(bsr); +EM_ASM_2W(bt); +EM_ASM_2W(bts); +EM_ASM_2W(btr); +EM_ASM_2W(btc); =20 EM_ASM_2R(cmp, cmp_r); =20 @@ -1093,7 +1102,7 @@ static int em_bsf_c(struct x86_emulate_c /* If src is zero, do not writeback, but update flags */ if (ctxt->src.val =3D=3D 0) ctxt->dst.type =3D OP_NONE; - return fastop(ctxt, em_bsf); + return em_bsf(ctxt); } =20 static int em_bsr_c(struct x86_emulate_ctxt *ctxt) @@ -1101,7 +1110,7 @@ static int em_bsr_c(struct x86_emulate_c /* If src is zero, do not writeback, but update flags */ if (ctxt->src.val =3D=3D 0) ctxt->dst.type =3D OP_NONE; - return fastop(ctxt, em_bsr); + return em_bsr(ctxt); } =20 static __always_inline u8 test_cc(unsigned int condition, unsigned long fl= ags) @@ -3221,7 +3230,7 @@ static int em_xchg(struct x86_emulate_ct static int em_imul_3op(struct x86_emulate_ctxt *ctxt) { ctxt->dst.val =3D ctxt->src2.val; - return fastop(ctxt, em_imul); + return em_imul(ctxt); } =20 static int em_cwd(struct x86_emulate_ctxt *ctxt) @@ -4135,10 +4144,10 @@ static const struct group_dual group7 =3D =20 static const struct opcode group8[] =3D { N, N, N, N, - F(DstMem | SrcImmByte | NoWrite, em_bt), - F(DstMem | SrcImmByte | Lock | PageTable, em_bts), - F(DstMem | SrcImmByte | Lock, em_btr), - F(DstMem | SrcImmByte | Lock | PageTable, em_btc), + I(DstMem | SrcImmByte | NoWrite, em_bt), + I(DstMem | SrcImmByte | Lock | PageTable, em_bts), + I(DstMem | SrcImmByte | Lock, em_btr), + I(DstMem | SrcImmByte | Lock | PageTable, em_btc), }; =20 /* @@ -4459,27 +4468,27 @@ static const struct opcode twobyte_table /* 0xA0 - 0xA7 */ I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg), II(ImplicitOps, em_cpuid, cpuid), - F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), + I(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N, /* 0xA8 - 0xAF */ I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg), II(EmulateOnUD | ImplicitOps, em_rsm, rsm), - F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), + I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), F(DstMem | SrcReg | Src2CL | ModRM, em_shrd), - GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul), + GD(0, &group15), I(DstReg | SrcMem | ModRM, em_imul), /* 0xB0 - 0xB7 */ I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg), I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg), - F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr), + I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr), I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg), I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg), D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov), /* 0xB8 - 0xBF */ N, N, G(BitOp, group8), - F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc), + I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc), I(DstReg | SrcMem | ModRM, em_bsf_c), I(DstReg | SrcMem | ModRM, em_bsr_c), D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov), From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0E853246332; Mon, 14 Jul 2025 10:45:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489907; cv=none; b=WS5JCpt2IMaoex0ypeFfYO5WsiNK+ZYNc08XycW53TXxgWvXePoyxnSnzqyoNOsy9tBn+7Ab+46hNzjFck6Is3T+Oke6pbyKIOjjHjKPVLhXa/ufmPH5A0AFsyosuREJxazUUgPqRQ4/99yBzk5/GmWDVA6HI/NDx5HcjhuOQ/0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489907; c=relaxed/simple; bh=Lm1PsQZFxdoWNTI1Q+RNhnY7JtPtzUkiyH/xNwDExCs=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=dU5twFUkBys3wfmb8cfN6Lu6Bb/t49QL/QoU8BK9kKoRaUV5IwkgzOelU7dHVz6q9oUtvE1CfJ4QNuthesIcUxzNp4/Nd4OGNUw5oM8DDGLJVuKVWrL2PI38mO2TzvTuIEGK4kB9YX0IRrRbeUknbS/fiepPe5DcnjyOZPBCaec= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=SC/GCyQG; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="SC/GCyQG" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=jotUtNQIA7oK1p4O3nX4QCvhhLgXNFqkMI8Hz+9cSMI=; b=SC/GCyQG0qK7KE5IvGLrynl9+3 utORzdyH2kgzEngucxQgV/d99JsjzqxS4cYYtI++lcN4FjPwAVxBF8ukjS20zTIgA3K1I9EXVIO7e 03CtxqKK+mJktahq0cpVCcfWALBFzVtaGvokkjeHfgAoRQMgDeri3dQMsCsyhK8IwDbbOl69Ef/by Gt21kvqNv32miMf2+0Wyl9N+OvQDtlbfrL3Hcqv+CwJQPxhBdBy3270pbWkbFVDvXJXUvILdSIk/A 5Lu/MXqPzp8fEHgW+/q7GgmfyJ/Li1nK2zRa7Im/41upaKNWhD+G4S1Rgn1I088zgGZoW07QL23+g Ri84vr5g==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg0-00000009kch-29AW; Mon, 14 Jul 2025 10:44:52 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id E2076301142; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.251039692@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:17 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 06/16] x86/kvm/emulate: Introduce EM_ASM_2CL References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the FASTOP2CL instructions. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -344,6 +344,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ EM_ASM_END =20 +/* 2-operand, using "a" (dst) and CL (src2) */ +#define EM_ASM_2CL(op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_2(op##b, al, cl); break; \ + case 2: __EM_ASM_2(op##w, ax, cl); break; \ + case 4: __EM_ASM_2(op##l, eax, cl); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \ + EM_ASM_END + /* * fastop functions have a special calling convention: * @@ -1080,13 +1089,13 @@ EM_ASM_1(neg); EM_ASM_1(inc); EM_ASM_1(dec); =20 -FASTOP2CL(rol); -FASTOP2CL(ror); -FASTOP2CL(rcl); -FASTOP2CL(rcr); -FASTOP2CL(shl); -FASTOP2CL(shr); -FASTOP2CL(sar); +EM_ASM_2CL(rol); +EM_ASM_2CL(ror); +EM_ASM_2CL(rcl); +EM_ASM_2CL(rcr); +EM_ASM_2CL(shl); +EM_ASM_2CL(shr); +EM_ASM_2CL(sar); =20 EM_ASM_2W(bsf); EM_ASM_2W(bsr); @@ -4079,14 +4088,14 @@ static const struct opcode group1A[] =3D { }; =20 static const struct opcode group2[] =3D { - F(DstMem | ModRM, em_rol), - F(DstMem | ModRM, em_ror), - F(DstMem | ModRM, em_rcl), - F(DstMem | ModRM, em_rcr), - F(DstMem | ModRM, em_shl), - F(DstMem | ModRM, em_shr), - F(DstMem | ModRM, em_shl), - F(DstMem | ModRM, em_sar), + I(DstMem | ModRM, em_rol), + I(DstMem | ModRM, em_ror), + I(DstMem | ModRM, em_rcl), + I(DstMem | ModRM, em_rcr), + I(DstMem | ModRM, em_shl), + I(DstMem | ModRM, em_shr), + I(DstMem | ModRM, em_shl), + I(DstMem | ModRM, em_sar), }; =20 static const struct opcode group3[] =3D { From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B7F591EF38F; Mon, 14 Jul 2025 10:45:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; cv=none; b=W3My2BGpepvGawReLrRTqnJU/iU9WVMLZdVX3Sutj6eLcoL3WcLFSzqoW6o83idHQONmPvrEUND4jYr+crGdJg2NpCBkVMi+3d7H7zAQktOavoq1wo8wj3Znk54Djan24hsasucV5a6WOF8V/NaM8g2744h5jz8cCI2w2vrIhwQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; c=relaxed/simple; bh=stTTWQ2PrnQ5pWJsZvXWpLEwFy8R389SiRh7T6dqJ9M=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=mbip/gVHumabJbd8nc6dDgBLKRrJW1rBv5M053VVeRPODEqkofG8fLo1QrS1y0d7gaqwu61stQvlAYCHj8zDmQwGyTLvSxmMBu9379whxJAun7y4ScL+TqlkGIskPYdLAHpE8pZu3Gps0WpJvLbyISXsdjdJ/JGPwXNpsyf75zE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=RWhND9nE; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="RWhND9nE" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=16Q2GjdSSsv4jMqrK08IswOlebpDVF3OJKCNgU2VGgI=; b=RWhND9nEVSv7C0Zf9o/T0zY6uM XgxZnIWWTix00DPc/6ZdQG8WelG9tLmHM8HvUU5Ia7TqEZctrdgqDzGyaX7r1FU1VbQ7Bus6J0eri hDncwt6VhrLBLI2jhgWF0dpOieev2fNc3dHpj2AnypKp8vE6B0K9UtyCq+TbKmBsKQTEmDBsAAYae NhpAH4Socj5jp7VcSLyt4sZCKZMHhHVvNuk0bRwJ6r1MiywZSM3AWyNkLDTmbFgH6WtgcbdgEWHxx OHBiyUB4lSa73pU5CQNcEcCHiwrOE2J4LDwcllRgS0d64v02TyECKn4KUb5WNU0ABu46xnNVFza06 TjHsAgQA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg0-00000009kci-27YT; Mon, 14 Jul 2025 10:44:52 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id E60A9302D3E; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.394654786@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:18 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 07/16] x86/kvm/emulate: Introduce EM_ASM_1SRC2 References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the FASTOP1SRC2*() instructions. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -317,6 +317,24 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END =20 +/* 1-operand, using "c" (src2) */ +#define EM_ASM_1SRC2(op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1(op##b, cl); break; \ + case 2: __EM_ASM_1(op##w, cx); break; \ + case 4: __EM_ASM_1(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END + +/* 1-operand, using "c" (src2) with exception */ +#define EM_ASM_1SRC2EX(op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1_EX(op##b, cl); break; \ + case 2: __EM_ASM_1_EX(op##w, cx); break; \ + case 4: __EM_ASM_1_EX(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END + /* 2-operand, using "a" (dst), "d" (src) */ #define EM_ASM_2(op) \ EM_ASM_START(op) \ @@ -1074,10 +1092,10 @@ EM_ASM_2(cmp); EM_ASM_2(test); EM_ASM_2(xadd); =20 -FASTOP1SRC2(mul, mul_ex); -FASTOP1SRC2(imul, imul_ex); -FASTOP1SRC2EX(div, div_ex); -FASTOP1SRC2EX(idiv, idiv_ex); +EM_ASM_1SRC2(mul, mul_ex); +EM_ASM_1SRC2(imul, imul_ex); +EM_ASM_1SRC2EX(div, div_ex); +EM_ASM_1SRC2EX(idiv, idiv_ex); =20 FASTOP3WCL(shld); FASTOP3WCL(shrd); @@ -4103,10 +4121,10 @@ static const struct opcode group3[] =3D { I(DstMem | SrcImm | NoWrite, em_test), I(DstMem | SrcNone | Lock, em_not), I(DstMem | SrcNone | Lock, em_neg), - F(DstXacc | Src2Mem, em_mul_ex), - F(DstXacc | Src2Mem, em_imul_ex), - F(DstXacc | Src2Mem, em_div_ex), - F(DstXacc | Src2Mem, em_idiv_ex), + I(DstXacc | Src2Mem, em_mul_ex), + I(DstXacc | Src2Mem, em_imul_ex), + I(DstXacc | Src2Mem, em_div_ex), + I(DstXacc | Src2Mem, em_idiv_ex), }; =20 static const struct opcode group4[] =3D { From nobody Tue Oct 7 05:19:45 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D2A4E23B63C; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489910; cv=none; b=L/2NHtUp3YPemxZw9Gr33wtutcId2g+eVs0KEp0BDKjIwQXCPqqtBa2YFFsIh4qeADWmzfZuIj0KfS1RYAgXZDqoNu5Mw/tGB7M0W/FT5Hf7qxZ5k6rs5fq2r5HbHc3GjrX33qE9vDhkgwp8zmMTKJgpV5gOKNKwWM+OUEMz7Mc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489910; c=relaxed/simple; bh=H5YBZf0fNnVsYlv27L5Q/Ob03FWPbKXTKA4uMtscfig=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=avwtI4SnIo4UrGGX/a2BwC+adhhocfEyFnunE9zk0s2MinV1bQpa2fqodlSnpfZjCHYXCQ2+aCw3OOCaPuSbfZf5OrX8YGvh6ju6iPIHFqYSVCBADfL+xaG4yJ+fSuFN9nhHyx/inbV1M9P9hdPMVHr71EireJLFqPM+fgdiEsA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=JKNMgt0N; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="JKNMgt0N" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=eNidn6T/kDeqMRXxeY6Z3Dw7McLh++ZG+chmKF2+4y4=; b=JKNMgt0Nzkt7+k2O9fjju3DVYL Rl4HloztCaLyxT2XugEncc9CSUmPd2MsGNwHSWXVuIbmelVcGC0v/ML0UPvF9TXqBPMoymdyO5GxF 1RuoWckKCHXPL2W84AiPHCybiaOTv3a3HPt4qgjLg866Z6NIQzyNEoWBFULqLyyaRxStYZRdFBMIT w99I+X1Uw+XI+hw44KI0jWQiSKfG6CezLKtclNVprvDC1zj6wcAaNSHB3ccU6pQ6akEmX1mloHcKJ uyesqNzU70O5DFHj2YFy/rThy9RuBg7MFF7eoh6DNcgnLPxH4FioyW9BGOwgflTCWDxQN282K3iuA yPrxUEsA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg0-00000006uKY-21wd; Mon, 14 Jul 2025 10:44:52 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id EA0F9302D42; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.513865075@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:19 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 08/16] x86/kvm/emulate: Introduce EM_ASM_3WCL References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the FASTOP3WCL instructions. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -302,6 +302,9 @@ static int em_##op(struct x86_emulate_ct #define __EM_ASM_2(op, dst, src) \ __EM_ASM(#op " %%" #src ", %%" #dst " \n\t") =20 +#define __EM_ASM_3(op, dst, src, src2) \ + __EM_ASM(#op " %%" #src2 ", %%" #src ", %%" #dst " \n\t") + #define EM_ASM_END \ } \ ctxt->eflags =3D (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \ @@ -371,6 +374,16 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \ EM_ASM_END =20 +/* 3-operand, using "a" (dst), "d" (src) and CL (src2) */ +#define EM_ASM_3WCL(op) \ + EM_ASM_START(op) \ + case 1: break; \ + case 2: __EM_ASM_3(op##w, ax, dx, cl); break; \ + case 4: __EM_ASM_3(op##l, eax, edx, cl); break; \ + ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \ + EM_ASM_END + + /* * fastop functions have a special calling convention: * @@ -1097,8 +1110,8 @@ EM_ASM_1SRC2(imul, imul_ex); EM_ASM_1SRC2EX(div, div_ex); EM_ASM_1SRC2EX(idiv, idiv_ex); =20 -FASTOP3WCL(shld); -FASTOP3WCL(shrd); +EM_ASM_3WCL(shld); +EM_ASM_3WCL(shrd); =20 EM_ASM_2W(imul); =20 @@ -4496,14 +4509,14 @@ static const struct opcode twobyte_table I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg), II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), - F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), - F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N, + I(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), + I(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N, /* 0xA8 - 0xAF */ I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg), II(EmulateOnUD | ImplicitOps, em_rsm, rsm), I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), - F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), - F(DstMem | SrcReg | Src2CL | ModRM, em_shrd), + I(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), + I(DstMem | SrcReg | Src2CL | ModRM, em_shrd), GD(0, &group15), I(DstReg | SrcMem | ModRM, em_imul), /* 0xB0 - 0xB7 */ I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg), From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7252845009; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; cv=none; b=LB7ZkVndjYN00Vo0YTUflUvg58lw/Om1VC0EUmcwHFodOggYKptNq1CRcpHUbdRZO01/LdF7b/igm5EoBlfDubeOSm38LqIq+mJuamvHdbcTATKgb6ktcohZnNsMqo2htjHSbXGTQ9RPq8Wh1O4YxeOeVpZq2Q/1/ssR70oBqBM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; c=relaxed/simple; bh=vpKuQizapCMKqH9WA11ms5jc5OISD3N0+Dznv7nE1MM=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=q7juthuc9VQxMnJUMfNGrYMj42hI+8+ue2PvLyASXaZmiG673SbA0JN//yYE9+8xz22dngo8tbZeYtDp+gXNPJB+HJjDyXs4QbW2J427cWPlYmdExoo0u1LzAZwiI0GK/Ss+0CDlp86BQZvyc9F35HTMzdcmzsXKtN/uWetCOJM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=Vw0v8/Qb; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="Vw0v8/Qb" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=e4AF8GQacEV/iOhFocsW7bzZKWyP7FsAsEjPpw7Nh70=; b=Vw0v8/Qbvk3pA+yzNp3T5erRn8 9U3n7yjcqOcL3jDyxibLBNo3oB35aTiYck29trmgT/TWR7hdFASz6QEtk+W+RI2tqJ4uxUXsukPZU zkUgGayQy9jRz7Jxcl1U9WzFIthSal8Uu6BkauxYH7qcet5qJc658oQrmutDfy3CPUv/k8urDsoKq nSFl+HUEiexUBhGx/qA3CDGsJuQ7WRIs9Sg8TO8cQvstMeOPSlgvzo6oYGhuMbsMYSAVEpMmcMTrV 8OonnGIu5MPqfNWeTUdAMeCJBDpAHjFO80SD9Ly8OjhLHqwbKeZseoZuvDJlk/gxu+Vjbgt8BYhKM jGGIELkA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000009kck-0hxd; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id EDDD7302D50; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.634145269@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:20 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 09/16] x86/kvm/emulate: Convert em_salc() to C References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the SALC (Set AL if Carry) instruction in C. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -529,11 +529,14 @@ static int fastop(struct x86_emulate_ctx ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END =20 -FOP_START(salc) -FOP_FUNC(salc) -"pushf; sbb %al, %al; popf \n\t" -FOP_RET(salc) -FOP_END; +static int em_salc(struct x86_emulate_ctxt *ctxt) +{ + /* + * Set AL 0xFF if CF is set, or 0x00 when clear. + */ + ctxt->dst.val =3D 0xFF * !!(ctxt->eflags & X86_EFLAGS_CF); + return X86EMUL_CONTINUE; +} =20 /* * XXX: inoutclob user must know where the argument is being expanded. @@ -4423,7 +4426,7 @@ static const struct opcode opcode_table[ G(Src2CL | ByteOp, group2), G(Src2CL, group2), I(DstAcc | SrcImmUByte | No64, em_aam), I(DstAcc | SrcImmUByte | No64, em_aad), - F(DstAcc | ByteOp | No64, em_salc), + I(DstAcc | ByteOp | No64, em_salc), I(DstAcc | SrcXLat | ByteOp, em_mov), /* 0xD8 - 0xDF */ N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N, From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 867BE149C51; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; cv=none; b=A8EcVpv4xFPSAPFXngICV3jqWVsF54enemcnQ8hRi3fGVMlJBntWSLp7TRc0RNOgaxFkLYAoppv5bZh4c9HgHLqzGWY8MPIxdW+DhNTFUXx3TG0zAEJy5ximTwBqHX/unbopGWsT8EhZpj68C9MtJLW41tIjXYdNenHrYqRtQwE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489908; c=relaxed/simple; bh=nOn0fX75ysPhTk8MlfTUlzFn+Ml7ySHuCtwKNQhCjKc=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=tLmOBBmaUJSMZA3jcBd3rWtT+lORSqBnjL0yle/7deqs8ZlYHCnZA3OQMcIdFM1Q29K2fc3wdMXeAKh7uupwlh9MvIncoLwVsuxA8a0qyC+stLlISAtaR4X1Q1ImCy43fcI6kWFQk07okYWeUv0fPMVLQEZ5MwJ9NlmknL8lwA4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=QVp/quoR; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="QVp/quoR" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=WEU+51TOfKmqbdqEWlsR9cikZMb6arGOO4YuD2naPts=; b=QVp/quoRgC71xk9rOmHRYf6Men hNU7YR+HBaU6PX68bohJ4aSazBnmfGyHGSLXfJmOaxpvXqn7hiCE0LSJE3UqBdGwe4ZTOCOHs8nza 1TjCalZSW+T88HuIMTVzaS+5INRf0MlP46nUACUsBupKnLHwJiCOC8ZGOK0AmZZGUQfSVvuuUR4rc XU2Bm9b1BDCQxpbzKT1gspMoCGWStEwrUn9a000b4P6KVJWXDas2AUQ4Pgw8KwiwrSrcR4Di0sUMY NkeH/l6fA1O60rXAvklY4zvL19f1X0FilnziXVQKhjm1RvlSvanSHS0GY/6yKMShsOERB7XtspLuM vNKDuSCg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000009kcl-0ksX; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id F20BB302D64; Mon, 14 Jul 2025 12:44:50 +0200 (CEST) Message-ID: <20250714103440.751192860@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:21 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 10/16] x86/kvm/emulate: Remove fastops References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" No more FASTOPs, remove the remains. Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 172 --------------------------------------------= ----- 1 file changed, 1 insertion(+), 171 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -167,7 +167,6 @@ #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) = */ #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */ #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. F= XSAVE) */ -#define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */ #define NoWrite ((u64)1 << 45) /* No writeback */ #define SrcWrite ((u64)1 << 46) /* Write back src operand */ #define NoMod ((u64)1 << 47) /* Mod field is ignored */ @@ -203,7 +202,6 @@ struct opcode { const struct escape *esc; const struct instr_dual *idual; const struct mode_dual *mdual; - void (*fastop)(struct fastop *fake); } u; int (*check_perm)(struct x86_emulate_ctxt *ctxt); }; @@ -383,152 +381,6 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \ EM_ASM_END =20 - -/* - * fastop functions have a special calling convention: - * - * dst: rax (in/out) - * src: rdx (in/out) - * src2: rcx (in) - * flags: rflags (in/out) - * ex: rsi (in:fastop pointer, out:zero if exception) - * - * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for - * different operand sizes can be reached by calculation, rather than a ju= mp - * table (which would be bigger than the code). - * - * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for END= BR - * and 1 for the straight line speculation INT3, leaves 7 bytes for the - * body of the function. Currently none is larger than 4. - */ -static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); - -#define FASTOP_SIZE 16 - -#define __FOP_FUNC(name) \ - ".align " __stringify(FASTOP_SIZE) " \n\t" \ - ".type " name ", @function \n\t" \ - name ":\n\t" \ - ASM_ENDBR \ - IBT_NOSEAL(name) - -#define FOP_FUNC(name) \ - __FOP_FUNC(#name) - -#define __FOP_RET(name) \ - "11: " ASM_RET \ - ".size " name ", .-" name "\n\t" - -#define FOP_RET(name) \ - __FOP_RET(#name) - -#define __FOP_START(op, align) \ - extern void em_##op(struct fastop *fake); \ - asm(".pushsection .text, \"ax\" \n\t" \ - ".global em_" #op " \n\t" \ - ".align " __stringify(align) " \n\t" \ - "em_" #op ":\n\t" - -#define FOP_START(op) __FOP_START(op, FASTOP_SIZE) - -#define FOP_END \ - ".popsection") - -#define __FOPNOP(name) \ - __FOP_FUNC(name) \ - __FOP_RET(name) - -#define FOPNOP() \ - __FOPNOP(__stringify(__UNIQUE_ID(nop))) - -#define FOP1E(op, dst) \ - __FOP_FUNC(#op "_" #dst) \ - "10: " #op " %" #dst " \n\t" \ - __FOP_RET(#op "_" #dst) - -#define FOP1EEX(op, dst) \ - FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi) - -#define FASTOP1(op) \ - FOP_START(op) \ - FOP1E(op##b, al) \ - FOP1E(op##w, ax) \ - FOP1E(op##l, eax) \ - ON64(FOP1E(op##q, rax)) \ - FOP_END - -/* 1-operand, using src2 (for MUL/DIV r/m) */ -#define FASTOP1SRC2(op, name) \ - FOP_START(name) \ - FOP1E(op, cl) \ - FOP1E(op, cx) \ - FOP1E(op, ecx) \ - ON64(FOP1E(op, rcx)) \ - FOP_END - -/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */ -#define FASTOP1SRC2EX(op, name) \ - FOP_START(name) \ - FOP1EEX(op, cl) \ - FOP1EEX(op, cx) \ - FOP1EEX(op, ecx) \ - ON64(FOP1EEX(op, rcx)) \ - FOP_END - -#define FOP2E(op, dst, src) \ - __FOP_FUNC(#op "_" #dst "_" #src) \ - #op " %" #src ", %" #dst " \n\t" \ - __FOP_RET(#op "_" #dst "_" #src) - -#define FASTOP2(op) \ - FOP_START(op) \ - FOP2E(op##b, al, dl) \ - FOP2E(op##w, ax, dx) \ - FOP2E(op##l, eax, edx) \ - ON64(FOP2E(op##q, rax, rdx)) \ - FOP_END - -/* 2 operand, word only */ -#define FASTOP2W(op) \ - FOP_START(op) \ - FOPNOP() \ - FOP2E(op##w, ax, dx) \ - FOP2E(op##l, eax, edx) \ - ON64(FOP2E(op##q, rax, rdx)) \ - FOP_END - -/* 2 operand, src is CL */ -#define FASTOP2CL(op) \ - FOP_START(op) \ - FOP2E(op##b, al, cl) \ - FOP2E(op##w, ax, cl) \ - FOP2E(op##l, eax, cl) \ - ON64(FOP2E(op##q, rax, cl)) \ - FOP_END - -/* 2 operand, src and dest are reversed */ -#define FASTOP2R(op, name) \ - FOP_START(name) \ - FOP2E(op##b, dl, al) \ - FOP2E(op##w, dx, ax) \ - FOP2E(op##l, edx, eax) \ - ON64(FOP2E(op##q, rdx, rax)) \ - FOP_END - -#define FOP3E(op, dst, src, src2) \ - __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \ - #op " %" #src2 ", %" #src ", %" #dst " \n\t"\ - __FOP_RET(#op "_" #dst "_" #src "_" #src2) - -/* 3-operand, word-only, src2=3Dcl */ -#define FASTOP3WCL(op) \ - FOP_START(op) \ - FOPNOP() \ - FOP3E(op##w, ax, dx, cl) \ - FOP3E(op##l, eax, edx, cl) \ - ON64(FOP3E(op##q, rax, rdx, cl)) \ - FOP_END - static int em_salc(struct x86_emulate_ctxt *ctxt) { /* @@ -4052,7 +3904,6 @@ static int check_perm_out(struct x86_emu #define MD(_f, _m) { .flags =3D ((_f) | ModeDual), .u.mdual =3D (_m) } #define E(_f, _e) { .flags =3D ((_f) | Escape | ModRM), .u.esc =3D (_e) } #define I(_f, _e) { .flags =3D (_f), .u.execute =3D (_e) } -#define F(_f, _e) { .flags =3D (_f) | Fastop, .u.fastop =3D (_e) } #define II(_f, _e, _i) \ { .flags =3D (_f)|Intercept, .u.execute =3D (_e), .intercept =3D x86_inte= rcept_##_i } #define IIP(_f, _e, _i, _p) \ @@ -5158,24 +5009,6 @@ static void fetch_possible_mmx_operand(s kvm_read_mmx_reg(op->addr.mm, &op->mm_val); } =20 -static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop) -{ - ulong flags =3D (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; - - if (!(ctxt->d & ByteOp)) - fop +=3D __ffs(ctxt->dst.bytes) * FASTOP_SIZE; - - asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n" - : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags), - [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT - : "c"(ctxt->src2.val)); - - ctxt->eflags =3D (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); - if (!fop) /* exception is returned in fop variable */ - return emulate_de(ctxt); - return X86EMUL_CONTINUE; -} - void init_decode_cache(struct x86_emulate_ctxt *ctxt) { /* Clear fields that are set conditionally but read without a guard. */ @@ -5340,10 +5173,7 @@ int x86_emulate_insn(struct x86_emulate_ ctxt->eflags &=3D ~X86_EFLAGS_RF; =20 if (ctxt->execute) { - if (ctxt->d & Fastop) - rc =3D fastop(ctxt, ctxt->fop); - else - rc =3D ctxt->execute(ctxt); + rc =3D ctxt->execute(ctxt); if (rc !=3D X86EMUL_CONTINUE) goto done; goto writeback; From nobody Tue Oct 7 05:19:45 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id EC050246771; Mon, 14 Jul 2025 10:45:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489913; cv=none; b=ANSMXJDhUyDFkLF4onUJA+fdpg+dGq/WsNmL7+rtNz3IVAM1y4salk7SK2foqq5xj3xyS7W3qgzwmky0XM6VTA8Mz+z5mI9qgSkeF72WKdeQ46X++pbbN4LkRwgg53k0VEm1gPDB1gTrQUjB2j1KcYFZajrI5BiheFagIW2iZ2A= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489913; c=relaxed/simple; bh=4BcP4fXrFBMBRLHjzTo4hB9zOH0/vrDJl5LSlbB8vJA=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=KZt2iA8v8Gwhdl171vnCW0g0CpAyw25RXbi9nNmOlHp+mvw74yCYue/oX0Sn4BPo9NG+165+THPFk0JmRb04gCHvUxEHV0Tn4sXsXwOttEhmVGIASUN924uJDITR6Xvhf7M7WxxDedEe1SYP4QCf2rFy574xcVPtRjXedOTBT10= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=ZBddkl3h; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="ZBddkl3h" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=MeZEORZBfZVp+I+bBuCjS6/pJWWHjDhfB/pRSpagjvs=; b=ZBddkl3hDaDiOnTDk1oFxpLdsj SAbQD4KMDwIT7P98DltzebQ4aB2ygKBRrVJvkpYmIJRALJV4zAVqMluXyUpTu1GZJSOyk4CvlT3rb SEQXYniPOxt+M0I0B8bfFPzUPNMttMAFR4B2gRAbU6gar7rKpI0PzYZ7A4IDm61+P2tqzMWTW2iS9 keFVLDvJAKGlKUSqnlWoBF3RsNvLhwspUSKr0Y7ikn9nNa00cBZqtd4hiPscMp2PG6h4LWlG4aIOR ot96ip9LZlGPWyDudZSesA9lOmMaPhV064XJYmR1p4ZL4s72NJaVOGWxQNvG1UrKdew0zh0R9Gb5F IZAp4RXg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000006uLE-0wad; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 01B35302D93; Mon, 14 Jul 2025 12:44:51 +0200 (CEST) Message-ID: <20250714103440.897136093@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:22 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org, Michael Kelley Subject: [PATCH v3 11/16] x86,hyperv: Clean up hv_do_hypercall() References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" What used to be a simple few instructions has turned into a giant mess (for x86_64). Not only does it use static_branch wrong, it mixes it with dynamic branches for no apparent reason. Notably it uses static_branch through an out-of-line function call, which completely defeats the purpose, since instead of a simple JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is absolutely idiotic. Add to that a dynamic test of hyperv_paravisor_present, something which is set once and never changed. Replace all this idiocy with a single direct function call to the right hypercall variant. Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Reviewed-by: Michael Kelley Acked-by: Sean Christopherson Acked-by: Wei Liu --- arch/x86/hyperv/hv_init.c | 20 +++++ arch/x86/hyperv/ivm.c | 15 ++++ arch/x86/include/asm/mshyperv.h | 137 +++++++++++------------------------= ----- arch/x86/kernel/cpu/mshyperv.c | 19 +++-- 4 files changed, 89 insertions(+), 102 deletions(-) --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -36,7 +36,27 @@ #include =20 void *hv_hypercall_pg; + +#ifdef CONFIG_X86_64 +u64 hv_std_hypercall(u64 control, u64 param1, u64 param2) +{ + u64 hv_status; + + if (!hv_hypercall_pg) + return U64_MAX; + + register u64 __r8 asm("r8") =3D param2; + asm volatile (CALL_NOSPEC + : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, + "+c" (control), "+d" (param1), "+r" (__r8) + : THUNK_TARGET(hv_hypercall_pg) + : "cc", "memory", "r9", "r10", "r11"); + + return hv_status; +} +#else EXPORT_SYMBOL_GPL(hv_hypercall_pg); +#endif =20 union hv_ghcb * __percpu *hv_ghcb_pg; =20 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -377,9 +377,23 @@ int hv_snp_boot_ap(u32 cpu, unsigned lon return ret; } =20 +u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) +{ + u64 hv_status; + + register u64 __r8 asm("r8") =3D param2; + asm volatile("vmmcall" + : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, + "+c" (control), "+d" (param1), "+r" (__r8) + : : "cc", "memory", "r9", "r10", "r11"); + + return hv_status; +} + #else static inline void hv_ghcb_msr_write(u64 msr, u64 value) {} static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {} +u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX= ; } #endif /* CONFIG_AMD_MEM_ENCRYPT */ =20 #ifdef CONFIG_INTEL_TDX_GUEST @@ -429,6 +443,7 @@ u64 hv_tdx_hypercall(u64 control, u64 pa #else static inline void hv_tdx_msr_write(u64 msr, u64 value) {} static inline void hv_tdx_msr_read(u64 msr, u64 *value) {} +u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX= ; } #endif /* CONFIG_INTEL_TDX_GUEST */ =20 #if defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST) --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -39,16 +40,21 @@ static inline unsigned char hv_get_nmi_r return 0; } =20 -#if IS_ENABLED(CONFIG_HYPERV) -extern bool hyperv_paravisor_present; +extern u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2); +extern u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2); +extern u64 hv_std_hypercall(u64 control, u64 param1, u64 param2); =20 +#if IS_ENABLED(CONFIG_HYPERV) extern void *hv_hypercall_pg; =20 extern union hv_ghcb * __percpu *hv_ghcb_pg; =20 bool hv_isolation_type_snp(void); bool hv_isolation_type_tdx(void); -u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2); + +#ifdef CONFIG_X86_64 +DECLARE_STATIC_CALL(hv_hypercall, hv_std_hypercall); +#endif =20 /* * DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA @@ -65,37 +71,15 @@ static inline u64 hv_do_hypercall(u64 co { u64 input_address =3D input ? virt_to_phys(input) : 0; u64 output_address =3D output ? virt_to_phys(output) : 0; - u64 hv_status; =20 #ifdef CONFIG_X86_64 - if (hv_isolation_type_tdx() && !hyperv_paravisor_present) - return hv_tdx_hypercall(control, input_address, output_address); - - if (hv_isolation_type_snp() && !hyperv_paravisor_present) { - __asm__ __volatile__("mov %[output_address], %%r8\n" - "vmmcall" - : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input_address) - : [output_address] "r" (output_address) - : "cc", "memory", "r8", "r9", "r10", "r11"); - return hv_status; - } - - if (!hv_hypercall_pg) - return U64_MAX; - - __asm__ __volatile__("mov %[output_address], %%r8\n" - CALL_NOSPEC - : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input_address) - : [output_address] "r" (output_address), - THUNK_TARGET(hv_hypercall_pg) - : "cc", "memory", "r8", "r9", "r10", "r11"); + return static_call_mod(hv_hypercall)(control, input_address, output_addre= ss); #else u32 input_address_hi =3D upper_32_bits(input_address); u32 input_address_lo =3D lower_32_bits(input_address); u32 output_address_hi =3D upper_32_bits(output_address); u32 output_address_lo =3D lower_32_bits(output_address); + u64 hv_status; =20 if (!hv_hypercall_pg) return U64_MAX; @@ -108,8 +92,8 @@ static inline u64 hv_do_hypercall(u64 co "D"(output_address_hi), "S"(output_address_lo), THUNK_TARGET(hv_hypercall_pg) : "cc", "memory"); -#endif /* !x86_64 */ return hv_status; +#endif /* !x86_64 */ } =20 /* Hypercall to the L0 hypervisor */ @@ -121,41 +105,23 @@ static inline u64 hv_do_nested_hypercall /* Fast hypercall with 8 bytes of input and no output */ static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1) { - u64 hv_status; - #ifdef CONFIG_X86_64 - if (hv_isolation_type_tdx() && !hyperv_paravisor_present) - return hv_tdx_hypercall(control, input1, 0); - - if (hv_isolation_type_snp() && !hyperv_paravisor_present) { - __asm__ __volatile__( - "vmmcall" - : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input1) - :: "cc", "r8", "r9", "r10", "r11"); - } else { - __asm__ __volatile__(CALL_NOSPEC - : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input1) - : THUNK_TARGET(hv_hypercall_pg) - : "cc", "r8", "r9", "r10", "r11"); - } + return static_call_mod(hv_hypercall)(control, input1, 0); #else - { - u32 input1_hi =3D upper_32_bits(input1); - u32 input1_lo =3D lower_32_bits(input1); - - __asm__ __volatile__ (CALL_NOSPEC - : "=3DA"(hv_status), - "+c"(input1_lo), - ASM_CALL_CONSTRAINT - : "A" (control), - "b" (input1_hi), - THUNK_TARGET(hv_hypercall_pg) - : "cc", "edi", "esi"); - } -#endif + u32 input1_hi =3D upper_32_bits(input1); + u32 input1_lo =3D lower_32_bits(input1); + u64 hv_status; + + __asm__ __volatile__ (CALL_NOSPEC + : "=3DA"(hv_status), + "+c"(input1_lo), + ASM_CALL_CONSTRAINT + : "A" (control), + "b" (input1_hi), + THUNK_TARGET(hv_hypercall_pg) + : "cc", "edi", "esi"); return hv_status; +#endif } =20 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) @@ -175,45 +141,24 @@ static inline u64 hv_do_fast_nested_hype /* Fast hypercall with 16 bytes of input */ static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 inp= ut2) { - u64 hv_status; - #ifdef CONFIG_X86_64 - if (hv_isolation_type_tdx() && !hyperv_paravisor_present) - return hv_tdx_hypercall(control, input1, input2); - - if (hv_isolation_type_snp() && !hyperv_paravisor_present) { - __asm__ __volatile__("mov %[input2], %%r8\n" - "vmmcall" - : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input1) - : [input2] "r" (input2) - : "cc", "r8", "r9", "r10", "r11"); - } else { - __asm__ __volatile__("mov %[input2], %%r8\n" - CALL_NOSPEC - : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input1) - : [input2] "r" (input2), - THUNK_TARGET(hv_hypercall_pg) - : "cc", "r8", "r9", "r10", "r11"); - } + return static_call_mod(hv_hypercall)(control, input1, input2); #else - { - u32 input1_hi =3D upper_32_bits(input1); - u32 input1_lo =3D lower_32_bits(input1); - u32 input2_hi =3D upper_32_bits(input2); - u32 input2_lo =3D lower_32_bits(input2); - - __asm__ __volatile__ (CALL_NOSPEC - : "=3DA"(hv_status), - "+c"(input1_lo), ASM_CALL_CONSTRAINT - : "A" (control), "b" (input1_hi), - "D"(input2_hi), "S"(input2_lo), - THUNK_TARGET(hv_hypercall_pg) - : "cc"); - } -#endif + u32 input1_hi =3D upper_32_bits(input1); + u32 input1_lo =3D lower_32_bits(input1); + u32 input2_hi =3D upper_32_bits(input2); + u32 input2_lo =3D lower_32_bits(input2); + u64 hv_status; + + __asm__ __volatile__ (CALL_NOSPEC + : "=3DA"(hv_status), + "+c"(input1_lo), ASM_CALL_CONSTRAINT + : "A" (control), "b" (input1_hi), + "D"(input2_hi), "S"(input2_lo), + THUNK_TARGET(hv_hypercall_pg) + : "cc"); return hv_status; +#endif } =20 static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2) --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -38,10 +38,6 @@ bool hv_nested; struct ms_hyperv_info ms_hyperv; =20 -/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyper= v.h */ -bool hyperv_paravisor_present __ro_after_init; -EXPORT_SYMBOL_GPL(hyperv_paravisor_present); - #if IS_ENABLED(CONFIG_HYPERV) static inline unsigned int hv_get_nested_msr(unsigned int reg) { @@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc old_restore_sched_clock_state =3D x86_platform.restore_sched_clock_state; x86_platform.restore_sched_clock_state =3D hv_restore_sched_clock_state; } + +#ifdef CONFIG_X86_64 +DEFINE_STATIC_CALL(hv_hypercall, hv_std_hypercall); +EXPORT_STATIC_CALL_TRAMP_GPL(hv_hypercall); +#define hypercall_update(hc) static_call_update(hv_hypercall, hc) +#endif #endif /* CONFIG_HYPERV */ =20 +#ifndef hypercall_update +#define hypercall_update(hc) (void)hc +#endif + static uint32_t __init ms_hyperv_platform(void) { u32 eax; @@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platfo ms_hyperv.shared_gpa_boundary =3D BIT_ULL(ms_hyperv.shared_gpa_boundary_bits); =20 - hyperv_paravisor_present =3D !!ms_hyperv.paravisor_present; - pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n", ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b); =20 =20 if (hv_get_isolation_type() =3D=3D HV_ISOLATION_TYPE_SNP) { static_branch_enable(&isolation_type_snp); + if (!ms_hyperv.paravisor_present) + hypercall_update(hv_snp_hypercall); } else if (hv_get_isolation_type() =3D=3D HV_ISOLATION_TYPE_TDX) { static_branch_enable(&isolation_type_tdx); =20 @@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platfo ms_hyperv.hints &=3D ~HV_X64_APIC_ACCESS_RECOMMENDED; =20 if (!ms_hyperv.paravisor_present) { + hypercall_update(hv_tdx_hypercall); /* * Mark the Hyper-V TSC page feature as disabled * in a TDX VM without paravisor so that the From nobody Tue Oct 7 05:19:45 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CFCB422836C; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; cv=none; b=TzDWn6VLnBX/4CLkvZWsCqgXgNC5u0ep1X1OlSTB+hwxZfP+X/DMSJ6FxBoURscFM+JokjWxesn1LNN5NnhyvkWlFECkwtrUhIEN4BCmqbuYGrt9HIlcSctAD+koTlOgAlR6bKUaTmuITnTHGeYUFxIURyfcJJKR3IYbyRDyx8M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; c=relaxed/simple; bh=QX0FuRDOAhBaSmYLOCWMM+5snfSl3DY0vGzX5svD2Qs=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=EWSBJLcK3JzSRHN08qncQQdsFYiFSyBeYEPOSoKgFwjG3bjULi9vcECkn9iw1sVVQfBGQpNcrb2j/qbH41YVX1HTlRrLfKMU0Zad5KnNOlj0So/1knJnIDuaNqqFQ76eK+57Vq2uCl2YPVh0SAJXNZPuJSlVJ4BqeFEC3hSHVZE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=r4a5wYdB; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="r4a5wYdB" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=xyqfgYFIZicYr7jarOLRNC4OG3o0XqiWBsV4JdioE+8=; b=r4a5wYdBxnZB8a/vFS4cDKxcTO 7oVK8XPBkbNVudpn22oc7xHDzTgfWEvqn/D+fkqGJY1X/Lb6DOjOuFDgvBBfrOkunXWat1NK2+sFA nULNlfr7b01z22k/KtThyq/iEkQR4p8SUyRAyjbP0TKFXNAcef0CzT8jeNdP3bYEUpzeyFRgeQRlD 6mYrf59tJewGMJg5T86VkMt588DxtXJPhGPXeG/5rbcWUdfmuo/eKuKmEyIiDN4cUrMRlJPmJgeFz Es/6/TJOMlbg98UvRMvAX+NK/rIe/BQGVzV1RoPTVin0OZvstDoar4lprgdKThavzQGA26uLKVCLs rdmdiMDw==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000006uLF-10Ax; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 0C74B300186; Mon, 14 Jul 2025 12:44:51 +0200 (CEST) Message-ID: <20250714103441.011387946@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:23 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org, Michael Kelley Subject: [PATCH v3 12/16] x86_64,hyperv: Use direct call to hypercall-page References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of using an indirect call to the hypercall page, use a direct call instead. This avoids all CFI problems, including the one where the hypercall page doesn't have IBT on. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Michael Kelley Acked-by: Sean Christopherson Acked-by: Wei Liu --- arch/x86/hyperv/hv_init.c | 61 ++++++++++++++++++++++-------------------= ----- 1 file changed, 30 insertions(+), 31 deletions(-) --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -38,23 +37,41 @@ void *hv_hypercall_pg; =20 #ifdef CONFIG_X86_64 +static u64 __hv_hyperfail(u64 control, u64 param1, u64 param2) +{ + return U64_MAX; +} + +DEFINE_STATIC_CALL(__hv_hypercall, __hv_hyperfail); + u64 hv_std_hypercall(u64 control, u64 param1, u64 param2) { u64 hv_status; =20 - if (!hv_hypercall_pg) - return U64_MAX; - register u64 __r8 asm("r8") =3D param2; - asm volatile (CALL_NOSPEC + asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall) : "=3Da" (hv_status), ASM_CALL_CONSTRAINT, "+c" (control), "+d" (param1), "+r" (__r8) - : THUNK_TARGET(hv_hypercall_pg) - : "cc", "memory", "r9", "r10", "r11"); + : : "cc", "memory", "r9", "r10", "r11"); =20 return hv_status; } + +typedef u64 (*hv_hypercall_f)(u64 control, u64 param1, u64 param2); + +static inline void hv_set_hypercall_pg(void *ptr) +{ + hv_hypercall_pg =3D ptr; + + if (!ptr) + ptr =3D &__hv_hyperfail; + static_call_update(__hv_hypercall, (hv_hypercall_f)ptr); +} #else +static inline void hv_set_hypercall_pg(void *ptr) +{ + hv_hypercall_pg =3D ptr; +} EXPORT_SYMBOL_GPL(hv_hypercall_pg); #endif =20 @@ -349,7 +366,7 @@ static int hv_suspend(void) * pointer is restored on resume. */ hv_hypercall_pg_saved =3D hv_hypercall_pg; - hv_hypercall_pg =3D NULL; + hv_set_hypercall_pg(NULL); =20 /* Disable the hypercall page in the hypervisor */ rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); @@ -375,7 +392,7 @@ static void hv_resume(void) vmalloc_to_pfn(hv_hypercall_pg_saved); wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); =20 - hv_hypercall_pg =3D hv_hypercall_pg_saved; + hv_set_hypercall_pg(hv_hypercall_pg_saved); hv_hypercall_pg_saved =3D NULL; =20 /* @@ -529,8 +546,8 @@ void __init hyperv_init(void) if (hv_isolation_type_tdx() && !ms_hyperv.paravisor_present) goto skip_hypercall_pg_init; =20 - hv_hypercall_pg =3D __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, - VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX, + hv_hypercall_pg =3D __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR, + MODULES_END, GFP_KERNEL, PAGE_KERNEL_ROX, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, __builtin_return_address(0)); if (hv_hypercall_pg =3D=3D NULL) @@ -568,27 +585,9 @@ void __init hyperv_init(void) wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); } =20 -skip_hypercall_pg_init: - /* - * Some versions of Hyper-V that provide IBT in guest VMs have a bug - * in that there's no ENDBR64 instruction at the entry to the - * hypercall page. Because hypercalls are invoked via an indirect call - * to the hypercall page, all hypercall attempts fail when IBT is - * enabled, and Linux panics. For such buggy versions, disable IBT. - * - * Fixed versions of Hyper-V always provide ENDBR64 on the hypercall - * page, so if future Linux kernel versions enable IBT for 32-bit - * builds, additional hypercall page hackery will be required here - * to provide an ENDBR32. - */ -#ifdef CONFIG_X86_KERNEL_IBT - if (cpu_feature_enabled(X86_FEATURE_IBT) && - *(u32 *)hv_hypercall_pg !=3D gen_endbr()) { - setup_clear_cpu_cap(X86_FEATURE_IBT); - pr_warn("Disabling IBT because of Hyper-V bug\n"); - } -#endif + hv_set_hypercall_pg(hv_hypercall_pg); =20 +skip_hypercall_pg_init: /* * hyperv_init() is called before LAPIC is initialized: see * apic_intr_mode_init() -> x86_platform.apic_post_init() and From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7978C1DF26A; Mon, 14 Jul 2025 10:45:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489911; cv=none; b=d56ldEGJbWbcq5+d0WVVD+3SbOLIysyaTtS355g99WL1jB818a0bLitg80Kpj+gNRdsTsZMJMwlpcx+6Z5VDregKshJHMcUnTKGl+ILU7cY3eWL6aPnEMpUFwhOub5iM7+aZyjwRVm5agegsB80M5mXrMKBsNiFMRIlki+/9oF4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489911; c=relaxed/simple; bh=bkHFi5EFOP73EpSepmQZ4Q9VkRfFvaXax+LB6lsSbCM=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=k1irD7ozCp3A6u2M6+IPE0v24hVjkvK2SyhBMzG9/OdI0m2ZlvTNoY74XvFpD+pwZiU+E1DbGy6Vtfq/nbul/6qqLDIzvfBqktMymO5Df3KfG+r0VVbYzYg70BabLtcUFTPWS2KMSOCjNCFkKrtcAiOPoHruw2FqWiFqRlLNzpE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=dZXGedzz; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="dZXGedzz" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=WIJjzE7IhtYykmJnVpMemXxfCZI1nRxY9Sjvw5cW+qs=; b=dZXGedzzBh/fghEtWOpO+lNJMl D8A2lujEejAlY0c9KFQpHgzgdUFlv7SFXkvIIy5g0Hvh9+HtIyNQ9shvhQVhk81+uE6faieRn9lPI H2ni3JWuNGXeePTfCXwkrlljP8xxMezV6McXMd51WhRUgGDIWWwxGMo/LCg7PxEMRcm9C6+Lk6K6L RQ7LEOaJbdseIHRixZbf2ke3w87fvWl74LmdphVdMd788Kxti7kM3MSAL6dQs0veAXnHt8KoTfBXe zKIlnWAO40HaAdLEd1ac/srXtkBEBal/Hr2e7Yo7/lsem3RsOBXAtO185jFhcUn+bD+cS7QMs6SKE txJBVI9w==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000009kcp-0z61; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 2101F302D9C; Mon, 14 Jul 2025 12:44:51 +0200 (CEST) Message-ID: <20250714103441.121251108@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:24 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 13/16] x86/fred: Install system vector handlers even if FRED isnt fully enabled References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Sean Christopherson Install the system vector IRQ handlers for FRED even if FRED isn't fully enabled in hardware. This will allow KVM to use the FRED IRQ path even on non-FRED hardware, which in turn will eliminate a non-CFI indirect CALL (KVM currently invokes the IRQ handler via an IDT lookup on the vector). [sean: extract from diff, drop stub, write changelog] Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/include/asm/idtentry.h | 9 ++------- arch/x86/kernel/irqinit.c | 6 ++++-- 2 files changed, 6 insertions(+), 9 deletions(-) --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -460,17 +460,12 @@ __visible noinstr void func(struct pt_re #endif =20 void idt_install_sysvec(unsigned int n, const void *function); - -#ifdef CONFIG_X86_FRED void fred_install_sysvec(unsigned int vector, const idtentry_t function); -#else -static inline void fred_install_sysvec(unsigned int vector, const idtentry= _t function) { } -#endif =20 #define sysvec_install(vector, function) { \ - if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + if (IS_ENABLED(CONFIG_X86_FRED)) \ fred_install_sysvec(vector, function); \ - else \ + if (!cpu_feature_enabled(X86_FEATURE_FRED)) \ idt_install_sysvec(vector, asm_##function); \ } =20 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -97,9 +97,11 @@ void __init native_init_IRQ(void) /* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init(); =20 - if (cpu_feature_enabled(X86_FEATURE_FRED)) + /* FRED's IRQ path may be used even if FRED isn't fully enabled. */ + if (IS_ENABLED(CONFIG_X86_FRED)) fred_complete_exception_setup(); - else + + if (!cpu_feature_enabled(X86_FEATURE_FRED)) idt_setup_apic_and_irq_gates(); =20 lapic_assign_system_vectors(); From nobody Tue Oct 7 05:19:45 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3BE401A38F9; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; cv=none; b=ZTLg0W+TlHdiFHA37bcwFBq/59g8l5Hao4HUziuVYbJpkozb0DcBJMRYCJrNJkV7a6qSixy+3NsRU1VUMjKd0+McvJHBqFBE4/yOI5J0eZRuERuOwixT33s3smMZGPzLgi02Ov+Es21vVnAhYYtrVxAnkyqSbXMcERe17PffPhM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; c=relaxed/simple; bh=HcFn8Q0xwnm6suBA2N/nvS0KE9mbW5Qxd19hx1mLQMw=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=GHFebNK5FE4tgBr+MKcCqYhdc0SGZ4z65wVGbICOpBaYRjwgDFWWtPyWxpCss8gu2x7LkOnQe6v4ex/d/5hGISIBGSGThB3vFxlrG4j+S9AVWYzQMru91pKZS272H+9NS9SXt+mU6WZezrAKKvGFTMeYXlW+3K4r/Hlgn8pJo2Q= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=B1MsDznr; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="B1MsDznr" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=52GVqrYarxAbay9vWyxzRq6hCVQnEhJYo4bGbdbqIGI=; b=B1MsDznrPqmZQe8ozWqn7fJJUQ /wHbnnqTlr5zDinO+VkTdreppaIHQeD4c8YG7TA0RVJO2dfTWdfDOKzb7qulXrsb4+DcP6KUspmRc CEG57COZEkQl8+yQLGHBGGOSBPV5kSPWPA49wlw5RzwtEFLHuBSYOpdR1gUYmK+W2fzVqDX8AKpQf FGkQpnbVgqPpDj8pQR1lYPtr4/+KjArZFU/ii5Y28hKZAbgIcukMnjjywYKGykLudfekPLz4+IZkl pu/QGi8JUS2zmNjeJlqEgRC6Iqd2RG3L4JfbcC9Z429ogyOYkDMv5Aza1XnOEqwcV8HTTiYtD4eM0 +YYXl+Cg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000006uLG-10BX; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 28779302E03; Mon, 14 Jul 2025 12:44:51 +0200 (CEST) Message-ID: <20250714103441.245417052@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:25 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 14/16] x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Josh Poimboeuf Modify asm_fred_entry_from_kvm() to allow it to be invoked by KVM even when FRED isn't fully enabled, e.g. when running with CONFIG_X86_FRED=3Dy on non-FRED hardware. This will allow forcing KVM to always use the FRED entry points for 64-bit kernels, which in turn will eliminate a rather gross non-CFI indirect call that KVM uses to trampoline IRQs by doing IDT lookups. The point of asm_fred_entry_from_kvm() is to bridge between C (vmx:handle_external_interrupt_irqoff()) and more C (__fred_entry_from_kvm()) while changing the calling context to appear like an interrupt (pt_regs). Making the whole thing bound by C ABI. All that remains for non-FRED hardware is to restore RSP (to undo the redzone and alignment). However the trivial change would result in code like: push %rbp mov %rsp, %rbp sub $REDZONE, %rsp and $MASK, %rsp PUSH_AND_CLEAR_REGS push %rbp POP_REGS pop %rbp <-- *objtool fail* mov %rbp, %rsp pop %rbp ret And this will confuse objtool something wicked -- it gets confused by the extra pop %rbp, not realizing the push and pop preserve the value. Rather than trying to each objtool about this, recognise that since the code is bound by C ABI on both ends and interrupts are not allowed to change pt_regs (only exceptions are) it is sufficient to PUSH_REGS in order to create pt_regs, but there is no reason to POP_REGS -- provided the callee-saved registers are preserved. So avoid clearing callee-saved regs and skip POP_REGS. [Original patch by Sean; much of this version by Josh; Changelog, comments and final form by Peterz] Originally-by: Sean Christopherson Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/entry/calling.h | 11 +++++------ arch/x86/entry/entry_64_fred.S | 33 ++++++++++++++++++++++++++------- arch/x86/kernel/asm-offsets.c | 1 + 3 files changed, 32 insertions(+), 13 deletions(-) --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -99,7 +99,7 @@ For 32-bit we have the following convent .endif .endm =20 -.macro CLEAR_REGS clear_bp=3D1 +.macro CLEAR_REGS clear_callee=3D1 /* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered @@ -113,20 +113,19 @@ For 32-bit we have the following convent xorl %r9d, %r9d /* nospec r9 */ xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ + .if \clear_callee xorl %ebx, %ebx /* nospec rbx */ - .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ - .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */ xorl %r15d, %r15d /* nospec r15 */ - + .endif .endm =20 -.macro PUSH_AND_CLEAR_REGS rdx=3D%rdx rcx=3D%rcx rax=3D%rax save_ret=3D0 c= lear_bp=3D1 unwind_hint=3D1 +.macro PUSH_AND_CLEAR_REGS rdx=3D%rdx rcx=3D%rcx rax=3D%rax save_ret=3D0 c= lear_callee=3D1 unwind_hint=3D1 PUSH_REGS rdx=3D\rdx, rcx=3D\rcx, rax=3D\rax, save_ret=3D\save_ret unwind= _hint=3D\unwind_hint - CLEAR_REGS clear_bp=3D\clear_bp + CLEAR_REGS clear_callee=3D\clear_callee .endm =20 .macro POP_REGS pop_rdi=3D1 --- a/arch/x86/entry/entry_64_fred.S +++ b/arch/x86/entry/entry_64_fred.S @@ -112,18 +112,37 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) push %rax /* Return RIP */ push $0 /* Error code, 0 for IRQ/NMI */ =20 - PUSH_AND_CLEAR_REGS clear_bp=3D0 unwind_hint=3D0 + PUSH_AND_CLEAR_REGS clear_callee=3D0 unwind_hint=3D0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + /* + * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx} + * are clobbered, which corresponds to: arguments, extra caller-saved + * and return. All registers a C function is allowed to clobber. + * + * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15} + * are untouched, with the exception of rbp, which carries the stack + * frame and will be restored before exit. + * + * Further calling another C function will not alter this state. + */ call __fred_entry_from_kvm /* Call the C entry point */ - POP_REGS - ERETS -1: + +1: /* + * When FRED, use ERETS to potentially clear NMIs, otherwise simply + * restore the stack pointer. + */ + ALTERNATIVE "nop; nop; mov %rbp, %rsp", \ + __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ + X86_FEATURE_FRED + /* - * Objtool doesn't understand what ERETS does, this hint tells it that - * yes, we'll reach here and with what stack state. A save/restore pair - * isn't strictly needed, but it's the simplest form. + * Objtool doesn't understand ERETS, and the cfi register state is + * different from initial_func_cfi due to PUSH_REGS. Tell it the state + * is similar to where UNWIND_HINT_SAVE is. */ UNWIND_HINT_RESTORE + pop %rbp RET =20 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -102,6 +102,7 @@ static void __used common(void) =20 BLANK(); DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); + OFFSET(C_PTREGS_SIZE, pt_regs, orig_ax); =20 /* TLB state for the entry code */ OFFSET(TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask); From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B813E1FDA94; Mon, 14 Jul 2025 10:45:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489907; cv=none; b=ekIyuIJK+CpSxuVJQL50Q6IiMcG9B5MgDObz0lRbCsgMY7Ejgwe2ljIgpVJop+jV0mMC24h80pJ8buNMGFbld6uvHAT8rMFWP2/NHK3D7YaAA/Z1FQ1Ewo3CnU7Jy/C5Np5+Rv9lO646Q1jdhznI9Mj64iyFHkPx2KwLA99sxHI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489907; c=relaxed/simple; bh=laH8zvY2NdqgddV1uxCUdzQgi9ZK8xvb1p3SXlHj50A=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=ii7Q7Q8aXUIOClMhppq/bJUX2hE45XM86TkXt8UXxteSdnlvwnF7+ur81HsH8aByvrPrhetl/b1+LPLmdTJzGIFIiPGJoQLxb4EDWZ/h5PX1AbpcjBpTVE9ubuxPjmNwNvxANw8qI/3kBlpWv26BesblrdRnPUbggbVMKhBU/Qk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=I3ByvgNl; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="I3ByvgNl" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=ewom6iS5EFylqeechOg2mSSAJjmRRMGPDBC4yQb3ICA=; b=I3ByvgNlQB8b9X1OcxuTh/ldip iqAFgQ1nHpABs/r9QProbboIHdgRWlAmk7uEyA6ZaV465SWovxk+vJhs/+6ZykwDm3C3WL4rEdEdE rXiR7exX0XoRUqL5WyRPS58q64mC37SssiuJ5gqYSur2ISIMDzo2hlYDVjc+pikxRpDJSUf3GX/Cs KtdRB5uhH10lNtDc2Y3puNdkh1DI/9jPwIrwn10GevDi2WhMAjMQxzZlecpsvNNW/kIZ7kM2OPV39 G5UXw1uSS3s22EMpcPbe5L09ryZkBH5Jqhi6IB3i2vmqtgF1CeKIeah/apYAAue5aCII6wJJuS92i UBl+PKCw==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000009kcq-15Ba; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 2D485302DA9; Mon, 14 Jul 2025 12:44:51 +0200 (CEST) Message-ID: <20250714103441.381946911@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:26 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 15/16] x86/fred: KVM: VMX: Always use FRED for IRQs when CONFIG_X86_FRED=y References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Sean Christopherson Now that FRED provides C-code entry points for handling IRQs, use the FRED infrastructure for forwarding IRQs even if FRED is fully disabled, e.g. isn't supported in hardware. Avoiding the non-FRED assembly trampolines into the IDT handlers for IRQs eliminates the associated non-CFI indirect call (KVM performs a CALL by doing a lookup on the IDT using the IRQ vector). Keep NMIs on the legacy IDT path, as the FRED NMI entry code relies on FRED's architectural behavior with respect to NMI blocking, i.e. doesn't jump through the myriad hoops needed to deal with IRET "unexpectedly" unmasking NMIs. KVM's NMI path already makes a direct CALL to C-code, i.e. isn't problematic for CFI. KVM does make a short detour through assembly code to build the stack frame, but the "FRED entry from KVM" path does the same. Force FRED for 64-bit kernels if KVM_INTEL is enabled, as the benefits of eliminating the IRQ trampoline usage far outwieghts the code overhead for FRED. Suggested-by: Peter Zijlstra Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Acked-by: Sean Christopherson --- arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/vmx/vmx.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -97,6 +97,7 @@ config KVM_INTEL depends on KVM && IA32_FEAT_CTL select KVM_GENERIC_PRIVATE_MEM if INTEL_TDX_HOST select KVM_GENERIC_MEMORY_ATTRIBUTES if INTEL_TDX_HOST + select X86_FRED if X86_64 help Provides support for KVM on processors equipped with Intel's VT extensions, a.k.a. Virtual Machine Extensions (VMX). --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6989,8 +6989,14 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; =20 + /* + * Invoke the kernel's IRQ handler for the vector. Use the FRED path + * when it's available even if FRED isn't fully enabled, e.g. even if + * FRED isn't supported in hardware, in order to avoid the indirect + * CALL in the non-FRED path. + */ kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - if (cpu_feature_enabled(X86_FEATURE_FRED)) + if (IS_ENABLED(CONFIG_X86_FRED)) fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector); else vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector)= ); From nobody Tue Oct 7 05:19:45 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3DDE5239570; Mon, 14 Jul 2025 10:45:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; cv=none; b=mlUlLOFesZbuZZV3f3vpCQ+2lFkuoUNavnKYS2nAD6LmiC1lEamfMOjIZrRj1CB3IntScnK8XCHzvfczzqQaw6afVpFety+DJGzQDnomw5Iq8iKDdz6k+O+YeJNiBeNsH3AzvM+cGgBiHmhbbwMS6mWeVE00+TM35fL+76RyG+I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752489909; c=relaxed/simple; bh=aGkfmjELao9k1FgDXHVaajQR1EjDMMt8/GCiX0e70PI=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=kyYYSIvHnXp8ChZ9UT2qIW0MZpNrCm5w1pOdKlT5+54c88AFPro3/loyrvsjil/q2OVMTwNjrzSRv+PeGy3v2nvXv/dTInC9plM3VNuUG1h9ATcqpuyXCjg3gfeCxdhYWbpswwZuKvrJA+LHvCyW+n0SGIRaiJnVGrqd3OvqmOs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=GTOfVI5d; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="GTOfVI5d" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=upWx64KkvtXvdkAHnKQ/kaQ2aUrWbLJGP5mR8fF7cdE=; b=GTOfVI5d/iVG/JrjZfks4uWkvv RZZ0SSg67o9t+w5RK0X08J58TaplwoKmdpklKLcbOYjhKxYDU/MRQbXC/CIvPYDwcNo7wAAoLa8gM KNtaLuU17i9pCrLyK6Vic9Qw92H9fMv8saUVsDtnpWQxuBPv4vyBTb8D0vSIJXRb8XBoStGx+sq2t QBYSo+e7Qaumoo+YAGYWsFyRLo19Y/I6UZP8Dh4ai1U8f+vFYXMqYNIrhlpooPth7fdV4+4y1YFoX oXyDWSFMClBQlEiOvve0FjqzQDltQG6xfoaTKgHS0I/OsP3BF21yIMNlKFxLvVpEl4+3ZNp70qqHO IvsTIcww==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98.2 #2 (Red Hat Linux)) id 1ubGg1-00000009kcr-15I1; Mon, 14 Jul 2025 10:44:53 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 330EA302E2E; Mon, 14 Jul 2025 12:44:51 +0200 (CEST) Message-ID: <20250714103441.496787279@infradead.org> User-Agent: quilt/0.68 Date: Mon, 14 Jul 2025 12:20:27 +0200 From: Peter Zijlstra To: x86@kernel.org Cc: kys@microsoft.com, haiyangz@microsoft.com, wei.liu@kernel.org, decui@microsoft.com, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, hpa@zytor.com, seanjc@google.com, pbonzini@redhat.com, ardb@kernel.org, kees@kernel.org, Arnd Bergmann , gregkh@linuxfoundation.org, jpoimboe@kernel.org, peterz@infradead.org, linux-hyperv@vger.kernel.org, linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-efi@vger.kernel.org, samitolvanen@google.com, ojeda@kernel.org Subject: [PATCH v3 16/16] objtool: Validate kCFI calls References: <20250714102011.758008629@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Validate that all indirect calls adhere to kCFI rules. Notably doing nocfi indirect call to a cfi function is broken. Apparently some Rust 'core' code violates this and explodes when ran with FineIBT. All the ANNOTATE_NOCFI_SYM sites are prime targets for attackers. - runtime EFI is especially henous because it also needs to disable IBT. Basically calling unknown code without CFI protection at runtime is a massice security issue. - Kexec image handover; if you can exploit this, you get to keep it :-) Signed-off-by: Peter Zijlstra (Intel) Acked-by: Josh Poimboeuf Acked-by: Sean Christopherson --- arch/x86/kernel/machine_kexec_64.c | 4 +++ arch/x86/kvm/vmx/vmenter.S | 4 +++ arch/x86/platform/efi/efi_stub_64.S | 4 +++ drivers/misc/lkdtm/perms.c | 5 ++++ include/linux/objtool.h | 10 ++++++++ include/linux/objtool_types.h | 1=20 tools/include/linux/objtool_types.h | 1=20 tools/objtool/check.c | 42 +++++++++++++++++++++++++++++++= +++++ tools/objtool/include/objtool/elf.h | 1=20 9 files changed, 72 insertions(+) --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c @@ -453,6 +453,10 @@ void __nocfi machine_kexec(struct kimage =20 __ftrace_enabled_restore(save_ftrace_enabled); } +/* + * Handover to the next kernel, no CFI concern. + */ +ANNOTATE_NOCFI_SYM(machine_kexec); =20 /* arch-dependent functionality related to kexec file-based syscall */ =20 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) =20 .section .text, "ax" =20 +#ifndef CONFIG_X86_FRED + SYM_FUNC_START(vmx_do_interrupt_irqoff) VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(vmx_do_interrupt_irqoff) + +#endif --- a/arch/x86/platform/efi/efi_stub_64.S +++ b/arch/x86/platform/efi/efi_stub_64.S @@ -11,6 +11,10 @@ #include =20 SYM_FUNC_START(__efi_call) + /* + * The EFI code doesn't have any CFI, annotate away the CFI violation. + */ + ANNOTATE_NOCFI_SYM pushq %rbp movq %rsp, %rbp and $~0xf, %rsp --- a/drivers/misc/lkdtm/perms.c +++ b/drivers/misc/lkdtm/perms.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include =20 @@ -86,6 +87,10 @@ static noinline __nocfi void execute_loc func(); pr_err("FAIL: func returned\n"); } +/* + * Explicitly doing the wrong thing for testing. + */ +ANNOTATE_NOCFI_SYM(execute_location); =20 static void execute_user_location(void *dst) { --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -184,6 +184,15 @@ * WARN using UD2. */ #define ANNOTATE_REACHABLE(label) __ASM_ANNOTATE(label, ANNOTYPE_REACHABLE) +/* + * This should not be used; it annotates away CFI violations. There are a = few + * valid use cases like kexec handover to the next kernel image, and there= is + * no security concern there. + * + * There are also a few real issues annotated away, like EFI because we ca= n't + * control the EFI code. + */ +#define ANNOTATE_NOCFI_SYM(sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOCFI)) =20 #else #define ANNOTATE_NOENDBR ANNOTATE type=3DANNOTYPE_NOENDBR @@ -194,6 +203,7 @@ #define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=3DANNOTYPE_INTRA_FUNCTI= ON_CALL #define ANNOTATE_UNRET_BEGIN ANNOTATE type=3DANNOTYPE_UNRET_BEGIN #define ANNOTATE_REACHABLE ANNOTATE type=3DANNOTYPE_REACHABLE +#define ANNOTATE_NOCFI_SYM ANNOTATE type=3DANNOTYPE_NOCFI #endif =20 #if defined(CONFIG_NOINSTR_VALIDATION) && \ --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -65,5 +65,6 @@ struct unwind_hint { #define ANNOTYPE_IGNORE_ALTS 6 #define ANNOTYPE_INTRA_FUNCTION_CALL 7 #define ANNOTYPE_REACHABLE 8 +#define ANNOTYPE_NOCFI 9 =20 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -65,5 +65,6 @@ struct unwind_hint { #define ANNOTYPE_IGNORE_ALTS 6 #define ANNOTYPE_INTRA_FUNCTION_CALL 7 #define ANNOTYPE_REACHABLE 8 +#define ANNOTYPE_NOCFI 9 =20 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2390,6 +2390,8 @@ static int __annotate_ifc(struct objtool =20 static int __annotate_late(struct objtool_file *file, int type, struct ins= truction *insn) { + struct symbol *sym; + switch (type) { case ANNOTYPE_NOENDBR: /* early */ @@ -2431,6 +2433,15 @@ static int __annotate_late(struct objtoo insn->dead_end =3D false; break; =20 + case ANNOTYPE_NOCFI: + sym =3D insn->sym; + if (!sym) { + ERROR_INSN(insn, "dodgy NOCFI annotation"); + return -1; + } + insn->sym->nocfi =3D 1; + break; + default: ERROR_INSN(insn, "Unknown annotation type: %d", type); return -1; @@ -4000,6 +4011,37 @@ static int validate_retpoline(struct obj warnings++; } =20 + if (!opts.cfi) + return warnings; + + /* + * kCFI call sites look like: + * + * movl $(-0x12345678), %r10d + * addl -4(%r11), %r10d + * jz 1f + * ud2 + * 1: cs call __x86_indirect_thunk_r11 + * + * Verify all indirect calls are kCFI adorned by checking for the + * UD2. Notably, doing __nocfi calls to regular (cfi) functions is + * broken. + */ + list_for_each_entry(insn, &file->retpoline_call_list, call_node) { + struct symbol *sym =3D insn->sym; + + if (sym && (sym->type =3D=3D STT_NOTYPE || + sym->type =3D=3D STT_FUNC) && !sym->nocfi) { + struct instruction *prev =3D + prev_insn_same_sym(file, insn); + + if (!prev || prev->type !=3D INSN_BUG) { + WARN_INSN(insn, "no-cfi indirect call!"); + warnings++; + } + } + } + return warnings; } =20 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -70,6 +70,7 @@ struct symbol { u8 local_label : 1; u8 frame_pointer : 1; u8 ignore : 1; + u8 nocfi : 1; struct list_head pv_target; struct reloc *relocs; struct section *group_sec;