From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B06821FE443 for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; cv=none; b=Jxy5aQg2D0JSsyYU0STv5lwFHQ4QXGtNkA771bWyyn2nJi7g97remAng0aknEMaTty1ljfet85x5qy0Dg8MQMdD1GYDNj7Z/l80XM6pBTCyO9576T2QzregIln1Q+A4HQzvEB51Hlo1N/O5bNGgm3X7Blu1lCKpr0iQvAU3jfzg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; c=relaxed/simple; bh=+DJ3eXlkCzlIieljPXX3Feo8Y2owvrTAtdRKuEyVaLQ=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=OP7yUrcSS8C/TFb3lEkPhi8VvGN7fu2XMg+xpq4rc1yrVXC1t6xie60wpPqJ4ZT0m3amjOIsHvikJJzNuxtphralAdCtxl81jSpAhJZZozfe1wqJOYoTFwEOjBx+D2EGXtPNmlH3DCRkZmVxPuB7bpzgNPpDs0LJ6gRQzmVDCXQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=Ze0zzoSJ; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="Ze0zzoSJ" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=3l3jBdGgQLX2gy+RZgHHu7XYzFFZN7WRc+HHnGo9En8=; b=Ze0zzoSJwCQiu2oBd76bpyVv8/ pVR27Fjg3gMDWyb4tVk4bC9wT02Y6dkXxXkWXB3OUZG6TpiCXn2hMICyHo70NMoJaV0DYasiZp55T t2iePANm7uwBr2yMlROge3fxWBYA8jpvdnhMBjQQ5h8KFVIzf7JnJkgcr9T/7j+zEI6BsTdP+MO3p WLbZ6vjSjsonKDPs/YUzaZ0obqnlXC/PQ7b8adT5/pYZ9ZdammB50dOo3hSRGmEVR3xL4uQ4a+rOf Bd1yzmf47S+NnjExgESYmwmQacU2h8vSCQ5a2dgYZa8pVUB0/Nh4lbGBJs8YHeWm30wz1oPNwQEo/ rnhdC3VA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn40-00000002HAH-0JNu; Wed, 19 Feb 2025 16:36:44 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id EA670300787; Wed, 19 Feb 2025 17:36:42 +0100 (CET) Message-ID: <20250219163514.364049576@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:08 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 01/10] x86/cfi: Add warn option References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Rebuilding with CFI_PERMISSIVE toggled is such a pain, esp. since clang is so slow. Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/include/asm/cfi.h | 1 + arch/x86/kernel/alternative.c | 4 ++++ arch/x86/kernel/cfi.c | 14 ++++++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -100,6 +100,7 @@ enum cfi_mode { }; =20 extern enum cfi_mode cfi_mode; +extern bool cfi_warn; =20 struct pt_regs; =20 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -916,6 +916,7 @@ void __init_or_module apply_seal_endbr(s #endif =20 enum cfi_mode cfi_mode __ro_after_init =3D __CFI_DEFAULT; +bool cfi_warn __ro_after_init =3D false; =20 #ifdef CONFIG_CFI_CLANG struct bpf_insn; @@ -1022,6 +1023,9 @@ static __init int cfi_parse_cmdline(char cfi_mode =3D CFI_FINEIBT; } else if (!strcmp(str, "norand")) { cfi_rand =3D false; + } else if (!strcmp(str, "warn")) { + pr_alert("CFI mismatch non-fatal!\n"); + cfi_warn =3D true; } else { pr_err("Ignoring unknown cfi option (%s).", str); } --- a/arch/x86/kernel/cfi.c +++ b/arch/x86/kernel/cfi.c @@ -67,16 +67,17 @@ static bool decode_cfi_insn(struct pt_re */ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { - unsigned long target; + unsigned long target, addr =3D regs->ip; + enum bug_trap_type btt; u32 type; =20 switch (cfi_mode) { case CFI_KCFI: - if (!is_cfi_trap(regs->ip)) + if (!is_cfi_trap(addr)) return BUG_TRAP_TYPE_NONE; =20 if (!decode_cfi_insn(regs, &target, &type)) - return report_cfi_failure_noaddr(regs, regs->ip); + return report_cfi_failure_noaddr(regs, addr); =20 break; =20 @@ -90,7 +91,12 @@ enum bug_trap_type handle_cfi_failure(st return BUG_TRAP_TYPE_NONE; } =20 - return report_cfi_failure(regs, regs->ip, &target, type); + btt =3D report_cfi_failure(regs, addr, &target, type); + if (btt =3D=3D BUG_TRAP_TYPE_BUG && cfi_warn) { + __warn(NULL, 0, (void *)addr, 0, regs, NULL); + btt =3D BUG_TRAP_TYPE_WARN; + } + return btt; } =20 /* From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 484521FF1DF for ; Wed, 19 Feb 2025 16:36:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983014; cv=none; b=HIBBORXFl5SiX5jdIVG3K94suZig6ooXzKqJxozHuSYjKiio5EWnyRrJsFOx/FRr0Igz8cCG8jcUB0Qo+7vDUn2sbHE338oAycsn2EfmmcqBz/q7s//r8IQqfoMaVSFVdEaKaG2DyXg/9Sjr1fniL5YJwoCDqMdcVKbiIAmeYDM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983014; c=relaxed/simple; bh=4qlEk+2cA0A2mCx18eN4QRncWR/yLUGSj1VHVWvZTQ4=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=IB7IOanNl2mmGsfdPAMNnw++HIJwsaF8lpbuQixxmKB2vO+DWH3ftPcE0gbGjTtXKUX02vgRQIbNzMjpfhSIt1pPp3ddnxZ0SbhoK4B9KVwKR5j4s2+LMged1U5aNPQPQ23p+emtOdXy9++CChILQmv/ppn1xvL5JV1lCjuuZjs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=bV36ePHT; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="bV36ePHT" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=TqS8RQT0Cc1AuRp0tf3Zy97LsglkPcKd8GoxoofXqeQ=; b=bV36ePHTSizscnBsrTl3o0/AT2 YzrotZKwL2xH3s3kpMH1TcOXXGzEmEFSDVVRYhFhiIPRScFpmyB5/1fxpR6yvFcL5hOjKoAWiixsE xpnBi+u31EG8bWfoQESJZvVLXq1y3zAbi9c2PpNcfVFiq6ahnF2DA5ARSKHh1JiHkH2vqLTfZMmAa mg/ZGkNZ+yATidWc0TcUVx3SUboEfjL9dYsad+MBjyP0x1oACdCz9vu7QbAzrLnXgpq2eSZlCV9CY zuErbquJi4Gl2ZCX6n5NDxM4Qcp5AxRoKaAylT55pypqdq106fnztulI9CmrZ9UH30sU2qfy+doPa 8MgTfJvg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn40-00000002HAF-0GxZ; Wed, 19 Feb 2025 16:36:44 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id EEADC3007A4; Wed, 19 Feb 2025 17:36:42 +0100 (CET) Message-ID: <20250219163514.475718637@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:09 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 02/10] x86/ibt: Add exact_endbr() helper References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" For when we want to exactly match ENDBR, and not everything that we can scribble it with. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- arch/x86/kernel/alternative.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -863,6 +863,17 @@ __noendbr bool is_endbr(u32 *val) return false; } =20 +static __noendbr bool exact_endbr(u32 *val) +{ + u32 endbr; + + __get_kernel_nofault(&endbr, val, u32, Efault); + return endbr =3D=3D gen_endbr(); + +Efault: + return false; +} + static void poison_cfi(void *addr); =20 static void __init_or_module poison_endbr(void *addr) @@ -1427,10 +1438,9 @@ static void poison_cfi(void *addr) bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 = *type) { unsigned long addr =3D regs->ip - fineibt_preamble_ud2; - u32 endbr, hash; + u32 hash; =20 - __get_kernel_nofault(&endbr, addr, u32, Efault); - if (endbr !=3D gen_endbr()) + if (!exact_endbr((void *)addr)) return false; =20 *target =3D addr + fineibt_preamble_size; From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2822F1F8F09 for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983011; cv=none; b=A/3iFDjOF3Mt/YqP/udQiVUw09X6el+xykMveNYB9J3yfSDfvX3v8C80zmRwYwBlTrFKwNR+/9aRbFd2ahb8B7dbqBwMvYQRKhbwm9sXVdVCdo+agNIdud/ZSmbGH01mo72Bpwzlg+Ub1mS+LsmEZzLTfY0FBgXQFhq2/U6Ovm4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983011; c=relaxed/simple; bh=l06kFxeKrcAhg/x/b2aTiCRIVs83ZHWC/2HlO0/Qs4s=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=lwyHwgtcIw+2juxhWbNyqVLJ6zHxTXUagMbwHq0OSeAY4VdtWaYx7atvg1tb2SZ++43rAI/nhaYhQHLhB1RgYXpSlyEys0H0/6gz9Vfq5OQR5CIiVDbVAIxd1KeBfy4zKCfCA//5ikhUbAjO9vXezN5/eexTNoXIedpVbYWvzKQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=JPOeqdbA; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="JPOeqdbA" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=gFdsCjt9iaR5QNqBsC/Zj16q/fjCzjNxQUrJ7toUTpQ=; b=JPOeqdbAAGa2KgldSFkG4S2ywV I7d31HyN+Szd0oatA29mYWdeV0mB0MP+ZXI4+KTX/9AdXfnUl/wWlG0AYpRCcxYNiwuT71G/p7pBd 76kpWjfCHADLfaMyUW3cSM+wivhNz+hbkr8edQc+//szQix9s2Z9AtlAB9KLUNZq6Mz1wv5lBX89L Ye+3pm0CXZ/oVcUPrmuqJUGL9ugbObFPMlB8SPN5hYCP4//7dz+mNcNuVyDpDQe5FyQHIJZIUFeZ1 8sSCCaTBaYWo/UoYai4Vnlb78ug0BbZb5ePffkSkffVHufiuCB84L2nTkqRYA7DJHwOtYml2L84Nd evmV1GzA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn40-00000002HAI-0JSg; Wed, 19 Feb 2025 16:36:44 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id F22CF3007CD; Wed, 19 Feb 2025 17:36:42 +0100 (CET) Message-ID: <20250219163514.581527735@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:10 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 03/10] x86/traps: Decode 0xEA #UD References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" FineIBT will start using 0xEA as #UD Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- arch/x86/include/asm/bug.h | 1 + arch/x86/kernel/traps.c | 12 ++++++++++++ 2 files changed, 13 insertions(+) --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -25,6 +25,7 @@ #define BUG_UD2 0xfffe #define BUG_UD1 0xfffd #define BUG_UD1_UBSAN 0xfffc +#define BUG_EA 0xffea =20 #ifdef CONFIG_GENERIC_BUG =20 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -96,6 +96,7 @@ __always_inline int is_valid_bugaddr(uns * Check for UD1 or UD2, accounting for Address Size Override Prefixes. * If it's a UD1, further decode to determine its use: * + * FineIBT: ea (bad) * UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax * UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax * static_call: 0f b9 cc ud1 %esp,%ecx @@ -113,6 +114,10 @@ __always_inline int decode_bug(unsigned v =3D *(u8 *)(addr++); if (v =3D=3D INSN_ASOP) v =3D *(u8 *)(addr++); + if (v =3D=3D 0xea) { + *len =3D addr - start; + return BUG_EA; + } if (v !=3D OPCODE_ESCAPE) return BUG_NONE; =20 @@ -308,6 +313,13 @@ static noinstr bool handle_bug(struct pt raw_local_irq_enable(); =20 switch (ud_type) { + case BUG_EA: + if (handle_cfi_failure(regs) =3D=3D BUG_TRAP_TYPE_WARN) { + regs->ip +=3D ud_len; + handled =3D true; + } + break; + case BUG_UD2: if (report_bug(regs->ip, regs) =3D=3D BUG_TRAP_TYPE_WARN || handle_cfi_failure(regs) =3D=3D BUG_TRAP_TYPE_WARN) { From nobody Wed Dec 17 04:15:36 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 31AAF1FDE12 for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983011; cv=none; b=XOEIVW23blzNQmVo4bw63Fqh5A8+O/TGFbpKvGCx89Szip29xpRiCzjHtRIKG8kusBjSICko/YsgukqqQP1tZLD1eiiKb0azsDTJMewXxFNPxctIy/Ej3Y7CukorZmhdSexAYjypYqyqB90MOYx5CGjAbHe8OHyYY9dA2JKsYp0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983011; c=relaxed/simple; bh=7/R9Q29okdEYiDknWp2ksDiZmvkLI7PssYu9BwnB2K0=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=VI5qMnvTEImaexXGiA626IobB+ahC8UAn/v1BEWBwqFN9+C7QXZ9ZN8mlwE8PvRjWZOK3Qpdh706IxiwT5Dj5FZ9dJCskOPF908G0rL/xlrGkYZnNVR9iKdQP4p+0d+KOrQSIwvYUA3R2V4VaYK4JwcF5iF1S+NK4JB7wtDtCzc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=XbGMXRZr; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="XbGMXRZr" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=U88omwiH5A+inj+uOYAaqUYCMp5kwoQ7OKMg92ce4XM=; b=XbGMXRZr9CESTWAEvEqgc2HyNd YCkSknwhnGk2i23J3Robj++By5p63VWH84RNGz88HXVdP0r65K/EXoBLRIT5I4I2qcxz+v12pTroI 07055jck3j3FqodDJ/8Zx+eic24HV3ENzWFOJQ/LAvzWIFa38X3AkEOp+c49v5EM3UJgDKKvF110i RhJABt7FJHQE5n6LylPMOMo2g6JD14RrhRIp5bXf7CMuP9LplQT5Q1/u26GO0EJ2DNhMrKAcr8xas /+6WmcYJJbNaCKdA2Z11Bq2gSv9wDujqiJrxBF3IojI63UCbfyAojqcBwRlTTaV2Rf1g1UCv5cNUn MfO7V8sg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn3z-00000006Whh-3pdO; Wed, 19 Feb 2025 16:36:44 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 024C63007F0; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163514.688460830@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:11 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 04/10] x86/traps: Allow custom fixups in handle_bug() References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The normal fixup in handle_bug() is simply continuing at the next instruction. However upcomming patches make this the wrong thing, so allow handlers (specifically handle_cfi_failure()) to over-ride regs->ip. The callchain is such that the fixup needs to be done before it is determined if the exception is fatal, as such, revert any changes in that case. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- arch/x86/kernel/traps.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -287,11 +287,12 @@ static inline void handle_invalid_op(str =20 static noinstr bool handle_bug(struct pt_regs *regs) { + unsigned long addr =3D regs->ip; bool handled =3D false; int ud_type, ud_len; s32 ud_imm; =20 - ud_type =3D decode_bug(regs->ip, &ud_imm, &ud_len); + ud_type =3D decode_bug(addr, &ud_imm, &ud_len); if (ud_type =3D=3D BUG_NONE) return handled; =20 @@ -315,7 +316,8 @@ static noinstr bool handle_bug(struct pt switch (ud_type) { case BUG_EA: if (handle_cfi_failure(regs) =3D=3D BUG_TRAP_TYPE_WARN) { - regs->ip +=3D ud_len; + if (regs->ip =3D=3D addr) + regs->ip +=3D ud_len; handled =3D true; } break; @@ -323,7 +325,8 @@ static noinstr bool handle_bug(struct pt case BUG_UD2: if (report_bug(regs->ip, regs) =3D=3D BUG_TRAP_TYPE_WARN || handle_cfi_failure(regs) =3D=3D BUG_TRAP_TYPE_WARN) { - regs->ip +=3D ud_len; + if (regs->ip =3D=3D addr) + regs->ip +=3D ud_len; handled =3D true; } break; @@ -340,6 +343,9 @@ static noinstr bool handle_bug(struct pt break; } =20 + if (!handled && regs->ip !=3D addr) + regs->ip =3D addr; + if (regs->flags & X86_EFLAGS_IF) raw_local_irq_disable(); instrumentation_end(); From nobody Wed Dec 17 04:15:36 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E9FF51FE45B for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; cv=none; b=cCIRkiUkS3YhHF2DAzsUCUv+I81n44Be268DpC6hwqmh3jrnqFkREmxg1PfYQWSrF+dJYR0O5utOy71TMpZubnYo5aXXZakkoef+xQPmQQDaZM/HJmpY80qLqjYGA9dfw1Xs2qry5D6qMtYbmrt8Pyi2IxU+buDbzpC4CZSTcCo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; c=relaxed/simple; bh=TM+i+OGrZK8ytIpm5H0KdmzYz4hrrB25z0z6K2huRys=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=uzauDdl3yUU/haItdb2yOM4Pii6S5JXD5JZSXVfzpYPJnVKGsEYgD33OYusosxCOOD/nZLTNEhT5FdfaVbXFNueVZ/voy6gy3J/EgxlRgZnV27KCyyDWYMyaZiu/txXjwljJk9OP/+hGdpkNu0zFoP8aI0KW2aW1ublu+31J18o= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=M/EwmvmN; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="M/EwmvmN" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=XiaeP0zY0E9OZG92iLfjRSZjSKwJIsUYCSKEWwoZUPM=; b=M/EwmvmNW8f6hgqPsH0JR69QWN rF1k4jEdB7NsrI3CtMeBVvrVo/Ci3aWhcgT4r9ur0P5EvXR/L4ScREo1AN2lPUH6PkVKiDLcrunud 1N6avw0GjPLiByBdCzZhyd8CPKcAsFiht1cLCWYG4w60EKRMaa2rGPUCPqUQky/S9ukFc2fnBuYmP KJJf9lz7/rUsAX0YV9eN5guYnYsH8Kb4FfJ9aHzz/pn2dyXtmxbw0P9Qmzzd4ufMBVdB57AVsVl5T 6qWp2XJ/yUKE8ICb1cV/bCuerOyjiqokJXjPT3pKi/Y/FaMoK7QlOxR5DXjDJnGunYLGCxZuMmSWQ XJuIFOeg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn40-00000006Whj-2QYr; Wed, 19 Feb 2025 16:36:44 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 06372300B40; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163514.791951626@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:12 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 05/10] x86/ibt: Optimize FineIBT sequence References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Scott notes that non-taken branches are faster. Abuse overlapping code that traps instead of explicit UD2 instructions. And LEA does not modify flags and will have less dependencies. Suggested-by: Scott Constable Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- arch/x86/kernel/alternative.c | 58 ++++++++++++++++++++++++++-----------= ----- arch/x86/net/bpf_jit_comp.c | 5 +-- 2 files changed, 39 insertions(+), 24 deletions(-) --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1054,9 +1054,9 @@ early_param("cfi", cfi_parse_cmdline); * __cfi_\func: __cfi_\func: * movl $0x12345678,%eax // 5 endbr64 // 4 * nop subl $0x12345678,%r10d // 7 - * nop jz 1f // 2 - * nop ud2 // 2 - * nop 1: nop // 1 + * nop jne __cfi_\func+6 // 2 + * nop nop3 // 3 + * nop * nop * nop * nop @@ -1068,37 +1068,47 @@ early_param("cfi", cfi_parse_cmdline); * * caller: caller: * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6 - * addl $-15(%r11),%r10d // 4 sub $16,%r11 // 4 + * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4 * je 1f // 2 nop4 // 4 * ud2 // 2 - * 1: call __x86_indirect_thunk_r11 // 5 call *%r11; nop2; // 5 + * 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6 * */ =20 -asm( ".pushsection .rodata \n" - "fineibt_preamble_start: \n" - " endbr64 \n" - " subl $0x12345678, %r10d \n" - " je fineibt_preamble_end \n" - "fineibt_preamble_ud2: \n" - " ud2 \n" - " nop \n" - "fineibt_preamble_end: \n" +/* + * : + * 0: f3 0f 1e fa endbr64 + * 4: 41 81 78 56 34 12 sub $0x12345678, %r10d + * b: 75 f9 jne 6 + * d: 0f 1f 00 nopl (%rax) + */ +asm( ".pushsection .rodata \n" + "fineibt_preamble_start: \n" + " endbr64 \n" + " subl $0x12345678, %r10d \n" + " jne fineibt_preamble_start+6 \n" + ASM_NOP3 + "fineibt_preamble_end: \n" ".popsection\n" ); =20 extern u8 fineibt_preamble_start[]; -extern u8 fineibt_preamble_ud2[]; extern u8 fineibt_preamble_end[]; =20 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_sta= rt) -#define fineibt_preamble_ud2 (fineibt_preamble_ud2 - fineibt_preamble_sta= rt) +#define fineibt_preamble_ud 6 #define fineibt_preamble_hash 7 =20 +/* + * : + * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d + * 6: 4d 8d 5b f0 lea -0x10(%r11), %r11 + * a: 0f 1f 40 00 nopl 0x0(%rax) + */ asm( ".pushsection .rodata \n" "fineibt_caller_start: \n" " movl $0x12345678, %r10d \n" - " sub $16, %r11 \n" + " lea -0x10(%r11), %r11 \n" ASM_NOP4 "fineibt_caller_end: \n" ".popsection \n" @@ -1429,15 +1439,15 @@ static void poison_cfi(void *addr) } =20 /* - * regs->ip points to a UD2 instruction, return true and fill out target a= nd - * type when this UD2 is from a FineIBT preamble. + * When regs->ip points to a 0xEA byte in the FineIBT preamble, + * return true and fill out target and type. * * We check the preamble by checking for the ENDBR instruction relative to= the - * UD2 instruction. + * 0xEA instruction. */ bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 = *type) { - unsigned long addr =3D regs->ip - fineibt_preamble_ud2; + unsigned long addr =3D regs->ip - fineibt_preamble_ud; u32 hash; =20 if (!exact_endbr((void *)addr)) @@ -1448,6 +1458,12 @@ bool decode_fineibt_insn(struct pt_regs __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); *type =3D (u32)regs->r10 + hash; =20 + /* + * Since regs->ip points to the middle of an instruction; it cannot + * continue with the normal fixup. + */ + regs->ip =3D *target; + return true; =20 Efault: --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -417,9 +417,8 @@ static void emit_fineibt(u8 **pprog, u32 =20 EMIT_ENDBR(); EMIT3_off32(0x41, 0x81, 0xea, hash); /* subl $hash, %r10d */ - EMIT2(0x74, 0x07); /* jz.d8 +7 */ - EMIT2(0x0f, 0x0b); /* ud2 */ - EMIT1(0x90); /* nop */ + EMIT2(0x75, 0xf9); /* jne.d8 .-7 */ + EMIT3(0x0f, 0x1f, 0x00); /* nop3 */ EMIT_ENDBR_POISON(); =20 *pprog =3D prog; From nobody Wed Dec 17 04:15:36 2025 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 31A531FDA9D for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983013; cv=none; b=iv2FKNAdBWQEBWN78rt3gJWMOZEULrVVmEjyzfjJFGXWql+KZp31fUOl9FvZ891Gv/9UOPWcCTxDW3+Web/VrugPJ4pqq0uEiQr8qJbuTL7aYhs9GS4sJvzouKMPYXfdk7iO4ZIMK1rNlDfvvjUPVF9EHer+4tTxr+xwfECNHbw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983013; c=relaxed/simple; bh=vDtfpVS7JsMghIsD5j58iOIqHJHa3Hky7Gqd6rSqidY=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=GTMw64P/Sh+W1giK1/D4zvZ0vTMJzWh4SiwjhYAmrcZb+zrgOHm3tsbILVCClgz8ewkf0KDPcas74gYD55SGWdgjWjfPE1JWNE53U2jhBDVK78rJXsCMCRmiNmzs92fx+JpqA2PWvscaxe6S0rV1aeXQK+V9aH1Ozlr8F2TZodI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=icstvHM6; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="icstvHM6" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=4gavA9yGarxIk7e0bFuvMl3MjG58bvR3QzZc+2BZfSQ=; b=icstvHM6RG32kIawRuDSqrrKVk 7FPvn/ZNbpra7ukMgtjgDQjMn/RLg9WgoKyRuQliLnVq+1A1e0pAQfY8nZ+Ib/jJWWoepFV6hnG4b pxcur5KAbvoyPkChjiQX6e8L6UFHhCas85NKSV+6Sp3MpEsH9wNBQmjgCNaQCvKBcyHo7cvwwnZGi hoG5S9WVyUZ4cl6X20b4+wCJWGMFxEg9F3UnRZ2MPHRBMS2lweLo4uRMekhzyEfmO24+HP/d1gUlt uuzzTwKQ19XEyxQraEOUheEtKDgTnfhSdv4vNEmvY6nQPuz8uWXhaPj+1SVDd8JWGD+UXmrS4rA1E 50XnRK8Q==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn40-00000006Whk-2OwH; Wed, 19 Feb 2025 16:36:44 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 0A57F3021DF; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163514.928125334@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:13 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 06/10] x86/traps: Decode LOCK Jcc.d8 #UD References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Because overlapping code sequences are all the rage. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- arch/x86/include/asm/bug.h | 2 ++ arch/x86/kernel/traps.c | 30 +++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -17,6 +17,7 @@ * In clang we have UD1s reporting UBSAN failures on X86, 64 and 32bit. */ #define INSN_ASOP 0x67 +#define INSN_LOCK 0xf0 #define OPCODE_ESCAPE 0x0f #define SECOND_BYTE_OPCODE_UD1 0xb9 #define SECOND_BYTE_OPCODE_UD2 0x0b @@ -26,6 +27,7 @@ #define BUG_UD1 0xfffd #define BUG_UD1_UBSAN 0xfffc #define BUG_EA 0xffea +#define BUG_LOCK 0xfff0 =20 #ifdef CONFIG_GENERIC_BUG =20 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -97,6 +97,7 @@ __always_inline int is_valid_bugaddr(uns * If it's a UD1, further decode to determine its use: * * FineIBT: ea (bad) + * FineIBT: 0f 75 f9 lock jne . - 6 * UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax * UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax * static_call: 0f b9 cc ud1 %esp,%ecx @@ -106,6 +107,7 @@ __always_inline int is_valid_bugaddr(uns __always_inline int decode_bug(unsigned long addr, s32 *imm, int *len) { unsigned long start =3D addr; + bool lock =3D false; u8 v; =20 if (addr < TASK_SIZE_MAX) @@ -114,12 +116,29 @@ __always_inline int decode_bug(unsigned v =3D *(u8 *)(addr++); if (v =3D=3D INSN_ASOP) v =3D *(u8 *)(addr++); - if (v =3D=3D 0xea) { + + if (v =3D=3D INSN_LOCK) { + lock =3D true; + v =3D *(u8 *)(addr++); + } + + switch (v) { + case 0x70 ... 0x7f: /* Jcc.d8 */ + addr +=3D 1; /* d8 */ + *len =3D addr - start; + WARN_ON_ONCE(!lock); + return BUG_LOCK; + + case 0xea: *len =3D addr - start; return BUG_EA; - } - if (v !=3D OPCODE_ESCAPE) + + case OPCODE_ESCAPE: + break; + + default: return BUG_NONE; + } =20 v =3D *(u8 *)(addr++); if (v =3D=3D SECOND_BYTE_OPCODE_UD2) { @@ -315,7 +334,8 @@ static noinstr bool handle_bug(struct pt =20 switch (ud_type) { case BUG_EA: - if (handle_cfi_failure(regs) =3D=3D BUG_TRAP_TYPE_WARN) { + case BUG_LOCK: + if (handle_cfi_failure(ud_type, regs) =3D=3D BUG_TRAP_TYPE_WARN) { if (regs->ip =3D=3D addr) regs->ip +=3D ud_len; handled =3D true; @@ -324,7 +344,7 @@ static noinstr bool handle_bug(struct pt =20 case BUG_UD2: if (report_bug(regs->ip, regs) =3D=3D BUG_TRAP_TYPE_WARN || - handle_cfi_failure(regs) =3D=3D BUG_TRAP_TYPE_WARN) { + handle_cfi_failure(ud_type, regs) =3D=3D BUG_TRAP_TYPE_WARN) { if (regs->ip =3D=3D addr) regs->ip +=3D ud_len; handled =3D true; From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1B9CE198E81 for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; cv=none; b=II9Pk9U4MwqyaTX/LdPW8Ps6zV9uM0cCSJjFr4Ea1Lp1x9/Yfsw/Tj73JHtN54xkX6uAPypDTEZmy5B95hRF0yuXz1DWoX6adxfPCosvdC7Ctfc5RX65bjH6o97NFaCyk9PXgfYmRVhB0q+DIaE2lNWdFc+NSADSMLpcOMmduQc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; c=relaxed/simple; bh=T+ErmRUTbrmeEaOqBtf+5hjN0fLX2iTXUYqwBSnsnbE=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=oben5lGne8HxcSdvpPTrS5RdybEpbR/33FE5znSFA3DLmNE38FtX3VbriNy11KAyIc+ocgvOlPWbEz5Lgep6WQlKTJVePfQqwrp8YzxcrHIW2AVf85/GdrGWl0su7pJeIRiqpkHuAPJYmLvI1S5uQV/++/z4+bg3N+WtziO4Ir8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=hnaZuDrb; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="hnaZuDrb" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=NNBJ3WzBo+K7wIvhm3+Ot5P/kY6iwqtzBK2LJIsIOww=; b=hnaZuDrb2Uq8vZvBIxG3I2y8/k /zDNHUTkoYf3GBNdn1ZaFnHGTrO5aEU9wk0szEM8u8nXMSde49qr2SPU04NCleUupsgsgLy3l5/Ix T60h+BWZMlGU++gSs1Yte8rG8851jwETEKL9O6ynun3KxFoAUZ/wr5KIciuAWBG2iUqjkWmbcSQCw PIDxq2jxaEw4RIbee1BOgGVnmWCMlJf8CepeL6DF1CC8gKWJNxVS7J92NTNXQIPbsEOkcAci3DUXX iX4HhDMnaduYVcQhOH0f4doXWYdOVyVYHXTmHyomeZkxRZKhWjF0D9bZcGp89Mspa9BL2Fjj0vtf3 31fuY4Kw==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn41-00000002HAW-0689; Wed, 19 Feb 2025 16:36:45 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 0E14230651E; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163515.040300905@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:14 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 07/10] x86/ibt: Add paranoid FineIBT mode References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Due to concerns about circumvention attacks against FineIBT on 'naked' ENDBR, add an additional caller side hash check to FineIBT. This should make it impossible to pivot over such a 'naked' ENDBR instruction at the cost of an additional load. The specific pivot reported was against the SYSCALL entry site and FRED will have all those holes fixed up. This specific fineibt_paranoid_start[] sequence was concocted by Scott. Reported-by: Jennifer Miller Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20250215210729.GA25168@noisy.programming.ki= cks-ass.net Reviewed-by: Kees Cook --- arch/x86/include/asm/cfi.h | 8 +- arch/x86/kernel/alternative.c | 143 +++++++++++++++++++++++++++++++++++++= +++-- arch/x86/kernel/cfi.c | 4 - 3 files changed, 143 insertions(+), 12 deletions(-) --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -105,7 +105,7 @@ extern bool cfi_warn; struct pt_regs; =20 #ifdef CONFIG_CFI_CLANG -enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +enum bug_trap_type handle_cfi_failure(int ud_type, struct pt_regs *regs); #define __bpfcall extern u32 cfi_bpf_hash; extern u32 cfi_bpf_subprog_hash; @@ -128,10 +128,10 @@ static inline int cfi_get_offset(void) extern u32 cfi_get_func_hash(void *func); =20 #ifdef CONFIG_FINEIBT -extern bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *targe= t, u32 *type); +extern bool decode_fineibt_insn(int ud_type, struct pt_regs *regs, unsigne= d long *target, u32 *type); #else static inline bool -decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) +decode_fineibt_insn(int ud_type, struct pt_regs *regs, unsigned long *targ= et, u32 *type) { return false; } @@ -139,7 +139,7 @@ decode_fineibt_insn(struct pt_regs *regs #endif =20 #else -static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) +static inline enum bug_trap_type handle_cfi_failure(int ud_type, struct pt= _regs *regs) { return BUG_TRAP_TYPE_NONE; } --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -741,6 +741,11 @@ void __init_or_module noinline apply_ret op2 =3D insn.opcode.bytes[1]; =20 switch (op1) { + case 0x70 ... 0x7f: /* Jcc.d8 */ + /* See cfi_paranoid. */ + WARN_ON_ONCE(cfi_mode !=3D CFI_FINEIBT); + continue; + case CALL_INSN_OPCODE: case JMP32_INSN_OPCODE: break; @@ -995,6 +1000,8 @@ u32 cfi_get_func_hash(void *func) static bool cfi_rand __ro_after_init =3D true; static u32 cfi_seed __ro_after_init; =20 +static bool cfi_paranoid __ro_after_init =3D false; + /* * Re-hash the CFI hash with a boot-time seed while making sure the result= is * not a valid ENDBR instruction. @@ -1037,6 +1044,12 @@ static __init int cfi_parse_cmdline(char } else if (!strcmp(str, "warn")) { pr_alert("CFI mismatch non-fatal!\n"); cfi_warn =3D true; + } else if (!strcmp(str, "paranoid")) { + if (cfi_mode =3D=3D CFI_FINEIBT) { + cfi_paranoid =3D true; + } else { + pr_err("Ignoring paranoid; depends on fineibt.\n"); + } } else { pr_err("Ignoring unknown cfi option (%s).", str); } @@ -1116,6 +1129,52 @@ extern u8 fineibt_caller_end[]; =20 #define fineibt_caller_jmp (fineibt_caller_size - 2) =20 +/* + * Since FineIBT does hash validation on the callee side it is prone to + * circumvention attacks where a 'naked' ENDBR instruction exists that + * is not part of the fineibt_preamble sequence. + * + * Notably the x86 entry points must be ENDBR and equally cannot be + * fineibt_preamble. + * + * The fineibt_paranoid caller sequence adds additional caller side + * hash validation. This stops such circumvetion attacks dead, but at the = cost + * of adding a load. + * + * : + * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d + * 6: 45 3b 53 f7 cmp -0x9(%r11), %r10d + * a: 4d 8d 5b lea -0x10(%r11), %r11 + * e: 75 fd jne d + * 10: 41 ff d3 call *%r11 + * 13: 90 nop + * + * Notably LEA does not modify flags and can be reordered with the CMP, + * avoiding a dependency. Again, using a non-taken (backwards) branch + * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for t= he + * JCC.d8, causing #UD. + */ +asm( ".pushsection .rodata \n" + "fineibt_paranoid_start: \n" + " movl $0x12345678, %r10d \n" + " cmpl -9(%r11), %r10d \n" + " lea -0x10(%r11), %r11 \n" + " jne fineibt_paranoid_start+0xd \n" + "fineibt_paranoid_ind: \n" + " call *%r11 \n" + " nop \n" + "fineibt_paranoid_end: \n" + ".popsection \n" +); + +extern u8 fineibt_paranoid_start[]; +extern u8 fineibt_paranoid_ind[]; +extern u8 fineibt_paranoid_end[]; + +#define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_sta= rt) +#define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_sta= rt) +#define fineibt_paranoid_ud 0xd + static u32 decode_preamble_hash(void *addr) { u8 *p =3D addr; @@ -1279,18 +1338,48 @@ static int cfi_rewrite_callers(s32 *star { s32 *s; =20 + BUG_ON(fineibt_paranoid_size !=3D 20); + for (s =3D start; s < end; s++) { void *addr =3D (void *)s + *s; + struct insn insn; + u8 bytes[20]; u32 hash; + int ret; + u8 op; =20 addr -=3D fineibt_caller_size; hash =3D decode_caller_hash(addr); - if (hash) { + if (!hash) + continue; + + if (!cfi_paranoid) { text_poke_early(addr, fineibt_caller_start, fineibt_caller_size); WARN_ON(*(u32 *)(addr + fineibt_caller_hash) !=3D 0x12345678); text_poke_early(addr + fineibt_caller_hash, &hash, 4); + /* rely on apply_retpolines() */ + continue; } - /* rely on apply_retpolines() */ + + /* cfi_paranoid */ + ret =3D insn_decode_kernel(&insn, addr + fineibt_caller_size); + if (WARN_ON_ONCE(ret < 0)) + continue; + + op =3D insn.opcode.bytes[0]; + if (op !=3D CALL_INSN_OPCODE && op !=3D JMP32_INSN_OPCODE) { + WARN_ON_ONCE(1); + continue; + } + + memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size); + memcpy(bytes + fineibt_caller_hash, &hash, 4); + + ret =3D emit_indirect(op, 11, bytes + fineibt_paranoid_ind); + if (WARN_ON_ONCE(ret !=3D 3)) + continue; + + text_poke_early(addr, bytes, fineibt_paranoid_size); } =20 return 0; @@ -1307,8 +1396,15 @@ static void __apply_fineibt(s32 *start_r =20 if (cfi_mode =3D=3D CFI_AUTO) { cfi_mode =3D CFI_KCFI; - if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) + if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) { + /* + * FRED has much saner context on exception entry and + * is less easy to take advantage of. + */ + if (!cpu_feature_enabled(X86_FEATURE_FRED)) + cfi_paranoid =3D true; cfi_mode =3D CFI_FINEIBT; + } } =20 /* @@ -1365,8 +1461,10 @@ static void __apply_fineibt(s32 *start_r /* now that nobody targets func()+0, remove ENDBR there */ cfi_rewrite_endbr(start_cfi, end_cfi); =20 - if (builtin) - pr_info("Using FineIBT CFI\n"); + if (builtin) { + pr_info("Using %sFineIBT CFI\n", + cfi_paranoid ? "paranoid " : ""); + } return; =20 default: @@ -1439,7 +1537,8 @@ static void poison_cfi(void *addr) * We check the preamble by checking for the ENDBR instruction relative to= the * 0xEA instruction. */ -bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 = *type) +static bool decode_fineibt_preamble(int ud_type, struct pt_regs *regs, + unsigned long *target, u32 *type) { unsigned long addr =3D regs->ip - fineibt_preamble_ud; u32 hash; @@ -1464,6 +1563,38 @@ bool decode_fineibt_insn(struct pt_regs return false; } =20 +/* + * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_= start[] + * sequence. + */ +static bool decode_fineibt_paranoid(int ud_type, struct pt_regs *regs, + unsigned long *target, u32 *type) +{ + unsigned long addr =3D regs->ip - fineibt_paranoid_ud; + u32 hash; + + __get_kernel_nofault(&hash, addr + fineibt_caller_hash, u32, Efault); + *target =3D regs->r11 + fineibt_preamble_size; + *type =3D regs->r10; + + /* + * Since the trapping instruction is the exact, but LOCK prefixed, + * Jcc.d8 that got us here, the normal fixup will work. + */ + return true; + +Efault: + return false; +} + +bool decode_fineibt_insn(int ud_type, struct pt_regs *regs, + unsigned long *target, u32 *type) +{ + if (ud_type =3D=3D BUG_LOCK) + return decode_fineibt_paranoid(ud_type, regs, target, type); + return decode_fineibt_preamble(ud_type, regs, target, type); +} + #else =20 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, --- a/arch/x86/kernel/cfi.c +++ b/arch/x86/kernel/cfi.c @@ -65,7 +65,7 @@ static bool decode_cfi_insn(struct pt_re * Checks if a ud2 trap is because of a CFI failure, and handles the trap * if needed. Returns a bug_trap_type value similarly to report_bug. */ -enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) +enum bug_trap_type handle_cfi_failure(int ud_type, struct pt_regs *regs) { unsigned long target, addr =3D regs->ip; enum bug_trap_type btt; @@ -82,7 +82,7 @@ enum bug_trap_type handle_cfi_failure(st break; =20 case CFI_FINEIBT: - if (!decode_fineibt_insn(regs, &target, &type)) + if (!decode_fineibt_insn(ud_type, regs, &target, &type)) return BUG_TRAP_TYPE_NONE; =20 break; From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2E6E51FCCFD for ; Wed, 19 Feb 2025 16:36:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983011; cv=none; b=TdL6vVNOpW8Xgn/dGbhISLkzn28CwR0s2rWF/IPHrskq3oqqDiEFpoGcxqcVf5obBnwAuA+iJ18/8NnybFOjgwH0V37s74NFpK/V1ymhN8fzuTcyl9zyt0qBObTN0VnetsHDfIrGOKgPsrsCU1yVdGRan06E77jXv3NzlkTo68Q= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983011; c=relaxed/simple; bh=Btti5HdLHNcBRd1KzoFWC5o4jtbV4UKRZPpvGQwFmYs=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=onjJCo509PTRHu36nFgV71igltE6sih1MeukfocFSvB7K03svwP/lPwOA6w6WXkJ/OC4Rx2jAQZUsMGcdrCTCxmShfLXq8W21vu0t2In7/VCXQC7nrXiFbjNsX0Vpa03ygUUoPZ9ufsjfmjdcq0996DNIUnrtYSTu8rhmZEl628= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=m6E47sfs; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="m6E47sfs" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=EO/UeWnTi587WDLa+KBtfMxZo5oSImtpAx4xhvYuAZk=; b=m6E47sfsI3mOxUXYn2MCWPsrlF QJY5qT/vVAzn6xd4Kw1vxZ3s3YXhtmsBHzJMUoagwX1CyZEU4HczkqcxYGnyAQJHf3mS2VZzhm7oC qw2SluBt2Xs8N72wXkZP5AfCGwrZ5mTuuRZcC3yCbnFvALzQqmAg/gIqBnjhDYNPeHOgvNlti1L6d Z0DL7DaLJWMlZfV5wvh317bF3yG4Qq5Z0jy2+lLSBOczouOkTDr6CFdJtMhQsqDuJGHZBapxQl8Ib ByjwaUYq55rZ7KjNruVrwUHRgMRMhfPOXfxvXqih7mknjv9cVfA2ktd5Blxkv5lLMQ7A8QXUzZWwq UXiNkDbg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn41-00000002HAU-065T; Wed, 19 Feb 2025 16:36:45 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 113C1306573; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163515.150619755@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:15 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 08/10] x86: BHI stubs References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/include/asm/cfi.h | 4 + arch/x86/lib/Makefile | 3=20 arch/x86/lib/bhi.S | 146 ++++++++++++++++++++++++++++++++++++++++= +++++ 3 files changed, 152 insertions(+), 1 deletion(-) --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -102,6 +102,10 @@ enum cfi_mode { extern enum cfi_mode cfi_mode; extern bool cfi_warn; =20 +typedef u8 bhi_thunk[32]; +extern bhi_thunk __bhi_args[]; +extern bhi_thunk __bhi_args_end[]; + struct pt_regs; =20 #ifdef CONFIG_CFI_CLANG --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -66,5 +66,6 @@ endif lib-y +=3D clear_page_64.o copy_page_64.o lib-y +=3D memmove_64.o memset_64.o lib-y +=3D copy_user_64.o copy_user_uncached_64.o - lib-y +=3D cmpxchg16b_emu.o + lib-y +=3D cmpxchg16b_emu.o + lib-y +=3D bhi.o endif --- /dev/null +++ b/arch/x86/lib/bhi.S @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include + +/* + * Notably, the FineIBT preamble calling these will have ZF set and r10 ze= ro. + * + * The very last element is in fact larger than 32 bytes, but since its the + * last element, this does not matter, + * + * There are 2 #UD sites, located between 0,1-2,3 and 4,5-6,7 such that th= ey + * can be reached using Jcc.d8, these elements (1 and 5) have sufficiently + * big alignment holes for this to not stagger the array. + */ + +.pushsection .noinstr.text, "ax" + + .align 32 +SYM_CODE_START(__bhi_args) + +#ifdef CONFIG_FINEIBT_BHI + + .align 32 +SYM_INNER_LABEL(__bhi_args_0, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_1 + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_1, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_1 + cmovne %r10, %rdi + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 8 + ANNOTATE_REACHABLE +.Lud_1: ud2 + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_2, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_1 + cmovne %r10, %rdi + cmovne %r10, %rsi + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_3, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_1 + cmovne %r10, %rdi + cmovne %r10, %rsi + cmovne %r10, %rdx + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_4, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_2 + cmovne %r10, %rdi + cmovne %r10, %rsi + cmovne %r10, %rdx + cmovne %r10, %rcx + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_5, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_2 + cmovne %r10, %rdi + cmovne %r10, %rsi + cmovne %r10, %rdx + cmovne %r10, %rcx + cmovne %r10, %r8 + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 8 + ANNOTATE_REACHABLE +.Lud_2: ud2 + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_6, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_2 + cmovne %r10, %rdi + cmovne %r10, %rsi + cmovne %r10, %rdx + cmovne %r10, %rcx + cmovne %r10, %r8 + cmovne %r10, %r9 + ANNOTATE_UNRET_SAFE + ret + int3 + + .align 32 +SYM_INNER_LABEL(__bhi_args_7, SYM_L_LOCAL) + ANNOTATE_NOENDBR + UNWIND_HINT_FUNC + jne .Lud_2 + cmovne %r10, %rdi + cmovne %r10, %rsi + cmovne %r10, %rdx + cmovne %r10, %rcx + cmovne %r10, %r8 + cmovne %r10, %r9 + cmovne %r10, %rsp + ANNOTATE_UNRET_SAFE + ret + int3 + +#endif /* CONFIG_FINEIBT_BHI */ + + .align 32 +SYM_INNER_LABEL(__bhi_args_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR +SYM_CODE_END(__bhi_args) + +.popsection From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2E7B81FDA7C for ; Wed, 19 Feb 2025 16:36:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; cv=none; b=dWrabR+dq/ICQj/34ZMJapsJKDfB5TAXKLUuIgcNy1qg/v5y48QYQcnY9K0lvveS8x8CbYJVsjVU+aQZVGNbjZm/oQ1ON9Y9r0kZF2573dzoo2ZB3/MyfaY4ex/Hytr+nWDvhPVC11mPKmWTKngQCojFXOV7soZt3L7hssIecW4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; c=relaxed/simple; bh=WWLgwcr1iviX+ssT6byJTCw29PdFNdzy9d/eq3z4WFM=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=Onzz1e4IuhVXO9iHvL9RlggwmiHPVNsC4qgpY0GGjD3Imvwv/Hfjp7RyhpMc55aBJu571jnlcluBbU4e7WNhGrdf9BSD4Lvr0JTaY6jBOrwUHjvBQfVg/SPoc/8OMi2UTuRUQOWQM2b/OeYihMQHRFTR4nT6iW0STU2daLNCCu0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=Fb9CpNa5; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="Fb9CpNa5" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=dVIbxD0kjnfglDLzgJxL3mSgF+P17AVA1k/bY9HdUKQ=; b=Fb9CpNa5hjmq0zZNdyYUaIB+qb xgLT8VEnHWUzdQGys6VhEq1NBTm6nFLUejw7bT49kXjaOzSw6ZzU1xUayEajYaDJMPUfutv4cL/kN TI73ZmzqgmwMukfrzVHxkWk+hPjYk1CCdZLe/jE8ENMnTCDkdGFqfg9CqFmW2L4aLp2OH21invlEN DCzC1m78LdSNGVnlSKuRn6WmHagvepAJIJ13K0OSZle/sLJcFaRLL/0WvuVVkmWtIMry+0/aBQcJ3 7Eas133TQo6i6XSxpnGzINxoqJNp0aQ+DVp5c/hFlajjYij1ZLhuC6tTtcGu1j1HjfqO8QWyClikO mumGcWLw==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn41-00000002HAV-04Vt; Wed, 19 Feb 2025 16:36:45 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 15D523078A4; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163515.259229338@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:16 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 09/10] x86/ibt: Implement FineIBT-BHI mitigation References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" While WAIT_FOR_ENDBR is specified to be a full speculation stop; it has been shown that some implementations are 'leaky' to such an extend that speculation can escape even the FineIBT preamble. To deal with this, add additional hardening to the FineIBT preamble. Notably, using a new LLVM feature: https://github.com/llvm/llvm-project/commit/e223485c9b38a5579991b8cebb6a2= 00153eee245 which encodes the number of arguments in the kCFI preamble's register. Using this register<->arity mapping, have the FineIBT preamble CALL into a stub clobbering the relevant argument registers in the speculative case. (This is where Scott goes and gives more details...) Suggested-by: Scott Constable Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- Makefile | 3 + arch/x86/Kconfig | 8 +++ arch/x86/include/asm/cfi.h | 6 ++ arch/x86/kernel/alternative.c | 101 +++++++++++++++++++++++++++++++++++++= ----- arch/x86/net/bpf_jit_comp.c | 29 ++++++++---- 5 files changed, 128 insertions(+), 19 deletions(-) --- a/Makefile +++ b/Makefile @@ -1014,6 +1014,9 @@ CC_FLAGS_CFI :=3D -fsanitize=3Dkcfi ifdef CONFIG_CFI_ICALL_NORMALIZE_INTEGERS CC_FLAGS_CFI +=3D -fsanitize-cfi-icall-experimental-normalize-integers endif +ifdef CONFIG_FINEIBT_BHI + CC_FLAGS_CFI +=3D -fsanitize-kcfi-arity +endif ifdef CONFIG_RUST # Always pass -Zsanitizer-cfi-normalize-integers as CONFIG_RUST selects # CONFIG_CFI_ICALL_NORMALIZE_INTEGERS. --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2473,6 +2473,10 @@ config CC_HAS_RETURN_THUNK config CC_HAS_ENTRY_PADDING def_bool $(cc-option,-fpatchable-function-entry=3D16,16) =20 +config CC_HAS_KCFI_ARITY + def_bool $(cc-option,-fsanitize=3Dkcfi -fsanitize-kcfi-arity) + depends on CC_IS_CLANG && !RUST + config FUNCTION_PADDING_CFI int default 59 if FUNCTION_ALIGNMENT_64B @@ -2498,6 +2502,10 @@ config FINEIBT depends on X86_KERNEL_IBT && CFI_CLANG && MITIGATION_RETPOLINE select CALL_PADDING =20 +config FINEIBT_BHI + def_bool y + depends on FINEIBT && CC_HAS_KCFI_ARITY + config HAVE_CALL_THUNKS def_bool y depends on CC_HAS_ENTRY_PADDING && MITIGATION_RETHUNK && OBJTOOL --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -101,6 +101,7 @@ enum cfi_mode { =20 extern enum cfi_mode cfi_mode; extern bool cfi_warn; +extern bool cfi_bhi; =20 typedef u8 bhi_thunk[32]; extern bhi_thunk __bhi_args[]; @@ -130,6 +131,7 @@ static inline int cfi_get_offset(void) #define cfi_get_offset cfi_get_offset =20 extern u32 cfi_get_func_hash(void *func); +extern int cfi_get_func_arity(void *func); =20 #ifdef CONFIG_FINEIBT extern bool decode_fineibt_insn(int ud_type, struct pt_regs *regs, unsigne= d long *target, u32 *type); @@ -153,6 +155,10 @@ static inline u32 cfi_get_func_hash(void { return 0; } +static inline int cfi_get_func_arity(void *func) +{ + return 0; +} #endif /* CONFIG_CFI_CLANG */ =20 #if HAS_KERNEL_IBT =3D=3D 1 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -933,6 +933,7 @@ void __init_or_module apply_seal_endbr(s =20 enum cfi_mode cfi_mode __ro_after_init =3D __CFI_DEFAULT; bool cfi_warn __ro_after_init =3D false; +bool cfi_bhi __ro_after_init =3D false; =20 #ifdef CONFIG_CFI_CLANG struct bpf_insn; @@ -993,6 +994,21 @@ u32 cfi_get_func_hash(void *func) =20 return hash; } + +int cfi_get_func_arity(void *func) +{ + bhi_thunk *target; + s32 disp; + + if (cfi_mode !=3D CFI_FINEIBT && !cfi_bhi) + return 0; + + if (get_kernel_nofault(disp, func - 4)) + return 0; + + target =3D func + disp; + return target - __bhi_args; +} #endif =20 #ifdef CONFIG_FINEIBT @@ -1050,6 +1066,12 @@ static __init int cfi_parse_cmdline(char } else { pr_err("Ignoring paranoid; depends on fineibt.\n"); } + } else if (!strcmp(str, "bhi")) { + if (cfi_mode =3D=3D CFI_FINEIBT) { + cfi_bhi =3D true; + } else { + pr_err("Ignoring bhi; depends on fineibt.\n"); + } } else { pr_err("Ignoring unknown cfi option (%s).", str); } @@ -1099,6 +1121,7 @@ asm( ".pushsection .rodata \n" "fineibt_preamble_start: \n" " endbr64 \n" " subl $0x12345678, %r10d \n" + "fineibt_preamble_bhi: \n" " jne fineibt_preamble_start+6 \n" ASM_NOP3 "fineibt_preamble_end: \n" @@ -1106,9 +1129,11 @@ asm( ".pushsection .rodata \n" ); =20 extern u8 fineibt_preamble_start[]; +extern u8 fineibt_preamble_bhi[]; extern u8 fineibt_preamble_end[]; =20 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_sta= rt) +#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_sta= rt) #define fineibt_preamble_ud 6 #define fineibt_preamble_hash 7 =20 @@ -1181,13 +1206,16 @@ extern u8 fineibt_paranoid_end[]; #define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_sta= rt) #define fineibt_paranoid_ud 0xd =20 -static u32 decode_preamble_hash(void *addr) +static u32 decode_preamble_hash(void *addr, int *reg) { u8 *p =3D addr; =20 - /* b8 78 56 34 12 mov $0x12345678,%eax */ - if (p[0] =3D=3D 0xb8) + /* b8+reg 78 56 34 12 movl $0x12345678,\reg */ + if (p[0] >=3D 0xb8 && p[0] < 0xc0) { + if (reg) + *reg =3D p[0] - 0xb8; return *(u32 *)(addr + 1); + } =20 return 0; /* invalid hash value */ } @@ -1196,11 +1224,11 @@ static u32 decode_caller_hash(void *addr { u8 *p =3D addr; =20 - /* 41 ba 78 56 34 12 mov $0x12345678,%r10d */ + /* 41 ba 88 a9 cb ed mov $(-0x12345678),%r10d */ if (p[0] =3D=3D 0x41 && p[1] =3D=3D 0xba) return -*(u32 *)(addr + 2); =20 - /* e8 0c 78 56 34 12 jmp.d8 +12 */ + /* e8 0c 88 a9 cb ed jmp.d8 +12 */ if (p[0] =3D=3D JMP8_INSN_OPCODE && p[1] =3D=3D fineibt_caller_jmp) return -*(u32 *)(addr + 2); =20 @@ -1265,7 +1293,7 @@ static int cfi_rand_preamble(s32 *start, void *addr =3D (void *)s + *s; u32 hash; =20 - hash =3D decode_preamble_hash(addr); + hash =3D decode_preamble_hash(addr, NULL); if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", addr, addr, 5, addr)) return -EINVAL; @@ -1283,6 +1311,7 @@ static int cfi_rewrite_preamble(s32 *sta =20 for (s =3D start; s < end; s++) { void *addr =3D (void *)s + *s; + int arity; u32 hash; =20 /* @@ -1293,7 +1322,7 @@ static int cfi_rewrite_preamble(s32 *sta if (!is_endbr(addr + 16)) continue; =20 - hash =3D decode_preamble_hash(addr); + hash =3D decode_preamble_hash(addr, &arity); if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", addr, addr, 5, addr)) return -EINVAL; @@ -1301,6 +1330,19 @@ static int cfi_rewrite_preamble(s32 *sta text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) !=3D 0x12345678); text_poke_early(addr + fineibt_preamble_hash, &hash, 4); + + WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity, + "kCFI preamble has wrong register at: %pS %*ph\n", + addr, 5, addr); + + if (!cfi_bhi || !arity) + continue; + + text_poke_early(addr + fineibt_preamble_bhi, + text_gen_insn(CALL_INSN_OPCODE, + addr + fineibt_preamble_bhi, + __bhi_args[arity]), + CALL_INSN_SIZE); } =20 return 0; @@ -1468,8 +1510,9 @@ static void __apply_fineibt(s32 *start_r cfi_rewrite_endbr(start_cfi, end_cfi); =20 if (builtin) { - pr_info("Using %sFineIBT CFI\n", - cfi_paranoid ? "paranoid " : ""); + pr_info("Using %sFineIBT%s CFI\n", + cfi_paranoid ? "paranoid " : "", + cfi_bhi ? "+BHI" : ""); } return; =20 @@ -1520,7 +1563,7 @@ static void poison_cfi(void *addr) /* * kCFI prefix should start with a valid hash. */ - if (!decode_preamble_hash(addr)) + if (!decode_preamble_hash(addr, NULL)) break; =20 /* @@ -1570,6 +1613,41 @@ static bool decode_fineibt_preamble(int } =20 /* + * regs->ip points to one of the UD2 in __bhi_args[]. + */ +static bool decode_fineibt_bhi(int ud_type, struct pt_regs *regs, + unsigned long *target, u32 *type) +{ + unsigned long addr; + u32 hash; + + /* + * Fetch the return address from the stack, this points to the + * FineIBT preamble. Since the CALL instruction is in the 5 last + * bytes of the preamble, the return address is in fact the target + * address. + */ + __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault); + *target =3D addr; + + addr -=3D fineibt_preamble_size; + if (!exact_endbr((void *)addr)) + return false; + + __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault); + *type =3D (u32)regs->r10 + hash; + + /* + * The UD2 sites are constructed with a RET immediately following, + * as such the non-fatal case can use the regular fixup. + */ + return true; + +Efault: + return false; +} + +/* * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_= start[] * sequence. */ @@ -1598,6 +1676,9 @@ bool decode_fineibt_insn(int ud_type, st { if (ud_type =3D=3D BUG_LOCK) return decode_fineibt_paranoid(ud_type, regs, target, type); + if (regs->ip > (unsigned long)__bhi_args && + regs->ip < (unsigned long)__bhi_args_end) + return decode_fineibt_bhi(ud_type, regs, target, type); return decode_fineibt_preamble(ud_type, regs, target, type); } =20 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -410,15 +410,20 @@ static void emit_nops(u8 **pprog, int le * Emit the various CFI preambles, see asm/cfi.h and the comments about Fi= neIBT * in arch/x86/kernel/alternative.c */ +static int emit_call(u8 **prog, void *func, void *ip); =20 -static void emit_fineibt(u8 **pprog, u32 hash) +static void emit_fineibt(u8 **pprog, u8 *ip, u32 hash, int arity) { u8 *prog =3D *pprog; =20 EMIT_ENDBR(); EMIT3_off32(0x41, 0x81, 0xea, hash); /* subl $hash, %r10d */ - EMIT2(0x75, 0xf9); /* jne.d8 .-7 */ - EMIT3(0x0f, 0x1f, 0x00); /* nop3 */ + if (cfi_bhi) { + emit_call(&prog, __bhi_args[arity], ip + 11); + } else { + EMIT2(0x75, 0xf9); /* jne.d8 .-7 */ + EMIT3(0x0f, 0x1f, 0x00); /* nop3 */ + } EMIT_ENDBR_POISON(); =20 *pprog =3D prog; @@ -447,13 +452,13 @@ static void emit_kcfi(u8 **pprog, u32 ha *pprog =3D prog; } =20 -static void emit_cfi(u8 **pprog, u32 hash) +static void emit_cfi(u8 **pprog, u8 *ip, u32 hash, int arity) { u8 *prog =3D *pprog; =20 switch (cfi_mode) { case CFI_FINEIBT: - emit_fineibt(&prog, hash); + emit_fineibt(&prog, ip, hash, arity); break; =20 case CFI_KCFI: @@ -504,13 +509,17 @@ static void emit_prologue_tail_call(u8 * * bpf_tail_call helper will skip the first X86_TAIL_CALL_OFFSET bytes * while jumping to another program */ -static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, +static void emit_prologue(u8 **pprog, u8 *ip, u32 stack_depth, bool ebpf_f= rom_cbpf, bool tail_call_reachable, bool is_subprog, bool is_exception_cb) { u8 *prog =3D *pprog; =20 - emit_cfi(&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash); + if (is_subprog) { + emit_cfi(&prog, ip, cfi_bpf_subprog_hash, 5); + } else { + emit_cfi(&prog, ip, cfi_bpf_hash, 1); + } /* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ @@ -1479,7 +1488,7 @@ static int do_jit(struct bpf_prog *bpf_p =20 detect_reg_usage(insn, insn_cnt, callee_regs_used); =20 - emit_prologue(&prog, stack_depth, + emit_prologue(&prog, image, stack_depth, bpf_prog_was_classic(bpf_prog), tail_call_reachable, bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb); /* Exception callback will clobber callee regs for its own use, and @@ -3046,7 +3055,9 @@ static int __arch_prepare_bpf_trampoline /* * Indirect call for bpf_struct_ops */ - emit_cfi(&prog, cfi_get_func_hash(func_addr)); + emit_cfi(&prog, image, + cfi_get_func_hash(func_addr), + cfi_get_func_arity(func_addr)); } else { /* * Direct-call fentry stub, as such it needs accounting for the From nobody Wed Dec 17 04:15:36 2025 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E9F351FE44E for ; Wed, 19 Feb 2025 16:36:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; cv=none; b=P92Ftb4JgxYI7W/FFTdoUxg/7Xv7L1LQVvSuMGkESk2DdtT1gR+lup+tpWvvkAE1UPXjxYdfq/E95mGMUmM5CeL17sLN3FOTAqFMuykwVWu7x/e6pL93L1oWqX5OwCdTVtw6aTAD9j6bYKMwUvF1WJanmKu2jV3+yJLxPrwzF4o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1739983012; c=relaxed/simple; bh=XH3MuXAiObT15pcf8fCzwAFGPy5FlpxKu3WYPUsMAT0=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=t/Gbn6oNjUyd43ZNpIAkzlDINVN4U6Qe6ulUAYI9haRq6JOC9Na7vcgwN3+cnmuJ4SONLpNSJoIIzAywNR0xLCyNrLuH3V+AVEJNmZWlx5Z11Egg1kxw1ZMnuM0/WYP/6NlbWgnuBIbu3i8xvo3WS2CA3cRU5vPb7lP6nPcHQ9c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=FWS30tB5; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="FWS30tB5" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=PiM728l6AQnzOFGFj6jdFclsn5t3z/jGFxhKy5mQQoA=; b=FWS30tB5oNIVLSFr17j7W9M5mv iejn8xIniPiuKybqGO7uDy2IqKRMZxKfngiyzaw86whWdPXpFm1C9G3EIeSkZlqEBplS/D5SWmG34 9imVnrG3fCKxmBQLYl1fxqMhUaXtkfafOv5oDbumapMo8JQBcaJi0j2psxJOcAX/RstOJ3EaVk48/ j8ny9lIqZckxqLjImA09XgpVGwuUG6lgWThpg5vRCvqdXVzTXGwhjklJJs6I1G7Or9kG79I+/9XJK TsggGE5n8hD6fxE+fveGA+H/OsKoZeJhphz8hqf6+GzvSL8ECJ6ZF7RqkR++c8aGh0PGm8dajMCAe OtuB0QVw==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.98 #2 (Red Hat Linux)) id 1tkn41-00000002HAX-065R; Wed, 19 Feb 2025 16:36:45 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 19081300201; Wed, 19 Feb 2025 17:36:43 +0100 (CET) Message-ID: <20250219163515.368430004@infradead.org> User-Agent: quilt/0.66 Date: Wed, 19 Feb 2025 17:21:17 +0100 From: Peter Zijlstra To: x86@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, alyssa.milburn@intel.com, scott.d.constable@intel.com, joao@overdrivepizza.com, andrew.cooper3@citrix.com, jpoimboe@kernel.org, jose.marchesi@oracle.com, hjl.tools@gmail.com, ndesaulniers@google.com, samitolvanen@google.com, nathan@kernel.org, ojeda@kernel.org, kees@kernel.org, alexei.starovoitov@gmail.com, mhiramat@kernel.org, jmill@asu.edu Subject: [PATCH v3 10/10] x86/ibt: Optimize fineibt-bhi arity 1 case References: <20250219162107.880673196@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Saves a CALL to an out-of-line thunk for the common case of 1 argument. Suggested-by: Scott Constable Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook --- arch/x86/include/asm/ibt.h | 4 ++ arch/x86/kernel/alternative.c | 61 +++++++++++++++++++++++++++++++++++--= ----- 2 files changed, 56 insertions(+), 9 deletions(-) --- a/arch/x86/include/asm/ibt.h +++ b/arch/x86/include/asm/ibt.h @@ -70,6 +70,10 @@ static inline bool __is_endbr(u32 val) if (val =3D=3D gen_endbr_poison()) return true; =20 + /* See cfi_fineibt_bhi_preamble() */ + if (IS_ENABLED(CONFIG_FINEIBT_BHI) && val =3D=3D 0x001f0ff5) + return true; + val &=3D ~0x01000000U; /* ENDBR32 -> ENDBR64 */ return val =3D=3D gen_endbr(); } --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1299,6 +1299,55 @@ static int cfi_rand_preamble(s32 *start, return 0; } =20 +static void cfi_fineibt_bhi_preamble(void *addr, int arity) +{ + bool warn =3D IS_ENABLED(CONFIG_CFI_PREMISSIVE) || cfi_warn; + + if (!arity) + return; + + if (!warn && arity =3D=3D 1) { + /* + * Crazy scheme to allow arity-1 inline: + * + * __cfi_foo: + * 0: f3 0f 1e fa endbr64 + * 4: 41 81 78 56 34 12 sub 0x12345678, %r10d + * b: 49 0f 45 fa cmovne %r10, %rdi + * f: 75 f5 jne __cfi_foo+6 + * 11: 0f 1f 00 nopl (%rax) + * + * Code that direct calls to foo()+0, decodes the tail end as: + * + * foo: + * 0: f5 cmc + * 1: 0f 1f 00 nopl (%rax) + * + * which clobbers CF, but does not affect anything ABI + * wise. + * + * Notably, this scheme is incompatible with permissive CFI + * because the cmov is unconditional and RDI will have been + * clobbered. + */ + const u8 magic[9] =3D { + 0x49, 0x0f, 0x45, 0xfa, + 0x75, 0xf5, + BYTES_NOP3, + }; + + text_poke_early(addr + fineibt_preamble_bhi, magic, 9); + + return; + } + + text_poke_early(addr + fineibt_preamble_bhi, + text_gen_insn(CALL_INSN_OPCODE, + addr + fineibt_preamble_bhi, + __bhi_args[arity]), + CALL_INSN_SIZE); +} + static int cfi_rewrite_preamble(s32 *start, s32 *end) { s32 *s; @@ -1329,14 +1378,8 @@ static int cfi_rewrite_preamble(s32 *sta "kCFI preamble has wrong register at: %pS %*ph\n", addr, 5, addr); =20 - if (!cfi_bhi || !arity) - continue; - - text_poke_early(addr + fineibt_preamble_bhi, - text_gen_insn(CALL_INSN_OPCODE, - addr + fineibt_preamble_bhi, - __bhi_args[arity]), - CALL_INSN_SIZE); + if (cfi_bhi) + cfi_fineibt_bhi_preamble(addr, arity); } =20 return 0; @@ -1349,7 +1392,7 @@ static void cfi_rewrite_endbr(s32 *start for (s =3D start; s < end; s++) { void *addr =3D (void *)s + *s; =20 - if (!is_endbr(addr + 16)) + if (!exact_endbr(addr + 16)) continue; =20 poison_endbr(addr + 16);