From nobody Wed Sep 17 15:42:36 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9A0C2C4332F for ; Sat, 17 Dec 2022 18:58:27 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230118AbiLQS6Y (ORCPT ); Sat, 17 Dec 2022 13:58:24 -0500 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:41118 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230125AbiLQSzu (ORCPT ); Sat, 17 Dec 2022 13:55:50 -0500 Received: from galois.linutronix.de (Galois.linutronix.de [IPv6:2a0a:51c0:0:12e:550::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 70E662708; Sat, 17 Dec 2022 10:55:40 -0800 (PST) Date: Sat, 17 Dec 2022 18:55:35 -0000 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linutronix.de; s=2020; t=1671303336; h=from:from:sender:sender:reply-to:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=sVZZ7CWMbBm6slTM99NemKcM0c2Hyx5EueEjneako/4=; b=PyQlXk9yYtv1Crp6Vclt9gZA1t2AMSLGV0oYbJCkXqApHNdZELiifZNeBKEDy9H30tU4+T qgeGpDSVVl7/uLNjMnMS1N/iL9cvQ29JhBMY8QQwiTQd3oauYeNYxAX79Ze3rOh4UxeRdB 1X2IbCpD7U2kwW0+iZdlv+5xpPzG56j0h/FKexbOc4wdwHpZEyLzUh9pqsM23ZU7vYBaFd cjjnp9ijIVL0VcdwICXk7xha5ODKYKiIa21SwlMpsbRJ7MmINPNQHFo0G6LBoNoOAzWt02 ihGKh2ryzlWYULgJB6KNa/dMF/n2YKsHrdq4yxvK4+Jk8dQ+bvTwJyTEhZ24rw== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=linutronix.de; s=2020e; t=1671303336; h=from:from:sender:sender:reply-to:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=sVZZ7CWMbBm6slTM99NemKcM0c2Hyx5EueEjneako/4=; b=ZyI7KXOjFXOWIEcYB7KHHhuRJtwjSuNEqh4+zIEMp0g0AUtSXnkq/MRLe+OADm2j6Xx+CA /GsdjIVa4IvdHMBQ== From: "tip-bot2 for Peter Zijlstra" Sender: tip-bot2@linutronix.de Reply-to: linux-kernel@vger.kernel.org To: linux-tip-commits@vger.kernel.org Subject: [tip: x86/mm] x86/mm: Randomize per-cpu entry area Cc: Seth Jenkins , Kees Cook , "Peter Zijlstra (Intel)" , Dave Hansen , Borislav Petkov , x86@kernel.org, linux-kernel@vger.kernel.org MIME-Version: 1.0 Message-ID: <167130333595.4906.6044970417672298869.tip-bot2@tip-bot2> Robot-ID: Robot-Unsubscribe: Contact to get blacklisted from these emails Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org The following commit has been merged into the x86/mm branch of tip: Commit-ID: 97e3d26b5e5f371b3ee223d94dd123e6c442ba80 Gitweb: https://git.kernel.org/tip/97e3d26b5e5f371b3ee223d94dd123e6c= 442ba80 Author: Peter Zijlstra AuthorDate: Thu, 27 Oct 2022 14:54:41 -07:00 Committer: Dave Hansen CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00 x86/mm: Randomize per-cpu entry area Seth found that the CPU-entry-area; the piece of per-cpu data that is mapped into the userspace page-tables for kPTI is not subject to any randomization -- irrespective of kASLR settings. On x86_64 a whole P4D (512 GB) of virtual address space is reserved for this structure, which is plenty large enough to randomize things a little. As such, use a straight forward randomization scheme that avoids duplicates to spread the existing CPUs over the available space. [ bp: Fix le build. ] Reported-by: Seth Jenkins Reviewed-by: Kees Cook Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Dave Hansen Signed-off-by: Borislav Petkov --- arch/x86/include/asm/cpu_entry_area.h | 4 +-- arch/x86/include/asm/pgtable_areas.h | 8 ++++- arch/x86/kernel/hw_breakpoint.c | 2 +- arch/x86/mm/cpu_entry_area.c | 46 +++++++++++++++++++++++--- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/cpu_entry_area.h b/arch/x86/include/asm/c= pu_entry_area.h index 75efc4c..462fc34 100644 --- a/arch/x86/include/asm/cpu_entry_area.h +++ b/arch/x86/include/asm/cpu_entry_area.h @@ -130,10 +130,6 @@ struct cpu_entry_area { }; =20 #define CPU_ENTRY_AREA_SIZE (sizeof(struct cpu_entry_area)) -#define CPU_ENTRY_AREA_ARRAY_SIZE (CPU_ENTRY_AREA_SIZE * NR_CPUS) - -/* Total size includes the readonly IDT mapping page as well: */ -#define CPU_ENTRY_AREA_TOTAL_SIZE (CPU_ENTRY_AREA_ARRAY_SIZE + PAGE_SIZE) =20 DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area); DECLARE_PER_CPU(struct cea_exception_stacks *, cea_exception_stacks); diff --git a/arch/x86/include/asm/pgtable_areas.h b/arch/x86/include/asm/pg= table_areas.h index d34cce1..4f056fb 100644 --- a/arch/x86/include/asm/pgtable_areas.h +++ b/arch/x86/include/asm/pgtable_areas.h @@ -11,6 +11,12 @@ =20 #define CPU_ENTRY_AREA_RO_IDT_VADDR ((void *)CPU_ENTRY_AREA_RO_IDT) =20 -#define CPU_ENTRY_AREA_MAP_SIZE (CPU_ENTRY_AREA_PER_CPU + CPU_ENTRY_AREA_= ARRAY_SIZE - CPU_ENTRY_AREA_BASE) +#ifdef CONFIG_X86_32 +#define CPU_ENTRY_AREA_MAP_SIZE (CPU_ENTRY_AREA_PER_CPU + \ + (CPU_ENTRY_AREA_SIZE * NR_CPUS) - \ + CPU_ENTRY_AREA_BASE) +#else +#define CPU_ENTRY_AREA_MAP_SIZE P4D_SIZE +#endif =20 #endif /* _ASM_X86_PGTABLE_AREAS_H */ diff --git a/arch/x86/kernel/hw_breakpoint.c b/arch/x86/kernel/hw_breakpoin= t.c index 668a4a6..bbb0f73 100644 --- a/arch/x86/kernel/hw_breakpoint.c +++ b/arch/x86/kernel/hw_breakpoint.c @@ -266,7 +266,7 @@ static inline bool within_cpu_entry(unsigned long addr,= unsigned long end) =20 /* CPU entry erea is always used for CPU entry */ if (within_area(addr, end, CPU_ENTRY_AREA_BASE, - CPU_ENTRY_AREA_TOTAL_SIZE)) + CPU_ENTRY_AREA_MAP_SIZE)) return true; =20 /* diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c index d7081b1..dff9001 100644 --- a/arch/x86/mm/cpu_entry_area.c +++ b/arch/x86/mm/cpu_entry_area.c @@ -16,16 +16,53 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_p= age, entry_stack_storage) #ifdef CONFIG_X86_64 static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stac= ks); DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks); -#endif =20 -#ifdef CONFIG_X86_32 +static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, _cea_offset); + +static __always_inline unsigned int cea_offset(unsigned int cpu) +{ + return per_cpu(_cea_offset, cpu); +} + +static __init void init_cea_offsets(void) +{ + unsigned int max_cea; + unsigned int i, j; + + max_cea =3D (CPU_ENTRY_AREA_MAP_SIZE - PAGE_SIZE) / CPU_ENTRY_AREA_SIZE; + + /* O(sodding terrible) */ + for_each_possible_cpu(i) { + unsigned int cea; + +again: + cea =3D prandom_u32_max(max_cea); + + for_each_possible_cpu(j) { + if (cea_offset(j) =3D=3D cea) + goto again; + + if (i =3D=3D j) + break; + } + + per_cpu(_cea_offset, i) =3D cea; + } +} +#else /* !X86_64 */ DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack); + +static __always_inline unsigned int cea_offset(unsigned int cpu) +{ + return cpu; +} +static inline void init_cea_offsets(void) { } #endif =20 /* Is called from entry code, so must be noinstr */ noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu) { - unsigned long va =3D CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE; + unsigned long va =3D CPU_ENTRY_AREA_PER_CPU + cea_offset(cpu) * CPU_ENTRY= _AREA_SIZE; BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE !=3D 0); =20 return (struct cpu_entry_area *) va; @@ -211,7 +248,6 @@ static __init void setup_cpu_entry_area_ptes(void) =20 /* The +1 is for the readonly IDT: */ BUILD_BUG_ON((CPU_ENTRY_AREA_PAGES+1)*PAGE_SIZE !=3D CPU_ENTRY_AREA_MAP_S= IZE); - BUILD_BUG_ON(CPU_ENTRY_AREA_TOTAL_SIZE !=3D CPU_ENTRY_AREA_MAP_SIZE); BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK); =20 start =3D CPU_ENTRY_AREA_BASE; @@ -227,6 +263,8 @@ void __init setup_cpu_entry_areas(void) { unsigned int cpu; =20 + init_cea_offsets(); + setup_cpu_entry_area_ptes(); =20 for_each_possible_cpu(cpu)