From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B6D2E511229 for ; Fri, 18 Sep 2026 16:14:19 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748062; cv=none; b=NVuRI3gUERcvWnAmvZmzy+/NGIjeHeYVujJU3H+ii03fp0/RolERadqpvhzOySI91fLOuWB4EYySf8hrg3hj30CvLtk9wfsoR7Heqm1fVle4k5awhuYQs3w5hdCYpqZATQzmI9EakMKyeNutTrklmHT/Q6Mp4K7/JSCvSAmcj70= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748062; c=relaxed/simple; bh=jEHOyThQNVQGH87GswmHtKYBQUn0el5qwv3lK3LootU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=GZ+MTTjrnaRsGd5hY/9O8eIg3nAKVLgTbpPQ6z0YXvf4kTtQ+C/WNRz7BoJ+RLGdwfIxVD9tbcdpuy6YaOPhTu/xUu1Wz73hAcHKQ++Lixy1XeRtx+MZpwgbWXcz64Ab5UtzGk5M7V0t7iTcCr4U0cjiV8fFXbtM5ApHZJimr9c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=iJ/U7tCM; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="iJ/U7tCM" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4F4BF1F00898; Fri, 18 Sep 2026 16:14:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748059; bh=g05Ard4UsBeemIxWUEUKYYZHiVLLsuvwUe2V0GrB124=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=iJ/U7tCMmdcODZval+eCkvTcE/eHYCD/sk5kfMZJcWywfaUd5spljQRtyBnx2LLcr mvOYukQVMrnMRhxDtFtAYpGoMnhyOa7KbHNsFb9umxVfLqPyWmp5/2Vu8aE5v2DDei 4M9OVXmoEijL8ztroDyvYwqaxwDMFKJu2KhtG39A3rhg19Isk/1Z6p0IuD81zB/wPT /8ysYLuZ90um3OuJCNfUbo159L37cfQ3cqkB2bAsIcRoFhlmGldQFENq0osA/ouRW+ xF5jmImyd7jDelonalvOm/5ryybQ6GNT/08JxX9Wo0BiifR4WxmcWmI6ABZRT4fGBw 8Q9a6i4QiIZFg== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier , Jinjie Ruan Subject: [PATCH v2 01/21] arm64: entry: Defer setting of TPIDRRO_EL0 until exit to userspace Date: Fri, 18 Sep 2026 17:13:45 +0100 Message-ID: <20260918161407.2300-2-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation for using TPIDRRO_EL0 to point at 'current' while running inside the kernel, defer its userspace initialisation from the context-switch patch to exception return. This has the added benefit of not having to worry about keeping the in-memory value and the register value in-sync during preemptible sections. Cc: Mark Rutland Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/asm-offsets.c | 1 + arch/arm64/kernel/entry.S | 4 +++- arch/arm64/kernel/process.c | 16 +--------------- arch/arm64/kernel/sys_compat.c | 7 ------- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offset= s.c index 9c853ed3ceab..6038ab3beb25 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -42,6 +42,7 @@ int main(void) #endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_conte= xt)); + DEFINE(THREAD_TP_VALUE, offsetof(struct task_struct, thread.uw.tp_value)= ); DEFINE(THREAD_SCTLR_USER, offsetof(struct task_struct, thread.sctlr_user= )); #ifdef CONFIG_ARM64_PTR_AUTH DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user)); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index f63049ac32dc..59f045e496ec 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -47,7 +47,6 @@ b .Lskip_tramp_vectors_cleanup\@ .if \regsize =3D=3D 64 mrs x30, tpidrro_el0 - msr tpidrro_el0, xzr .else mov x30, xzr .endif @@ -360,9 +359,12 @@ alternative_else_nop_endif .if \el =3D=3D 0 ldr x23, [sp, #S_SP] // load return stack pointer msr sp_el0, x23 + msr tpidrro_el0, xzr tst x22, #PSR_MODE32_BIT // native task? b.eq 3f =20 + ldr x0, [tsk, #THREAD_TP_VALUE] + msr tpidrro_el0, x0 #ifdef CONFIG_ARM64_ERRATUM_845719 alternative_if ARM64_WORKAROUND_845719 #ifdef CONFIG_PID_IN_CONTEXTIDR diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 581f80e9b9b7..bfdc12166895 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -256,17 +256,8 @@ static void tls_thread_flush(void) if (system_supports_tpidr2()) write_sysreg_s(0, SYS_TPIDR2_EL0); =20 - if (is_compat_task()) { + if (is_compat_task()) current->thread.uw.tp_value =3D 0; - - /* - * We need to ensure ordering between the shadow state and the - * hardware state, so that we don't corrupt the hardware state - * with a stale shadow state during context switch. - */ - barrier(); - write_sysreg(0, tpidrro_el0); - } } =20 static void flush_tagged_addr_state(void) @@ -531,11 +522,6 @@ static void tls_thread_switch(struct task_struct *next) { tls_preserve_current_state(); =20 - if (is_compat_thread(task_thread_info(next))) - write_sysreg(next->thread.uw.tp_value, tpidrro_el0); - else - write_sysreg(0, tpidrro_el0); - write_sysreg(*task_user_tls(next), tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(next->thread.tpidr2_el0, SYS_TPIDR2_EL0); diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 0451f96c2c3f..59eee301c6f4 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -89,13 +89,6 @@ long compat_arm_syscall(struct pt_regs *regs, int scno) =20 case __ARM_NR_compat_set_tls: current->thread.uw.tp_value =3D regs->regs[0]; - - /* - * Protect against register corruption from context switch. - * See comment in tls_thread_flush. - */ - barrier(); - write_sysreg(regs->regs[0], tpidrro_el0); return 0; =20 default: --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6186E502D66 for ; Fri, 18 Sep 2026 16:14:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748064; cv=none; b=KzXVplG6vLOJrXgdE6n3L+Gi20nTDWdOwgcrn4F3Mb0i+CMIQWgN6Wu7PFKxWkIuOEKby/hB46S1xrGmo/ulR/be2txoa91fDth4I3FfEdSzCI9vdrJUZuP3xS4eSurzPoIAv1axiiEYpvjnOd84dtjsxc1jVX/EHcVlBcoMz44= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748064; c=relaxed/simple; bh=GmxteVjU9qYXSGVJEqSdzMCASmIFnZ0iHlTvWcHzzTM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ldfszVIV8ZF2MTgX93lDrBMgvN2wf7ykl/t5f21g1rup8BfARALdVhuW4yfPbMRQKOzcVh3TLOFSldehXR7qQb22/Ax/2HtoB2YrDq3IuG0n3HVz1PkGjzd0+9AY3kDjOk8j83cT/aY49jOI94jNnPxTUNEBaxcRUiry6hiGI+E= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=iO+7mL8M; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="iO+7mL8M" Received: by smtp.kernel.org (Postfix) with ESMTPSA id ECA0B1F000FF; Fri, 18 Sep 2026 16:14:19 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748062; bh=i5WKKe+LbffkRYv2t0YaLabQ2JHUtS1RbnFSIgfXllw=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=iO+7mL8MmOAWCnP2p3vRnFMM5ydmWe2icNPmTsVcjHbvOU4rLqVzrXS1I1k9+MNA/ stDBU1VLeerXsSEA79KxaRJfyuseqKhc/45qU5CE8iArvQfC1cex21xq4Ln+N7GlkO AI+9Du1UY0pbV2grKzZGatPldtNLu9TRu2KXF9Oy5rp6aobHqB+OeDZW+0JPi7yp7R CWpM7VuSdumgLkcopP9B4/1sMj3oKRkahBkCJCb1REtr5b5FW40SjT8wUAlXCwFNrf hYh04brISd5qnnXlIVIOC2dohJl4KOUP0BFyvi00G5LYlpq/6vqN2VXUPQ3Lm2V2gd Hh8dZa1uEyOrA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 02/21] arm64: entry: Only check for stack overflow on exceptions from EL1 Date: Fri, 18 Sep 2026 17:13:46 +0100 Message-ID: <20260918161407.2300-3-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Checking for kernel stack overflow on entry from EL0 is largely pointless and gets in the way of using SPSel to switch to the overflow stack. If the kernel stack overflows on entry from userspace, that implies that it was unbalanced on the last exception return which means we can't sensibly reason about the state of the system given that we could've returned to an arbitrary virtual address in an arbitrary mode. Remove the stack overflow check from the EL0 exception entry path. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/kernel/entry.S | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 59f045e496ec..b5d8277f608a 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -54,6 +54,8 @@ .endif =20 sub sp, sp, #PT_REGS_SIZE + + .if \el =3D=3D 1 /* * Test whether the SP has overflowed, without corrupting a GPR. * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT) @@ -64,8 +66,11 @@ tbnz x0, #THREAD_SHIFT, 0f sub x0, sp, x0 // x0'' =3D sp' - x0' =3D (sp + x0) - sp =3D x0 sub sp, sp, x0 // sp'' =3D sp' - x0 =3D (sp + x0) - x0 =3D sp + .endif + b el\el\ht\()_\regsize\()_\label =20 + .if \el =3D=3D 1 0: /* * Either we've just detected an overflow, or we've taken an exception @@ -96,6 +101,7 @@ sub sp, sp, x0 mrs x0, tpidrro_el0 b el\el\ht\()_\regsize\()_\label + .endif .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm =20 --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B25F251477F for ; Fri, 18 Sep 2026 16:14:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748067; cv=none; b=SOveDLpVgkP/OdSgCpWfRri7j9xYSkcRrMbPrR5I5icPdarmG9fl4Y9J/DtZNdk99nFPut6Cmr6ql23Pc/yelaDbxX+3nGoOQFbDsz6E4stvUdzphD/U7rieMQCVDA6f87bjRIyGHBFz8YfmBzdFrk3B3HJrsjxlRnapXbbiOEQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748067; c=relaxed/simple; bh=SjZ18QX/2N+ATPjK2ecF/18E9fYVn7LnNkzfiyyaSpc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=LFc+6hB3s0rm7NJP44o4vwJK1nPMAjuH2lLF/eXfnWihYq1qkenGDbUcswb1+ae8Oh9oRhpe7VRQEAeHzvPEeFPeQt3g0XRN2WgjPiMIC8eDq60PyPkxXwDf7HemN+zw5mERoj28CeJI4GmH/ldS4w++2Ux3VUBEo2tNXLfCvAg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=LYefk78/; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="LYefk78/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6D3291F0089B; Fri, 18 Sep 2026 16:14:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748066; bh=lbrobJz80vYx6iQgmyTnPoDjm5gh3FY+/PSyqHaMdQM=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=LYefk78/b4y9UWbTdqqO/keF41dagr6o96GY1C4WOOV7R67uxQXxrUKS8tTbN/Kyu Hs6DLLoPzOkR+P+iHexhfDJL4GDd8X4us/9HbIL6hODwQ0GH3KIqd7bORm6xD7K9Xe vkOCUDo1X/RElORdScxyUerbJgh8vLKurXlLstcVK51hh58CwCQ/8JDo3kqOD2qzPN SoJeozoy2NERbXs8ia1k+PF6RqrdK4PTEoGCGiYzDaoVyX9Z1gUg5cFc+VYxY3mfV5 IXaZRyqIBQsbtDclwqjpAPldl0sXXetkDkloc2e+UcVoke37rJc25s+DzLWXD3qb+Z 9Nzu88fR7MCwA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 03/21] arm64: stackprotector: Temporarily disable per-task stackprotector Date: Fri, 18 Sep 2026 17:13:47 +0100 Message-ID: <20260918161407.2300-4-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Per-task stackprotector relies on telling the compiler about the system register used to hold a pointer to the current task. As we're about to change that around, temporarily disable per-task stackprotector for now. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5a51b0ef944..b8270b301809 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2455,6 +2455,7 @@ config CC_HAVE_STACKPROTECTOR_SYSREG config STACKPROTECTOR_PER_TASK def_bool y depends on STACKPROTECTOR && CC_HAVE_STACKPROTECTOR_SYSREG + depends on BROKEN =20 config UNWIND_PATCH_PAC_INTO_SCS bool "Enable shadow call stack dynamically using code patching" --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5099B51588D for ; Fri, 18 Sep 2026 16:14:30 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748071; cv=none; b=nnWEHy8UWXcEgBuOc4OaqiIbzOAJh2Mw+nPaLWx9/Pyp2WIrZkMmf4+Vj9JCQHbV+aM9OhakdQTZh0LDhnGhvEfi74/V66EfJ0vzUHUmbWqwvJ0lZz8jpD+Y4zit0I5Lv2wFhAA+3MQepR0i3c1kxT5RcMSei1U4C4ezuvPo2+4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748071; c=relaxed/simple; bh=qskTDOTZ3X/eC+QtrSnYkRmfA8VXtYLdU8LszGaACHQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=FFbvKjB8SqmHExhPZM7W5dUxwOD5xTI8y8eneLRhI0wcqLyTHzlYruY/l8C/Do3uGmf5vp0xgL2ZQ9mQqtyaFbKg5iVXoSG3ZpSgCw8omLOpNJ/GrLK8flT01esyDnorqsnQiZvZMXYEm7N8sw/Q+Gf93IbyjUnn0i/RZRMnxyY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Edp8R+bB; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Edp8R+bB" Received: by smtp.kernel.org (Postfix) with ESMTPSA id F07361F000FF; Fri, 18 Sep 2026 16:14:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748069; bh=aHMoz1vySQvYeQW+hJXZKhFp+T//bo0AG/CTIQJNNJo=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Edp8R+bBPQqImx1+AnsYjmOnuRIOuZBNv9WinTHaoT2DaiptqJx27nbkcCY8+1zLU peekFHkhOw2J1WKN44ohH3bDuRpgH1lqV8GkyKcFKG1KjapgQHnyVDf0Sslq++JjFa 2kRTowqnTeUg37ewHZQ7yuWYaGpnjk4GrQ6UN/jeec3jsd/wCXHG8NF475UmF3JIPI +NqXQs5Mow6EX4NSCPmfNjqJmyTm5paYlW2mbLzrjMxhw/HWMXejN/5sONfC8hsm9g qKS8tHj98Eh8MQxxT0DUmDbARLdqk3QEXt+tCJIfBgSM+KErRj8J243E/rwLS/pYfG tY0NDIM1gutHg== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 04/21] arm64: bpf: Add support for generating reads of TPIDRRO_EL0 Date: Fri, 18 Sep 2026 17:13:48 +0100 Message-ID: <20260918161407.2300-5-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation for storing 'current' in TPIDRRO_EL0, add support for the instruction encoding of "MRS Rt, TPIDRRO_EL0" to the BPF JIT. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/include/asm/insn.h | 1 + arch/arm64/net/bpf_jit.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 1ce75a442638..0e739740513d 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -136,6 +136,7 @@ enum aarch64_insn_special_register { }; =20 enum aarch64_insn_system_register { + AARCH64_INSN_SYSREG_TPIDRRO_EL0 =3D 0x5e83, AARCH64_INSN_SYSREG_TPIDR_EL1 =3D 0x4684, AARCH64_INSN_SYSREG_TPIDR_EL2 =3D 0x6682, AARCH64_INSN_SYSREG_SP_EL0 =3D 0x4208, diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index b2fe6e6dcf44..df469a005373 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -331,6 +331,8 @@ aarch64_insn_gen_adr(0, offset, Rd, AARCH64_INSN_ADR_TYPE_ADR) =20 /* MRS */ +#define A64_MRS_TPIDRRO_EL0(Rt) \ + aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDRRO_EL0) #define A64_MRS_TPIDR_EL1(Rt) \ aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_TPIDR_EL1) #define A64_MRS_TPIDR_EL2(Rt) \ --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0C393513550 for ; Fri, 18 Sep 2026 16:14:33 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748075; cv=none; b=m205Bf57szdzwELfQuUvC33mFjVPc4XEIilaA7KtSAoGTSVzJjIsiVQyTsvO5kkN0rG1pYyD5+4rJ4ijnXSuLIk8R0R11PzhdkHetH09xB7ZS/fXv8TsFj2nvwapab8W+Ub5tFS1q+f+L31rxCrutkSwiht56EkyZuGuF4YcEfI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748075; c=relaxed/simple; bh=/MNpjrsAdeuGBr8lafXjVrXztBQBNQ9r8wMeiWGBFZI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ggqpM5b0xU5X5iBzBLPtkOTIxeq92Fmff/DBIQeQ84w+Zn8hY4eITJs0pY0kNGfR7/I5j14zXgEBnlx4q3js2ppNH+LE/4jdzg+sNCPa7UwGMBn+w9wVNyYtoUgxZRROK0UMeMRn/qrTLxOELBPpZAzWXLH91jLdxtFizsK3Wgo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Z2FrRP/u; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Z2FrRP/u" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 76C651F0089A; Fri, 18 Sep 2026 16:14:30 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748073; bh=2tB1KkXkaU5QmWVylv9lGv4ky5kgG3UAVpPmpa8Uw3I=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Z2FrRP/uuCgfSvknfub0VLAy0QKjf0HMfhws0VgGmRM40vLVDGdAQfPs0Fc8Ql4H9 xi1hQrIyyIi3VgRo+Qx0mLrMkRRauAt+aKSCSILzyoi+pbWICaAE8D33XDWiZ5DBBB DsofAeVnKxVJ0hcMm0LypJhxiy1eVgUwiMdpzPSVKUbQHiWbELkxUANc/+6rMR98Jm vKoJYf10/mdQ8lYdLlZoC//ub3yliJaIgYDssRyayNTAqO6Z4Jd/A3zgY+qlXT2IoU fVf13W1KBziWhUy2WtTRZVBDsAju8ljxebcLGO8/fCOQ+2clCC2RLvfqvcch8m6OzL TKsVdmCuJf5iQ== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 05/21] KVM: arm64: Protect TPIDRRO_EL0 across guest entry/exit Date: Fri, 18 Sep 2026 17:13:49 +0100 Message-ID: <20260918161407.2300-6-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Mostafa Saleh To prepare for using TPIDRRO_EL0 to hold 'current' and repurposing SP_EL0 as an overflow stack pointer, eagerly switch both registers when entering and exiting the guest. Signed-off-by: Mostafa Saleh Signed-off-by: Will Deacon --- arch/arm64/include/asm/kvm_asm.h | 10 ++++++++++ arch/arm64/kernel/asm-offsets.c | 1 + arch/arm64/kvm/hyp/entry.S | 12 ++++++++---- arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 2 -- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_= asm.h index e5b92ac09e69..1cfe9216a2ff 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -408,6 +408,16 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr,= u64 spsr, u64 elr_virt, msr sp_el0, \tmp .endm =20 +.macro save_tpidrro_el0 ctxt, tmp + mrs \tmp, tpidrro_el0 + str \tmp, [\ctxt, #CPU_TPIDRRO_EL0_OFFSET] +.endm + +.macro restore_tpidrro_el0 ctxt, tmp + ldr \tmp, [\ctxt, #CPU_TPIDRRO_EL0_OFFSET] + msr tpidrro_el0, \tmp +.endm + #endif =20 #endif /* __ARM_KVM_ASM_H__ */ diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offset= s.c index 6038ab3beb25..cb8448cc5f51 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -108,6 +108,7 @@ int main(void) DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2)); DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs)); DEFINE(CPU_ELR_EL2, offsetof(struct kvm_cpu_context, sys_regs[ELR_EL2])= ); + DEFINE(CPU_TPIDRRO_EL0_OFFSET, offsetof(struct kvm_cpu_context, sys_regs= [TPIDRRO_EL0])); DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1= ])); DEFINE(CPU_GCR_EL1, offsetof(struct kvm_cpu_context, sys_regs[GCR_EL1])= ); DEFINE(CPU_APIAKEYLO_EL1, offsetof(struct kvm_cpu_context, sys_regs[APIA= KEYLO_EL1])); diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 308100ed25de..4c89931a6a92 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -30,8 +30,9 @@ SYM_FUNC_START(__guest_enter) // Store the hyp regs save_callee_saved_regs x1 =20 - // Save hyp's sp_el0 + // Save hyp's sp_el0 and tpidrro_el0 save_sp_el0 x1, x2 + save_tpidrro_el0 x1, x2 =20 // Now the hyp state is stored if we have a pending RAS SError it must // affect the host or hyp. If any asynchronous exception is pending we @@ -66,8 +67,9 @@ alternative_else_nop_endif // when this feature is enabled for kernel code. ptrauth_switch_to_guest x29, x0, x1, x2 =20 - // Restore the guest's sp_el0 + // Restore the guest's sp_el0 and tpidrro_el0 restore_sp_el0 x29, x0 + restore_tpidrro_el0 x29, x0 =20 // Restore guest regs x0-x17 ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)] @@ -146,8 +148,9 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL) // Store the guest regs x18-x29, lr save_callee_saved_regs x1 =20 - // Store the guest's sp_el0 + // Store the guest's sp_el0 and tpidrro_el0 save_sp_el0 x1, x2 + save_tpidrro_el0 x1, x2 =20 adr_this_cpu x2, kvm_hyp_ctxt, x3 =20 @@ -161,8 +164,9 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL) // mte_switch_to_hyp(g_ctxt, h_ctxt, reg1) mte_switch_to_hyp x1, x2, x3 =20 - // Restore hyp's sp_el0 + // Restore hyp's sp_el0 and tpidrro_el0 restore_sp_el0 x2, x3 + restore_tpidrro_el0 x2, x3 =20 // Now restore the hyp regs restore_callee_saved_regs x2 diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hy= p/include/hyp/sysreg-sr.h index c382848d3194..b5f7ef8e6371 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -66,7 +66,6 @@ static inline void __sysreg_save_common_state(struct kvm_= cpu_context *ctxt) static inline void __sysreg_save_user_state(struct kvm_cpu_context *ctxt) { ctxt_sys_reg(ctxt, TPIDR_EL0) =3D read_sysreg(tpidr_el0); - ctxt_sys_reg(ctxt, TPIDRRO_EL0) =3D read_sysreg(tpidrro_el0); } =20 static inline bool ctxt_has_mte(struct kvm_cpu_context *ctxt) @@ -209,7 +208,6 @@ static inline void __sysreg_restore_common_state(struct= kvm_cpu_context *ctxt) static inline void __sysreg_restore_user_state(struct kvm_cpu_context *ctx= t) { write_sysreg(ctxt_sys_reg(ctxt, TPIDR_EL0), tpidr_el0); - write_sysreg(ctxt_sys_reg(ctxt, TPIDRRO_EL0), tpidrro_el0); } =20 static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt, --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 36D43513576 for ; Fri, 18 Sep 2026 16:14:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748078; cv=none; b=bCIjKxz7N2Pltg7m3rOzoTgEbOEWsrbnNPzjU0087WeJ/iw5o3mmyYQItZK6/Gy5vPbsLvp3r34f3Qf1vU70rtROPPaNVjEEDnkjWPJ5jfTyfWGApHbvtFme3yqY5RQlgJVfDJGCyl+DjHlT/MU83kU8xR3+0hVXRbJV707fvRE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748078; c=relaxed/simple; bh=rdrfb96Fkz98P++HGolCmgeoVwevAMX6ja0kNnSFm7Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Gw6m+f2Kw5wYMy9ecbWFqJ50mLnFB2GAKCiD6wohef9Po2H+oP2uot3UX77e8g8NPSWX1ChM8JtoJJerap68WdmYOglg7038/xot29SHu6p7nK5WkgAkbFYiVBi9TwzOQ+EqFalG0RETwZe2FLxWo6PjsQKblmQUWTLdrD1jTNk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=mBzSpswB; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="mBzSpswB" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E92211F000FF; Fri, 18 Sep 2026 16:14:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748076; bh=Yj3uQfPL5QpQaijVWcdGBfiFx0mvGXpme5FKU6Df8+c=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=mBzSpswBxzlDApkp72n3j6wu4B2CAB+jfjCHubUEJXTyehUqxSy2kpHopSw1bN1IH nZvtzJAkLglbinbIcSexhc4AbjWsR7chcrWqX5zEg9hdor6sJZpE95wNy3tVGOZ8+E sDZuGO2tEl1AWYsvbyWb2vaKwV6OtGijeH4tsgP33gpGGxY7slQNRqjmvSajeEKp1z Wunafa1SHHEmaAZIZoOVy1IMk6ikNjyTPTPADbAFXJTWneJzGQ9cuYpQb89rG8LF5/ ue+JBUX5RXJkC2JEwTLesyrDLKVFzTZSwXAsnYsPrNO+8f6fydGm4qDZDAp5Sc1pIO g8+23RlLf/1vg== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 06/21] arm64: Store 'current' in TPIDRRO_EL0 instead of SP_EL0 Date: Fri, 18 Sep 2026 17:13:50 +0100 Message-ID: <20260918161407.2300-7-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" To make SP_EL0 available to point at the per-cpu kernel overflow stack, we must relocate 'current' elsewhere. With TPIDRRO_EL0 now restored late in the return-to-user path, we can otherwise use it to hold 'current' and update the various accessors accordingly. Move 'current' from SP_EL0 to TPIDRRO_EL0. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/include/asm/assembler.h | 2 +- arch/arm64/include/asm/current.h | 6 +++--- arch/arm64/kernel/entry.S | 20 ++++++++++---------- arch/arm64/kernel/head.S | 2 +- arch/arm64/kernel/process.c | 5 +++-- arch/arm64/mm/proc.S | 6 +++++- arch/arm64/net/bpf_jit_comp.c | 4 ++-- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/as= sembler.h index 0b58b550e8dc..fd1ae5337935 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -587,7 +587,7 @@ alternative_else_nop_endif * Return the current task_struct. */ .macro get_current_task, rd - mrs \rd, sp_el0 + mrs \rd, tpidrro_el0 .endm =20 /* diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/curr= ent.h index c92912eaf186..8c7efac05394 100644 --- a/arch/arm64/include/asm/current.h +++ b/arch/arm64/include/asm/current.h @@ -14,11 +14,11 @@ struct task_struct; */ static __always_inline struct task_struct *get_current(void) { - unsigned long sp_el0; + unsigned long tpidrro_el0; =20 - asm ("mrs %0, sp_el0" : "=3Dr" (sp_el0)); + asm ("mrs %0, tpidrro_el0" : "=3Dr" (tpidrro_el0)); =20 - return (struct task_struct *)sp_el0; + return (struct task_struct *)tpidrro_el0; } =20 #define current get_current() diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index b5d8277f608a..a45be0a837c8 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -81,9 +81,9 @@ /* Stash the original SP (minus PT_REGS_SIZE) in tpidr_el0. */ msr tpidr_el0, x0 =20 - /* Recover the original x0 value and stash it in tpidrro_el0 */ + /* Recover the original x0 value and stash it in sp_el0 */ sub x0, sp, x0 - msr tpidrro_el0, x0 + msr sp_el0, x0 =20 /* Switch to the overflow stack */ adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 @@ -99,7 +99,7 @@ =20 /* We were already on the overflow stack. Restore sp/x0 and carry on. */ sub sp, sp, x0 - mrs x0, tpidrro_el0 + mrs x0, sp_el0 b el\el\ht\()_\regsize\()_\label .endif .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? @@ -226,7 +226,7 @@ alternative_cb_end clear_gp_regs mrs x21, sp_el0 ldr_this_cpu tsk, __entry_task, x20 - msr sp_el0, tsk + msr tpidrro_el0, tsk =20 /* * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions @@ -548,7 +548,7 @@ SYM_CODE_START_LOCAL(__bad_stack) */ =20 /* Restore the original x0 value */ - mrs x0, tpidrro_el0 + mrs x0, sp_el0 =20 /* * Store the original GPRs to the new stack. The orginal SP (minus @@ -843,7 +843,7 @@ SYM_FUNC_START(cpu_switch_to) ldp x29, x9, [x8], #16 ldr lr, [x8] mov sp, x9 - msr sp_el0, x1 + msr tpidrro_el0, x1 ptrauth_keys_install_kernel x1, x8, x9, x10 scs_save x0 scs_load_current @@ -1031,11 +1031,11 @@ SYM_CODE_START(__sdei_asm_handler) =20 /* * We may have interrupted userspace, or a guest, or exit-from or - * return-to either of these. We can't trust sp_el0, restore it. + * return-to either of these. We can't trust tpidrro_el0, restore it. */ - mrs x28, sp_el0 + mrs x28, tpidrro_el0 ldr_this_cpu dst=3Dx0, sym=3D__entry_task, tmp=3Dx1 - msr sp_el0, x0 + msr tpidrro_el0, x0 =20 /* If we interrupted the kernel point to the previous stack/frame. */ and x0, x3, #0xc @@ -1051,7 +1051,7 @@ SYM_CODE_START(__sdei_asm_handler) mov x1, x19 bl __sdei_handler =20 - msr sp_el0, x28 + msr tpidrro_el0, x28 /* restore regs >x17 that we clobbered */ mov x4, x19 // keep x4 for __sdei_asm_exit_trampoline ldp x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14] diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 87a822e5c4ca..c33551ee6572 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -193,7 +193,7 @@ SYM_CODE_END(preserve_boot_args) * for consistency with user tasks and kthreads. */ .macro init_cpu_task tsk, tmp1, tmp2 - msr sp_el0, \tsk + msr tpidrro_el0, \tsk =20 ldr \tmp1, [\tsk, #TSK_STACK] add sp, \tmp1, #THREAD_SIZE diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index bfdc12166895..79a19be88ced 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -551,8 +551,9 @@ static void ssbs_thread_switch(struct task_struct *next) } =20 /* - * We store our current task in sp_el0, which is clobbered by userspace. K= eep a - * shadow copy so that we can restore this upon entry from userspace. + * We store our current task in tpidrro_el0, which is clobbered when retur= ning + * to userspace. Keep a shadow copy so that we can restore this upon entry= from + * userspace. * * This is *only* for exception entry from EL0, and is not valid until we * __switch_to() a user task. diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 12aacc74f764..ad4965d34978 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -129,6 +129,10 @@ SYM_FUNC_START(cpu_do_resume) ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] ldp x13, x14, [x0, #80] + + /* Move 'current' somewhere safe */ + mov x15, x3 + /* * Restore x18, as it may be used as a platform register, and clear * the buffer to minimize the risk of exposure when used for shadow @@ -176,7 +180,7 @@ alternative_if ARM64_HAS_GIC_PRIO_MASKING alternative_else_nop_endif #endif =20 - ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 + ptrauth_keys_install_kernel_nosync x15, x1, x2, x3 isb ret SYM_FUNC_END(cpu_do_resume) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41db..51d853b92443 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1694,7 +1694,7 @@ static int build_insn(const struct bpf_verifier_env *= env, const struct bpf_insn if (insn->src_reg =3D=3D 0 && insn->imm =3D=3D BPF_FUNC_get_smp_processo= r_id) { cpu_offset =3D offsetof(struct thread_info, cpu); =20 - emit(A64_MRS_SP_EL0(tmp), ctx); + emit(A64_MRS_TPIDRRO_EL0(tmp), ctx); if (is_lsi_offset(cpu_offset, 2)) { emit(A64_LDR32I(r0, tmp, cpu_offset), ctx); } else { @@ -1707,7 +1707,7 @@ static int build_insn(const struct bpf_verifier_env *= env, const struct bpf_insn /* Implement helper call to bpf_get_current_task/_btf() inline */ if (insn->src_reg =3D=3D 0 && (insn->imm =3D=3D BPF_FUNC_get_current_tas= k || insn->imm =3D=3D BPF_FUNC_get_current_task_btf)) { - emit(A64_MRS_SP_EL0(r0), ctx); + emit(A64_MRS_TPIDRRO_EL0(r0), ctx); break; } =20 --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C80A6515890 for ; Fri, 18 Sep 2026 16:14:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748081; cv=none; b=nqvb0dEMk74XFegJUtFHkj0JzEONk8uEbc1UwBDcIEGaW1Xe2BWjJa0S2yFOHmoHjhYwEyavw8hUTjz7ALMcP67/GEhR9aoOau8n5LJ6LLfmFQItyEgS1lkodVTwE5ODtv7ltmfi/MOh4WHVhbAbcmNVRwYsaGNjb+TVNcbP1Sk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748081; c=relaxed/simple; bh=BPHwYC+/3zydmxdp+H0MV5n7mum1DHw+RAMwW9cldiQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=IoTZoXDQziDTYw/3pyWNEq+yqy2S/FBnRJ0LGhsvXNIcTW98vPDRCLm3Y18gKE1GOL914jJT/8Yc1Et7XWeNMUVdMTuHCYXj2DI4pZ30aMRK8gIaRbtZ8V32WOLT2HfJHk88j2jHbEvvstOXWd1yk6dE8Svw8/IHU9u7jTjDn80= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=UMSBMPqO; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="UMSBMPqO" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6397D1F00898; Fri, 18 Sep 2026 16:14:37 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748080; bh=QbYLgua3LTRCkHC2dRvKQSbxhGctAFKekAK0EDDmZSU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=UMSBMPqO8Wnq9czmGejil4XXGqU+RtD33pykqoNceiGbGOR3FhEfWGsEdR6/+YYp5 MvMwHsljDqr5Ym7RSNgHZtmw/Wf4B9ErzLVO7yq+CIHY6GQtMTr6Wz059JmuCyB1Dj isxtlJevCFivSWcKZcPW8KsZvucON1RxNznHLSdIiYVB6w+ocxv6RlLZLSww7/tlus /3Em1SfAUDKdy0UVnLR9/ZaNNWgYGDLcMFWrHXaa1/mDQWEmbtXPmNPi/n5VjN9Trg 2JOmMon8k+7OSoUD1jQtJSB/juWVd40zsEeQwtaOt08SWCD6AD8mGz164psJ6qTA9C 4YdkwCHepl+/w== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 07/21] selftests/bpf: arm64: Use TPIDRRO_EL0 instead of SP_EL0 for 'current' Date: Fri, 18 Sep 2026 17:13:51 +0100 Message-ID: <20260918161407.2300-8-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The current task pointer now lives in TPIDRRO_EL0 instead of SP_EL0, so update the bpf selftests with the new register allocation. Cc: Mark Rutland Signed-off-by: Will Deacon --- tools/testing/selftests/bpf/progs/verifier_jit_inline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tool= s/testing/selftests/bpf/progs/verifier_jit_inline.c index 02e562f56f9d..a3d6c095acc3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c @@ -9,7 +9,7 @@ __success __retval(0) __arch_x86_64 __jited(" addq %gs:{{.*}}, %rax") __arch_arm64 -__jited(" mrs x8, SP_EL0") +__jited(" mrs x8, TPIDRRO_EL0") __arch_riscv64 __jited(" mv a5, tp") __arch_loongarch --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 46E6C514768 for ; Fri, 18 Sep 2026 16:14:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748085; cv=none; b=I0BPYdsJurOOVOlj3iJUOFYmBjin79G2gXksEstIr44pc0liCoM+EmrGYWoGtRlwPm9alcrljDEiTj/B5dcc5w/BkTQUr1fh721LYmGUh+8kNGYrLuPQI0gBiWur1KGiYzR4laxoBrxGquoOzCmdCZegKJUCF/4yFy4PQ9pmR90= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748085; c=relaxed/simple; bh=3sRKcASsIcsNqAQEzNR/dM3ekLNXwXnihrjiprdTOjU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=qiLo2tvkIQ5tt379GduGO4khaIzpRJulvgjFsJyXCFI1eGTLH50jJw28iLXuAd9wAhtTiVFDwBp55LoK6i0jES6lEPDc7XsZet5jF/MFAG4m2+Rn1wabO2O7YHm3re797t6T90AIeTmKk8JgGRXtMbfeZavO/VErocqoww8I2Y0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ZTfdxI/T; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ZTfdxI/T" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E22DF1F000FF; Fri, 18 Sep 2026 16:14:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748083; bh=mq6/KTZTkKh5H1ZNq8kaadqg9w0eqgaS+tnd4DM5ipU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ZTfdxI/T41330h7hOs9oLlabWGQj6ipTXbgPriHVmHvk71Wlp7PeCePz04WCyZaP8 wyvskyGxz9QMIB0KsVPPzIuFF/1J+Ublo2aWOy9uXnJJY2WS7+0eKMh4sMQGYonxyx D3gM6BLQ8PxWakQ6wbZrW2JcxXBPYT9EVJVF9NrSdNvZU8MAegdiXxrbnt29AyrAtB nkDPDvbSKtoQXwOgyKmxMRKQqw6YqhiOpxly/XUuu0ptIFrnRoIyChWkDRfwamGf75 8sfLH+hPb8jKn+pDtKsp7G0lTrmGueKzpK3MawloUhVZZfpoLPHmhn/yCBfC9t+Vcj hD9t6H8/MiD/w== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 08/21] scripts/gdb: arm64: Use TPIDRRO_EL0 instead of SP_EL0 for 'current' Date: Fri, 18 Sep 2026 17:13:52 +0100 Message-ID: <20260918161407.2300-9-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The current task pointer now lives in TPIDRRO_EL0 instead of SP_EL0, so update the gdb 'cpus.py' helper script with the new register allocation. Cc: Mark Rutland Signed-off-by: Will Deacon --- scripts/gdb/linux/cpus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gdb/linux/cpus.py b/scripts/gdb/linux/cpus.py index 6edf4ef61636..547a623280ae 100644 --- a/scripts/gdb/linux/cpus.py +++ b/scripts/gdb/linux/cpus.py @@ -185,7 +185,7 @@ def get_current_task(cpu): var_ptr =3D gdb.parse_and_eval("¤t_task") return per_cpu(var_ptr, cpu).dereference() elif utils.is_target_arch("aarch64"): - current_task_addr =3D gdb.parse_and_eval("(unsigned long)$SP_EL0") + current_task_addr =3D gdb.parse_and_eval("(unsigned long)$TPIDRRO_= EL0") if (current_task_addr >> 63) !=3D 0: current_task =3D current_task_addr.cast(task_ptr_type) return current_task.dereference() --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 93404516176 for ; Fri, 18 Sep 2026 16:14:47 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748088; cv=none; b=pL+mw5ybao+AaZpXSz0Ksm5e530qRcOF8pXOfkNYfY228KCidUOBdvijW93Fwe3zfM7d6lwT8xglqIQ0hflvch25m4Dw64cWwkZtimJ2UzZimezxCvH7gAHRO04LQSpno26gYtDs+Amg+DI4q1ZuXYi7Vtz7lrfXSEfnmruQTkU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748088; c=relaxed/simple; bh=JmhJGumalitAnkG+Of4OIu81PtkNzTEqVBPSWVBLvmo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=IBZmVSog9XduiHZMPDJl2K1d5fuiqccC0rfyJVFlRXw2vmAVBL2K/JQ46SjyrxviYzvGaQO+iuUwU/I1RGFFNDCxGMo7obq8cnLzTNxhPgaA1lIGgdH5iLYYoOoNNmoeT7T/L6MqPgICpHxoPWxipio81XwODJZpVW1a5NgjSrE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Qb0YxHlt; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Qb0YxHlt" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 650CE1F00898; Fri, 18 Sep 2026 16:14:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748087; bh=dtOLc80ZchH1SrJFHNt1KR4Dm3KMyagivA2zXxrs1go=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Qb0YxHlt3q2DkGZ8Sdf5eTPdlGv44KUoKGREksMF7S+ufmYvYiQYiXZeo/ew3aUr3 4BtWfDhb/8qtxjgh1uAA7jqIAUiCI3k1MCNgIQRA9D/zNy1p8vG9szNXJod811+iX2 ltFzvp0/byPOhWY9lnSUdbO0nKnElVNG1C4vEh3LqYbuZNVFemTx1ddedSqmccvTdI mUXS9g5py+KBJnKBFbz+FQkejtujW0lysjn7/7mqKoSL2BU+dd+IvncGgSmOECsljt dV5UihvAzdi+mCEBychIY6zIeobzLCtiwlsZZ181aUFQA8iolhFMhQJtp+Kftj2Wbp HVOW7lca2e+YA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 09/21] arm64: stackprotector: Re-enable per-task stackprotector Date: Fri, 18 Sep 2026 17:13:53 +0100 Message-ID: <20260918161407.2300-10-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Now that 'current' resides in TPIDRRO_EL0, update the per-task stackprotector compiler invocation to use the new register. Note that this requires a very recent build of clang [1], as (unlike GCC), it historically rejected all registers other than SP_EL0. Cc: Mark Rutland Link: https://github.com/llvm/llvm-project/pull/188054 [1] Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 3 +-- arch/arm64/Makefile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b8270b301809..020fcaf36b1b 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2450,12 +2450,11 @@ config RANDOMIZE_MODULE_REGION_FULL exhaustion, modules might be able to fall back to a larger 2GB area. =20 config CC_HAVE_STACKPROTECTOR_SYSREG - def_bool $(cc-option,-mstack-protector-guard=3Dsysreg -mstack-protector-g= uard-reg=3Dsp_el0 -mstack-protector-guard-offset=3D0) + def_bool $(cc-option,-mstack-protector-guard=3Dsysreg -mstack-protector-g= uard-reg=3Dtpidrro_el0 -mstack-protector-guard-offset=3D0) =20 config STACKPROTECTOR_PER_TASK def_bool y depends on STACKPROTECTOR && CC_HAVE_STACKPROTECTOR_SYSREG - depends on BROKEN =20 config UNWIND_PATCH_PAC_INTO_SCS bool "Enable shadow call stack dynamically using code patching" diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 6b005c8fef70..a293ab67818f 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -72,7 +72,7 @@ ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y) prepare: stack_protector_prepare stack_protector_prepare: prepare0 $(eval KBUILD_CFLAGS +=3D -mstack-protector-guard=3Dsysreg \ - -mstack-protector-guard-reg=3Dsp_el0 \ + -mstack-protector-guard-reg=3Dtpidrro_el0 \ -mstack-protector-guard-offset=3D$(shell \ awk '{if ($$2 =3D=3D "TSK_STACK_CANARY") print $$3;}' \ $(objtree)/include/generated/asm-offsets.h)) --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E56195172CA for ; Fri, 18 Sep 2026 16:14:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748092; cv=none; b=FH4dK+Np+Ck2sr8kjaa9VfOOaMBD+MDDLodb9caJLtO+aZIc/2fYWthmWqedhDBa5Kbren2TEtIpEK4cllMrzUnUZgxFpF7tYiQuJjmANBuJisrZ0lHnDnuL3zzPuoXtG1CQXpTaCwp48697vXh39wGJB6Dt/FUZv+H2fwIe0EQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748092; c=relaxed/simple; bh=KDJKK7II+/aQDt8adWTZDLdrnfW60Q7byX6OKOMNkP0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=B3OdZ7TebpQgAqQJeVUdqZ++RWEy/DDyn51BpwGWdhsYyxpDcr9o3z2/GtdNvJkE9IUU5GUdXYbLn4AtDzZlVDlXNX7zuw8N2ua+IiBXXo38EYMOss3pgtXEDX/5N/PnAOpwhJXqOzCeIuPT91eKnTCbsp4FtusjXOZJTKHp4ik= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Vu7rNH92; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Vu7rNH92" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D7F0C1F000FF; Fri, 18 Sep 2026 16:14:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748090; bh=FPZbgiJbb83ig3UORAUcXElGA2dwvm2nNhAABZ7YfkI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Vu7rNH92TOdtZLduMK8rLA3uEFDZQDwq8l7LDQGEm82PkkYHhb86gyupURzKebUvF hHvg2fsEhEDNNYqY4eXcbzxU0pCGqEYz5BqAOBtZy4wRH77oZWdhyqngl2tLW2mGlW W1a0NIE0Nv6sARQUcDG5zRLMNqMCQtCyKPl27MJ3NVn0ZRR/NXQs9wj4o+4ew8AdBf WYNbXAGRwW2X7y4W16Hy3ZTZ5LBnZSV5AXsK+qE5zhM/gPWkSZvLINVQDrDuN7TD2F TuLGcL11ABTZf0lw16y0xkD4kyjmTzGi32aappZ4zy99iGnDm83xoSJvVCxI/GsCAE V0PUERIiREpwg== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 10/21] arm64: percpu: Specialise set_my_cpu_offset() for the primary CPU Date: Fri, 18 Sep 2026 17:13:54 +0100 Message-ID: <20260918161407.2300-11-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" set_my_cpu_offset() is only called from smp_prepare_boot_cpu() which runs once on the primary CPU and before alternatives have been applied. Rework the function so that it operates only on the boot CPU offset, drop the unused alternative code sequence and internalise it into smp.c alongside its only caller. Signed-off-by: Will Deacon --- arch/arm64/include/asm/percpu.h | 8 -------- arch/arm64/kernel/smp.c | 8 +++++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percp= u.h index b57b2bb00967..3dc7291b16e9 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -12,14 +12,6 @@ #include #include =20 -static inline void set_my_cpu_offset(unsigned long off) -{ - asm volatile(ALTERNATIVE("msr tpidr_el1, %0", - "msr tpidr_el2, %0", - ARM64_HAS_VIRT_HOST_EXTN) - :: "r" (off) : "memory"); -} - static inline unsigned long __hyp_my_cpu_offset(void) { /* diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index a61dc3016a11..702dee8d0db5 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -446,6 +446,12 @@ void __init smp_cpus_done(unsigned int max_cpus) mark_linear_text_alias_ro(); } =20 +static void __init set_boot_cpu_offset(void) +{ + asm volatile("msr tpidr_el1, %0" + :: "r" (per_cpu_offset(0)) : "memory"); +} + void __init smp_prepare_boot_cpu(void) { /* @@ -453,7 +459,7 @@ void __init smp_prepare_boot_cpu(void) * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be * freed shortly, so we must move over to the runtime per-cpu area. */ - set_my_cpu_offset(per_cpu_offset(smp_processor_id())); + set_boot_cpu_offset(); =20 cpuinfo_store_boot_cpu(); setup_boot_cpu_features(); --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6552B5172F2 for ; Fri, 18 Sep 2026 16:14:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748096; cv=none; b=L3fgP9IW1RrhOTPu1O/gX427IAAf9LeCU+xMZPe0pzx9P/w6jzK0QXEdzNDqbFTgH9POcmkGFSv1kJPbs617+d59uB91VSIOIPti8XAOHgYn/n1y2QiVjxAryiV+2CvKRlUzV/ac0Uo5pQLIXuVAHMGBDd8ZIl0ksMgemg2oDcw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748096; c=relaxed/simple; bh=0UvJYpoEn/TY4aVO9CwMQRLah5Bz+jNMBvjhtIhu+tc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=QzhOPE9K3s9i3tXoumXSOtn0fzYj7FF+nrqmxlAYmHcwsx+FknVg2ZuZBq7q8SP+MxKNVItCD8Xjm/UiZhwf3EaD4uQthsWzzGarlFmMsGwV/rGLRNitJnw9QsfyoevbLVjyEWdxF4lr1ljxb21osD/qIN7lKyBbnMc7pOLhFq0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=U9ViQKP6; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="U9ViQKP6" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 57F871F00898; Fri, 18 Sep 2026 16:14:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748094; bh=aA8vLI7MAtOEMjRIQWw1T8YssBKmDitylE36RcPCelE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=U9ViQKP605sp41Kl+OYIXFTMHNkjsPaN2gmL80+h0GHT4bi0XhukSKAuCSKudzw6/ mNbemGeBBrsxCsLMXXsm6OTFprr7uZGpyNidrH8gkohXkQgyfCSw9azU3Owq1X01pZ alemikg424ug7GwLvdrtHnzWS8vmSx88qc3pvpjedEiXydpPjvY5UZOwvyN1iyIShg bCoYirL5Dx6eXBb8efux4ufgjE34cX11PprEfTcSMG4+kNSDIDCPJCL2H239Gaho+E AZbEBytGNJBIo0AnucpJq525EwCsHX07AFmmPOyz2QCzPGYwoB8aMKeRD/Z4cczfkq mPcYCVUXFieYA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 11/21] arm64: percpu: Annotate __kern_my_cpu_offset() as '__always_inline' Date: Fri, 18 Sep 2026 17:13:55 +0100 Message-ID: <20260918161407.2300-12-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Allow the use of per-cpu variables from noinstr functions by marking __kern_my_cpu_offset() as '__always_inline'. This will later be relied upon to detect stack overflow of the overflow stack from noinstr C code. Signed-off-by: Will Deacon --- arch/arm64/include/asm/percpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percp= u.h index 3dc7291b16e9..cb9ae8a1ae87 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -21,7 +21,7 @@ static inline unsigned long __hyp_my_cpu_offset(void) return read_sysreg(tpidr_el2); } =20 -static inline unsigned long __kern_my_cpu_offset(void) +static __always_inline unsigned long __kern_my_cpu_offset(void) { unsigned long off; =20 --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CD287515895 for ; Fri, 18 Sep 2026 16:14:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748099; cv=none; b=CgGtXmnIHd0nOepWr95Csf27HZGwUnBR6Y+uZyeOYjrbCr+i6sIzgLyqP78IG/rOS7IXZu/j0rpUOodXMYg6kqBZlj4HSCunQ6Z/2naPyF1DjyH5qFkrz4IC/u/XKvv44wZB5OITML1cSYTTf89i8EN8Wt9oiUquNjC1s6R09J4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748099; c=relaxed/simple; bh=Vr7db0umz6Dh0P5DWn6iaRKwA4WHAyRAxkpJpgZhULY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gwhY2M5E0GBhWlRH30N6f9I+G6D+wLM0e2tSar8GFO3XZV6ksf2SgKBVHKMKItEisNz9LUqZugLHVldeUCzCkxAOl0OAOGsIgy59/ZlzMBYHaH6+q4n3UUxAcKG+/DrG7I4FjhrwsJeGYkjOJpxxZHaYVUESy4fhLXUNnSCbS/Y= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=KjrujF6I; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="KjrujF6I" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CF6371F00899; Fri, 18 Sep 2026 16:14:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748097; bh=OPmtMvErDChPnAkncAvX9/4lDdnZXVCa66ctdGe0lWk=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=KjrujF6I8y3AN56jW7bSdjLKy8uypi9i70m0KIMGyZzzAZOpzq6tSGxK4t1tdRAml j4isO93RUuPAhEifUXnYxsswhXO4mAj1efZp0KL+1wnE47A1YQcZwkN43GqxvstJ5s wed0kDmZZwVnlLug6QXUj2vwe/lpG6HT5LU4TtMHOpn3q54KSDwvaCEmB7JzJOvSxY 3D2EWtoM5pjXNy4ZFI91JATLqj+mSK+cKzU5H60uiRzkkhxalzeQ2UEk4w1iAhqPQ1 WSFKUUfUSXJtSP/EtGUdrszLLl+2ce2vYjHztTN2vPLJdGhNQYwwwSMD3pYUwziXg6 PHiiyoYRz7N5g== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 12/21] KVM: arm64: Preserve handler/thread bit of EL1 mode in __finalise_el2() Date: Fri, 18 Sep 2026 17:13:56 +0100 Message-ID: <20260918161407.2300-13-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" __finalise_el2() unconditionally returns to EL2h when VHE is being used. In preparation for handling callers from both EL1h and EL1t, preserve the handler/thread bit of the calling mode when rewriting the exception level in the SPSR. Signed-off-by: Will Deacon --- arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/hyp-stub.S | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi= /asm/ptrace.h index 15649a253a57..25d9d9a0a6f7 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -37,6 +37,7 @@ #define PSR_MODE_EL3t 0x0000000c #define PSR_MODE_EL3h 0x0000000d #define PSR_MODE_MASK 0x0000000f +#define PSR_MODE_h_BIT 0x00000001 =20 /* AArch32 CPSR bits */ #define PSR_MODE32_BIT 0x00000010 diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 37c6976e44a4..f61778cccd3c 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -149,10 +149,11 @@ SYM_CODE_START_LOCAL(__finalise_el2) =20 isb =20 - // Hack the exception return to stay at EL2 + // Hack the exception return to stay at EL2 ... mrs x0, spsr_el1 - and x0, x0, #~PSR_MODE_MASK - mov x1, #PSR_MODE_EL2h + // ... preserving the 'h' bit so we don't swizzle SP. + and x0, x0, #(~PSR_MODE_MASK | PSR_MODE_h_BIT) + mov x1, #PSR_MODE_EL2t orr x0, x0, x1 msr spsr_el1, x0 =20 --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D8C2D517BCE for ; Fri, 18 Sep 2026 16:15:01 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748103; cv=none; b=hvsoarY8twsm/hjzQ0Kb0M5/Sm4GYO4KlM9FJBXSxwh1G9uqDv+esyKZjrjxVDK16ywDckMOLkc7Ub79ECC36lpbcvyr6CnTNZJt/KCERpTI0/Q4ljjJIhOZxRgzKVEjqHGUr5aggXYJe7p806qRzbWfX6WPArX1ETprRcUsrTw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748103; c=relaxed/simple; bh=DK9Y/UMpGFKL933fN/ZhyfaYUwcJNs/SMB/QpAPXVLo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=NSRPR7wSmw2I5f1p43NwBKuVOZ/oKh6bCEOT4FnJGyqmblG4cbCmnJuX/oFJPscG++KRCcQ8C3PjClvl0OAFjKkAltnf/vEgMGBz1DnO4rgAwKWiVSwYNkceJDSC3zDYCxK/yzVxP9/TiIO9EqauXNxVLOBzVt97q9/sNvK3omI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=BpMwUcXt; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="BpMwUcXt" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4DF501F000FF; Fri, 18 Sep 2026 16:14:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748101; bh=Tc8GR7rTTo27bA7yHyZhX9hh9yph2GkWra5UE2xqfbs=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=BpMwUcXtxZXvVSt7IywpT3I566I5Wb9vA+b1HRQ0SPTXq31VO5Co5iAq1W1lssnp+ IRFyEi/tH1AwRv4wOS9f00oUX4wA4BGBgQYEFHmVRtbVwkEa7E9ynmxDJoyxPQHoza FqFdiKbeyxonu1drcwS0+otlkvWbj/Cm3sc6R9u9TW3VMtSLcqnugO3TFIKBJ659P3 VIBg6/F5AoGPaTLb0A27uHciXsGIElTW00LAeCsI5a/w372TjfMF4FF4S2wb4RvnxS h0aTrKhu6AtkY1O02DW3VjZqu+oUnMuAFU7prdjm+52PTI1QjUvir49tbHIunMVEVO w1JYAxdBenYQg== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier , kernel test robot Subject: [PATCH v2 13/21] arm64: sdei: Guard most of asm/sdei.h with CONFIG_ARM_SDE_INTERFACE Date: Fri, 18 Sep 2026 17:13:57 +0100 Message-ID: <20260918161407.2300-14-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The SDEI header includes are a bit of a mess: linux/arm_sdei.h only includes asm/sdei.h if CONFIG_ARM_SDE_INTERFACE=3Dy and then goes on to provide a definition of the sdei_arch_get_entry_point() macro if it hasn't already been defined. asm/sdei.h, however, defines the same macro unconditionally and so this breaks the build if CONFIG_ARM_SDE_INTERFACE is not set and asm/sdei.h is included after linux/arm_sdei.h. The kbuild robot ran into this situation with randconfig testing, thanks to the transitive inclusion of asm/sdei.h via asm/stacktrace.h. Guard most of asm/sdei.h with CONFIG_ARM_SDE_INTERFACE to avoid redefining macros provided by linux/arm_sdei.h. Reported-by: kernel test robot Signed-off-by: Will Deacon --- arch/arm64/include/asm/sdei.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h index b2248bd3cb58..b96eb9e22d4b 100644 --- a/arch/arm64/include/asm/sdei.h +++ b/arch/arm64/include/asm/sdei.h @@ -9,6 +9,7 @@ =20 #define SDEI_STACK_SIZE IRQ_STACK_SIZE =20 +#ifdef CONFIG_ARM_SDE_INTERFACE #ifndef __ASSEMBLER__ =20 #include @@ -50,4 +51,5 @@ unsigned long sdei_arch_get_entry_point(int conduit); #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x) =20 #endif /* __ASSEMBLER__ */ +#endif /* CONFIG_ARM_SDE_INTERFACE */ #endif /* __ASM_SDEI_H */ --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3E2C751812F for ; Fri, 18 Sep 2026 16:15:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748106; cv=none; b=savKRZf2bw9aQcB73Bd+YnZ4JT1WfkfOSMQXgRwfeD71le5wcTbQke1MbRMuVhKzXKFObNkRb3bCzi0HDuzxVJfaDg5kjYoAjxRrdoti8QY34Y1c5CSbXlxy3Q+OfHI2ULvkJFb2eVkBodsgDNVpxEKSpg/g6xrdAE4vgJuPRKM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748106; c=relaxed/simple; bh=IGlAsYPqJLBdGQbuzb+CS793IwgUKqQ5O1eHtGwRLxU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=dkX7Dcrbk8CN+TCeZb5D5wqwF+78XnjGluwymmVQCIsqd+0C4CArlcb3TB3sCBs8qcVZi0NwjptDUUqG02OK4jVweNsf/nJJjgUKxLFVytdyIW9pYb+EVrv1otTax05NQNYUoAfUq5Tb0xVbpOB7jwg69DrMEtzWkexh+j/wNZ0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Z/ffdFAM; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Z/ffdFAM" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0CD191F00898; Fri, 18 Sep 2026 16:15:01 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748105; bh=8nBLQSKemxDIktlR563dkf+XaG9PZ7B4rjq2+jvC9t0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=Z/ffdFAMxmWA9YF2dCYlP4WVx8iG4pj2rNzJqqF7scRBaWTu8PUGnJ/RLD+kbIzDc 50DJpkxeP1moeVXfp135B87JcBo0uHGmXQWF2XKtqE8eL4xHRuAWOVbcKcx9av58Tp BEVVhahBxZNv3dbKArxMlqTDUaWgwgFsx1qAYCJoHIBcAhdFqSJUPei3Bk5cu2Cynt alJs+g5kiqed+FY6dL0LmkdsqqhwtG5O6qSMjqTNuiA+ZxtKmLHwty7AP7ML5f6okU B+RhZ1XlPlYxzQqYg1cGypFhkVKuSEc7L6aAFLu15Yc86AHTtCcghi5l/LDZL5e0fA XfWAtM9QciATA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 14/21] arm64: sdei: Support SDEI events from kernel handler and thread modes Date: Fri, 18 Sep 2026 17:13:58 +0100 Message-ID: <20260918161407.2300-15-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Section 5.2.2 ("Event resume context") of the SDEI specification states that, as a result of an SDEI_EVENT_COMPLETE_AND_RESUME call: | SPSR_ELc is set to the PSTATE when the event was taken In preparation for making use of both thread and handler modes in the kernel, adjust the SDEI event handler so that we return to the correct kernel vector for the interrupted mode. Signed-off-by: Will Deacon --- arch/arm64/kernel/sdei.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index 778f2a1faac8..6337ce905593 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -208,7 +208,7 @@ unsigned long __kprobes do_sdei_event(struct pt_regs *r= egs, int i, err =3D 0; int clobbered_registers =3D 4; u64 elr =3D read_sysreg(elr_el1); - u32 kernel_mode =3D read_sysreg(CurrentEL) | 1; /* +SPSel */ + u32 kernel_mode =3D read_sysreg(CurrentEL); unsigned long vbar =3D read_sysreg(vbar_el1); =20 if (arm64_kernel_unmapped_at_el0()) @@ -233,7 +233,7 @@ unsigned long __kprobes do_sdei_event(struct pt_regs *r= egs, pr_warn("unsafe: exception during handler\n"); } =20 - mode =3D regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK); + mode =3D regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK) & ~PSR_MODE_h_BI= T; =20 /* * If we interrupted the kernel with interrupts masked, we always go @@ -251,7 +251,7 @@ unsigned long __kprobes do_sdei_event(struct pt_regs *r= egs, * address'. */ if (mode =3D=3D kernel_mode) - return vbar + 0x280; + return vbar + (regs->pstate & PSR_MODE_h_BIT ? 0x280 : 0x80); else if (mode & PSR_MODE32_BIT) return vbar + 0x680; =20 --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C675151814F for ; Fri, 18 Sep 2026 16:15:08 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748110; cv=none; b=dT467J5kpKgXcGkdu8FCotQCGPNweVbzyJwDl/jJUoIyHiSZHQoSQ92JBqrxWhjP/rSPzskiLCIYt91rSwZjPukYq93uZKhHzDizsOHb/MBg0w9w/HDVKAP0HOxgZzLjBmIKjGl7vWZXw1SDzxU3qr5BeKgD9a84MHdRKINLhgs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748110; c=relaxed/simple; bh=llOvWVYViqY0vmfDVhIh84dvZVHJjkdP1y8KQLkPkZk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=uCSanxQhzR/rCMnmtKM5Uc4YkwQ4C+mdq3Nv5xXQElww89uIYH6lGPTjQeNeKgTbAKIg65AlTG7PxV7O3kCmuVzo/Zr3Ag8xssyQja1Lk33/9pPASBCakndJSurzPxjlufnhbzbOBOWCQlif7U6WasIXwno5fqireLxbPD6+qVM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=i1bL8DTy; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="i1bL8DTy" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 8A6C51F000FF; Fri, 18 Sep 2026 16:15:05 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748108; bh=Eo1Mu3/UqB3iPcOR9znRzzkYsVC8kR8uT7JIdm2G87o=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=i1bL8DTyxaWwPU8TmGAoCePCspJVY5PvZJnYDmNVOGLub5VO4F6tsWTvDd1WS/AkC 94V05/JqBZRNJDkHkd5DzYnOdbBcVuLaxspjaPJqk/97qxUHkYs17+qq14oapAeo8P KIcaGIOomO9VflToJe9jdtuve71vCN8mpBlKkOfJemv1zI2uIBjlJUJaRTa5xMWtNo 85IMk1+2akR7OhmPUGmVYjj5NB5xVqrwDXqQYlzie/ycIcdcTwoVx+Au08Qj/2AFfw k3bNy9qpww/NLJ+RWZ907wCvj/QquOD3pwDWmRg7M/caCZG2gXNLm0GBtXf4gkTFMb aKX/Tb8Vw1slA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 15/21] arm64: entry: Point SP_EL0 at the overflow stack Date: Fri, 18 Sep 2026 17:13:59 +0100 Message-ID: <20260918161407.2300-16-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" With SP_EL0 now available for use in the kernel, point it at the overflow stack so that exception entry from EL1h will later be able to switch to the overflow stack by writing to SPSel. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/kernel/entry.S | 2 ++ arch/arm64/kernel/head.S | 3 +++ arch/arm64/kernel/smp.c | 12 ++++++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index a45be0a837c8..6958ee238649 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -227,6 +227,8 @@ alternative_cb_end mrs x21, sp_el0 ldr_this_cpu tsk, __entry_task, x20 msr tpidrro_el0, tsk + adr_this_cpu x19, overflow_stack + OVERFLOW_STACK_SIZE, x20 + msr sp_el0, x19 =20 /* * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index c33551ee6572..27a19e1a0ee6 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -210,6 +210,9 @@ SYM_CODE_END(preserve_boot_args) ldr w\tmp2, [\tsk, #TSK_TI_CPU] ldr \tmp1, [\tmp1, \tmp2, lsl #3] set_this_cpu_offset \tmp1 + + adr_this_cpu \tmp1, overflow_stack + OVERFLOW_STACK_SIZE, \tmp2 + msr sp_el0, \tmp1 .endm =20 /* diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 702dee8d0db5..4c122d6598de 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -448,8 +449,15 @@ void __init smp_cpus_done(unsigned int max_cpus) =20 static void __init set_boot_cpu_offset(void) { - asm volatile("msr tpidr_el1, %0" - :: "r" (per_cpu_offset(0)) : "memory"); + u64 ovf_sp =3D (u64)raw_cpu_ptr(overflow_stack) + OVERFLOW_STACK_SIZE; + + asm volatile( + " msr tpidr_el1, %1\n" + " add %0, %0, %1\n" + " msr sp_el0, %0" /* Update the overflow stack pointer */ + : "+r" (ovf_sp) + : "r" (per_cpu_offset(0)) + : "memory"); } =20 void __init smp_prepare_boot_cpu(void) --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0D90C514758 for ; Fri, 18 Sep 2026 16:15:12 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748113; cv=none; b=UdxH85hSHUiaQsCxA00+KJc4QaWpEp+GWoT8CRiQ9bwxDLz4vSzxDidL4NDvfll9mvpshWiI794VSuIWFVEBYvU9yKgGEW2KyI6yXYUPOsYapmPC71bXtzke/nkSHZkqXxmV4kn9wRg0EYVW/6kYxcYm/S+Kb0XR7+adWtj9kY8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748113; c=relaxed/simple; bh=8N+dMro6IY5WMzrJLEVXWe787chZDhO/60ie5X0SGSY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=eARocZg8xRWXrHkfxYohClZrgeQ9kr0fcT1Kzg+N2xaFE3zehNzg7c7vVByvWLIG8zSIF5mheVXyL5PHZgSzdH7luM7RWMHSyNWLjdNo129BR6Hmy0FF+bmJYwbPIfHcomAwOOl1Pk3Uv504Tl3Sdiq5HuwkSWyKOYIFoKQVbIQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=AW4leaLe; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="AW4leaLe" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0B7941F00898; Fri, 18 Sep 2026 16:15:08 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748111; bh=PHdOX9xdJ/boaqPZ/GwtRSXh8g+Ri88d8K76J4PQiZA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=AW4leaLe7j4HIbFH2dIJD4sVJv8bNmmtDS2WAAo4HLGUrfMxov+PMkjuIVqHRFL9b 3jTIF/zEjLp6fQwqp3r5PMg+RuxqLVDmnkvctgfNzjflj60eejygFy6fiBGIl+ByvM v7VkBY2mCW7HjhCo2TjPOhn8P3JeaTUd8TXi89dNH4KKwlHR0GWPjoOCSsD2naCidU FIUXzPe+EmQGG420NDLeSTXAYovB/SJOeJuYBhWvQzbx0TdD0IvOG2WNGUeBfJM2J/ FcBUNsduS9xkFTa3yjwuaYZgQVEFWhm3ZqUqQDXQHbQNDO8ySpEY/0BG/cFzhE/K/9 77wWm0LhbKWdA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 16/21] arm64: entry: Implement EL1t exception handlers for overflow stack Date: Fri, 18 Sep 2026 17:14:00 +0100 Message-ID: <20260918161407.2300-17-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation for using EL1t to handle exceptions taken whilst running on the overflow stack, implement some simple wrappers around the EL1h handlers which perform a best-effort stack overflow check before proceeding. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/kernel/entry-common.c | 43 +++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-com= mon.c index 72c03ccea59f..9738142780df 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -327,10 +327,45 @@ static void debug_exception_exit(struct pt_regs *regs) } NOKPROBE_SYMBOL(debug_exception_exit); =20 -UNHANDLED(el1t, 64, sync) -UNHANDLED(el1t, 64, irq) -UNHANDLED(el1t, 64, fiq) -UNHANDLED(el1t, 64, error) +static void noinstr el1t_64_check_overflow_stack(struct pt_regs *regs) +{ + unsigned long sp =3D kernel_stack_pointer(regs) - sizeof(*regs); + unsigned long ovf_stack =3D (unsigned long)this_cpu_ptr(overflow_stack); + + /* + * We're in big trouble if we've overflowed the overflow stack + * so perform a best-effort check before we proceed. If our SP + * is outside of the overflow stack for this CPU then presumably + * we're already corrupting memory, so park ourselves here in an + * attempt to contain the damage. + */ + if (sp < ovf_stack || sp > ovf_stack + OVERFLOW_STACK_SIZE) + cpu_park_loop(); +} + +asmlinkage void noinstr el1t_64_sync_handler(struct pt_regs *regs) +{ + el1t_64_check_overflow_stack(regs); + el1h_64_sync_handler(regs); +} + +asmlinkage void noinstr el1t_64_irq_handler(struct pt_regs *regs) +{ + el1t_64_check_overflow_stack(regs); + el1h_64_irq_handler(regs); +} + +asmlinkage void noinstr el1t_64_fiq_handler(struct pt_regs *regs) +{ + el1t_64_check_overflow_stack(regs); + el1h_64_fiq_handler(regs); +} + +asmlinkage void noinstr el1t_64_error_handler(struct pt_regs *regs) +{ + el1t_64_check_overflow_stack(regs); + el1h_64_error_handler(regs); +} =20 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) { --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 887285187EA for ; Fri, 18 Sep 2026 16:15:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748117; cv=none; b=tUOF6i3iqhL7kFJrORIjxrNc8dunEjUq2RZb8PX44QYf8syrSB6FB9ETv7yFaBY+p3aDZm7Thm5xmfa6rDgMb0zlOe6RK3zgLyH/bASg4nYwyjnZTQJK0Z1/urCWggD2tpLCB1naKZCjz3ErA/Hpa+QF4gG6Rg8lg6p9guR+umA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748117; c=relaxed/simple; bh=4H8nq6aN5191R1wLSk/mNRCy+uaEBAGMhwj0bVdfAxo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=VceumLk6PrEXxMgmDfLrLc+LIBcC3sLjQekROECRz/EfW9DtUDSgv0GwOlesMeM9xHrbJFvcEHuWwm7lMrrMaVugRymRs627WlMFGWTrGpA2BEITj/5tPMLatKf/w/NnTOOsHgjzJyZZIdOGp1edxmH0dxvu7yY73giezgzAWUs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=jtCEb6pB; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="jtCEb6pB" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 8179F1F000FF; Fri, 18 Sep 2026 16:15:12 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748115; bh=O4WlNdEIdKBpgRVlNn9eY6c+SeO9ekMqgtGrusaeyd8=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=jtCEb6pB1QhXR5GjRKnPXcGta5jkaxic0bs+1/lBhmhZpIXV+PNrFy8CjgacFxm++ PUy4J4jF4ok0N/IX52sLgWM8qU3KTAqMOIvOUuxX4Cj1DNs7BC1rcC/k1zjCBv3l5p NEoCygbAed/r3GOd0v7pGSGDdLezqr8Qtv/hfMU3K0PfWvzyf/I+E1UdM4kS21ntPG vQPbPQSidbgM7N250L2i9MQ8UQ/Pm8NNGb2MAw2/Mg8Gpl6WYScphFzxIoErZHGlfE Zv16DarCVy3RLMvEZoTMtLNx5E2m08aUR9kLRmSmT2neYQjf7nNL3jo2/Xa4LYTXmz X31DJ/ldCeuVA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 17/21] arm64: entry: Use SPSel to switch to overflow stack Date: Fri, 18 Sep 2026 17:14:01 +0100 Message-ID: <20260918161407.2300-18-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When detecting a stack overflow on exception entry from EL1, use SPSel to switch to the overflow stack without corrupting any GPRs. Not only is this simpler than the previous logic, but it also opens the door to more complicated overflow checks (for example, based on per-task stack sizes or stacks which are not aligned to a specific page order) as well as the possibility of returning from a stack fault if we were able to resolve it. Cc: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/kernel/entry.S | 69 ++++++++++++--------------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 6958ee238649..b4df2f23ebe7 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -51,11 +51,11 @@ mov x30, xzr .endif .Lskip_tramp_vectors_cleanup\@: - .endif - + sub sp, sp, #PT_REGS_SIZE + .else /* \el =3D=3D 1 */ + .ifc \ht, h sub sp, sp, #PT_REGS_SIZE =20 - .if \el =3D=3D 1 /* * Test whether the SP has overflowed, without corrupting a GPR. * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT) @@ -63,45 +63,16 @@ */ add sp, sp, x0 // sp' =3D sp + x0 sub x0, sp, x0 // x0' =3D sp' - x0 =3D (sp + x0) - x0 =3D sp - tbnz x0, #THREAD_SHIFT, 0f + tbnz x0, #THREAD_SHIFT, __bad_stack sub x0, sp, x0 // x0'' =3D sp' - x0' =3D (sp + x0) - sp =3D x0 sub sp, sp, x0 // sp'' =3D sp' - x0 =3D (sp + x0) - x0 =3D sp + .else /* EL1t */ + msr spsel, #0 // Stay on the overflow stack + sub sp, sp, #PT_REGS_SIZE + .endif .endif =20 b el\el\ht\()_\regsize\()_\label - - .if \el =3D=3D 1 -0: - /* - * Either we've just detected an overflow, or we've taken an exception - * while on the overflow stack. Either way, we won't return to - * userspace, and can clobber EL0 registers to free up GPRs. - */ - - /* Stash the original SP (minus PT_REGS_SIZE) in tpidr_el0. */ - msr tpidr_el0, x0 - - /* Recover the original x0 value and stash it in sp_el0 */ - sub x0, sp, x0 - msr sp_el0, x0 - - /* Switch to the overflow stack */ - adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 - - /* - * Check whether we were already on the overflow stack. This may happen - * after panic() re-enables interrupts. - */ - mrs x0, tpidr_el0 // sp of interrupted context - sub x0, sp, x0 // delta with top of overflow stack - tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range? - b.ne __bad_stack // no? -> bad stack pointer - - /* We were already on the overflow stack. Restore sp/x0 and carry on. */ - sub sp, sp, x0 - mrs x0, sp_el0 - b el\el\ht\()_\regsize\()_\label - .endif .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm =20 @@ -544,22 +515,24 @@ SYM_CODE_END(vectors) =20 SYM_CODE_START_LOCAL(__bad_stack) /* - * We detected an overflow in kernel_ventry, which switched to the - * overflow stack. Stash the exception regs, and head to our overflow - * handler. + * We detected an overflow in kernel_ventry. + * Restore SP and X0. */ + sub x0, sp, x0 + sub sp, sp, x0 + add sp, sp, #PT_REGS_SIZE =20 - /* Restore the original x0 value */ - mrs x0, sp_el0 + /* Switch to the overflow stack */ + msr spsel, #0 =20 - /* - * Store the original GPRs to the new stack. The orginal SP (minus - * PT_REGS_SIZE) was stashed in tpidr_el0 by kernel_ventry. - */ + /* Stash the exception regs */ sub sp, sp, #PT_REGS_SIZE kernel_entry 1 - mrs x0, tpidr_el0 - add x0, x0, #PT_REGS_SIZE + + /* Fix-up the saved SP */ + msr spsel, #1 + mov x0, sp + msr spsel, #0 str x0, [sp, #S_SP] =20 /* Stash the regs for handle_bad_stack */ --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B9F0851597D for ; Fri, 18 Sep 2026 16:15:19 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748122; cv=none; b=sn6dURQ9zhEq9fP4SfIv5hrgilIYHJoozstSpepwaDUUlWh9GE6QoBecrDQ3EALpZBAmqi+Eq5fadzyKtx0rHgJ3DWfoidL326sU2LdB74gYNB2RdPzqRx3hIHzR/iqDj8YVYhf1zh1JCXXPThFNlOFeAMlpcEUWIQBpuVpI9Cc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748122; c=relaxed/simple; bh=6NR6+Zh/GpAz+hIS+YmPj16Sh+A3Ww9Id+QKN9qMaGQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=lYGRBK7YnVKb3LTBVtjg0J47Nncvld5BwNC6tMbbo3dmFuvKmibyO/ilvU8CRwf4xlpU1WXaxT11lKtXV4RlM0QYdtT0AwE4yNPX2LqFBAxs6ygZX+4BV5HB3l4zKNLvQET43Ixe0Sob3jGEIgZRVCZdqMfLOTsnsLatskx6BEY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=g5rqkejV; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="g5rqkejV" Received: by smtp.kernel.org (Postfix) with ESMTPSA id F2ECA1F00898; Fri, 18 Sep 2026 16:15:15 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748118; bh=kfG+kuuPyslNrsAkX7LF5I0fyep8WFQ1NzuAK6CzMzQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=g5rqkejVYdUdMoiDminapFdGn8UZKRPghhfEdAHDNIIJh8mQeEPS90eBpC77m7Vl5 V5EgImJ3N50lroLBZoIL0lJdlmM6OUeMx6vMGUande27V3lnZd4H3iM+qlwcVwns92 5gF6y38Fh2Cj/ALk7KmMjRaQBbHQCz9tMsLnsTt8wTpM0N8GyAwXD1aiRNKzNAJqYy 4eVsD95h7xLGCQOzFjXSyyCWtBalX3hC5LKV4AcqW4LAYZVMGZdZICzWjKFKPtrDOm 366L3FJ1BPQWP+5zkwN0JP+lMTvDAvbFRYTSamQP0TJ911vcoEZPuwvDiS5H3WRDyV wdoknujOpAGJA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 18/21] arm64: entry: Split up kernel_ventry macro into separate helper macros Date: Fri, 18 Sep 2026 17:14:02 +0100 Message-ID: <20260918161407.2300-19-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Handling entry from EL0, EL1h and EL1t in the 'kernel_ventry' assembly macro is getting a bit unwieldy as the control flow becomes increasingly dependent on the macro arguments. Split 'kernel_ventry' into three leaf helper macros for EL0, EL1h and EL1t, which are then dispatched by simple wrappers. Signed-off-by: Will Deacon --- arch/arm64/kernel/entry.S | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index b4df2f23ebe7..38f9327e6a0a 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -36,10 +36,7 @@ .endr .endm =20 - .macro kernel_ventry, el:req, ht:req, regsize:req, label:req - .align 7 -.Lventry_start\@: - .if \el =3D=3D 0 + .macro kernel_ventry_el0, ht:req, regsize:req, label:req /* * This must be the first instruction of the EL0 vector entries. It is * skipped by the trampoline vectors, to trigger the cleanup. @@ -52,8 +49,10 @@ .endif .Lskip_tramp_vectors_cleanup\@: sub sp, sp, #PT_REGS_SIZE - .else /* \el =3D=3D 1 */ - .ifc \ht, h + b el0\ht\()_\regsize\()_\label + .endm + + .macro kernel_ventry_el1h, regsize:req, label:req sub sp, sp, #PT_REGS_SIZE =20 /* @@ -66,13 +65,23 @@ tbnz x0, #THREAD_SHIFT, __bad_stack sub x0, sp, x0 // x0'' =3D sp' - x0' =3D (sp + x0) - sp =3D x0 sub sp, sp, x0 // sp'' =3D sp' - x0 =3D (sp + x0) - x0 =3D sp - .else /* EL1t */ + b el1h_\regsize\()_\label + .endm + + .macro kernel_ventry_el1t, regsize:req, label:req msr spsel, #0 // Stay on the overflow stack sub sp, sp, #PT_REGS_SIZE - .endif - .endif + b el1t_\regsize\()_\label + .endm =20 - b el\el\ht\()_\regsize\()_\label + .macro kernel_ventry, el:req, ht:req, regsize:req, label:req + .align 7 +.Lventry_start\@: + .if \el =3D=3D 0 + kernel_ventry_el0 \ht, \regsize, \label + .else + kernel_ventry_el1\ht \regsize, \label + .endif .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm =20 --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D07DE51354D for ; Fri, 18 Sep 2026 16:15:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748125; cv=none; b=KaIJEiaPFXmoO+DkRtoYSNfROI4hoAwGByJfEU2FGBkEGQhHg741Yy00w6yckWErdpCF7BvVOHkkcw+wEdIpvEbTWD3Y9XNQOraLUKhdsQpbSvyrbsD0+cCzN/4NjOQEcqYV+COFHzAFBYcRKwELxguWOrEf66e43h/NY+klNv4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748125; c=relaxed/simple; bh=yFSglgfwwVFsCbPPMwLVI7K+suJCXr7OqYy1NCXd5NM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=luMcJZrXu/AfTGkZ5a2R5PcutIsW1REsAZFOssxkwHl/qzY0m+GmRFU1f6DpolBUDiMRZuYmeIh2us4cqE0+JCd90NI9/AKu5GmgrT1yQUIgLkWgVUjC+Bowa8wHNkZoX5jdi46YTBC9hb/7vhk1Po1vtczHYNjVRDva+/DRiMo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=gVfV/Yli; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="gVfV/Yli" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 798031F0089B; Fri, 18 Sep 2026 16:15:19 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748122; bh=BGQmYnb6QmPK7Dalbu5acCmGDnjMEGrMKannhaz7hEo=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=gVfV/YliCqgFe3j7/EqyE331jMQcIkWkHEMOW44yLOSkTCvChlgCLoPssporg/DBx e+MkTwXJpiH/V2/PqLJNyxInU84fAxMNCcTapDNZg/2OqDzKiyrPeW/8sxaUNQ+2Yj aPMjm14eDX/9G73Z7hrMnVaiPlGZtryR10nzL8Gy7eJA9P2HvyqxP0Kf9x/D3iUYmh odISHHY0J1XTDuc+A1WmV0tGeil9e5Pr4EFMWMGGi016pX1j9IN/hrTA+kW3onlIow MquMiIfS/pZ4QWtQDO3q6IaVH0Y2wZbuMpur4ZBhFP7S+g8RjKYxSW9o6bJHbycqZ5 ykFRIXHIGa7DA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 19/21] arm64: entry: The great stack switcheroo Date: Fri, 18 Sep 2026 17:14:03 +0100 Message-ID: <20260918161407.2300-20-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" With the kernel stack pointer in SP_EL1 and the overflow stack pointer in SP_EL0, it is now straightforward to switch between the two on exception entry from EL1 by writing to SPSel. However, since exception entry sets PSTATE.SP to 1 (selecting SP_EL1 as the stack pointer), repurposing the overflow stack as a more general kernel exception stack would require writing to SPSel on every exception entry from the kernel. Switch things around so that the overflow stack resides in SP_EL1, with the kernel stack residing in SP_EL0. Signed-off-by: Will Deacon --- arch/arm64/include/asm/kvm_asm.h | 16 +++++++++++++++ arch/arm64/kernel/entry-common.c | 34 ++++++++++++++++---------------- arch/arm64/kernel/entry.S | 26 +++++++++++++----------- arch/arm64/kernel/head.S | 5 ++++- arch/arm64/kernel/sleep.S | 3 ++- arch/arm64/kernel/smp.c | 8 +++++--- arch/arm64/kvm/hyp/entry.S | 3 +++ arch/arm64/mm/proc.S | 10 ++++++++-- 8 files changed, 70 insertions(+), 35 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_= asm.h index 1cfe9216a2ff..bde046c3ca79 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -398,6 +398,22 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr,= u64 spsr, u64 elr_virt, ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)] .endm =20 +#ifdef __KVM_VHE_HYPERVISOR__ +.macro activate_exception_stack +msr spsel, #1 +.endm + +.macro deactivate_exception_stack +msr spsel, #0 +.endm +#else +.macro activate_exception_stack +.endm + +.macro deactivate_exception_stack +.endm +#endif + .macro save_sp_el0 ctxt, tmp mrs \tmp, sp_el0 str \tmp, [\ctxt, #CPU_SP_EL0_OFFSET] diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-com= mon.c index 9738142780df..9d734cd09f62 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -327,7 +327,7 @@ static void debug_exception_exit(struct pt_regs *regs) } NOKPROBE_SYMBOL(debug_exception_exit); =20 -static void noinstr el1t_64_check_overflow_stack(struct pt_regs *regs) +static void noinstr el1h_64_check_overflow_stack(struct pt_regs *regs) { unsigned long sp =3D kernel_stack_pointer(regs) - sizeof(*regs); unsigned long ovf_stack =3D (unsigned long)this_cpu_ptr(overflow_stack); @@ -343,28 +343,28 @@ static void noinstr el1t_64_check_overflow_stack(stru= ct pt_regs *regs) cpu_park_loop(); } =20 -asmlinkage void noinstr el1t_64_sync_handler(struct pt_regs *regs) +asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) { - el1t_64_check_overflow_stack(regs); - el1h_64_sync_handler(regs); + el1h_64_check_overflow_stack(regs); + el1t_64_sync_handler(regs); } =20 -asmlinkage void noinstr el1t_64_irq_handler(struct pt_regs *regs) +asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) { - el1t_64_check_overflow_stack(regs); - el1h_64_irq_handler(regs); + el1h_64_check_overflow_stack(regs); + el1t_64_irq_handler(regs); } =20 -asmlinkage void noinstr el1t_64_fiq_handler(struct pt_regs *regs) +asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) { - el1t_64_check_overflow_stack(regs); - el1h_64_fiq_handler(regs); + el1h_64_check_overflow_stack(regs); + el1t_64_fiq_handler(regs); } =20 -asmlinkage void noinstr el1t_64_error_handler(struct pt_regs *regs) +asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) { - el1t_64_check_overflow_stack(regs); - el1h_64_error_handler(regs); + el1h_64_check_overflow_stack(regs); + el1t_64_error_handler(regs); } =20 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) @@ -494,7 +494,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsi= gned long esr) arm64_exit_to_kernel_mode(regs, state); } =20 -asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) +asmlinkage void noinstr el1t_64_sync_handler(struct pt_regs *regs) { unsigned long esr =3D read_sysreg(esr_el1); =20 @@ -578,17 +578,17 @@ static void noinstr el1_interrupt(struct pt_regs *reg= s, __el1_irq(regs, handler); } =20 -asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) +asmlinkage void noinstr el1t_64_irq_handler(struct pt_regs *regs) { el1_interrupt(regs, handle_arch_irq); } =20 -asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) +asmlinkage void noinstr el1t_64_fiq_handler(struct pt_regs *regs) { el1_interrupt(regs, handle_arch_fiq); } =20 -asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) +asmlinkage void noinstr el1t_64_error_handler(struct pt_regs *regs) { unsigned long esr =3D read_sysreg(esr_el1); irqentry_state_t state; diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 38f9327e6a0a..afcd84510daf 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -54,6 +54,12 @@ =20 .macro kernel_ventry_el1h, regsize:req, label:req sub sp, sp, #PT_REGS_SIZE + b el1h_\regsize\()_\label + .endm + + .macro kernel_ventry_el1t, regsize:req, label:req + msr spsel, #0 // Stay on the kernel stack + sub sp, sp, #PT_REGS_SIZE =20 /* * Test whether the SP has overflowed, without corrupting a GPR. @@ -65,12 +71,6 @@ tbnz x0, #THREAD_SHIFT, __bad_stack sub x0, sp, x0 // x0'' =3D sp' - x0' =3D (sp + x0) - sp =3D x0 sub sp, sp, x0 // sp'' =3D sp' - x0 =3D (sp + x0) - x0 =3D sp - b el1h_\regsize\()_\label - .endm - - .macro kernel_ventry_el1t, regsize:req, label:req - msr spsel, #0 // Stay on the overflow stack - sub sp, sp, #PT_REGS_SIZE b el1t_\regsize\()_\label .endm =20 @@ -208,7 +208,10 @@ alternative_cb_end ldr_this_cpu tsk, __entry_task, x20 msr tpidrro_el0, tsk adr_this_cpu x19, overflow_stack + OVERFLOW_STACK_SIZE, x20 - msr sp_el0, x19 + mov x20, sp + msr sp_el0, x20 + mov sp, x19 + msr spsel, #0 =20 /* * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions @@ -346,6 +349,9 @@ alternative_else_nop_endif =20 .if \el =3D=3D 0 ldr x23, [sp, #S_SP] // load return stack pointer + mov x0, sp + msr spsel, #1 + mov sp, x0 msr sp_el0, x23 msr tpidrro_el0, xzr tst x22, #PSR_MODE32_BIT // native task? @@ -532,16 +538,14 @@ SYM_CODE_START_LOCAL(__bad_stack) add sp, sp, #PT_REGS_SIZE =20 /* Switch to the overflow stack */ - msr spsel, #0 + msr spsel, #1 =20 /* Stash the exception regs */ sub sp, sp, #PT_REGS_SIZE kernel_entry 1 =20 /* Fix-up the saved SP */ - msr spsel, #1 - mov x0, sp - msr spsel, #0 + mrs x0, sp_el0 str x0, [sp, #S_SP] =20 /* Stash the regs for handle_bad_stack */ diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 27a19e1a0ee6..7419e1d8b7ec 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -211,8 +211,11 @@ SYM_CODE_END(preserve_boot_args) ldr \tmp1, [\tmp1, \tmp2, lsl #3] set_this_cpu_offset \tmp1 =20 - adr_this_cpu \tmp1, overflow_stack + OVERFLOW_STACK_SIZE, \tmp2 + mov \tmp1, sp msr sp_el0, \tmp1 + adr_this_cpu \tmp1, overflow_stack + OVERFLOW_STACK_SIZE, \tmp2 + mov sp, \tmp1 + msr spsel, #0 .endm =20 /* diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index f093cdf71be1..facf3f1cc3b1 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -133,7 +133,8 @@ SYM_FUNC_START(_cpu_resume) add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS /* load sp from context */ ldr x2, [x0, #CPU_CTX_SP] - mov sp, x2 + msr sp_el0, x2 + /* * cpu_do_resume expects x0 to contain context address pointer */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 4c122d6598de..b2e778aae881 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -447,14 +447,16 @@ void __init smp_cpus_done(unsigned int max_cpus) mark_linear_text_alias_ro(); } =20 -static void __init set_boot_cpu_offset(void) +static void __init update_boot_cpu_offset_and_overflow_stack(void) { u64 ovf_sp =3D (u64)raw_cpu_ptr(overflow_stack) + OVERFLOW_STACK_SIZE; =20 asm volatile( " msr tpidr_el1, %1\n" " add %0, %0, %1\n" - " msr sp_el0, %0" /* Update the overflow stack pointer */ + " msr spsel, #1\n" + " mov sp, %0\n" /* Update the overflow stack pointer */ + " msr spsel, #0" : "+r" (ovf_sp) : "r" (per_cpu_offset(0)) : "memory"); @@ -467,7 +469,7 @@ void __init smp_prepare_boot_cpu(void) * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be * freed shortly, so we must move over to the runtime per-cpu area. */ - set_boot_cpu_offset(); + update_boot_cpu_offset_and_overflow_stack(); =20 cpuinfo_store_boot_cpu(); setup_boot_cpu_features(); diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 4c89931a6a92..4d1205d04383 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -31,6 +31,7 @@ SYM_FUNC_START(__guest_enter) save_callee_saved_regs x1 =20 // Save hyp's sp_el0 and tpidrro_el0 + activate_exception_stack save_sp_el0 x1, x2 save_tpidrro_el0 x1, x2 =20 @@ -50,6 +51,7 @@ alternative_else_nop_endif // that would usually be synchonized by the ERET. isb mov x0, #ARM_EXCEPTION_IRQ + deactivate_exception_stack ret =20 1: @@ -167,6 +169,7 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL) // Restore hyp's sp_el0 and tpidrro_el0 restore_sp_el0 x2, x3 restore_tpidrro_el0 x2, x3 + deactivate_exception_stack =20 // Now restore the hyp regs restore_callee_saved_regs x2 diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index ad4965d34978..bec7f858fae9 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -87,6 +87,7 @@ * This must be kept in sync with struct cpu_suspend_ctx in . */ SYM_FUNC_START(cpu_do_suspend) + msr spsel, #1 mrs x2, tpidr_el0 mrs x3, tpidrro_el0 mrs x4, contextidr_el1 @@ -98,7 +99,7 @@ SYM_FUNC_START(cpu_do_suspend) mrs x10, oslsr_el1 mrs x11, sctlr_el1 get_this_cpu_offset x12 - mrs x13, sp_el0 + mov x13, sp // SP_EL1 stp x2, x3, [x0] stp x4, x5, [x0, #16] stp x6, x7, [x0, #32] @@ -114,6 +115,7 @@ alternative_if ARM64_HAS_TCR2 mrs x2, REG_TCR2_EL1 str x2, [x0, #104] alternative_else_nop_endif + msr spsel, #0 ret SYM_FUNC_END(cpu_do_suspend) =20 @@ -121,6 +123,8 @@ SYM_FUNC_END(cpu_do_suspend) * cpu_do_resume - restore CPU register context * * x0: Address of context pointer + * + * Entered with SPSel =3D=3D 1, returns with SPSel =3D=3D 0. */ SYM_FUNC_START(cpu_do_resume) ldp x2, x3, [x0] @@ -159,7 +163,8 @@ alternative_else_nop_endif =20 msr sctlr_el1, x12 set_this_cpu_offset x13 - msr sp_el0, x14 + mov sp, x14 // SP_EL1 + /* * Restore oslsr_el1 by writing oslar_el1 */ @@ -182,6 +187,7 @@ alternative_else_nop_endif =20 ptrauth_keys_install_kernel_nosync x15, x1, x2, x3 isb + msr spsel, #0 ret SYM_FUNC_END(cpu_do_resume) #endif --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5B721515985 for ; Fri, 18 Sep 2026 16:15:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748130; cv=none; b=qnE4S3KnK29a85EjsM70mEz2avsR1YdxD1LNX9o9FnJ5dJ2KMTRSm6pQFmNWAwdrFwCoyRgifr9EMcsHSskpDNimrzkhUNzqFVOmE1H3u6oYzUekbR0ST0xLxWeFOYv+umVcK1dajUnIwxGNhS0/UqSldtg7KxBJa7W0szgU324= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748130; c=relaxed/simple; bh=plIVOrd3fTl4D3hxRhJU17B1fyw4S/NMTdnntp9Zj1A=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=o23lpnCSdSWX21vIyeCvD/YcyoxbK0ntmJWXt2l+idKIntzUUN4ge8sWvnAZl6VjzwDQ7r3nsMQ/FAgdgDtRfwL27cbaxLxM/vWklsY2YgI8DKPL7BkyKFViuIolSldfbgR+W8P80jveda2khIfUgmQO4Jao6cgUGmr/N/uwtIA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=OET/BcYv; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="OET/BcYv" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1B4161F00898; Fri, 18 Sep 2026 16:15:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748126; bh=/mfIbyB+KiTZj/ummjBpFJweWa08M4vk6DdFzOZra/Y=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=OET/BcYvAijGn3Z+EJenGnwlssJv3P030zZZexQ37E4gCoO2NAKf5wH6uvLZPo3V0 7sX8CHwaxf/9tM8pxb83Bmjqqq1dZUJeh2zuw/S0YI/LXvTjzg62MFByp7SZc/NPCO DOjfIMzKVMiMvGZ1p1fFPIPs8ujtGd7Hh5DAJVPRG5Dvf8re2hONSRyr39h5iEzI1v QYRmOdU3ouaAC9sSpHV1y0U0Ojp0V9+P1+oBFSVBaQ7cl4CteT9srIcwHV+t1lZCK0 aqWDlJZCo0OZTTJTCpXKQzV0NeYONwzvmW8tXmYCKAWN8XAVoFioWCCqfe8rtPd6Mv +MlSxVi4AOaRw== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 20/21] arm64: tracing: Advertise a mode of EL1t in synthetic kernel regs Date: Fri, 18 Sep 2026 17:14:04 +0100 Message-ID: <20260918161407.2300-21-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When synthesising a kernel regs structure for ftrace or perf, advertise a mode of EL1t to reflect the mode in which the kernel runs for the vast majority of the time. Signed-off-by: Will Deacon --- arch/arm64/include/asm/ftrace.h | 4 ++-- arch/arm64/include/asm/perf_event.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrac= e.h index 1621c84f44b3..d55a019cdf0a 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -153,7 +153,7 @@ ftrace_partial_regs(const struct ftrace_regs *fregs, st= ruct pt_regs *regs) regs->pc =3D afregs->pc; regs->regs[29] =3D afregs->fp; regs->regs[30] =3D afregs->lr; - regs->pstate =3D PSR_MODE_EL1h; + regs->pstate =3D PSR_MODE_EL1t; return regs; } =20 @@ -161,7 +161,7 @@ ftrace_partial_regs(const struct ftrace_regs *fregs, st= ruct pt_regs *regs) (_regs)->pc =3D arch_ftrace_regs(fregs)->pc; \ (_regs)->regs[29] =3D arch_ftrace_regs(fregs)->fp; \ (_regs)->sp =3D arch_ftrace_regs(fregs)->sp; \ - (_regs)->pstate =3D PSR_MODE_EL1h; \ + (_regs)->pstate =3D PSR_MODE_EL1t; \ } while (0) =20 int ftrace_regs_query_register_offset(const char *name); diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/p= erf_event.h index ee45b4e77347..b3e7c24a5aab 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -17,7 +17,7 @@ (regs)->pc =3D (__ip); \ (regs)->regs[29] =3D (unsigned long) __builtin_frame_address(0); \ (regs)->sp =3D current_stack_pointer; \ - (regs)->pstate =3D PSR_MODE_EL1h; \ + (regs)->pstate =3D PSR_MODE_EL1t; \ } =20 #endif --=20 2.55.0.1082.g2b9226bbc0-goog From nobody Thu Sep 24 23:31:38 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7705A519926 for ; Fri, 18 Sep 2026 16:15:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748134; cv=none; b=EIGWA57583YAwBOF8Iqg6cWhRkElfMZLT27zhrrpGBDyYt9kG0rEnobvDRvpVirSrAGi972oRi2JQVMCjNy8/ld8XsTXLDsSo+XFZw5s5S5hgzfTO6tbsATT04b0wNJP+jHqd0UqrAhPiinKbtrApz6avLKZg2YBpXKZQRdsp/Q= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1789748134; c=relaxed/simple; bh=4uF2fd0q4Y/XJYsItfcyzse7AzMOcwY4m7eAHOXFNp0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XXaUlp/FJzoWL7hRJKbsfM8jk/UvFrbdB7Eu1JQ68c447+QNyeCKbpXlM0/u3bjT2PAVoNnc3ssarg8cnnqhQTxZy9SCbbxpX5EVrXAhnyRrMyosGXs5z8xkUptxA3c0mB/t8JOhw6STuOSceEc0zhHr/bL8jti+UhH64k2mTr4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=aOFRNg+d; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="aOFRNg+d" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9504C1F000FF; Fri, 18 Sep 2026 16:15:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1789748129; bh=PE4cCu1o37AUDU5SflXup6qIm7aL2qk9d7s5UIDZgZA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=aOFRNg+dgEZrk9frbheUHwy7evCHTAcjPX6vwsXAEV4X0XneSepCS+YIHrBeI5g7N OSqJQ8VloAPnn/vvnzH1fQ89Wj4csKZEgqnSBkg6zP+/W5YjStrYDJUrNxCJcIjolS UoI7zj169gjJaz6AsZH5RWZdP0Rchi+QsNcFatvqKyXGw97sWdUKT09CdTIN2qv7D7 9bA+46Ea81rLzIfAweRurzZziVu1z7FEV01j6Eg4G3KGlRg0HOcaZ0/6h9qayTGGTN CLLo8zrxWrctnR5avCxnSXo49zK41mlovj5pb3p2kX3HlCe8jp7UdsMtYWUvXynw08 XqIlm7LFE3GFA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Arnd Bergmann , Ard Biesheuvel , Ada Couprie Diaz , David Hildenbrand , Catalin Marinas , Vladimir Murzin , Mark Rutland , Mostafa Saleh , Lorenzo Stoakes , Oliver Upton , Linus Walleij , Marc Zyngier Subject: [PATCH v2 21/21] arm64: Rename 'overflow_stack' and OVERFLOW_STACK_SIZE Date: Fri, 18 Sep 2026 17:14:05 +0100 Message-ID: <20260918161407.2300-22-will@kernel.org> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260918161407.2300-1-will@kernel.org> References: <20260918161407.2300-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Now that the kernel "overflow" stack resides in SP_EL1, it is the active stack for any exception taken from the kernel. Even though we currently switch immediately to the kernel stack in the EL1t entry code, rename the "overflow_stack" to "kernel_exception_stack" to better reflect its usage. Introduce NVHE_OVERFLOW_STACK_SIZE to keep the EL2 overflow stack size distinct from the EL1 kernel exception stack size. Signed-off-by: Will Deacon --- arch/arm64/include/asm/memory.h | 10 ++++++---- arch/arm64/include/asm/stacktrace.h | 9 +++++---- arch/arm64/include/asm/stacktrace/nvhe.h | 3 +-- arch/arm64/kernel/entry-common.c | 18 +++++++++--------- arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/head.S | 2 +- arch/arm64/kernel/smp.c | 9 +++++---- arch/arm64/kernel/stacktrace.c | 2 +- arch/arm64/kernel/traps.c | 14 +++++++------- arch/arm64/kvm/hyp/nvhe/host.S | 2 +- arch/arm64/kvm/hyp/nvhe/stacktrace.c | 4 ++-- arch/arm64/kvm/stacktrace.c | 4 ++-- 12 files changed, 41 insertions(+), 38 deletions(-) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memor= y.h index a2b7a33966ff..93ce6ef65573 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -139,10 +139,11 @@ =20 #define IRQ_STACK_SIZE THREAD_SIZE =20 -#define OVERFLOW_STACK_SIZE SZ_4K +#define KERNEL_EXC_STACK_SIZE SZ_4K =20 -#define NVHE_STACK_SHIFT PAGE_SHIFT -#define NVHE_STACK_SIZE (UL(1) << NVHE_STACK_SHIFT) +#define NVHE_OVERFLOW_STACK_SIZE SZ_4K +#define NVHE_STACK_SHIFT PAGE_SHIFT +#define NVHE_STACK_SIZE (UL(1) << NVHE_STACK_SHIFT) =20 /* * With the minimum frame size of [x29, x30], exactly half the combined @@ -150,7 +151,8 @@ * save the unwinded stacktrace; plus an additional entry to delimit the * end. */ -#define NVHE_STACKTRACE_SIZE ((OVERFLOW_STACK_SIZE + NVHE_STACK_SIZE) / 2 = + sizeof(long)) +#define NVHE_STACKTRACE_SIZE ((NVHE_OVERFLOW_STACK_SIZE + NVHE_STACK_SIZE)= \ + / 2 + sizeof(long)) =20 /* * Alignment of kernel segments (e.g. .text, .data). diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/s= tacktrace.h index 6d3280932bf5..eb18d7cee62f 100644 --- a/arch/arm64/include/asm/stacktrace.h +++ b/arch/arm64/include/asm/stacktrace.h @@ -59,12 +59,13 @@ static inline bool on_task_stack(const struct task_stru= ct *tsk, =20 #define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1= )) =20 -DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow= _stack); +DECLARE_PER_CPU(unsigned long [KERNEL_EXC_STACK_SIZE/sizeof(long)], + kernel_exception_stack); =20 -static inline struct stack_info stackinfo_get_overflow(void) +static inline struct stack_info stackinfo_get_kernel_exception(void) { - unsigned long low =3D (unsigned long)raw_cpu_ptr(overflow_stack); - unsigned long high =3D low + OVERFLOW_STACK_SIZE; + unsigned long low =3D (unsigned long)raw_cpu_ptr(kernel_exception_stack); + unsigned long high =3D low + KERNEL_EXC_STACK_SIZE; =20 return (struct stack_info) { .low =3D low, diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/= asm/stacktrace/nvhe.h index 171f9edef49f..629dd2d2caa3 100644 --- a/arch/arm64/include/asm/stacktrace/nvhe.h +++ b/arch/arm64/include/asm/stacktrace/nvhe.h @@ -44,8 +44,7 @@ static inline void kvm_nvhe_unwind_init(struct unwind_sta= te *state, * In non-protected mode, the unwinding is done from kernel proper context * (by the host in EL1). */ - -DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],= overflow_stack); +DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_OVERFLOW_STACK_SIZE/sizeof(lo= ng)], overflow_stack); DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_i= nfo); DECLARE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base); =20 diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-com= mon.c index 9d734cd09f62..7e9b26cf1bdc 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -327,43 +327,43 @@ static void debug_exception_exit(struct pt_regs *regs) } NOKPROBE_SYMBOL(debug_exception_exit); =20 -static void noinstr el1h_64_check_overflow_stack(struct pt_regs *regs) +static void noinstr el1h_64_check_exception_stack(struct pt_regs *regs) { unsigned long sp =3D kernel_stack_pointer(regs) - sizeof(*regs); - unsigned long ovf_stack =3D (unsigned long)this_cpu_ptr(overflow_stack); + unsigned long exc_stack =3D (unsigned long)this_cpu_ptr(kernel_exception_= stack); =20 /* - * We're in big trouble if we've overflowed the overflow stack + * We're in big trouble if we've overflowed the exception stack * so perform a best-effort check before we proceed. If our SP - * is outside of the overflow stack for this CPU then presumably + * is outside of the exceptoin stack for this CPU then presumably * we're already corrupting memory, so park ourselves here in an * attempt to contain the damage. */ - if (sp < ovf_stack || sp > ovf_stack + OVERFLOW_STACK_SIZE) + if (sp < exc_stack || sp > exc_stack + KERNEL_EXC_STACK_SIZE) cpu_park_loop(); } =20 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) { - el1h_64_check_overflow_stack(regs); + el1h_64_check_exception_stack(regs); el1t_64_sync_handler(regs); } =20 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) { - el1h_64_check_overflow_stack(regs); + el1h_64_check_exception_stack(regs); el1t_64_irq_handler(regs); } =20 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) { - el1h_64_check_overflow_stack(regs); + el1h_64_check_exception_stack(regs); el1t_64_fiq_handler(regs); } =20 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) { - el1h_64_check_overflow_stack(regs); + el1h_64_check_exception_stack(regs); el1t_64_error_handler(regs); } =20 diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index afcd84510daf..a31ef890a2ee 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -207,7 +207,7 @@ alternative_cb_end mrs x21, sp_el0 ldr_this_cpu tsk, __entry_task, x20 msr tpidrro_el0, tsk - adr_this_cpu x19, overflow_stack + OVERFLOW_STACK_SIZE, x20 + adr_this_cpu x19, kernel_exception_stack + KERNEL_EXC_STACK_SIZE, x20 mov x20, sp msr sp_el0, x20 mov sp, x19 diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 7419e1d8b7ec..794cefaf8482 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -213,7 +213,7 @@ SYM_CODE_END(preserve_boot_args) =20 mov \tmp1, sp msr sp_el0, \tmp1 - adr_this_cpu \tmp1, overflow_stack + OVERFLOW_STACK_SIZE, \tmp2 + adr_this_cpu \tmp1, kernel_exception_stack + KERNEL_EXC_STACK_SIZE, \tmp2 mov sp, \tmp1 msr spsel, #0 .endm diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index b2e778aae881..5aaeda5362f7 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -447,9 +447,10 @@ void __init smp_cpus_done(unsigned int max_cpus) mark_linear_text_alias_ro(); } =20 -static void __init update_boot_cpu_offset_and_overflow_stack(void) +static void __init update_boot_cpu_offset_and_exception_stack(void) { - u64 ovf_sp =3D (u64)raw_cpu_ptr(overflow_stack) + OVERFLOW_STACK_SIZE; + u64 exc_sp =3D (u64)raw_cpu_ptr(kernel_exception_stack) + + KERNEL_EXC_STACK_SIZE; =20 asm volatile( " msr tpidr_el1, %1\n" @@ -457,7 +458,7 @@ static void __init update_boot_cpu_offset_and_overflow_= stack(void) " msr spsel, #1\n" " mov sp, %0\n" /* Update the overflow stack pointer */ " msr spsel, #0" - : "+r" (ovf_sp) + : "+r" (exc_sp) : "r" (per_cpu_offset(0)) : "memory"); } @@ -469,7 +470,7 @@ void __init smp_prepare_boot_cpu(void) * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be * freed shortly, so we must move over to the runtime per-cpu area. */ - update_boot_cpu_offset_and_overflow_stack(); + update_boot_cpu_offset_and_exception_stack(); =20 cpuinfo_store_boot_cpu(); setup_boot_cpu_features(); diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 3ebcf8c53fb0..8e65b81c7502 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -337,7 +337,7 @@ kunwind_stack_walk(kunwind_consume_fn consume_state, struct stack_info stacks[] =3D { stackinfo_get_task(task), STACKINFO_CPU(irq), - STACKINFO_CPU(overflow), + STACKINFO_CPU(kernel_exception), #if defined(CONFIG_ARM_SDE_INTERFACE) STACKINFO_SDEI(normal), STACKINFO_SDEI(critical), diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 914282016069..ce0e213d772d 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -897,14 +897,14 @@ void bad_el0_sync(struct pt_regs *regs, int reason, u= nsigned long esr) "Bad EL0 synchronous exception"); } =20 -DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_= stack) - __aligned(16); +DEFINE_PER_CPU(unsigned long [KERNEL_EXC_STACK_SIZE/sizeof(long)], + kernel_exception_stack) __aligned(16); =20 void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, u= nsigned long far) { unsigned long tsk_stk =3D (unsigned long)current->stack; unsigned long irq_stk =3D (unsigned long)this_cpu_read(irq_stack_ptr); - unsigned long ovf_stk =3D (unsigned long)this_cpu_ptr(overflow_stack); + unsigned long exc_stk =3D (unsigned long)this_cpu_ptr(kernel_exception_st= ack); =20 console_verbose(); pr_emerg("Insufficient stack space to handle exception!"); @@ -912,12 +912,12 @@ void __noreturn panic_bad_stack(struct pt_regs *regs,= unsigned long esr, unsigne pr_emerg("ESR: 0x%016lx -- %s\n", esr, esr_get_class_string(esr)); pr_emerg("FAR: 0x%016lx\n", far); =20 - pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", + pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", tsk_stk, tsk_stk + THREAD_SIZE); - pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", + pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", irq_stk, irq_stk + IRQ_STACK_SIZE); - pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", - ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); + pr_emerg("Exception stack: [0x%016lx..0x%016lx]\n", + exc_stk, exc_stk + KERNEL_EXC_STACK_SIZE); =20 __show_regs(regs); =20 diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 9393fe3ea6a1..b05211a55074 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -205,7 +205,7 @@ SYM_FUNC_END(__host_hvc) =20 .L__hyp_sp_overflow\@: /* Switch to the overflow stack */ - adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 + adr_this_cpu sp, overflow_stack + NVHE_OVERFLOW_STACK_SIZE, x0 =20 b hyp_panic_bad_stack ASM_BUG() diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe= /stacktrace.c index 7c832d60d22b..92fe8b6d4de0 100644 --- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c +++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c @@ -9,7 +9,7 @@ #include #include =20 -DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_= stack) +DEFINE_PER_CPU(unsigned long [NVHE_OVERFLOW_STACK_SIZE/sizeof(long)], over= flow_stack) __aligned(16); =20 DEFINE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); @@ -42,7 +42,7 @@ DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof= (long)], pkvm_stacktrac static struct stack_info stackinfo_get_overflow(void) { unsigned long low =3D (unsigned long)this_cpu_ptr(overflow_stack); - unsigned long high =3D low + OVERFLOW_STACK_SIZE; + unsigned long high =3D low + NVHE_OVERFLOW_STACK_SIZE; =20 return (struct stack_info) { .low =3D low, diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c index 9724c320126b..e5b878f959fc 100644 --- a/arch/arm64/kvm/stacktrace.c +++ b/arch/arm64/kvm/stacktrace.c @@ -27,7 +27,7 @@ static struct stack_info stackinfo_get_overflow(void) struct kvm_nvhe_stacktrace_info *stacktrace_info =3D this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); unsigned long low =3D (unsigned long)stacktrace_info->overflow_stack_base; - unsigned long high =3D low + OVERFLOW_STACK_SIZE; + unsigned long high =3D low + NVHE_OVERFLOW_STACK_SIZE; =20 return (struct stack_info) { .low =3D low, @@ -38,7 +38,7 @@ static struct stack_info stackinfo_get_overflow(void) static struct stack_info stackinfo_get_overflow_kern_va(void) { unsigned long low =3D (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack= ); - unsigned long high =3D low + OVERFLOW_STACK_SIZE; + unsigned long high =3D low + NVHE_OVERFLOW_STACK_SIZE; =20 return (struct stack_info) { .low =3D low, --=20 2.55.0.1082.g2b9226bbc0-goog