From nobody Sat Dec 28 00:08:45 2024 Received: from szxga04-in.huawei.com (szxga04-in.huawei.com [45.249.212.190]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 070DC202F86 for ; Fri, 6 Dec 2024 10:18:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=45.249.212.190 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1733480301; cv=none; b=kTfeWOFRm0OM+i2qqFgsP6RvPaekZPZjEEMgJPfddzyuCE5TFVjqYuBZODUnlct3IwmgIpY3uDQjyxaM+BUSWphGA+SxcvMfBEn8wkhF0CsUT4BPkbEB9L5SUmmHgXZ8m27cdHHCy0/GuMlhPByF86LAlWViXJOx5A3yy9JvSIc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1733480301; c=relaxed/simple; bh=8iUmtnZixLddFQQtY6jhMRrmEOT2mXVSsMXGHal0LmY=; h=From:To:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=gT/r1EhtQPC7NTdIA6/E1mX3ObQtAY2rQgd9RlkDNejNJ+n9gctldh7tExj7q7nyNt32Azulrm4z1kvkbgyHWeEN1SyF/9OO6glxLEXBwmzx90RmzghTEfBgUXGxovVoX9SFFPmsy6qk9CFpf9SN0dIGaRcUlIOgow+L/6gw+/s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com; spf=pass smtp.mailfrom=huawei.com; arc=none smtp.client-ip=45.249.212.190 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=huawei.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=huawei.com Received: from mail.maildlp.com (unknown [172.19.162.112]) by szxga04-in.huawei.com (SkyGuard) with ESMTP id 4Y4RxV0z8nz21mdM; Fri, 6 Dec 2024 18:16:30 +0800 (CST) Received: from kwepemg200008.china.huawei.com (unknown [7.202.181.35]) by mail.maildlp.com (Postfix) with ESMTPS id 8A77C1401DC; Fri, 6 Dec 2024 18:18:09 +0800 (CST) Received: from huawei.com (10.90.53.73) by kwepemg200008.china.huawei.com (7.202.181.35) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1544.11; Fri, 6 Dec 2024 18:18:08 +0800 From: Jinjie Ruan To: , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , Subject: [PATCH -next v5 02/22] arm64: entry: Refactor the entry and exit for exceptions from EL1 Date: Fri, 6 Dec 2024 18:17:24 +0800 Message-ID: <20241206101744.4161990-3-ruanjinjie@huawei.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20241206101744.4161990-1-ruanjinjie@huawei.com> References: <20241206101744.4161990-1-ruanjinjie@huawei.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-ClientProxiedBy: dggems706-chm.china.huawei.com (10.3.19.183) To kwepemg200008.china.huawei.com (7.202.181.35) Content-Type: text/plain; charset="utf-8" The generic entry code uses irqentry_state_t to track lockdep and RCU state across exception entry and return. For historical reasons, arm64 embeds similar fields within its pt_regs structure. In preparation for moving arm64 over to the generic entry code, pull these fields out of arm64's pt_regs, and use a separate structure, matching the style of the generic entry code. No functional changes. Suggested-by: Mark Rutland Signed-off-by: Jinjie Ruan --- arch/arm64/include/asm/ptrace.h | 4 - arch/arm64/kernel/entry-common.c | 136 +++++++++++++++++++------------ 2 files changed, 85 insertions(+), 55 deletions(-) diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrac= e.h index bcfa96880377..e90dfc9982aa 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -169,10 +169,6 @@ struct pt_regs { =20 u64 sdei_ttbr1; struct frame_record_meta stackframe; - - /* Only valid for some EL1 exceptions. */ - u64 lockdep_hardirqs; - u64 exit_rcu; }; =20 /* For correct stack alignment, pt_regs has to be a multiple of 16 bytes. = */ diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-com= mon.c index c547e70428d3..1687627b2ecf 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -28,6 +28,13 @@ #include #include =20 +typedef struct irqentry_state { + union { + bool exit_rcu; + bool lockdep; + }; +} irqentry_state_t; + /* * Handle IRQ/context state management when entering from kernel mode. * Before this function is called it is not safe to call regular kernel co= de, @@ -36,29 +43,36 @@ * This is intended to match the logic in irqentry_enter(), handling the k= ernel * mode transitions only. */ -static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs) +static __always_inline irqentry_state_t __enter_from_kernel_mode(struct pt= _regs *regs) { - regs->exit_rcu =3D false; + irqentry_state_t state =3D { + .exit_rcu =3D false, + }; =20 if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { lockdep_hardirqs_off(CALLER_ADDR0); ct_irq_enter(); trace_hardirqs_off_finish(); =20 - regs->exit_rcu =3D true; - return; + state.exit_rcu =3D true; + return state; } =20 lockdep_hardirqs_off(CALLER_ADDR0); rcu_irq_enter_check_tick(); trace_hardirqs_off_finish(); + + return state; } =20 -static void noinstr enter_from_kernel_mode(struct pt_regs *regs) +static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *reg= s) { - __enter_from_kernel_mode(regs); + irqentry_state_t state =3D __enter_from_kernel_mode(regs); + mte_check_tfsr_entry(); mte_disable_tco_entry(current); + + return state; } =20 /* @@ -69,12 +83,13 @@ static void noinstr enter_from_kernel_mode(struct pt_re= gs *regs) * This is intended to match the logic in irqentry_exit(), handling the ke= rnel * mode transitions only, and with preemption handled elsewhere. */ -static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) +static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { lockdep_assert_irqs_disabled(); =20 if (!regs_irqs_disabled(regs)) { - if (regs->exit_rcu) { + if (state.exit_rcu) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); ct_irq_exit(); @@ -84,15 +99,16 @@ static __always_inline void __exit_to_kernel_mode(struc= t pt_regs *regs) =20 trace_hardirqs_on(); } else { - if (regs->exit_rcu) + if (state.exit_rcu) ct_irq_exit(); } } =20 -static void noinstr exit_to_kernel_mode(struct pt_regs *regs) +static void noinstr exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { mte_check_tfsr_exit(); - __exit_to_kernel_mode(regs); + __exit_to_kernel_mode(regs, state); } =20 /* @@ -190,9 +206,11 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct p= t_regs *regs) * mode. Before this function is called it is not safe to call regular ker= nel * code, instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_enter_nmi(struct pt_regs *regs) +static noinstr irqentry_state_t arm64_enter_nmi(struct pt_regs *regs) { - regs->lockdep_hardirqs =3D lockdep_hardirqs_enabled(); + irqentry_state_t state; + + state.lockdep =3D lockdep_hardirqs_enabled(); =20 __nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); @@ -201,6 +219,8 @@ static void noinstr arm64_enter_nmi(struct pt_regs *reg= s) =20 trace_hardirqs_off_finish(); ftrace_nmi_enter(); + + return state; } =20 /* @@ -208,19 +228,18 @@ static void noinstr arm64_enter_nmi(struct pt_regs *r= egs) * mode. After this function returns it is not safe to call regular kernel * code, instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_nmi(struct pt_regs *regs) +static void noinstr arm64_exit_nmi(struct pt_regs *regs, + irqentry_state_t state) { - bool restore =3D regs->lockdep_hardirqs; - ftrace_nmi_exit(); - if (restore) { + if (state.lockdep) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); } =20 ct_nmi_exit(); lockdep_hardirq_exit(); - if (restore) + if (state.lockdep) lockdep_hardirqs_on(CALLER_ADDR0); __nmi_exit(); } @@ -230,14 +249,18 @@ static void noinstr arm64_exit_nmi(struct pt_regs *re= gs) * kernel mode. Before this function is called it is not safe to call regu= lar * kernel code, instrumentable code, or any code which may trigger an exce= ption. */ -static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) +static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) { - regs->lockdep_hardirqs =3D lockdep_hardirqs_enabled(); + irqentry_state_t state; + + state.lockdep =3D lockdep_hardirqs_enabled(); =20 lockdep_hardirqs_off(CALLER_ADDR0); ct_nmi_enter(); =20 trace_hardirqs_off_finish(); + + return state; } =20 /* @@ -245,17 +268,16 @@ static void noinstr arm64_enter_el1_dbg(struct pt_reg= s *regs) * kernel mode. After this function returns it is not safe to call regular * kernel code, instrumentable code, or any code which may trigger an exce= ption. */ -static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) +static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, + irqentry_state_t state) { - bool restore =3D regs->lockdep_hardirqs; - - if (restore) { + if (state.lockdep) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); } =20 ct_nmi_exit(); - if (restore) + if (state.lockdep) lockdep_hardirqs_on(CALLER_ADDR0); } =20 @@ -426,78 +448,86 @@ UNHANDLED(el1t, 64, error) static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) { unsigned long far =3D read_sysreg(far_el1); + irqentry_state_t state; =20 - enter_from_kernel_mode(regs); + state =3D enter_from_kernel_mode(regs); local_daif_inherit(regs); do_mem_abort(far, esr, regs); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) { unsigned long far =3D read_sysreg(far_el1); + irqentry_state_t state; =20 - enter_from_kernel_mode(regs); + state =3D enter_from_kernel_mode(regs); local_daif_inherit(regs); do_sp_pc_abort(far, esr, regs); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) { - enter_from_kernel_mode(regs); + irqentry_state_t state =3D enter_from_kernel_mode(regs); + local_daif_inherit(regs); do_el1_undef(regs, esr); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) { - enter_from_kernel_mode(regs); + irqentry_state_t state =3D enter_from_kernel_mode(regs); + local_daif_inherit(regs); do_el1_bti(regs, esr); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) { - enter_from_kernel_mode(regs); + irqentry_state_t state =3D enter_from_kernel_mode(regs); + local_daif_inherit(regs); do_el1_gcs(regs, esr); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) { - enter_from_kernel_mode(regs); + irqentry_state_t state =3D enter_from_kernel_mode(regs); + local_daif_inherit(regs); do_el1_mops(regs, esr); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr) { unsigned long far =3D read_sysreg(far_el1); + irqentry_state_t state; =20 - arm64_enter_el1_dbg(regs); + state =3D arm64_enter_el1_dbg(regs); if (!cortex_a76_erratum_1463225_debug_handler(regs)) do_debug_exception(far, esr, regs); - arm64_exit_el1_dbg(regs); + arm64_exit_el1_dbg(regs, state); } =20 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) { - enter_from_kernel_mode(regs); + irqentry_state_t state =3D enter_from_kernel_mode(regs); + local_daif_inherit(regs); do_el1_fpac(regs, esr); local_daif_mask(); - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } =20 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) @@ -546,15 +576,16 @@ asmlinkage void noinstr el1h_64_sync_handler(struct p= t_regs *regs) static __always_inline void __el1_pnmi(struct pt_regs *regs, void (*handler)(struct pt_regs *)) { - arm64_enter_nmi(regs); + irqentry_state_t state =3D arm64_enter_nmi(regs); + do_interrupt_handler(regs, handler); - arm64_exit_nmi(regs); + arm64_exit_nmi(regs, state); } =20 static __always_inline void __el1_irq(struct pt_regs *regs, void (*handler)(struct pt_regs *)) { - enter_from_kernel_mode(regs); + irqentry_state_t state =3D enter_from_kernel_mode(regs); =20 irq_enter_rcu(); do_interrupt_handler(regs, handler); @@ -562,7 +593,7 @@ static __always_inline void __el1_irq(struct pt_regs *r= egs, =20 arm64_preempt_schedule_irq(); =20 - exit_to_kernel_mode(regs); + exit_to_kernel_mode(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) @@ -588,11 +619,12 @@ asmlinkage void noinstr el1h_64_fiq_handler(struct pt= _regs *regs) asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) { unsigned long esr =3D read_sysreg(esr_el1); + irqentry_state_t state; =20 local_daif_restore(DAIF_ERRCTX); - arm64_enter_nmi(regs); + state =3D arm64_enter_nmi(regs); do_serror(regs, esr); - arm64_exit_nmi(regs); + arm64_exit_nmi(regs, state); } =20 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) @@ -855,12 +887,13 @@ asmlinkage void noinstr el0t_64_fiq_handler(struct pt= _regs *regs) static void noinstr __el0_error_handler_common(struct pt_regs *regs) { unsigned long esr =3D read_sysreg(esr_el1); + irqentry_state_t state; =20 enter_from_user_mode(regs); local_daif_restore(DAIF_ERRCTX); - arm64_enter_nmi(regs); + state =3D arm64_enter_nmi(regs); do_serror(regs, esr); - arm64_exit_nmi(regs); + arm64_exit_nmi(regs, state); local_daif_restore(DAIF_PROCCTX); exit_to_user_mode(regs); } @@ -968,6 +1001,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(st= ruct pt_regs *regs) asmlinkage noinstr unsigned long __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) { + irqentry_state_t state; unsigned long ret; =20 /* @@ -992,9 +1026,9 @@ __sdei_handler(struct pt_regs *regs, struct sdei_regis= tered_event *arg) else if (cpu_has_pan()) set_pstate_pan(0); =20 - arm64_enter_nmi(regs); + state =3D arm64_enter_nmi(regs); ret =3D do_sdei_event(regs, arg); - arm64_exit_nmi(regs); + arm64_exit_nmi(regs, state); =20 return ret; } --=20 2.34.1