From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id B6D96333759; Mon, 15 Dec 2025 15:03:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811025; cv=none; b=Nf/JT2gEKsHRhe+LeSJahcQ17GKHhBIAYuZYOkzVKQT2/zvu+3ukoLvpne+6JTQ+aIKdvPcW1fjTuYm/EaGG2EDKU9N0IgUmyD82NhRGKBXZ1KkePQz+i4zLHXWXk9sNRVnWXRXAmjrfd+K8Y8tOmBLWSyw9wjIjacGEWBpmjbI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811025; c=relaxed/simple; bh=1Hqp0ZYR8uiHYoIgzrsy2ko5sWn0pLiL7Zrif8yQuuw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=nA6Djx+ImBGu5ZumevnOojyRsfwoE1zHKH3GyjGTFyDxMlD4VSea94L6iyijqYlP6H4+TZRla2tw2ouMH1469OaSmQ5dz3z84EPQS/CpSMaICptURbj0ArBPKYMlryOhVp7E/9/2/NT/Jz2Mvhu5ylaYTikFfv4DCoMriqL4GlM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E12E5FEC; Mon, 15 Dec 2025 07:03:35 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id C274B3F73B; Mon, 15 Dec 2025 07:03:37 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 01/14] powerpc/64s: Do not re-activate batched TLB flush Date: Mon, 15 Dec 2025 15:03:10 +0000 Message-ID: <20251215150323.2218608-2-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Alexander Gordeev Since commit b9ef323ea168 ("powerpc/64s: Disable preemption in hash lazy mmu mode") a task can not be preempted while in lazy MMU mode. Therefore, the batch re-activation code is never called, so remove it. Signed-off-by: Alexander Gordeev Reviewed-by: David Hildenbrand Reviewed-by: Ritesh Harjani (IBM) Reviewed-by: Ryan Roberts Tested-by: Venkat Rao Bagalkote Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/powerpc/include/asm/thread_info.h | 2 -- arch/powerpc/kernel/process.c | 25 ------------------------- 2 files changed, 27 deletions(-) diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/= asm/thread_info.h index b0f200aba2b3..97f35f9b1a96 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -154,12 +154,10 @@ void arch_setup_new_exec(void); /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */ #define TLF_NAPPING 0 /* idle thread enabled NAP mode */ #define TLF_SLEEPING 1 /* suspend code enabled SLEEP mode */ -#define TLF_LAZY_MMU 3 /* tlb_batch is active */ #define TLF_RUNLATCH 4 /* Is the runlatch enabled? */ =20 #define _TLF_NAPPING (1 << TLF_NAPPING) #define _TLF_SLEEPING (1 << TLF_SLEEPING) -#define _TLF_LAZY_MMU (1 << TLF_LAZY_MMU) #define _TLF_RUNLATCH (1 << TLF_RUNLATCH) =20 #ifndef __ASSEMBLER__ diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index a45fe147868b..a15d0b619b1f 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1281,9 +1281,6 @@ struct task_struct *__switch_to(struct task_struct *p= rev, { struct thread_struct *new_thread, *old_thread; struct task_struct *last; -#ifdef CONFIG_PPC_64S_HASH_MMU - struct ppc64_tlb_batch *batch; -#endif =20 new_thread =3D &new->thread; old_thread =3D ¤t->thread; @@ -1291,14 +1288,6 @@ struct task_struct *__switch_to(struct task_struct *= prev, WARN_ON(!irqs_disabled()); =20 #ifdef CONFIG_PPC_64S_HASH_MMU - batch =3D this_cpu_ptr(&ppc64_tlb_batch); - if (batch->active) { - current_thread_info()->local_flags |=3D _TLF_LAZY_MMU; - if (batch->index) - __flush_tlb_pending(batch); - batch->active =3D 0; - } - /* * On POWER9 the copy-paste buffer can only paste into * foreign real addresses, so unprivileged processes can not @@ -1369,20 +1358,6 @@ struct task_struct *__switch_to(struct task_struct *= prev, */ =20 #ifdef CONFIG_PPC_BOOK3S_64 -#ifdef CONFIG_PPC_64S_HASH_MMU - /* - * This applies to a process that was context switched while inside - * arch_enter_lazy_mmu_mode(), to re-activate the batch that was - * deactivated above, before _switch(). This will never be the case - * for new tasks. - */ - if (current_thread_info()->local_flags & _TLF_LAZY_MMU) { - current_thread_info()->local_flags &=3D ~_TLF_LAZY_MMU; - batch =3D this_cpu_ptr(&ppc64_tlb_batch); - batch->active =3D 1; - } -#endif - /* * Math facilities are masked out of the child MSR in copy_thread. * A new task does not need to restore_math because it will --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 4E0AC332902; Mon, 15 Dec 2025 15:03:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811030; cv=none; b=YhwR8sP4NeixgYPkSbuaH/keBRu7Df3CJiorCrr9xppPP05GJWkNMMk5tdqyxJxl+a2sUtaGVcQ8k5Tfv7XL+mjL/YIeOvu/dcKR9uSHf+d02ct4UIjO8n27V+bw5dkO7HzcsrpSBz7PQW3/9kXBuA3x+bpBC8syqVVtThcSLBU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811030; c=relaxed/simple; bh=O+wDRR3PpTS+EXXQ4+pAZKQW946eyAv3UVy/U7rFN8Q=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MZY15nJQL6h4DX0LJjV3NFdUcJnvnqRk1nn2hg+U4TNMUQ4lokZFQeLZp5M7Snb0D1Qn/s4mWA3MYOtvO0LdivYia/vwQtYdV8ln4h6TayTaIQRHSNAJH8Qu/dLK6u9V3RdlCPBJDl6Ak82TDTwC3PVXIZ57Bm9iZSb6OdMLbXs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 80DEC1655; Mon, 15 Dec 2025 07:03:41 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 5F7113F73B; Mon, 15 Dec 2025 07:03:43 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 02/14] x86/xen: simplify flush_lazy_mmu() Date: Mon, 15 Dec 2025 15:03:11 +0000 Message-ID: <20251215150323.2218608-3-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" arch_flush_lazy_mmu_mode() is called when outstanding batched pgtable operations must be completed immediately. There should however be no need to leave and re-enter lazy MMU completely. The only part of that sequence that we really need is xen_mc_flush(); call it directly. Reviewed-by: David Hildenbrand Reviewed-by: Ryan Roberts Reviewed-by: Juergen Gross Reviewed-by: Anshuman Khandual Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/x86/xen/mmu_pv.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 2a4a8deaf612..7a35c3393df4 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -2139,10 +2139,8 @@ static void xen_flush_lazy_mmu(void) { preempt_disable(); =20 - if (xen_get_lazy_mode() =3D=3D XEN_LAZY_MMU) { - arch_leave_lazy_mmu_mode(); - arch_enter_lazy_mmu_mode(); - } + if (xen_get_lazy_mode() =3D=3D XEN_LAZY_MMU) + xen_mc_flush(); =20 preempt_enable(); } --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id F3FE8332902; Mon, 15 Dec 2025 15:03:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811036; cv=none; b=cywF5qMUI6qotBHIZtS1l/HyDd7Yi3KREmWCvYOh1u/hd03+Epb6CxSLg/vKUNPAxkx8t7QnnSzZiowzIt2hc44/g8pmsPGSeBckVUAlEjkhHZzSQ96DRKSeq0HsCo4T3hWHXGXqLDIs+b5uDqK2e3AFlsDRrzu2HIPB9HAi6BU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811036; c=relaxed/simple; bh=b3ghA7FymwAlaXb+hvBmBZL3S7aJ6Kwth1qbl25PPeI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=PjOkvpFpzxT+ZFTjTKTSFjlwLT84FdSew4Bpoa/UNkjEq9mjvFY7jYNQc1rOXWT555FCr0SjUucisXy6Pm/zNNEKcrvAsndW+INJPuwTRLrj4MLbK4Gf4Hj+tqbqU2xxlVknzHWW4AsFxDlfUmo5x6J7QyFjOwwj7/oyTxUNPYo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1DD31165C; Mon, 15 Dec 2025 07:03:47 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id F099A3F73B; Mon, 15 Dec 2025 07:03:48 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 03/14] powerpc/mm: implement arch_flush_lazy_mmu_mode() Date: Mon, 15 Dec 2025 15:03:12 +0000 Message-ID: <20251215150323.2218608-4-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Upcoming changes to the lazy_mmu API will cause arch_flush_lazy_mmu_mode() to be called when leaving a nested lazy_mmu section. Move the relevant logic from arch_leave_lazy_mmu_mode() to arch_flush_lazy_mmu_mode() and have the former call the latter. The radix_enabled() check is required in both as arch_flush_lazy_mmu_mode() will be called directly from the generic layer in a subsequent patch. Note: the additional this_cpu_ptr() and radix_enabled() calls on the arch_leave_lazy_mmu_mode() path will be removed in a subsequent patch. Acked-by: David Hildenbrand Tested-by: Venkat Rao Bagalkote Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- .../powerpc/include/asm/book3s/64/tlbflush-hash.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powe= rpc/include/asm/book3s/64/tlbflush-hash.h index 146287d9580f..2d45f57df169 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -41,7 +41,7 @@ static inline void arch_enter_lazy_mmu_mode(void) batch->active =3D 1; } =20 -static inline void arch_leave_lazy_mmu_mode(void) +static inline void arch_flush_lazy_mmu_mode(void) { struct ppc64_tlb_batch *batch; =20 @@ -51,12 +51,21 @@ static inline void arch_leave_lazy_mmu_mode(void) =20 if (batch->index) __flush_tlb_pending(batch); +} + +static inline void arch_leave_lazy_mmu_mode(void) +{ + struct ppc64_tlb_batch *batch; + + if (radix_enabled()) + return; + batch =3D this_cpu_ptr(&ppc64_tlb_batch); + + arch_flush_lazy_mmu_mode(); batch->active =3D 0; preempt_enable(); } =20 -#define arch_flush_lazy_mmu_mode() do {} while (0) - extern void hash__tlbiel_all(unsigned int action); =20 extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 4504F332EDA; Mon, 15 Dec 2025 15:04:00 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811049; cv=none; b=YP/I1OauHf6orvxezGUSQLfQFgbVuOBKQVTJOjEqr4T6NdYYC7OTNpKalwHcqedmyUSocngl5mUPOd3bt3mENBNr2FPl81/5aHas/mKGcWFObaWc3uDso3tVdZkxTEwOqehRrWMi4P4H6dZ6fQvKCXAJuk6lnPSB6NstClT/cT0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811049; c=relaxed/simple; bh=sJHaIRnx36dzxhiGdyM8d4HIhQLew91HQNz728XfFxM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ppquthszs8GItp2wyZrVv8hFHDLJAiHzazCruw0c3ccYTnhARnWryI3yePaH3vefczpzH8M12+GjfsMOK1bCeAoi159jmphYOEZfpCyzxfLiZC18XESfirQ20wpclvINooHs+4Em6GY9bKvEU9nNoeAWLkOEbeHv6eA6AEtOrsw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id ABDB91684; Mon, 15 Dec 2025 07:03:52 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 8D5F43F73B; Mon, 15 Dec 2025 07:03:54 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 04/14] sparc/mm: implement arch_flush_lazy_mmu_mode() Date: Mon, 15 Dec 2025 15:03:13 +0000 Message-ID: <20251215150323.2218608-5-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Upcoming changes to the lazy_mmu API will cause arch_flush_lazy_mmu_mode() to be called when leaving a nested lazy_mmu section. Move the relevant logic from arch_leave_lazy_mmu_mode() to arch_flush_lazy_mmu_mode() and have the former call the latter. Note: the additional this_cpu_ptr() call on the arch_leave_lazy_mmu_mode() path will be removed in a subsequent patch. Acked-by: David Hildenbrand Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/sparc/include/asm/tlbflush_64.h | 2 +- arch/sparc/mm/tlb.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/= tlbflush_64.h index 8b8cdaa69272..925bb5d7a4e1 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -43,8 +43,8 @@ void flush_tlb_kernel_range(unsigned long start, unsigned= long end); =20 void flush_tlb_pending(void); void arch_enter_lazy_mmu_mode(void); +void arch_flush_lazy_mmu_mode(void); void arch_leave_lazy_mmu_mode(void); -#define arch_flush_lazy_mmu_mode() do {} while (0) =20 /* Local cpu only. */ void __flush_tlb_all(void); diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index a35ddcca5e76..7b5dfcdb1243 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -59,12 +59,19 @@ void arch_enter_lazy_mmu_mode(void) tb->active =3D 1; } =20 -void arch_leave_lazy_mmu_mode(void) +void arch_flush_lazy_mmu_mode(void) { struct tlb_batch *tb =3D this_cpu_ptr(&tlb_batch); =20 if (tb->tlb_nr) flush_tlb_pending(); +} + +void arch_leave_lazy_mmu_mode(void) +{ + struct tlb_batch *tb =3D this_cpu_ptr(&tlb_batch); + + arch_flush_lazy_mmu_mode(); tb->active =3D 0; preempt_enable(); } --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id DA03B33375C; Mon, 15 Dec 2025 15:04:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811047; cv=none; b=QFiGQIPjmXYFUBAMj118JFy81f1p1vSb4d8gc8w69YTupM2fzcwI1K4MblJHnutt2ARncyc5nrzOpjY484fsCEbNLRXI8tcMqhmwFLOP5FoQapHZkqhCPEk5PIA8tMzVXB1X8emBUoscJru1AAm44b9KtNrRR8xyUwKyyQdTREo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811047; c=relaxed/simple; bh=ZQ1iti6W8jPqW5TKHNxB7bKJihvjMjTws70qh7QOsa0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tLzHh71q4MNxTL+jfDSzkS8wWkD6BiCWINsZjPkpd7cn0xXkHP2q0tF5os2TQYr70Pc1zsAUnqV2dQKJf2FxLVu2Ap7t8Kj+vIBGj3/wBOhpgxTUbc/vZ/+j6xVW1qmdPj6me9qDvN2EvcHHPjo+DoAFrvye8XsrMMfZXahaSt4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4B751168F; Mon, 15 Dec 2025 07:03:58 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 2A0493F73B; Mon, 15 Dec 2025 07:04:00 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 05/14] mm: clarify lazy_mmu sleeping constraints Date: Mon, 15 Dec 2025 15:03:14 +0000 Message-ID: <20251215150323.2218608-6-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The lazy MMU mode documentation makes clear that an implementation should not assume that preemption is disabled or any lock is held upon entry to the mode; however it says nothing about what code using the lazy MMU interface should expect. In practice sleeping is forbidden (for generic code) while the lazy MMU mode is active: say it explicitly. Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- include/linux/pgtable.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 652f287c1ef6..1abc4a1c3d72 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -225,11 +225,15 @@ static inline int pmd_dirty(pmd_t pmd) * up to date. * * In the general case, no lock is guaranteed to be held between entry and= exit - * of the lazy mode. So the implementation must assume preemption may be e= nabled - * and cpu migration is possible; it must take steps to be robust against = this. - * (In practice, for user PTE updates, the appropriate page table lock(s) = are - * held, but for kernel PTE updates, no lock is held). Nesting is not perm= itted - * and the mode cannot be used in interrupt context. + * of the lazy mode. (In practice, for user PTE updates, the appropriate p= age + * table lock(s) are held, but for kernel PTE updates, no lock is held). + * The implementation must therefore assume preemption may be enabled upon + * entry to the mode and cpu migration is possible; it must take steps to = be + * robust against this. An implementation may handle this by disabling + * preemption, as a consequence generic code may not sleep while the lazy = MMU + * mode is active. + * + * Nesting is not permitted and the mode cannot be used in interrupt conte= xt. */ #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE static inline void arch_enter_lazy_mmu_mode(void) {} --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id DEC243321B9; Mon, 15 Dec 2025 15:04:11 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811053; cv=none; b=IbDerJwfm5LltxlMpRQ4GF6XHUnXryozUjXOvZ129z21bsMpYKaGh0H5hFxjShUB9NPLdo0z/Hi6Iy3k4mPWiejsByk8MzKMr7LCCEJXg3ibIOkkyRNfT4vOd2SCy6IuISW5M1l08fG0z96coo7vN/P6KoyDdbxT0BPgPPEEr9M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811053; c=relaxed/simple; bh=7moG9g3ulZ2QsmkrLmW5GE57HQ/QXz8V1OONYSnQQE4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RqFC3Z6gDMUW5bQJWwJeQU2UdXuqhmGjf//znmL+WSHCacEBXHPWqqbC59kODXB4HGjgaF7ArF6mpx+sfx2oYy6IbIaipC6H6qrrePMIoCJJHRBamvDoSHhzEtUowSj6MkY/18BjsXxiMGg0aTrfNvfOAbZ1xKUnaqarcVRO2Hg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id DD0151691; Mon, 15 Dec 2025 07:04:03 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id BAB283F73B; Mon, 15 Dec 2025 07:04:05 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 06/14] mm: introduce CONFIG_ARCH_HAS_LAZY_MMU_MODE Date: Mon, 15 Dec 2025 15:03:15 +0000 Message-ID: <20251215150323.2218608-7-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Architectures currently opt in for implementing lazy_mmu helpers by defining __HAVE_ARCH_ENTER_LAZY_MMU_MODE. In preparation for introducing a generic lazy_mmu layer that will require storage in task_struct, let's switch to a cleaner approach: instead of defining a macro, select a CONFIG option. This patch introduces CONFIG_ARCH_HAS_LAZY_MMU_MODE and has each arch select it when it implements lazy_mmu helpers. __HAVE_ARCH_ENTER_LAZY_MMU_MODE is removed and relies on the new CONFIG instead. On x86, lazy_mmu helpers are only implemented if PARAVIRT_XXL is selected. This creates some complications in arch/x86/boot/, because a few files manually undefine PARAVIRT* options. As a result does not define the lazy_mmu helpers, but this breaks the build as only defines them if !CONFIG_ARCH_HAS_LAZY_MMU_MODE. There does not seem to be a clean way out of this - let's just undefine that new CONFIG too. Acked-by: David Hildenbrand Reviewed-by: Ritesh Harjani (IBM) Reviewed-by: Ryan Roberts Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 2 -- arch/powerpc/platforms/Kconfig.cputype | 1 + arch/sparc/Kconfig | 1 + arch/sparc/include/asm/tlbflush_64.h | 2 -- arch/x86/Kconfig | 1 + arch/x86/boot/compressed/misc.h | 1 + arch/x86/boot/startup/sme.c | 1 + arch/x86/include/asm/paravirt.h | 1 - include/linux/pgtable.h | 2 +- mm/Kconfig | 7 +++++++ 12 files changed, 14 insertions(+), 7 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 93173f0a09c7..3fb4603c0e16 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -35,6 +35,7 @@ config ARM64 select ARCH_HAS_KCOV select ARCH_HAS_KERNEL_FPU_SUPPORT if KERNEL_MODE_NEON select ARCH_HAS_KEEPINITRD + select ARCH_HAS_LAZY_MMU_MODE select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_MEM_ENCRYPT select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 64d5f1d9cce9..f7d66c261347 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -80,7 +80,6 @@ static inline void queue_pte_barriers(void) } } =20 -#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE static inline void arch_enter_lazy_mmu_mode(void) { /* diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powe= rpc/include/asm/book3s/64/tlbflush-hash.h index 2d45f57df169..565c1b7c3eae 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -24,8 +24,6 @@ DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); =20 extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); =20 -#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE - static inline void arch_enter_lazy_mmu_mode(void) { struct ppc64_tlb_batch *batch; diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platform= s/Kconfig.cputype index 4c321a8ea896..f399917c17bd 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -93,6 +93,7 @@ config PPC_BOOK3S_64 select IRQ_WORK select PPC_64S_HASH_MMU if !PPC_RADIX_MMU select KASAN_VMALLOC if KASAN + select ARCH_HAS_LAZY_MMU_MODE =20 config PPC_BOOK3E_64 bool "Embedded processors" diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index a630d373e645..2bad14744ca4 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -112,6 +112,7 @@ config SPARC64 select NEED_PER_CPU_PAGE_FIRST_CHUNK select ARCH_SUPPORTS_SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_MC if SMP + select ARCH_HAS_LAZY_MMU_MODE =20 config ARCH_PROC_KCORE_TEXT def_bool y diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/= tlbflush_64.h index 925bb5d7a4e1..4e1036728e2f 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -39,8 +39,6 @@ static inline void flush_tlb_range(struct vm_area_struct = *vma, =20 void flush_tlb_kernel_range(unsigned long start, unsigned long end); =20 -#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE - void flush_tlb_pending(void); void arch_enter_lazy_mmu_mode(void); void arch_flush_lazy_mmu_mode(void); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 80527299f859..2427a66cb0fe 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -808,6 +808,7 @@ config PARAVIRT config PARAVIRT_XXL bool depends on X86_64 + select ARCH_HAS_LAZY_MMU_MODE =20 config PARAVIRT_DEBUG bool "paravirt-ops debugging" diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/mis= c.h index fd855e32c9b9..4f86c5903e03 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -11,6 +11,7 @@ #undef CONFIG_PARAVIRT #undef CONFIG_PARAVIRT_XXL #undef CONFIG_PARAVIRT_SPINLOCKS +#undef CONFIG_ARCH_HAS_LAZY_MMU_MODE #undef CONFIG_KASAN #undef CONFIG_KASAN_GENERIC =20 diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c index e7ea65f3f1d6..b76a7c95dfe1 100644 --- a/arch/x86/boot/startup/sme.c +++ b/arch/x86/boot/startup/sme.c @@ -24,6 +24,7 @@ #undef CONFIG_PARAVIRT #undef CONFIG_PARAVIRT_XXL #undef CONFIG_PARAVIRT_SPINLOCKS +#undef CONFIG_ARCH_HAS_LAZY_MMU_MODE =20 /* * This code runs before CPU feature bits are set. By default, the diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravir= t.h index b5e59a7ba0d0..13f9cd31c8f8 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -526,7 +526,6 @@ static inline void arch_end_context_switch(struct task_= struct *next) PVOP_VCALL1(cpu.end_context_switch, next); } =20 -#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE static inline void arch_enter_lazy_mmu_mode(void) { PVOP_VCALL0(mmu.lazy_mode.enter); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 1abc4a1c3d72..d46d86959bd6 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -235,7 +235,7 @@ static inline int pmd_dirty(pmd_t pmd) * * Nesting is not permitted and the mode cannot be used in interrupt conte= xt. */ -#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE +#ifndef CONFIG_ARCH_HAS_LAZY_MMU_MODE static inline void arch_enter_lazy_mmu_mode(void) {} static inline void arch_leave_lazy_mmu_mode(void) {} static inline void arch_flush_lazy_mmu_mode(void) {} diff --git a/mm/Kconfig b/mm/Kconfig index bd0ea5454af8..62073bd61544 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1464,6 +1464,13 @@ config PT_RECLAIM config FIND_NORMAL_PAGE def_bool n =20 +config ARCH_HAS_LAZY_MMU_MODE + bool + help + The architecture uses the lazy MMU mode. This allows changes to + MMU-related architectural state to be deferred until the mode is + exited. See for details. + source "mm/damon/Kconfig" =20 endmenu --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 29DB13321B5; Mon, 15 Dec 2025 15:04:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811059; cv=none; b=C2fGDQWEhXW5Zyrx3JO3Fz/ZyMJEmTQDPk2NS/UMzcOHYNN55hVbkYrcS7yETvtyhZZ7hcy6Gqouxdl7ugBY2UZfmVICl4+8I/0JR8vK3v/iAkI8VHNKMygcqXvYAZHFLNmbnAypfbl0E73U1cpG9MHcQgjqhO1Lu9sz7lES0WU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811059; c=relaxed/simple; bh=I9rLeX1ktfRJqKp4y/YgATX4KxnZ9DY7l8PHy4nYRtE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pimvt4llwE44iRfAzlij8Qby/ufiVhJ0YbsS0GAClZ+AQjNbjGkZHsGlxESO7V6VnfwqBHsHQg0WV248ekwwV0YrXkaCfpeowK7oe8fO0EGwtR9FnLFRlqUiCDvInfZsNRUyxy03yxzqUBv2HENb6ElYWB+ohyLvRJIV8UZnNOQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 953FD1692; Mon, 15 Dec 2025 07:04:09 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 57F423F73B; Mon, 15 Dec 2025 07:04:11 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 07/14] mm: introduce generic lazy_mmu helpers Date: Mon, 15 Dec 2025 15:03:16 +0000 Message-ID: <20251215150323.2218608-8-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The implementation of the lazy MMU mode is currently entirely arch-specific; core code directly calls arch helpers: arch_{enter,leave}_lazy_mmu_mode(). We are about to introduce support for nested lazy MMU sections. As things stand we'd have to duplicate that logic in every arch implementing lazy_mmu - adding to a fair amount of logic already duplicated across lazy_mmu implementations. This patch therefore introduces a new generic layer that calls the existing arch_* helpers. Two pair of calls are introduced: * lazy_mmu_mode_enable() ... lazy_mmu_mode_disable() This is the standard case where the mode is enabled for a given block of code by surrounding it with enable() and disable() calls. * lazy_mmu_mode_pause() ... lazy_mmu_mode_resume() This is for situations where the mode is temporarily disabled by first calling pause() and then resume() (e.g. to prevent any batching from occurring in a critical section). The documentation in will be updated in a subsequent patch. No functional change should be introduced at this stage. The implementation of enable()/resume() and disable()/pause() is currently identical, but nesting support will change that. Most of the call sites have been updated using the following Coccinelle script: @@ @@ { ... - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); ... - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); ... } @@ @@ { ... - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_pause(); ... - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_resume(); ... } A couple of notes regarding x86: * Xen is currently the only case where explicit handling is required for lazy MMU when context-switching. This is purely an implementation detail and using the generic lazy_mmu_mode_* functions would cause trouble when nesting support is introduced, because the generic functions must be called from the current task. For that reason we still use arch_leave() and arch_enter() there. * x86 calls arch_flush_lazy_mmu_mode() unconditionally in a few places, but only defines it if PARAVIRT_XXL is selected, and we are removing the fallback in . Add a new fallback definition to to keep things building. Acked-by: David Hildenbrand Reviewed-by: Anshuman Khandual Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/arm64/mm/mmu.c | 8 +++---- arch/arm64/mm/pageattr.c | 4 ++-- arch/powerpc/mm/book3s64/hash_tlb.c | 8 +++---- arch/powerpc/mm/book3s64/subpage_prot.c | 4 ++-- arch/x86/include/asm/pgtable.h | 1 + fs/proc/task_mmu.c | 4 ++-- include/linux/pgtable.h | 29 +++++++++++++++++++++---- mm/kasan/shadow.c | 8 +++---- mm/madvise.c | 18 +++++++-------- mm/memory.c | 16 +++++++------- mm/migrate_device.c | 8 +++---- mm/mprotect.c | 4 ++-- mm/mremap.c | 4 ++-- mm/userfaultfd.c | 4 ++-- mm/vmalloc.c | 12 +++++----- mm/vmscan.c | 12 +++++----- 16 files changed, 83 insertions(+), 61 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 9ae7ce00a7ef..76d359f02982 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -801,7 +801,7 @@ int split_kernel_leaf_mapping(unsigned long start, unsi= gned long end) return -EINVAL; =20 mutex_lock(&pgtable_split_lock); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 /* * The split_kernel_leaf_mapping_locked() may sleep, it is not a @@ -823,7 +823,7 @@ int split_kernel_leaf_mapping(unsigned long start, unsi= gned long end) ret =3D split_kernel_leaf_mapping_locked(end); } =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); mutex_unlock(&pgtable_split_lock); return ret; } @@ -884,10 +884,10 @@ static int range_split_to_ptes(unsigned long start, u= nsigned long end, gfp_t gfp { int ret; =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); ret =3D walk_kernel_page_table_range_lockless(start, end, &split_to_ptes_ops, NULL, &gfp); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); =20 return ret; } diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index f0e784b963e6..508986608b49 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -110,7 +110,7 @@ static int update_range_prot(unsigned long start, unsig= ned long size, if (WARN_ON_ONCE(ret)) return ret; =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 /* * The caller must ensure that the range we are operating on does not @@ -119,7 +119,7 @@ static int update_range_prot(unsigned long start, unsig= ned long size, */ ret =3D walk_kernel_page_table_range_lockless(start, start + size, &pageattr_ops, NULL, &data); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); =20 return ret; } diff --git a/arch/powerpc/mm/book3s64/hash_tlb.c b/arch/powerpc/mm/book3s64= /hash_tlb.c index 21fcad97ae80..787f7a0e27f0 100644 --- a/arch/powerpc/mm/book3s64/hash_tlb.c +++ b/arch/powerpc/mm/book3s64/hash_tlb.c @@ -205,7 +205,7 @@ void __flush_hash_table_range(unsigned long start, unsi= gned long end) * way to do things but is fine for our needs here. */ local_irq_save(flags); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); for (; start < end; start +=3D PAGE_SIZE) { pte_t *ptep =3D find_init_mm_pte(start, &hugepage_shift); unsigned long pte; @@ -217,7 +217,7 @@ void __flush_hash_table_range(unsigned long start, unsi= gned long end) continue; hpte_need_flush(&init_mm, start, ptep, pte, hugepage_shift); } - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); local_irq_restore(flags); } =20 @@ -237,7 +237,7 @@ void flush_hash_table_pmd_range(struct mm_struct *mm, p= md_t *pmd, unsigned long * way to do things but is fine for our needs here. */ local_irq_save(flags); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); start_pte =3D pte_offset_map(pmd, addr); if (!start_pte) goto out; @@ -249,6 +249,6 @@ void flush_hash_table_pmd_range(struct mm_struct *mm, p= md_t *pmd, unsigned long } pte_unmap(start_pte); out: - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); local_irq_restore(flags); } diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book= 3s64/subpage_prot.c index ec98e526167e..07c47673bba2 100644 --- a/arch/powerpc/mm/book3s64/subpage_prot.c +++ b/arch/powerpc/mm/book3s64/subpage_prot.c @@ -73,13 +73,13 @@ static void hpte_flush_range(struct mm_struct *mm, unsi= gned long addr, pte =3D pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) return; - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); for (; npages > 0; --npages) { pte_update(mm, addr, pte, 0, 0, 0); addr +=3D PAGE_SIZE; ++pte; } - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); } =20 diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index e33df3da6980..2842fa1f7a2c 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -118,6 +118,7 @@ extern pmdval_t early_pmd_flags; #define __pte(x) native_make_pte(x) =20 #define arch_end_context_switch(prev) do {} while(0) +static inline void arch_flush_lazy_mmu_mode(void) {} #endif /* CONFIG_PARAVIRT_XXL */ =20 static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 81dfc26bfae8..480db575553e 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2739,7 +2739,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigne= d long start, return 0; } =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ @@ -2809,7 +2809,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigne= d long start, if (flush_end) flush_tlb_range(vma, start, addr); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); =20 cond_resched(); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index d46d86959bd6..116a18b7916c 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -235,10 +235,31 @@ static inline int pmd_dirty(pmd_t pmd) * * Nesting is not permitted and the mode cannot be used in interrupt conte= xt. */ -#ifndef CONFIG_ARCH_HAS_LAZY_MMU_MODE -static inline void arch_enter_lazy_mmu_mode(void) {} -static inline void arch_leave_lazy_mmu_mode(void) {} -static inline void arch_flush_lazy_mmu_mode(void) {} +#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE +static inline void lazy_mmu_mode_enable(void) +{ + arch_enter_lazy_mmu_mode(); +} + +static inline void lazy_mmu_mode_disable(void) +{ + arch_leave_lazy_mmu_mode(); +} + +static inline void lazy_mmu_mode_pause(void) +{ + arch_leave_lazy_mmu_mode(); +} + +static inline void lazy_mmu_mode_resume(void) +{ + arch_enter_lazy_mmu_mode(); +} +#else +static inline void lazy_mmu_mode_enable(void) {} +static inline void lazy_mmu_mode_disable(void) {} +static inline void lazy_mmu_mode_pause(void) {} +static inline void lazy_mmu_mode_resume(void) {} #endif =20 #ifndef pte_batch_hint diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 29a751a8a08d..c1433d5cc5db 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -305,7 +305,7 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsi= gned long addr, pte_t pte; int index; =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_pause(); =20 index =3D PFN_DOWN(addr - data->start); page =3D data->pages[index]; @@ -319,7 +319,7 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsi= gned long addr, } spin_unlock(&init_mm.page_table_lock); =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_resume(); =20 return 0; } @@ -471,7 +471,7 @@ static int kasan_depopulate_vmalloc_pte(pte_t *ptep, un= signed long addr, pte_t pte; int none; =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_pause(); =20 spin_lock(&init_mm.page_table_lock); pte =3D ptep_get(ptep); @@ -483,7 +483,7 @@ static int kasan_depopulate_vmalloc_pte(pte_t *ptep, un= signed long addr, if (likely(!none)) __free_page(pfn_to_page(pte_pfn(pte))); =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_resume(); =20 return 0; } diff --git a/mm/madvise.c b/mm/madvise.c index b617b1be0f53..6bf7009fa5ce 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -453,7 +453,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (!start_pte) return 0; flush_tlb_batched_pending(mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); for (; addr < end; pte +=3D nr, addr +=3D nr * PAGE_SIZE) { nr =3D 1; ptent =3D ptep_get(pte); @@ -461,7 +461,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (++batch_count =3D=3D SWAP_CLUSTER_MAX) { batch_count =3D 0; if (need_resched()) { - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); cond_resched(); goto restart; @@ -497,7 +497,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (!folio_trylock(folio)) continue; folio_get(folio); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); start_pte =3D NULL; err =3D split_folio(folio); @@ -508,7 +508,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (!start_pte) break; flush_tlb_batched_pending(mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); if (!err) nr =3D 0; continue; @@ -556,7 +556,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, } =20 if (start_pte) { - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); } if (pageout) @@ -675,7 +675,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned = long addr, if (!start_pte) return 0; flush_tlb_batched_pending(mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); for (; addr !=3D end; pte +=3D nr, addr +=3D PAGE_SIZE * nr) { nr =3D 1; ptent =3D ptep_get(pte); @@ -724,7 +724,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned = long addr, if (!folio_trylock(folio)) continue; folio_get(folio); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); start_pte =3D NULL; err =3D split_folio(folio); @@ -735,7 +735,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned = long addr, if (!start_pte) break; flush_tlb_batched_pending(mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); if (!err) nr =3D 0; continue; @@ -775,7 +775,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned = long addr, if (nr_swap) add_mm_counter(mm, MM_SWAPENTS, nr_swap); if (start_pte) { - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); } cond_resched(); diff --git a/mm/memory.c b/mm/memory.c index 2a55edc48a65..ee15303c4041 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1256,7 +1256,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct= vm_area_struct *src_vma, spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); orig_src_pte =3D src_pte; orig_dst_pte =3D dst_pte; - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 do { nr =3D 1; @@ -1325,7 +1325,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct= vm_area_struct *src_vma, } while (dst_pte +=3D nr, src_pte +=3D nr, addr +=3D PAGE_SIZE * nr, addr !=3D end); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(orig_src_pte, src_ptl); add_mm_rss_vec(dst_mm, rss); pte_unmap_unlock(orig_dst_pte, dst_ptl); @@ -1842,7 +1842,7 @@ static unsigned long zap_pte_range(struct mmu_gather = *tlb, return addr; =20 flush_tlb_batched_pending(mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); do { bool any_skipped =3D false; =20 @@ -1874,7 +1874,7 @@ static unsigned long zap_pte_range(struct mmu_gather = *tlb, direct_reclaim =3D try_get_and_clear_pmd(mm, pmd, &pmdval); =20 add_mm_rss_vec(mm, rss); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); =20 /* Do the actual TLB flush before dropping ptl */ if (force_flush) { @@ -2813,7 +2813,7 @@ static int remap_pte_range(struct mm_struct *mm, pmd_= t *pmd, mapped_pte =3D pte =3D pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte) return -ENOMEM; - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); if (!pfn_modify_allowed(pfn, prot)) { @@ -2823,7 +2823,7 @@ static int remap_pte_range(struct mm_struct *mm, pmd_= t *pmd, set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); pfn++; } while (pte++, addr +=3D PAGE_SIZE, addr !=3D end); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err; } @@ -3174,7 +3174,7 @@ static int apply_to_pte_range(struct mm_struct *mm, p= md_t *pmd, return -EINVAL; } =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 if (fn) { do { @@ -3187,7 +3187,7 @@ static int apply_to_pte_range(struct mm_struct *mm, p= md_t *pmd, } *mask |=3D PGTBL_PTE_MODIFIED; =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); =20 if (mm !=3D &init_mm) pte_unmap_unlock(mapped_pte, ptl); diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 23379663b1e1..0346c2d7819f 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -271,7 +271,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, ptep =3D pte_offset_map_lock(mm, pmdp, start, &ptl); if (!ptep) goto again; - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); ptep +=3D (addr - start) / PAGE_SIZE; =20 for (; addr < end; addr +=3D PAGE_SIZE, ptep++) { @@ -313,7 +313,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (folio_test_large(folio)) { int ret; =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret =3D migrate_vma_split_folio(folio, migrate->fault_page); @@ -356,7 +356,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (folio && folio_test_large(folio)) { int ret; =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret =3D migrate_vma_split_folio(folio, migrate->fault_page); @@ -485,7 +485,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (unmapped) flush_tlb_range(walk->vma, start, end); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(ptep - 1, ptl); =20 return 0; diff --git a/mm/mprotect.c b/mm/mprotect.c index 283889e4f1ce..c0571445bef7 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -233,7 +233,7 @@ static long change_pte_range(struct mmu_gather *tlb, is_private_single_threaded =3D vma_is_single_threaded_private(vma); =20 flush_tlb_batched_pending(vma->vm_mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); do { nr_ptes =3D 1; oldpte =3D ptep_get(pte); @@ -379,7 +379,7 @@ static long change_pte_range(struct mmu_gather *tlb, } } } while (pte +=3D nr_ptes, addr +=3D nr_ptes * PAGE_SIZE, addr !=3D end); - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); =20 return pages; diff --git a/mm/mremap.c b/mm/mremap.c index 672264807db6..8275b9772ec1 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -260,7 +260,7 @@ static int move_ptes(struct pagetable_move_control *pmc, if (new_ptl !=3D old_ptl) spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); flush_tlb_batched_pending(vma->vm_mm); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 for (; old_addr < old_end; old_ptep +=3D nr_ptes, old_addr +=3D nr_ptes *= PAGE_SIZE, new_ptep +=3D nr_ptes, new_addr +=3D nr_ptes * PAGE_SIZE) { @@ -305,7 +305,7 @@ static int move_ptes(struct pagetable_move_control *pmc, } } =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); if (force_flush) flush_tlb_range(vma, old_end - len, old_end); if (new_ptl !=3D old_ptl) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index e6dfd5f28acd..b11f81095fa5 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1103,7 +1103,7 @@ static long move_present_ptes(struct mm_struct *mm, /* It's safe to drop the reference now as the page-table is holding one. = */ folio_put(*first_src_folio); *first_src_folio =3D NULL; - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 while (true) { orig_src_pte =3D ptep_get_and_clear(mm, src_addr, src_pte); @@ -1140,7 +1140,7 @@ static long move_present_ptes(struct mm_struct *mm, break; } =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); if (src_addr > src_start) flush_tlb_range(src_vma, src_start, src_addr); =20 diff --git a/mm/vmalloc.c b/mm/vmalloc.c index ecbac900c35f..1dea299fbb5a 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -108,7 +108,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long add= r, unsigned long end, if (!pte) return -ENOMEM; =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 do { if (unlikely(!pte_none(ptep_get(pte)))) { @@ -134,7 +134,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long add= r, unsigned long end, pfn++; } while (pte +=3D PFN_DOWN(size), addr +=3D size, addr !=3D end); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); *mask |=3D PGTBL_PTE_MODIFIED; return 0; } @@ -366,7 +366,7 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long = addr, unsigned long end, unsigned long size =3D PAGE_SIZE; =20 pte =3D pte_offset_kernel(pmd, addr); - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 do { #ifdef CONFIG_HUGETLB_PAGE @@ -385,7 +385,7 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long = addr, unsigned long end, WARN_ON(!pte_none(ptent) && !pte_present(ptent)); } while (pte +=3D (size >> PAGE_SHIFT), addr +=3D size, addr !=3D end); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); *mask |=3D PGTBL_PTE_MODIFIED; } =20 @@ -533,7 +533,7 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned lo= ng addr, if (!pte) return -ENOMEM; =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 do { struct page *page =3D pages[*nr]; @@ -555,7 +555,7 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned lo= ng addr, (*nr)++; } while (pte++, addr +=3D PAGE_SIZE, addr !=3D end); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); *mask |=3D PGTBL_PTE_MODIFIED; =20 return err; diff --git a/mm/vmscan.c b/mm/vmscan.c index 670fe9fae5ba..ab7ca55ca0f3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3516,7 +3516,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long = start, unsigned long end, return false; } =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); restart: for (i =3D pte_index(start), addr =3D start; addr !=3D end; i++, addr += =3D PAGE_SIZE) { unsigned long pfn; @@ -3557,7 +3557,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long = start, unsigned long end, if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &= end)) goto restart; =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); pte_unmap_unlock(pte, ptl); =20 return suitable_to_scan(total, young); @@ -3598,7 +3598,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigne= d long addr, struct vm_area if (!spin_trylock(ptl)) goto done; =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 do { unsigned long pfn; @@ -3645,7 +3645,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigne= d long addr, struct vm_area =20 walk_update_folio(walk, last, gen, dirty); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); spin_unlock(ptl); done: *first =3D -1; @@ -4244,7 +4244,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk = *pvmw) } } =20 - arch_enter_lazy_mmu_mode(); + lazy_mmu_mode_enable(); =20 pte -=3D (addr - start) / PAGE_SIZE; =20 @@ -4278,7 +4278,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk = *pvmw) =20 walk_update_folio(walk, last, gen, dirty); =20 - arch_leave_lazy_mmu_mode(); + lazy_mmu_mode_disable(); =20 /* feedback from rmap walkers to page table walkers */ if (mm_state && suitable_to_scan(i, young)) --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 0CBDF335072; Mon, 15 Dec 2025 15:04:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811064; cv=none; b=GsAFTReyjjAbVTe/68ht0CEaQcUZ91Fjy1ByEK1aKEOt4i3YQe+2yoGTFa+ZqB6rH8iil4NYMguILYfL0aKEqPm5zvHyYvpQB3m2RXBxLiGQDRvtMFW2RLZ0GHBW4gFtopoTgeQy3KFO9uLAqyQ+dilvAxZv4v3rtNJpwt4CJvc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811064; c=relaxed/simple; bh=sL6c39TdEJ8Aw3B9BcNJN0zaqRYTxZG99sYPCVwmgO4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jY5n/mDL3ikknYE1qW3bZftF5WlRK+gqyGLtWWlB/89+bnMLbmECj4ay3lR2JSEo+k1fNI0jU/VwJk2mZk3jUtUVpsxLIg7vyS+LqumQCMGcK97SkTYKAYE/k9NOYDqTvLzqmXhaCZi6OnT7+OuzcQkWgxaE6/OSw5spr8H6m4E= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 52004497; Mon, 15 Dec 2025 07:04:15 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 134D63F73B; Mon, 15 Dec 2025 07:04:16 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org, "David Hildenbrand (Red Hat)" Subject: [PATCH v6 08/14] mm: bail out of lazy_mmu_mode_* in interrupt context Date: Mon, 15 Dec 2025 15:03:17 +0000 Message-ID: <20251215150323.2218608-9-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The lazy MMU mode cannot be used in interrupt context. This is documented in , but isn't consistently handled across architectures. arm64 ensures that calls to lazy_mmu_mode_* have no effect in interrupt context, because such calls do occur in certain configurations - see commit b81c688426a9 ("arm64/mm: Disable barrier batching in interrupt contexts"). Other architectures do not check this situation, most likely because it hasn't occurred so far. Let's handle this in the new generic lazy_mmu layer, in the same fashion as arm64: bail out of lazy_mmu_mode_* if in_interrupt(). Also remove the arm64 handling that is now redundant. Both arm64 and x86/Xen also ensure that any lazy MMU optimisation is disabled while in interrupt (see queue_pte_barriers() and xen_get_lazy_mode() respectively). This will be handled in the generic layer in a subsequent patch. Acked-by: David Hildenbrand (Red Hat) Reviewed-by: Anshuman Khandual Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/arm64/include/asm/pgtable.h | 9 --------- include/linux/pgtable.h | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index f7d66c261347..bf9178902bdb 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -94,26 +94,17 @@ static inline void arch_enter_lazy_mmu_mode(void) * keeps tracking simple. */ =20 - if (in_interrupt()) - return; - set_thread_flag(TIF_LAZY_MMU); } =20 static inline void arch_flush_lazy_mmu_mode(void) { - if (in_interrupt()) - return; - if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING)) emit_pte_barriers(); } =20 static inline void arch_leave_lazy_mmu_mode(void) { - if (in_interrupt()) - return; - arch_flush_lazy_mmu_mode(); clear_thread_flag(TIF_LAZY_MMU); } diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 116a18b7916c..dddde6873d1e 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -233,26 +233,41 @@ static inline int pmd_dirty(pmd_t pmd) * preemption, as a consequence generic code may not sleep while the lazy = MMU * mode is active. * - * Nesting is not permitted and the mode cannot be used in interrupt conte= xt. + * The mode is disabled in interrupt context and calls to the lazy_mmu API= have + * no effect. + * + * Nesting is not permitted. */ #ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE static inline void lazy_mmu_mode_enable(void) { + if (in_interrupt()) + return; + arch_enter_lazy_mmu_mode(); } =20 static inline void lazy_mmu_mode_disable(void) { + if (in_interrupt()) + return; + arch_leave_lazy_mmu_mode(); } =20 static inline void lazy_mmu_mode_pause(void) { + if (in_interrupt()) + return; + arch_leave_lazy_mmu_mode(); } =20 static inline void lazy_mmu_mode_resume(void) { + if (in_interrupt()) + return; + arch_enter_lazy_mmu_mode(); } #else --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id D1D8133506C; Mon, 15 Dec 2025 15:04:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811070; cv=none; b=QGCCihWul5sxLCKBhqTPRaBoLofhkoGdXlD3lDaqYlPH3UEfos+D9CPp0wef8OtOxTo4U55thUPblrZ+yzJ01wsooNdBjmgyWaPKhaCRicZGpbjwBMKWZolawCG0Cumo0ZguruyxnBXDFmwTvbaE9RPf6ComcW/5RMhZcSL0yGI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811070; c=relaxed/simple; bh=wzXHpnEJHPAKZdRMTRYpcuc2yCIhFJGa9S3kB+uj1d8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=VhvFswh4G9SNJsPLPdwk6v0gLXmc5P8Ikg1FIww5DzYJ/zuUGUQvC1v71HRq9wKnWsSD7ZMk4YBKqGGdjGEwfMR1LB0H9kSLDeQwMWy2C7LGkpmfrxU1W0Cil7ULPOrLoPygeTG2rISOkHHTnzrtHVa0Sp0UtaKbFROxYLPt0dk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0C467169C; Mon, 15 Dec 2025 07:04:21 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id C3A423F73B; Mon, 15 Dec 2025 07:04:22 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org, "David Hildenbrand (Red Hat)" Subject: [PATCH v6 09/14] mm: enable lazy_mmu sections to nest Date: Mon, 15 Dec 2025 15:03:18 +0000 Message-ID: <20251215150323.2218608-10-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Despite recent efforts to prevent lazy_mmu sections from nesting, it remains difficult to ensure that it never occurs - and in fact it does occur on arm64 in certain situations (CONFIG_DEBUG_PAGEALLOC). Commit 1ef3095b1405 ("arm64/mm: Permit lazy_mmu_mode to be nested") made nesting tolerable on arm64, but without truly supporting it: the inner call to leave() disables the batching optimisation before the outer section ends. This patch actually enables lazy_mmu sections to nest by tracking the nesting level in task_struct, in a similar fashion to e.g. pagefault_{enable,disable}(). This is fully handled by the generic lazy_mmu helpers that were recently introduced. lazy_mmu sections were not initially intended to nest, so we need to clarify the semantics w.r.t. the arch_*_lazy_mmu_mode() callbacks. This patch takes the following approach: * The outermost calls to lazy_mmu_mode_{enable,disable}() trigger calls to arch_{enter,leave}_lazy_mmu_mode() - this is unchanged. * Nested calls to lazy_mmu_mode_{enable,disable}() are not forwarded to the arch via arch_{enter,leave} - lazy MMU remains enabled so the assumption is that these callbacks are not relevant. However, existing code may rely on a call to disable() to flush any batched state, regardless of nesting. arch_flush_lazy_mmu_mode() is therefore called in that situation. A separate interface was recently introduced to temporarily pause the lazy MMU mode: lazy_mmu_mode_{pause,resume}(). pause() fully exits the mode *regardless of the nesting level*, and resume() restores the mode at the same nesting level. pause()/resume() are themselves allowed to nest, so we actually store two nesting levels in task_struct: enable_count and pause_count. A new helper is_lazy_mmu_mode_active() is introduced to determine whether we are currently in lazy MMU mode; this will be used in subsequent patches to replace the various ways arch's currently track whether the mode is enabled. In summary (enable/pause represent the values *after* the call): lazy_mmu_mode_enable() -> arch_enter() enable=3D1 pause=3D0 lazy_mmu_mode_enable() -> =C3=B8 enable=3D2 pause=3D0 lazy_mmu_mode_pause() -> arch_leave() enable=3D2 pause=3D1 lazy_mmu_mode_resume() -> arch_enter() enable=3D2 pause=3D0 lazy_mmu_mode_disable() -> arch_flush() enable=3D1 pause=3D0 lazy_mmu_mode_disable() -> arch_leave() enable=3D0 pause=3D0 Note: is_lazy_mmu_mode_active() is added to to allow arch headers included by to use it. Acked-by: David Hildenbrand (Red Hat) Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/arm64/include/asm/pgtable.h | 12 ---- include/linux/mm_types_task.h | 5 ++ include/linux/pgtable.h | 114 +++++++++++++++++++++++++++++-- include/linux/sched.h | 45 ++++++++++++ 4 files changed, 157 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index bf9178902bdb..7f528c36d53c 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -82,18 +82,6 @@ static inline void queue_pte_barriers(void) =20 static inline void arch_enter_lazy_mmu_mode(void) { - /* - * lazy_mmu_mode is not supposed to permit nesting. But in practice this - * does happen with CONFIG_DEBUG_PAGEALLOC, where a page allocation - * inside a lazy_mmu_mode section (such as zap_pte_range()) will change - * permissions on the linear map with apply_to_page_range(), which - * re-enters lazy_mmu_mode. So we tolerate nesting in our - * implementation. The first call to arch_leave_lazy_mmu_mode() will - * flush and clear the flag such that the remainder of the work in the - * outer nest behaves as if outside of lazy mmu mode. This is safe and - * keeps tracking simple. - */ - set_thread_flag(TIF_LAZY_MMU); } =20 diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h index a82aa80c0ba4..11bf319d78ec 100644 --- a/include/linux/mm_types_task.h +++ b/include/linux/mm_types_task.h @@ -88,4 +88,9 @@ struct tlbflush_unmap_batch { #endif }; =20 +struct lazy_mmu_state { + u8 enable_count; + u8 pause_count; +}; + #endif /* _LINUX_MM_TYPES_TASK_H */ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index dddde6873d1e..2f0dd3a4ace1 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -236,39 +236,139 @@ static inline int pmd_dirty(pmd_t pmd) * The mode is disabled in interrupt context and calls to the lazy_mmu API= have * no effect. * - * Nesting is not permitted. + * The lazy MMU mode is enabled for a given block of code using: + * + * lazy_mmu_mode_enable(); + * + * lazy_mmu_mode_disable(); + * + * Nesting is permitted: may itself use an enable()/disable() pair. + * A nested call to enable() has no functional effect; however disable() c= auses + * any batched architectural state to be flushed regardless of nesting. Af= ter a + * call to disable(), the caller can therefore rely on all previous page t= able + * modifications to have taken effect, but the lazy MMU mode may still be + * enabled. + * + * In certain cases, it may be desirable to temporarily pause the lazy MMU= mode. + * This can be done using: + * + * lazy_mmu_mode_pause(); + * + * lazy_mmu_mode_resume(); + * + * pause() ensures that the mode is exited regardless of the nesting level; + * resume() re-enters the mode at the same nesting level. Any call to the + * lazy_mmu_mode_* API between those two calls has no effect. In particula= r, + * this means that pause()/resume() pairs may nest. + * + * is_lazy_mmu_mode_active() can be used to check whether the lazy MMU mod= e is + * currently enabled. */ #ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE +/** + * lazy_mmu_mode_enable() - Enable the lazy MMU mode. + * + * Enters a new lazy MMU mode section; if the mode was not already enabled, + * enables it and calls arch_enter_lazy_mmu_mode(). + * + * Must be paired with a call to lazy_mmu_mode_disable(). + * + * Has no effect if called: + * - While paused - see lazy_mmu_mode_pause() + * - In interrupt context + */ static inline void lazy_mmu_mode_enable(void) { - if (in_interrupt()) + struct lazy_mmu_state *state =3D ¤t->lazy_mmu_state; + + if (in_interrupt() || state->pause_count > 0) return; =20 - arch_enter_lazy_mmu_mode(); + VM_WARN_ON_ONCE(state->enable_count =3D=3D U8_MAX); + + if (state->enable_count++ =3D=3D 0) + arch_enter_lazy_mmu_mode(); } =20 +/** + * lazy_mmu_mode_disable() - Disable the lazy MMU mode. + * + * Exits the current lazy MMU mode section. If it is the outermost section, + * disables the mode and calls arch_leave_lazy_mmu_mode(). Otherwise (nest= ed + * section), calls arch_flush_lazy_mmu_mode(). + * + * Must match a call to lazy_mmu_mode_enable(). + * + * Has no effect if called: + * - While paused - see lazy_mmu_mode_pause() + * - In interrupt context + */ static inline void lazy_mmu_mode_disable(void) { - if (in_interrupt()) + struct lazy_mmu_state *state =3D ¤t->lazy_mmu_state; + + if (in_interrupt() || state->pause_count > 0) return; =20 - arch_leave_lazy_mmu_mode(); + VM_WARN_ON_ONCE(state->enable_count =3D=3D 0); + + if (--state->enable_count =3D=3D 0) + arch_leave_lazy_mmu_mode(); + else /* Exiting a nested section */ + arch_flush_lazy_mmu_mode(); + } =20 +/** + * lazy_mmu_mode_pause() - Pause the lazy MMU mode. + * + * Pauses the lazy MMU mode; if it is currently active, disables it and ca= lls + * arch_leave_lazy_mmu_mode(). + * + * Must be paired with a call to lazy_mmu_mode_resume(). Calls to the + * lazy_mmu_mode_* API have no effect until the matching resume() call. + * + * Has no effect if called: + * - While paused (inside another pause()/resume() pair) + * - In interrupt context + */ static inline void lazy_mmu_mode_pause(void) { + struct lazy_mmu_state *state =3D ¤t->lazy_mmu_state; + if (in_interrupt()) return; =20 - arch_leave_lazy_mmu_mode(); + VM_WARN_ON_ONCE(state->pause_count =3D=3D U8_MAX); + + if (state->pause_count++ =3D=3D 0 && state->enable_count > 0) + arch_leave_lazy_mmu_mode(); } =20 +/** + * lazy_mmu_mode_resume() - Resume the lazy MMU mode. + * + * Resumes the lazy MMU mode; if it was active at the point where the matc= hing + * call to lazy_mmu_mode_pause() was made, re-enables it and calls + * arch_enter_lazy_mmu_mode(). + * + * Must match a call to lazy_mmu_mode_pause(). + * + * Has no effect if called: + * - While paused (inside another pause()/resume() pair) + * - In interrupt context + */ static inline void lazy_mmu_mode_resume(void) { + struct lazy_mmu_state *state =3D ¤t->lazy_mmu_state; + if (in_interrupt()) return; =20 - arch_enter_lazy_mmu_mode(); + VM_WARN_ON_ONCE(state->pause_count =3D=3D 0); + + if (--state->pause_count =3D=3D 0 && state->enable_count > 0) + arch_enter_lazy_mmu_mode(); } #else static inline void lazy_mmu_mode_enable(void) {} diff --git a/include/linux/sched.h b/include/linux/sched.h index d395f2810fac..f2fc6584c790 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1419,6 +1419,10 @@ struct task_struct { =20 struct page_frag task_frag; =20 +#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE + struct lazy_mmu_state lazy_mmu_state; +#endif + #ifdef CONFIG_TASK_DELAY_ACCT struct task_delay_info *delays; #endif @@ -1702,6 +1706,47 @@ static inline char task_state_to_char(struct task_st= ruct *tsk) return task_index_to_char(task_state_index(tsk)); } =20 +#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE +/** + * __task_lazy_mmu_mode_active() - Test the lazy MMU mode state for a task. + * @tsk: The task to check. + * + * Test whether @tsk has its lazy MMU mode state set to active (i.e. enabl= ed + * and not paused). + * + * This function only considers the state saved in task_struct; to test wh= ether + * current actually is in lazy MMU mode, is_lazy_mmu_mode_active() should = be + * used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline bool __task_lazy_mmu_mode_active(struct task_struct *tsk) +{ + struct lazy_mmu_state *state =3D &tsk->lazy_mmu_state; + + return state->enable_count > 0 && state->pause_count =3D=3D 0; +} + +/** + * is_lazy_mmu_mode_active() - Test whether we are currently in lazy MMU m= ode. + * + * Test whether the current context is in lazy MMU mode. This is true if b= oth: + * 1. We are not in interrupt context + * 2. Lazy MMU mode is active for the current task + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline bool is_lazy_mmu_mode_active(void) +{ + if (in_interrupt()) + return false; + + return __task_lazy_mmu_mode_active(current); +} +#endif + extern struct pid *cad_pid; =20 /* --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 9B27733506C; Mon, 15 Dec 2025 15:04:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811076; cv=none; b=DfLTdWIV20S13xPrhmobAIkxNZ8R6c1yV10zWeY3UoDWIRrXlh/tI65KoWaOyX+QBT2CzsEV44HD1DDEBR4WsgMstbL+WJI5U0etpBYixS9pzBbE1o7Q+s+2rjqdJp6TGcKZ/qSbrxyHeIcJCvjIFJ6NQYa3Rt5r0P4qDAKKBE0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811076; c=relaxed/simple; bh=QSRLToNPICgxd4B8egyyaNEuSX5TixzxTiA3TqJyQx8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=VSqXs+7M19DgQe1DH3mo1Bikz6WDGH0r/lEteq02Zh7Bbhw86XYQH8soNXaUgQyydNqC/6CdWB9Zu6X2tlC+97HULJFL4lWhMLw6v8rMqbB+CiYUwTaZd67Km85ZD6fjb+xFGnFl1iSZ4AS8zqEl4PCI/bq6uzcQCJunOgW5sZQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id BB8BF169E; Mon, 15 Dec 2025 07:04:26 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 7E8EC3F73B; Mon, 15 Dec 2025 07:04:28 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org, "David Hildenbrand (Red Hat)" Subject: [PATCH v6 10/14] arm64: mm: replace TIF_LAZY_MMU with is_lazy_mmu_mode_active() Date: Mon, 15 Dec 2025 15:03:19 +0000 Message-ID: <20251215150323.2218608-11-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The generic lazy_mmu layer now tracks whether a task is in lazy MMU mode. As a result we no longer need a TIF flag for that purpose - let's use the new is_lazy_mmu_mode_active() helper instead. The explicit check for in_interrupt() is no longer necessary either as is_lazy_mmu_mode_active() always returns false in interrupt context. Acked-by: David Hildenbrand (Red Hat) Reviewed-by: Anshuman Khandual Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/arm64/include/asm/pgtable.h | 19 +++---------------- arch/arm64/include/asm/thread_info.h | 3 +-- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 7f528c36d53c..445e18e92221 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -62,28 +62,16 @@ static inline void emit_pte_barriers(void) =20 static inline void queue_pte_barriers(void) { - unsigned long flags; - - if (in_interrupt()) { - emit_pte_barriers(); - return; - } - - flags =3D read_thread_flags(); - - if (flags & BIT(TIF_LAZY_MMU)) { + if (is_lazy_mmu_mode_active()) { /* Avoid the atomic op if already set. */ - if (!(flags & BIT(TIF_LAZY_MMU_PENDING))) + if (!test_thread_flag(TIF_LAZY_MMU_PENDING)) set_thread_flag(TIF_LAZY_MMU_PENDING); } else { emit_pte_barriers(); } } =20 -static inline void arch_enter_lazy_mmu_mode(void) -{ - set_thread_flag(TIF_LAZY_MMU); -} +static inline void arch_enter_lazy_mmu_mode(void) {} =20 static inline void arch_flush_lazy_mmu_mode(void) { @@ -94,7 +82,6 @@ static inline void arch_flush_lazy_mmu_mode(void) static inline void arch_leave_lazy_mmu_mode(void) { arch_flush_lazy_mmu_mode(); - clear_thread_flag(TIF_LAZY_MMU); } =20 #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/= thread_info.h index a803b887b0b4..e7cd017b07c8 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -84,8 +84,7 @@ void arch_setup_new_exec(void); #define TIF_SME_VL_INHERIT 28 /* Inherit SME vl_onexec across exec */ #define TIF_KERNEL_FPSTATE 29 /* Task is in a kernel mode FPSIMD section */ #define TIF_TSC_SIGSEGV 30 /* SIGSEGV on counter-timer access */ -#define TIF_LAZY_MMU 31 /* Task in lazy mmu mode */ -#define TIF_LAZY_MMU_PENDING 32 /* Ops pending for lazy mmu mode exit */ +#define TIF_LAZY_MMU_PENDING 31 /* Ops pending for lazy mmu mode exit */ =20 #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) --=20 2.51.2 From nobody Mon Dec 15 23:36:16 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 2509A335089; Mon, 15 Dec 2025 15:04:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811082; cv=none; b=nughVdnSRNCTQ2RVk0SaJQS8m3+i8aC64v2FdWzLabD5/5+v4aMhGyiZDGqjswYbrhRCQZDngzpZUcyHTWSyDGWacZN7/4VmAYWuvyKFaqIuKF0Qvp1k8HFj4xxE0Z54equ2HFSsqYM/cFVRAwYS/wpQ/xyXESr/vfgpUSATer4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811082; c=relaxed/simple; bh=MJUyBXKBAaeuwDqu799pG7TZ2957TO9Ka30me3CmT4g=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jpWyohuPNue0Li9sXFpG3hu7asMbTBDeVJQVvodaDX84HuPpSgLOba82Oezz5BZ0NGNEtsZ1J1eGv5d+vMZomdV8Ml/jSXY45MCHs4yGT1TtakkmPmrpOMe7nUTF5uj/9YAffnbbdO20PKYdmFOGRoaHlZgGHpw0k/e+N0yqyho= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 5B0D916A3; Mon, 15 Dec 2025 07:04:32 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 3A02B3F73B; Mon, 15 Dec 2025 07:04:34 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 11/14] powerpc/mm: replace batch->active with is_lazy_mmu_mode_active() Date: Mon, 15 Dec 2025 15:03:20 +0000 Message-ID: <20251215150323.2218608-12-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" A per-CPU batch struct is activated when entering lazy MMU mode; its lifetime is the same as the lazy MMU section (it is deactivated when leaving the mode). Preemption is disabled in that interval to ensure that the per-CPU reference remains valid. The generic lazy_mmu layer now tracks whether a task is in lazy MMU mode. We can therefore use the generic helper is_lazy_mmu_mode_active() to tell whether a batch struct is active instead of tracking it explicitly. Acked-by: David Hildenbrand Reviewed-by: Ritesh Harjani (IBM) Tested-by: Venkat Rao Bagalkote Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 9 --------- arch/powerpc/mm/book3s64/hash_tlb.c | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powe= rpc/include/asm/book3s/64/tlbflush-hash.h index 565c1b7c3eae..6cc9abcd7b3d 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -12,7 +12,6 @@ #define PPC64_TLB_BATCH_NR 192 =20 struct ppc64_tlb_batch { - int active; unsigned long index; struct mm_struct *mm; real_pte_t pte[PPC64_TLB_BATCH_NR]; @@ -26,8 +25,6 @@ extern void __flush_tlb_pending(struct ppc64_tlb_batch *b= atch); =20 static inline void arch_enter_lazy_mmu_mode(void) { - struct ppc64_tlb_batch *batch; - if (radix_enabled()) return; /* @@ -35,8 +32,6 @@ static inline void arch_enter_lazy_mmu_mode(void) * operating on kernel page tables. */ preempt_disable(); - batch =3D this_cpu_ptr(&ppc64_tlb_batch); - batch->active =3D 1; } =20 static inline void arch_flush_lazy_mmu_mode(void) @@ -53,14 +48,10 @@ static inline void arch_flush_lazy_mmu_mode(void) =20 static inline void arch_leave_lazy_mmu_mode(void) { - struct ppc64_tlb_batch *batch; - if (radix_enabled()) return; - batch =3D this_cpu_ptr(&ppc64_tlb_batch); =20 arch_flush_lazy_mmu_mode(); - batch->active =3D 0; preempt_enable(); } =20 diff --git a/arch/powerpc/mm/book3s64/hash_tlb.c b/arch/powerpc/mm/book3s64= /hash_tlb.c index 787f7a0e27f0..fbdeb8981ae7 100644 --- a/arch/powerpc/mm/book3s64/hash_tlb.c +++ b/arch/powerpc/mm/book3s64/hash_tlb.c @@ -100,7 +100,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned lon= g addr, * Check if we have an active batch on this CPU. If not, just * flush now and return. */ - if (!batch->active) { + if (!is_lazy_mmu_mode_active()) { flush_hash_page(vpn, rpte, psize, ssize, mm_is_thread_local(mm)); put_cpu_var(ppc64_tlb_batch); return; --=20 2.51.2 From nobody Mon Dec 15 23:36:17 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id D1AA4335089; Mon, 15 Dec 2025 15:04:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811087; cv=none; b=VBvfBaSQFe+D39/1BojoOXyeIQRLTxFIp5kaZ5jogQeRdsJr2WwZJjPIuQ3zN9bteJbDo7pVJUFv6e0BrelC0Pfi2t+Tlwie6OusBlGBFqjsSbwwJLlQdUpXFG9zvfoXh059wz46hwba9jZKvvjOgGpbWx23PwrKQuNwT+SMlWk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811087; c=relaxed/simple; bh=PY2BoVw7LD0zrC8pdiqNv3FrvXmiKm4QDYVHtVdAaw8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=LNeaYLnulL/aONwhaNLoLz2fJHCGRNosml+wytiqMP1oIc0iTgkrA20PNk8rj5bG/+9/EClYHf9i6kmg0vgYjKn7iN35BvTGYJAYslQ6HuwE2yvGVpGZURJtpvwzcziR29DJVpKtO2bWHmWw6Ikir114ePJqm4TuDbRLMUOrXN4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 12E5216F2; Mon, 15 Dec 2025 07:04:38 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id CA2463F73B; Mon, 15 Dec 2025 07:04:39 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org, "David Hildenbrand (Red Hat)" Subject: [PATCH v6 12/14] sparc/mm: replace batch->active with is_lazy_mmu_mode_active() Date: Mon, 15 Dec 2025 15:03:21 +0000 Message-ID: <20251215150323.2218608-13-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" A per-CPU batch struct is activated when entering lazy MMU mode; its lifetime is the same as the lazy MMU section (it is deactivated when leaving the mode). Preemption is disabled in that interval to ensure that the per-CPU reference remains valid. The generic lazy_mmu layer now tracks whether a task is in lazy MMU mode. We can therefore use the generic helper is_lazy_mmu_mode_active() to tell whether a batch struct is active instead of tracking it explicitly. Acked-by: David Hildenbrand (Red Hat) Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/sparc/include/asm/tlbflush_64.h | 1 - arch/sparc/mm/tlb.c | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/= tlbflush_64.h index 4e1036728e2f..6133306ba59a 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -12,7 +12,6 @@ struct tlb_batch { unsigned int hugepage_shift; struct mm_struct *mm; unsigned long tlb_nr; - unsigned long active; unsigned long vaddrs[TLB_BATCH_NR]; }; =20 diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index 7b5dfcdb1243..3a852071d260 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -52,11 +52,7 @@ void flush_tlb_pending(void) =20 void arch_enter_lazy_mmu_mode(void) { - struct tlb_batch *tb; - preempt_disable(); - tb =3D this_cpu_ptr(&tlb_batch); - tb->active =3D 1; } =20 void arch_flush_lazy_mmu_mode(void) @@ -69,10 +65,7 @@ void arch_flush_lazy_mmu_mode(void) =20 void arch_leave_lazy_mmu_mode(void) { - struct tlb_batch *tb =3D this_cpu_ptr(&tlb_batch); - arch_flush_lazy_mmu_mode(); - tb->active =3D 0; preempt_enable(); } =20 @@ -93,7 +86,7 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsig= ned long vaddr, nr =3D 0; } =20 - if (!tb->active) { + if (!is_lazy_mmu_mode_active()) { flush_tsb_user_page(mm, vaddr, hugepage_shift); global_flush_tlb_page(mm, vaddr); goto out; --=20 2.51.2 From nobody Mon Dec 15 23:36:17 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 9D976334373; Mon, 15 Dec 2025 15:04:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811093; cv=none; b=XORvgWHq9+cbvBdPDtFmAXufRPvaS493tongIkCUNd6mvDFbOxiktyBL8aZCeCSTmnGnZR8n4AJ7GXMiQs42tR/QyqF9TWV4Ly2WJ0yJYIB5Yi7EQ3Im4LmX0WMLxX1hELPFUgQGLUjsgjt+K7sdo2bBIjBYMpRDBt3x87KCKAc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811093; c=relaxed/simple; bh=SkQRSo4quAm87O9InyuAZz0iR7CSa/+9rhyfGhbC5WE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=g5GYnCbgkRZtQat2KZ+22NReOZnf1pmZtd4jdGZfHoysOcFifyQHOXYH9DmiTOivvcqDossXs8WZo/f9MmvJq4PeAVFx+U1MnmdWmjrDpywPFrsRuaOLtCZvV6RQhGgDy5K42s1BVb+eRMwJKttbHzs3CVdcohhI6UqLeE5PkEM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C3DCB16F3; Mon, 15 Dec 2025 07:04:43 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 858633F73B; Mon, 15 Dec 2025 07:04:45 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org, "David Hildenbrand (Red Hat)" Subject: [PATCH v6 13/14] x86/xen: use lazy_mmu_state when context-switching Date: Mon, 15 Dec 2025 15:03:22 +0000 Message-ID: <20251215150323.2218608-14-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" We currently set a TIF flag when scheduling out a task that is in lazy MMU mode, in order to restore it when the task is scheduled again. The generic lazy_mmu layer now tracks whether a task is in lazy MMU mode in task_struct::lazy_mmu_state. We can therefore check that state when switching to the new task, instead of using a separate TIF flag. Acked-by: David Hildenbrand (Red Hat) Reviewed-by: Juergen Gross Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- arch/x86/include/asm/thread_info.h | 4 +--- arch/x86/xen/enlighten_pv.c | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thre= ad_info.h index e71e0e8362ed..0067684afb5b 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -100,8 +100,7 @@ struct thread_info { #define TIF_FORCED_TF 24 /* true if TF in eflags artificially */ #define TIF_SINGLESTEP 25 /* reenable singlestep on user return*/ #define TIF_BLOCKSTEP 26 /* set when we want DEBUGCTLMSR_BTF */ -#define TIF_LAZY_MMU_UPDATES 27 /* task is updating the mmu lazily */ -#define TIF_ADDR32 28 /* 32-bit address space on 64 bits */ +#define TIF_ADDR32 27 /* 32-bit address space on 64 bits */ =20 #define _TIF_SSBD BIT(TIF_SSBD) #define _TIF_SPEC_IB BIT(TIF_SPEC_IB) @@ -114,7 +113,6 @@ struct thread_info { #define _TIF_FORCED_TF BIT(TIF_FORCED_TF) #define _TIF_BLOCKSTEP BIT(TIF_BLOCKSTEP) #define _TIF_SINGLESTEP BIT(TIF_SINGLESTEP) -#define _TIF_LAZY_MMU_UPDATES BIT(TIF_LAZY_MMU_UPDATES) #define _TIF_ADDR32 BIT(TIF_ADDR32) =20 /* flags to check in __switch_to() */ diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 4806cc28d7ca..98dbb6a61087 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -426,7 +426,6 @@ static void xen_start_context_switch(struct task_struct= *prev) =20 if (this_cpu_read(xen_lazy_mode) =3D=3D XEN_LAZY_MMU) { arch_leave_lazy_mmu_mode(); - set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); } enter_lazy(XEN_LAZY_CPU); } @@ -437,7 +436,7 @@ static void xen_end_context_switch(struct task_struct *= next) =20 xen_mc_flush(); leave_lazy(XEN_LAZY_CPU); - if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UP= DATES)) + if (__task_lazy_mmu_mode_active(next)) arch_enter_lazy_mmu_mode(); } =20 --=20 2.51.2 From nobody Mon Dec 15 23:36:17 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id ECC353385B8; Mon, 15 Dec 2025 15:04:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811102; cv=none; b=ZVgjaoPXjrspYGK9paKPldFb82PTywgWuh14/z6Z3a8crdVguEQ3EvMNY+mBJCIFwCJU1BCRqtWs8/sZSvbSRSx120UptHvxFuXZVVwouxoccqTAkRkk9mn2J/wGRUSA/gKBIlit1o0pgcrDh3QdVNmIWRzBj7GQph1cXlCZgE0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765811102; c=relaxed/simple; bh=Ce76RNBgMRqCzmVaPKcHwIxkfgsnqnEGf99tuVD0wv4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=QL8kZcML+KciHir8QCOVbIYsZvERh1K6h7oN0MKsoJ0pfwjmm4VnK8iyBn3/htSBLXp7zQlabBXCP4g0NkXfJmgsWVVHUGa386KPxVF43SJ9kgFRbj/qEtHgLd2rmZOJOWfsjf4/PwZdtOiH+W4RKskh4FXYc61mPI1zjxF3wB8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 62D991713; Mon, 15 Dec 2025 07:04:49 -0800 (PST) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 41F803F73B; Mon, 15 Dec 2025 07:04:51 -0800 (PST) From: Kevin Brodsky To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Alexander Gordeev , Andreas Larsson , Andrew Morton , Anshuman Khandual , Boris Ostrovsky , Borislav Petkov , Catalin Marinas , Christophe Leroy , Dave Hansen , David Hildenbrand , "David S. Miller" , David Woodhouse , "H. Peter Anvin" , Ingo Molnar , Jann Horn , Juergen Gross , "Liam R. Howlett" , Lorenzo Stoakes , Madhavan Srinivasan , Michael Ellerman , Michal Hocko , Mike Rapoport , Nicholas Piggin , Peter Zijlstra , "Ritesh Harjani (IBM)" , Ryan Roberts , Suren Baghdasaryan , Thomas Gleixner , Venkat Rao Bagalkote , Vlastimil Babka , Will Deacon , Yeoreum Yun , linux-arm-kernel@lists.infradead.org, linuxppc-dev@lists.ozlabs.org, sparclinux@vger.kernel.org, xen-devel@lists.xenproject.org, x86@kernel.org Subject: [PATCH v6 14/14] mm: Add basic tests for lazy_mmu Date: Mon, 15 Dec 2025 15:03:23 +0000 Message-ID: <20251215150323.2218608-15-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.51.2 In-Reply-To: <20251215150323.2218608-1-kevin.brodsky@arm.com> References: <20251215150323.2218608-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add basic KUnit tests for the generic aspects of the lazy MMU mode: ensure that it appears active when it should, depending on how enable/disable and pause/resume pairs are nested. Signed-off-by: Kevin Brodsky Reviewed-by: Yeoreum Yun --- mm/Kconfig | 12 ++++++ mm/Makefile | 1 + mm/tests/lazy_mmu_mode_kunit.c | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 mm/tests/lazy_mmu_mode_kunit.c diff --git a/mm/Kconfig b/mm/Kconfig index 62073bd61544..ac48deb44884 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1471,6 +1471,18 @@ config ARCH_HAS_LAZY_MMU_MODE MMU-related architectural state to be deferred until the mode is exited. See for details. =20 +config LAZY_MMU_MODE_KUNIT_TEST + tristate "KUnit tests for the lazy MMU mode" if !KUNIT_ALL_TESTS + depends on ARCH_HAS_LAZY_MMU_MODE + depends on KUNIT + default KUNIT_ALL_TESTS + help + Enable this option to check that the lazy MMU mode interface behaves + as expected. Only tests for the generic interface are included (not + architecture-specific behaviours). + + If unsure, say N. + source "mm/damon/Kconfig" =20 endmenu diff --git a/mm/Makefile b/mm/Makefile index 2d0570a16e5b..9175f8cc6565 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) +=3D shrinker_debug.o obj-$(CONFIG_EXECMEM) +=3D execmem.o obj-$(CONFIG_TMPFS_QUOTA) +=3D shmem_quota.o obj-$(CONFIG_PT_RECLAIM) +=3D pt_reclaim.o +obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) +=3D tests/lazy_mmu_mode_kunit.o diff --git a/mm/tests/lazy_mmu_mode_kunit.c b/mm/tests/lazy_mmu_mode_kunit.c new file mode 100644 index 000000000000..2720eb995714 --- /dev/null +++ b/mm/tests/lazy_mmu_mode_kunit.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +static void expect_not_active(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, is_lazy_mmu_mode_active()); +} + +static void expect_active(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, is_lazy_mmu_mode_active()); +} + +static void lazy_mmu_mode_active(struct kunit *test) +{ + expect_not_active(test); + + lazy_mmu_mode_enable(); + expect_active(test); + + { + /* Nested section */ + lazy_mmu_mode_enable(); + expect_active(test); + + lazy_mmu_mode_disable(); + expect_active(test); + } + + { + /* Paused section */ + lazy_mmu_mode_pause(); + expect_not_active(test); + + { + /* No effect (paused) */ + lazy_mmu_mode_enable(); + expect_not_active(test); + + lazy_mmu_mode_disable(); + expect_not_active(test); + + lazy_mmu_mode_pause(); + expect_not_active(test); + + lazy_mmu_mode_resume(); + expect_not_active(test); + } + + lazy_mmu_mode_resume(); + expect_active(test); + } + + lazy_mmu_mode_disable(); + expect_not_active(test); +} + +static struct kunit_case lazy_mmu_mode_test_cases[] =3D { + KUNIT_CASE(lazy_mmu_mode_active), + {} +}; + +static struct kunit_suite lazy_mmu_mode_test_suite =3D { + .name =3D "lazy_mmu_mode", + .test_cases =3D lazy_mmu_mode_test_cases, +}; +kunit_test_suite(lazy_mmu_mode_test_suite); + +MODULE_DESCRIPTION("Tests for the lazy MMU mode"); +MODULE_LICENSE("GPL"); --=20 2.51.2