From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4DBAB2EF2BE for ; Fri, 11 Jul 2025 16:17:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250664; cv=none; b=gbovGby6AWd2QGJUg0ZIPUKNWq0i4P8oGh5Bt5UN2hOH0HBAsw23JYAwaSP4VHcby9OQE1fqDq13WSAp4n5o5rzOOurLYEapAQqeTE8Yilqzc0LHa733kKSQkQT6FmMg4O3jtxA4QXob58zQArX5r8zPEB3peDq2PQb2Wv3LLDk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250664; c=relaxed/simple; bh=OxLUcaS+mva8YPmoE0SCu30mpSKl3MfgIv/3sz5pLSc=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=faQk4QFKW7lbSH4QnZpiOUkzuy3UWctnIJxVEeI4XumlFiC7OcAft5GFGrax5QaSU2vJZYz+5jgF5d7lp9BGQqw/gFqXgc+Y3q+ldNc1rjiMkU7T/kre0TIobll7paRtL6SNJ9vh6F1a53o+uQ2vGBn0WheJJ7RN5E3Z950cscI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Pu3CDGI7; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Pu3CDGI7" Received: by smtp.kernel.org (Postfix) with ESMTPSA id DA647C4CEED; Fri, 11 Jul 2025 16:17:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250663; bh=OxLUcaS+mva8YPmoE0SCu30mpSKl3MfgIv/3sz5pLSc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Pu3CDGI7gtFmF+qiVxu4UuGFf0kYyLet7UFwqfJtrlybo5L+Aw5JYH6JhSUi4TUEP y5m2Ke1W0ppaxpDSdnJd+acjFjWKxp20q3eKPIb2ftjpJ4NfUbDSvfVDz435hOGCFS wovLcYQWByRaS+uoaEMWxkRlwfWVvENvWMxbuCbb8YVniNQ6dIcp8VPEoGVFy+Ts3F xJE7oJ9B+6wIrDfopjzrVq9Yg+fBm+8LVmFrZVe7UHCKo3RlaUugw2L7J1N3ZdF/Zc SVyFGzIfFH+xtnqHFxhj3AT52CJraXOPLlUb8bs+QVjk6nGv8YhbzAZG7/FilvET8E 6BuzS8K5pzyYA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 01/10] arm64: mm: Introduce a C wrapper for by-level TLB invalidation helpers Date: Fri, 11 Jul 2025 17:17:23 +0100 Message-Id: <20250711161732.384-2-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation for reducing our reliance on complex preprocessor macros for TLB invalidation routines, introduce a new C wrapper for by-level TLB invalidation helpers which can be used instead of the __tlbi() macro and can additionally be called from C code. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index aa9efee17277..1c7548ec6cb7 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -88,6 +88,16 @@ static inline unsigned long get_trans_granule(void) } } =20 +enum tlbi_op { + vae1is, + vae2is, + vale1is, + vale2is, + vaale1is, + ipas2e1, + ipas2e1is, +}; + /* * Level-based TLBI operations. * @@ -105,6 +115,27 @@ static inline unsigned long get_trans_granule(void) =20 #define TLBI_TTL_UNKNOWN INT_MAX =20 +#define __GEN_TLBI_OP_CASE(op) \ + case op: \ + __tlbi(op, arg); \ + break + +static __always_inline void __tlbi_level_op(const enum tlbi_op op, u64 arg) +{ + switch (op) { + __GEN_TLBI_OP_CASE(vae1is); + __GEN_TLBI_OP_CASE(vae2is); + __GEN_TLBI_OP_CASE(vale1is); + __GEN_TLBI_OP_CASE(vale2is); + __GEN_TLBI_OP_CASE(vaale1is); + __GEN_TLBI_OP_CASE(ipas2e1); + __GEN_TLBI_OP_CASE(ipas2e1is); + default: + BUILD_BUG(); + } +} +#undef __GEN_TLBI_OP_CASE + #define __tlbi_level(op, addr, level) do { \ u64 arg =3D addr; \ \ @@ -116,7 +147,7 @@ static inline unsigned long get_trans_granule(void) arg |=3D FIELD_PREP(TLBI_TTL_MASK, ttl); \ } \ \ - __tlbi(op, arg); \ + __tlbi_level_op(op, arg); \ } while(0) =20 #define __tlbi_user_level(op, arg, level) do { \ --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 954632EF66B for ; Fri, 11 Jul 2025 16:17:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250666; cv=none; b=Jc5dn1zSznuH/vS5Ex4d3eogauW4A0MgfYCyOFqTxMw2AiNjorQ+0hynEKuUMOxpQAS13eUfeWd8kl5/Km5hxp4+v2IrUIWGmJr2HhQDsieaiABOO4aTHGIaEfsfa90ggowZY9nY2Nthm1jStfODwE2TgWjeCtowWrEwGQJR8Y8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250666; c=relaxed/simple; bh=q5+HuLJh1JU1y/46sorbdFwVVHM1smyj615SFqQ/6i0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=HqywUxiAf5BQ1QUXgeFZKaKpvWaKWjumuW8lnHxOHnw/OI1WJgDDHpCuvnoXHawNf+bRIBjmexmxXsYkyP+wdtJRXBLviottyNg/BDhC8WQG+LkaD5hq/zcpEdoFDa5uLJUvjVhjPA9/bQSstCu/DB4mPOYHgJX6VwaL4H3MGOs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=t4MVo+eE; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="t4MVo+eE" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4805CC4CEF5; Fri, 11 Jul 2025 16:17:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250666; bh=q5+HuLJh1JU1y/46sorbdFwVVHM1smyj615SFqQ/6i0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=t4MVo+eEZhW6ZtkMaOr5iS7MraQkoRsWptpg+/xXCDh9nyCeKzpJ3qzBhljEOupxN X8tGW8EPr5PUWkVFX7h1YMA3+g7GDp7V53CuIPkhGzp5eK98l8OmFAF1RRmkerE55R JhcXBJr5t/9WR9V7BV9j2kLXUl3olanDE+L2yoWCehqigvku5iab7x2b700RcAvnRD fn9izzwO8Z7VEmu2z/lBWNyfxmxCU9SdRpK/CiBmZI5WtlEGiQY3S9+ZMgsj/HAKAT kqiqykYPCVHzVF0nbEFiV0cOWKtvXgvYprHDKx4EKAQxo0LcVtaSDeuc5AW0eBuERM A0eebe0uqA7Ww== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 02/10] arm64: mm: Introduce a C wrapper for by-range TLB invalidation helpers Date: Fri, 11 Jul 2025 17:17:24 +0100 Message-Id: <20250711161732.384-3-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation for reducing our reliance on complex preprocessor macros for TLB invalidation routines, introduce a new C wrapper for by-range TLB invalidation helpers which can be used instead of the __tlbi() macro and can additionally be called from C code. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 1c7548ec6cb7..4408aeebf4d5 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -418,6 +418,24 @@ static inline void arch_tlbbatch_flush(struct arch_tlb= flush_unmap_batch *batch) * operations can only span an even number of pages. We save this for l= ast to * ensure 64KB start alignment is maintained for the LPA2 case. */ +#define __GEN_TLBI_OP_CASE(op) \ + case op: \ + __tlbi(r ## op, arg); \ + break + +static __always_inline void __tlbi_range(const enum tlbi_op op, u64 arg) +{ + switch (op) { + __GEN_TLBI_OP_CASE(vae1is); + __GEN_TLBI_OP_CASE(vale1is); + __GEN_TLBI_OP_CASE(vaale1is); + __GEN_TLBI_OP_CASE(ipas2e1is); + default: + BUILD_BUG(); + } +} +#undef __GEN_TLBI_OP_CASE + #define __flush_tlb_range_op(op, start, pages, stride, \ asid, tlb_level, tlbi_user, lpa2) \ do { \ @@ -445,7 +463,7 @@ do { \ if (num >=3D 0) { \ addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ scale, num, tlb_level); \ - __tlbi(r##op, addr); \ + __tlbi_range(op, addr); \ if (tlbi_user) \ __tlbi_user(r##op, addr); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1C2652EF9B6 for ; Fri, 11 Jul 2025 16:17:48 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250669; cv=none; b=Pg/mibayqAEIgFVT7TXkgV4qOAp6FEt+jLK3HqYL1N6HVD8NUJQo4tld0iKP+0CgCtu7uunYUfcGYh196FkbIwTp5UbTzQ/AOq+otCZ8WkTLGP7BI+hPZttjx5Sj0GE7NKiBLGooRkQuBpEhPVkC6WSQDFpSgpaHoJeqdFi5j9M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250669; c=relaxed/simple; bh=omASSf5AodAZJuWCX0tb4hiAUdVWa3UIK9zu/g/yffM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=kh5xFaUReZx9C0hR/tsJ9ZDCDrH3FzGNM+u4X2IpHMrM22imbZJPqjRZYrhZoVopstBLfhmIK9zVWaKWV7I7AKKqSb49BHfOsrDQYg+akymo9gjmfLeA9CuW6A49/gp5g3sI5Aix2L7ClELtDk2u9WvT0IFbgay2grTVC/ACq7A= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Re+Ll/b2; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Re+Ll/b2" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A9494C4CEF6; Fri, 11 Jul 2025 16:17:46 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250668; bh=omASSf5AodAZJuWCX0tb4hiAUdVWa3UIK9zu/g/yffM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Re+Ll/b20qk+1VLbHfZ5OgRv+ruiurQeg34El3ijBhRwVEuLWOzCvdykuUxfM5ZBz jTntl/TgmN96C2gaTttrPyIJNq0WeKoWLArhVnfPg55PzxH0HdKFmIMY7RZobP3rSG nchrEZc+GzmEe8lyrGGXd1SCqKNZZketyEosEYXL4Ak/RBhJjpCUWHcsJj6NQF3wlG IyWd5cQjZJIwFb0vlTURT5D5fsLVLnVqg5FKIK3vDopE16AYmRG3nkRGA/awhxnYAW KddjqQU4qYmWvEITZCztmNASX02ty/ZgBGK4xldyubNezUMbOkMbyGxe0B1ER1nU3C GEL4T+GhNpXLQ== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 03/10] arm64: mm: Implicitly invalidate user ASID based on TLBI operation Date: Fri, 11 Jul 2025 17:17:25 +0100 Message-Id: <20250711161732.384-4-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When kpti is enabled, separate ASIDs are used for userspace and kernelspace, requiring ASID-qualified TLB invalidation by virtual address to invalidate both of them. Push the logic for invalidating the two ASIDs down into the low-level __tlbi_level_op() function based on the TLBI operation and remove the burden from the caller to handle the kpti-specific behaviour. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 45 ++++++++++++++++++------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 4408aeebf4d5..08e509f37b28 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -115,17 +115,25 @@ enum tlbi_op { =20 #define TLBI_TTL_UNKNOWN INT_MAX =20 -#define __GEN_TLBI_OP_CASE(op) \ +#define ___GEN_TLBI_OP_CASE(op) \ case op: \ - __tlbi(op, arg); \ + __tlbi(op, arg) + +#define __GEN_TLBI_OP_ASID_CASE(op) \ + ___GEN_TLBI_OP_CASE(op); \ + __tlbi_user(op, arg); \ + break + +#define __GEN_TLBI_OP_CASE(op) \ + ___GEN_TLBI_OP_CASE(op); \ break =20 static __always_inline void __tlbi_level_op(const enum tlbi_op op, u64 arg) { switch (op) { - __GEN_TLBI_OP_CASE(vae1is); + __GEN_TLBI_OP_ASID_CASE(vae1is); __GEN_TLBI_OP_CASE(vae2is); - __GEN_TLBI_OP_CASE(vale1is); + __GEN_TLBI_OP_ASID_CASE(vale1is); __GEN_TLBI_OP_CASE(vale2is); __GEN_TLBI_OP_CASE(vaale1is); __GEN_TLBI_OP_CASE(ipas2e1); @@ -134,7 +142,8 @@ static __always_inline void __tlbi_level_op(const enum = tlbi_op op, u64 arg) BUILD_BUG(); } } -#undef __GEN_TLBI_OP_CASE +#undef __GEN_TLBI_OP_ASID_CASE +#undef ___GEN_TLBI_OP_CASE =20 #define __tlbi_level(op, addr, level) do { \ u64 arg =3D addr; \ @@ -150,11 +159,6 @@ static __always_inline void __tlbi_level_op(const enum= tlbi_op op, u64 arg) __tlbi_level_op(op, arg); \ } while(0) =20 -#define __tlbi_user_level(op, arg, level) do { \ - if (arm64_kernel_unmapped_at_el0()) \ - __tlbi_level(op, (arg | USER_ASID_FLAG), level); \ -} while (0) - /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: @@ -418,22 +422,28 @@ static inline void arch_tlbbatch_flush(struct arch_tl= bflush_unmap_batch *batch) * operations can only span an even number of pages. We save this for l= ast to * ensure 64KB start alignment is maintained for the LPA2 case. */ -#define __GEN_TLBI_OP_CASE(op) \ +#define ___GEN_TLBI_OP_CASE(op) \ case op: \ - __tlbi(r ## op, arg); \ + __tlbi(r ## op, arg) + +#define __GEN_TLBI_OP_ASID_CASE(op) \ + ___GEN_TLBI_OP_CASE(op); \ + __tlbi_user(r ## op, arg); \ break =20 static __always_inline void __tlbi_range(const enum tlbi_op op, u64 arg) { switch (op) { - __GEN_TLBI_OP_CASE(vae1is); - __GEN_TLBI_OP_CASE(vale1is); + __GEN_TLBI_OP_ASID_CASE(vae1is); + __GEN_TLBI_OP_ASID_CASE(vale1is); __GEN_TLBI_OP_CASE(vaale1is); __GEN_TLBI_OP_CASE(ipas2e1is); default: BUILD_BUG(); } } +#undef __GEN_TLBI_OP_ASID_CASE +#undef ___GEN_TLBI_OP_CASE #undef __GEN_TLBI_OP_CASE =20 #define __flush_tlb_range_op(op, start, pages, stride, \ @@ -452,8 +462,6 @@ do { \ (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ addr =3D __TLBI_VADDR(__flush_start, asid); \ __tlbi_level(op, addr, tlb_level); \ - if (tlbi_user) \ - __tlbi_user_level(op, addr, tlb_level); \ __flush_start +=3D stride; \ __flush_pages -=3D stride >> PAGE_SHIFT; \ continue; \ @@ -464,8 +472,6 @@ do { \ addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ scale, num, tlb_level); \ __tlbi_range(op, addr); \ - if (tlbi_user) \ - __tlbi_user(r##op, addr); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ } \ @@ -584,6 +590,7 @@ static inline void arch_tlbbatch_add_pending(struct arc= h_tlbflush_unmap_batch *b { __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); } -#endif =20 +#undef __tlbi_user +#endif #endif --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 551F62EF9DC for ; Fri, 11 Jul 2025 16:17:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250671; cv=none; b=Mk+9UKGeRGBUf5QnVcfNFZ7dQz9bSZKp+Gdaz/cgTR15wnGtgaFq5tKiqXenZA2Tbunj14JLWCKCD0LNxMUKygYe4953arxCm5dJNCxey9AH3Ys+2x0q1HwA305XQvt5bJH3jOia2ri4VmMn+CvK1z2a+N5+3XmjxjCUg5Q4xXc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250671; c=relaxed/simple; bh=srwRxatt1MHTQ063/JWIF6OtD3mvYAP76oIaQp4djOM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=FOyBStOnFPs8FmFg+85L2zB8/RH6SW+kHxOk3ObdJKtYd6hcZ/GMYv42WCPR6vlux8OKbJInxVCcONzQYXjChOUDfkPMY+QdB6U9zZLNzzqWHoNjiAGayX1SELGw0eFBzqrzn3zBROjjXNqpGJN5NSs/zE7va/tvDvJd5e+2UG4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=d1HVFDmh; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="d1HVFDmh" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 12AAEC4CEED; Fri, 11 Jul 2025 16:17:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250671; bh=srwRxatt1MHTQ063/JWIF6OtD3mvYAP76oIaQp4djOM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=d1HVFDmh76WvWr0qWzacxcWvB9yIcL9py/Z4Fag1+P39XUrlCCOzRyMV7xXrLy6df IuAnt3WLFD1KalgCwcK1DtlOozoV2AuCeikXbjWHxRh3GnylKq1xQsLj/Sl3hXEOI8 L4MUswDDY/RcH3zKr0DvP+v7XuVFSpBLi3Z5oG7n4sZtaNcHzQ8diTIR0VymqoUzji p8QM6RSpd08sopjO9aLJbPT+kHzZWi3hjA+09eJCX8+mIVomFNlFXGnTVXVZINoeQg nMR64LxIIsAQfM4HKr3lQlJhOyhR0KrQdBxwbqaeYSA51kEvJVhsqYYkcTkLH2W1Gr GuGjYR7ccqjbw== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 04/10] arm64: mm: Remove unused 'tlbi_user' argument from __flush_tlb_range_op() Date: Fri, 11 Jul 2025 17:17:26 +0100 Message-Id: <20250711161732.384-5-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The 'tlbi_user' argument to __flush_tlb_range_op() is unused. Drop it. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 08e509f37b28..728b00f3e1f4 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -398,8 +398,6 @@ static inline void arch_tlbbatch_flush(struct arch_tlbf= lush_unmap_batch *batch) * @stride: Flush granularity * @asid: The ASID of the task (0 for IPA instructions) * @tlb_level: Translation Table level hint, if known - * @tlbi_user: If 'true', call an additional __tlbi_user() - * (typically for user ASIDs). 'flase' for IPA instructions * @lpa2: If 'true', the lpa2 scheme is used as set out below * * When the CPU does not support TLB range operations, flush the TLB @@ -447,7 +445,7 @@ static __always_inline void __tlbi_range(const enum tlb= i_op op, u64 arg) #undef __GEN_TLBI_OP_CASE =20 #define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, tlbi_user, lpa2) \ + asid, tlb_level, lpa2) \ do { \ typeof(start) __flush_start =3D start; \ typeof(pages) __flush_pages =3D pages; \ @@ -480,7 +478,7 @@ do { \ } while (0) =20 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ - __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_l= pa2_is_enabled()); + __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); =20 static inline bool __flush_tlb_range_limit_excess(unsigned long start, unsigned long end, unsigned long pages, unsigned long stride) @@ -520,10 +518,10 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, =20 if (last_level) __flush_tlb_range_op(vale1is, start, pages, stride, asid, - tlb_level, true, lpa2_is_enabled()); + tlb_level, lpa2_is_enabled()); else __flush_tlb_range_op(vae1is, start, pages, stride, asid, - tlb_level, true, lpa2_is_enabled()); + tlb_level, lpa2_is_enabled()); =20 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -566,7 +564,7 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end =20 dsb(ishst); __flush_tlb_range_op(vaale1is, start, pages, stride, 0, - TLBI_TTL_UNKNOWN, false, lpa2_is_enabled()); + TLBI_TTL_UNKNOWN, lpa2_is_enabled()); dsb(ish); isb(); } --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D00C92EF2B7 for ; Fri, 11 Jul 2025 16:17:53 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250673; cv=none; b=FcyVm9KDoJ10IVDqtT9e5rR7+6LWIcxQHZLnV6YeU63Mgc6mnH7aiNX7InM21rDCqNkpqyI+HyDZr++C7gTQxHzsLeuMelaBmQaExj6aiUFjoLXCLe2Gyb3Jz+OOUzy8QFjsTJSuc/3yTvooPZdOwhFgwYs8E8AXbbEollfSFXc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250673; c=relaxed/simple; bh=7sV42ef3qnlm42vIFFwnfcJvwECcndeyiu5SfLlYqu4=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Y5uYxlG1buyWGKf7MVgb2EMP0j6h/nvxiFWnL9CYLzJHfpsThKjpcrWFkQ8uldr35PXHkbNPpywfnZU5fRee3zcgKqXEsLpJ4rFEoj56eCN2mCpk2f0SNP2vsVNqdDP9YIhdDNeed4xK9E/2/McszSIb1BNgtP8FZRNf303jW+A= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=cHUsYN57; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="cHUsYN57" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 75292C4CEF0; Fri, 11 Jul 2025 16:17:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250673; bh=7sV42ef3qnlm42vIFFwnfcJvwECcndeyiu5SfLlYqu4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=cHUsYN57p1WCIQfYJxnAcETmaWxcXbEabSHhQaNdnDM8HofRM9KnCq1IdbaSnmyrf J7InC+Ann/tJcPuIv1jpvuORX0zhlsYKdRnfWof9E2FsJwPzeDUN1b5SEWUdAUoldZ 0mP7m/GtRFrsTg7XjJGQCrOsEer5oCE2sfGAf9oW294bYXa4E4dNgqj9HouLH2JIhG 1KvVdLEGDj9KxTL/vBCwMUfYD1psoaQ9Um8A/NB4KGs9z3Q930aiw2jn+czdJHDj5Z lmqpwikWby6DauZlUIzFKKAr+np+pHFCmhHlyOqAA0xvlahFnddcJO8kSscThWe8mX tBgfVZ+2LEpUQ== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 05/10] arm64: mm: Re-implement the __tlbi_level macro in C Date: Fri, 11 Jul 2025 17:17:27 +0100 Message-Id: <20250711161732.384-6-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" __tlbi_level() is just a simple macro around __tlbi_level_op(), so merge the two into a single C function. Drop the redundant comparison of 'u32 level' against 0 and tidy up the code a little while we're at it. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 728b00f3e1f4..ddd77e92b268 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -128,8 +128,17 @@ enum tlbi_op { ___GEN_TLBI_OP_CASE(op); \ break =20 -static __always_inline void __tlbi_level_op(const enum tlbi_op op, u64 arg) +static __always_inline void __tlbi_level(const enum tlbi_op op, u64 addr, = u32 level) { + u64 arg =3D addr; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <=3D 3) { + u64 ttl =3D level | (get_trans_granule() << 2); + + arg &=3D ~TLBI_TTL_MASK; + arg |=3D FIELD_PREP(TLBI_TTL_MASK, ttl); + } + switch (op) { __GEN_TLBI_OP_ASID_CASE(vae1is); __GEN_TLBI_OP_CASE(vae2is); @@ -145,20 +154,6 @@ static __always_inline void __tlbi_level_op(const enum= tlbi_op op, u64 arg) #undef __GEN_TLBI_OP_ASID_CASE #undef ___GEN_TLBI_OP_CASE =20 -#define __tlbi_level(op, addr, level) do { \ - u64 arg =3D addr; \ - \ - if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && \ - level >=3D 0 && level <=3D 3) { \ - u64 ttl =3D level & 3; \ - ttl |=3D get_trans_granule() << 2; \ - arg &=3D ~TLBI_TTL_MASK; \ - arg |=3D FIELD_PREP(TLBI_TTL_MASK, ttl); \ - } \ - \ - __tlbi_level_op(op, arg); \ -} while(0) - /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 026D22F0E42 for ; Fri, 11 Jul 2025 16:17:55 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250676; cv=none; b=l1/yf3kYRYU0OiO8I2iItjI5ZhJ670IQOX93Yikbf1YlTlbyiFGzdHCx2olZ7hw6iBIEyTwzd7gdJFFlB80T2ZxCqvRrxnO017BUfpTxORLfxdPiNi0o2zPNXb86RFs7GobMJ2fSFiiQZdVBJJN26FcpuChHUeDBEqAS0u4l154= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250676; c=relaxed/simple; bh=Xuzx6uwSk0yNzSsVAXXMPAY9gCduvpNUj0iAmpfZdno=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=VKTEF6Dyi32juiqKIqyguuh/DkHBXGV9XGt6FTRg6+esaCDaMrkMM9zTwf7h9alGxFh4SUwsjreWw3Y+1be4+FH3ZqO01VCSUjwTwwGE/FI9IRpC2UwWIUxKziUZjJkLmydLe1ZxZBQYFVk+jVMjBdCY5SYViL5MkCi5kOhEpnM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=E8EoeJ8N; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="E8EoeJ8N" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D755DC4CEED; Fri, 11 Jul 2025 16:17:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250675; bh=Xuzx6uwSk0yNzSsVAXXMPAY9gCduvpNUj0iAmpfZdno=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=E8EoeJ8N9T8OaOoWAHbFraEkEQd6Ze+Ty6mhEeSmdVVdVQ2vIHINyKH8HtEz+v5dJ wpSPpI1Gg/2plhs+yU0fhXRgHc3G5g2Nh4T7LprrsfqHKouSZxOXFX1CrsSYKjgY9q uWbIqK6mopmsNhh522MlTaftjZrATL2gvSI5M+2aycE/kKAmZbDEQGb8hvBJmuk3C7 pZo1B23Nbpn7Hz4e1dgrcxkhzxw6tPYIN5bhVjKdFgdcoAf/CFrcb5hdhF5jRVP8xr zTVbW0FjrddOl6kc72mbDxT25XCWW+gF4xmV4OWRO9t+zrLK8kaiFtdUpzndhNKD+h CtQDfARzY3FfA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 06/10] arm64: mm: Simplify __TLBI_RANGE_NUM() macro Date: Fri, 11 Jul 2025 17:17:28 +0100 Message-Id: <20250711161732.384-7-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit e2768b798a19 ("arm64/mm: Modify range-based tlbi to decrement scale"), we don't need to clamp the 'pages' argument to fit the range for the specified 'scale' as we know that the upper bits will have been processed in a prior iteration. Drop the clamping and simplify the __TLBI_RANGE_NUM() macro. Signed-off-by: Will Deacon Reviewed-by: Dev Jain Reviewed-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index ddd77e92b268..a8d21e52ef3a 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -205,11 +205,7 @@ static __always_inline void __tlbi_level(const enum tl= bi_op op, u64 addr, u32 le * range. */ #define __TLBI_RANGE_NUM(pages, scale) \ - ({ \ - int __pages =3D min((pages), \ - __TLBI_RANGE_PAGES(31, (scale))); \ - (__pages >> (5 * (scale) + 1)) - 1; \ - }) + (((pages) >> (5 * (scale) + 1)) - 1) =20 /* * TLB Invalidation --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B04BB21C186 for ; Fri, 11 Jul 2025 16:17:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250678; cv=none; b=Hi+LXTW5wrhdcQO3AmEYlKpt2mGJN4C7kBajJaEjNOQXLU7JR5b1Z2IU2qwOEsMbgNZ7M7HxsUF69thYJNIQVQUtB06zJBoSfKrmYHGFAgWWG1U0d5yPcIEwyczrpmQgCsgiR7R3CpPDEjiu4VQ4eMXiSIbGAqpSmHIZrmTt1y8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250678; c=relaxed/simple; bh=LMvMiVr8imwSxfMPioAfx2yF722cKWt6LDwVcW9+tc8=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Dsc9Qxq3GtWG5aSqGF8n7C6ZW1vmSwdScRl2G9lZ+n04bPz0QcY2YoUqSsctdUYJTfOOe8/sA93KyOQDxOsYfZUEHcyBnqoUND40GlXSLO91v8/AZ7Gr56lG3SoamiO8/KgosHUYvvzsT9akZ3Kj60wLIqCivFyzMlM9AUfprrA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=irmQpPh0; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="irmQpPh0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4AA6CC4CEF5; Fri, 11 Jul 2025 16:17:56 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250678; bh=LMvMiVr8imwSxfMPioAfx2yF722cKWt6LDwVcW9+tc8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=irmQpPh0ukhYGhD3/z2Ca8EgS4PotewgF87LD8sxI5OTQ9i7FjNqoGmH0xDEqTQGg DIzUYkrPh17m+lP1WCMHSt76s7ILKZUv0IgQ6M47f0A2AVrrEbBZJ6A/DmZTKexfT1 RM+Xvt656cmaKYDw+F+erL1lMiMn8X3TTatTk7sO0Mi7oFffUWyoSPkWS6WJYMRETk qohSOsmIYEMuzfGEiHTp/n56JYCSEoWc3yQyxB9vPNPwqDY13DCLRc56lRmn/pmIpH JWACO2zveIEBTiuUBLfM/pOdOt52/U9m+paPaXup0u/y77RgwQSO9K1QDWyhCsaJlo 6bt+Aw4Gn4ccg== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 07/10] arm64: mm: Push __TLBI_VADDR() into __tlbi_level() Date: Fri, 11 Jul 2025 17:17:29 +0100 Message-Id: <20250711161732.384-8-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The __TLBI_VADDR() macro takes an ASID and an address and converts them into a single argument formatted correctly for a TLB invalidation instruction. Rather than have callers worry about this (especially in the case where the ASID is zero), push the macro down into __tlbi_level() via a new __tlbi_level_asid() helper. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 14 ++++++++++---- arch/arm64/kernel/sys_compat.c | 2 +- arch/arm64/kvm/hyp/nvhe/mm.c | 2 +- arch/arm64/kvm/hyp/pgtable.c | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index a8d21e52ef3a..434b9fdb340a 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -128,9 +128,10 @@ enum tlbi_op { ___GEN_TLBI_OP_CASE(op); \ break =20 -static __always_inline void __tlbi_level(const enum tlbi_op op, u64 addr, = u32 level) +static __always_inline void __tlbi_level_asid(const enum tlbi_op op, u64 a= ddr, + u32 level, u16 asid) { - u64 arg =3D addr; + u64 arg =3D __TLBI_VADDR(addr, asid); =20 if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <=3D 3) { u64 ttl =3D level | (get_trans_granule() << 2); @@ -154,6 +155,11 @@ static __always_inline void __tlbi_level(const enum tl= bi_op op, u64 addr, u32 le #undef __GEN_TLBI_OP_ASID_CASE #undef ___GEN_TLBI_OP_CASE =20 +static inline void __tlbi_level(const enum tlbi_op op, u64 addr, u32 level) +{ + __tlbi_level_asid(op, addr, level, 0); +} + /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: @@ -449,8 +455,7 @@ do { \ if (!system_supports_tlb_range() || \ __flush_pages =3D=3D 1 || \ (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ - addr =3D __TLBI_VADDR(__flush_start, asid); \ - __tlbi_level(op, addr, tlb_level); \ + __tlbi_level_asid(op, __flush_start, tlb_level, asid); \ __flush_start +=3D stride; \ __flush_pages -=3D stride >> PAGE_SHIFT; \ continue; \ @@ -580,6 +585,7 @@ static inline void arch_tlbbatch_add_pending(struct arc= h_tlbflush_unmap_batch *b __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); } =20 +#undef __TLBI_VADDR #undef __tlbi_user #endif #endif diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 4a609e9b65de..ad4857df4830 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -36,7 +36,7 @@ __do_compat_cache_op(unsigned long start, unsigned long e= nd) * The workaround requires an inner-shareable tlbi. * We pick the reserved-ASID to minimise the impact. */ - __tlbi(aside1is, __TLBI_VADDR(0, 0)); + __tlbi(aside1is, 0UL); dsb(ish); } =20 diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index ae8391baebc3..581385b21826 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -270,7 +270,7 @@ static void fixmap_clear_slot(struct hyp_fixmap_slot *s= lot) * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#m= f10dfbaf1eaef9274c581b81c53758918c1d0f03 */ dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level); + __tlbi_level(vale2is, addr, level); dsb(ish); isb(); } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index c351b4abd5db..540691987e13 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -472,14 +472,14 @@ static int hyp_unmap_walker(const struct kvm_pgtable_= visit_ctx *ctx, =20 kvm_clear_pte(ctx->ptep); dsb(ishst); - __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), TLBI_TTL_UNKNOWN); + __tlbi_level(vae2is, ctx->addr, TLBI_TTL_UNKNOWN); } else { if (ctx->end - ctx->addr < granule) return -EINVAL; =20 kvm_clear_pte(ctx->ptep); dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level); + __tlbi_level(vale2is, ctx->addr, ctx->level); *unmapped +=3D granule; } =20 --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DEF5D2F19B6 for ; Fri, 11 Jul 2025 16:18:00 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250681; cv=none; b=qIfyqa5ibqYZyT+1qZdLtTn/LcfDNoQJJfK+iWaKcunbYtrVbiEp/P6D9qS58pd845buFqo/cpbAqiaNm0ctrAh36uumoEJdNjqD6czIzc6yP5U0CtYWsP1EeRuoWpJG2c2/8rtWBvZxUMzRK6efI8K1DTAW83DMwbYEpJbQ7Kw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250681; c=relaxed/simple; bh=buXKCURD0rDg9LCeJR6ajdKTgsrB7TdGsFPHuLOxDsM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=jemMX5nF6kdc1ZI9z3EgYGNBFI1bKUmBmlZTbgNc9a+PIs/osCFidhtWUdCejuZkLpMUaGek1VddAogvOS3NTJeXVC04yjB0e+o1ZhYwrHRfDrxixPdnqcsOIjQ4S2VCQQ4c3CIeuelZLyv8E0PTsTLB5kMQl3z8RtcR4mM8kx0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=pK+Ugg/R; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="pK+Ugg/R" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A8AC8C4CEED; Fri, 11 Jul 2025 16:17:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250680; bh=buXKCURD0rDg9LCeJR6ajdKTgsrB7TdGsFPHuLOxDsM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=pK+Ugg/ROm1qkMd7IQzB6dWQZbn/qW5fG74SQ6NyZHiYOBAjT8B6TB7oEI8/XPysF Y46suYS74Kyqm88XLKPHlOta0+rbDOjtiyV4oPyOuxH4VBNhWu8mZIbM+DYfxDSitA Fg4bbtpMosIb+YB1qCgAeOInjmsfJEL4DZmg0i3DYxxXc3Phjf36cr1w2FLqNQayNn 89Qfc3uYCIQJtaXNWDIfv/8eV7uZUiiP38f58VzBoBYtGPYRwnmdisAXWhUZXZpTXO NDuQPkpM/S+FV836EOWXeJ19qS6oDPZcDsBOjWEbhHXLq+Nwne6ZqhCQwYHhrfSWwc TRAsmm4j8HUNw== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 08/10] arm64: mm: Inline __TLBI_VADDR_RANGE() into __tlbi_range() Date: Fri, 11 Jul 2025 17:17:30 +0100 Message-Id: <20250711161732.384-9-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The __TLBI_VADDR_RANGE() macro is only used in one place and isn't something that's generally useful outside of the low-level range invalidation gubbins. Inline __TLBI_VADDR_RANGE() into the __tlbi_range() function so that the macro can be removed entirely. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 32 +++++++++++++------------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 434b9fdb340a..8618a85d5cd3 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -185,19 +185,6 @@ static inline void __tlbi_level(const enum tlbi_op op,= u64 addr, u32 level) #define TLBIR_TTL_MASK GENMASK_ULL(38, 37) #define TLBIR_BADDR_MASK GENMASK_ULL(36, 0) =20 -#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl) \ - ({ \ - unsigned long __ta =3D 0; \ - unsigned long __ttl =3D (ttl >=3D 1 && ttl <=3D 3) ? ttl : 0; \ - __ta |=3D FIELD_PREP(TLBIR_BADDR_MASK, baddr); \ - __ta |=3D FIELD_PREP(TLBIR_TTL_MASK, __ttl); \ - __ta |=3D FIELD_PREP(TLBIR_NUM_MASK, num); \ - __ta |=3D FIELD_PREP(TLBIR_SCALE_MASK, scale); \ - __ta |=3D FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); \ - __ta |=3D FIELD_PREP(TLBIR_ASID_MASK, asid); \ - __ta; \ - }) - /* These macros are used by the TLBI RANGE feature. */ #define __TLBI_RANGE_PAGES(num, scale) \ ((unsigned long)((num) + 1) << (5 * (scale) + 1)) @@ -426,8 +413,19 @@ static inline void arch_tlbbatch_flush(struct arch_tlb= flush_unmap_batch *batch) __tlbi_user(r ## op, arg); \ break =20 -static __always_inline void __tlbi_range(const enum tlbi_op op, u64 arg) +static __always_inline void __tlbi_range(const enum tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2) { + u64 arg =3D 0; + + arg |=3D FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |=3D FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); + arg |=3D FIELD_PREP(TLBIR_NUM_MASK, num); + arg |=3D FIELD_PREP(TLBIR_SCALE_MASK, scale); + arg |=3D FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |=3D FIELD_PREP(TLBIR_ASID_MASK, asid); + switch (op) { __GEN_TLBI_OP_ASID_CASE(vae1is); __GEN_TLBI_OP_ASID_CASE(vale1is); @@ -448,8 +446,6 @@ do { \ typeof(pages) __flush_pages =3D pages; \ int num =3D 0; \ int scale =3D 3; \ - int shift =3D lpa2 ? 16 : PAGE_SHIFT; \ - unsigned long addr; \ \ while (__flush_pages > 0) { \ if (!system_supports_tlb_range() || \ @@ -463,9 +459,7 @@ do { \ \ num =3D __TLBI_RANGE_NUM(__flush_pages, scale); \ if (num >=3D 0) { \ - addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ - scale, num, tlb_level); \ - __tlbi_range(op, addr); \ + __tlbi_range(op, __flush_start, asid, scale, num, tlb_level, lpa2); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ } \ --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 913052EF664 for ; Fri, 11 Jul 2025 16:18:03 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250683; cv=none; b=rUkpp5NmQW9MlII+Tp39B6WLiZWZwBPB5pInaVem0md8MQehV877AoMUg7UvC8PU1FfCzcMbVOzcstjtW33TnzkXFMFuNzwDb3X/0ziDfvjvFc1u1705/O3EoykEpn+3xyLk+skCBMXoNrjlUNUcHSISSMlcH94/B7OvYyVtNv0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250683; c=relaxed/simple; bh=AuZ9fTTOEu8jUPZc0v6I8pAQSkyfRml4JgY7xt6+Jko=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=CrsrEWq50cKHPh6qRCWQU+xGWZhOCgdVd9DEG2DQtEpprQBVkcnTstH3oQahBnEEc6alAwzpiZjM3vxFgyagSH1AO8rsr461ekvq67ILu4zsBjAHe77ddDOkeO06eSOMBcHvso75h5dadmiNX38nd7TFYK12ccAFanmdggRR/0M= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=T5oVx7eR; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="T5oVx7eR" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4090BC4CEED; Fri, 11 Jul 2025 16:18:01 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250683; bh=AuZ9fTTOEu8jUPZc0v6I8pAQSkyfRml4JgY7xt6+Jko=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=T5oVx7eRlm5cJmNhjSbPJySGMgrCGkb69GM1qLknnOUZ5hzqKNSC/xi5T+5LPaloF mMGGIuXqQih0GCiKWUSfnscKG3nRV2nKK7LDsQDTW5Qa0FX7E7qvM1Z1ICnNE5Y7P2 j8EaB0ysrbG+2Uuh+siMhtWr70gDV6faF78BdlvtOPKogPnuGbMFF9wwBxeWA/hsdL ByXJMljfJA6KDwQh1MFNkbKoTd7HQ+kFwYDTmw/esY5+dC+6jBw4YfHrkuYVS24hJS EpnfGuno197YnU/hnJNzk9L6F2gFcD57MjQEfcUh2RPa/yhAELYe2cZQFVqFJrizzb 4uR4+DhwT6AwA== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 09/10] arm64: mm: Simplify __flush_tlb_range_limit_excess() Date: Fri, 11 Jul 2025 17:17:31 +0100 Message-Id: <20250711161732.384-10-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" __flush_tlb_range_limit_excess() is unnecessarily complicated: - It takes a 'start', 'end' and 'pages' argument, whereas it only needs 'pages' (which the caller has computed from the other two arguments!). - It erroneously compares 'pages' with MAX_TLBI_RANGE_PAGES when the system doesn't support range-based invalidation but the range to be invalidated would result in fewer than MAX_DVM_OPS invalidations. Simplify the function so that it no longer takes the 'start' and 'end' arguments and only considers the MAX_TLBI_RANGE_PAGES threshold on systems that implement range-based invalidation. Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlbflush.h | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 8618a85d5cd3..2541863721af 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -470,21 +470,13 @@ do { \ #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); =20 -static inline bool __flush_tlb_range_limit_excess(unsigned long start, - unsigned long end, unsigned long pages, unsigned long stride) +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) { - /* - * When the system does not support TLB range based flush - * operation, (MAX_DVM_OPS - 1) pages can be handled. But - * with TLB range based operation, MAX_TLBI_RANGE_PAGES - * pages can be handled. - */ - if ((!system_supports_tlb_range() && - (end - start) >=3D (MAX_DVM_OPS * stride)) || - pages > MAX_TLBI_RANGE_PAGES) + if (system_supports_tlb_range() && pages > MAX_TLBI_RANGE_PAGES) return true; =20 - return false; + return pages >=3D (MAX_DVM_OPS * stride) >> PAGE_SHIFT; } =20 static inline void __flush_tlb_range_nosync(struct mm_struct *mm, @@ -498,7 +490,7 @@ static inline void __flush_tlb_range_nosync(struct mm_s= truct *mm, end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 - if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { + if (__flush_tlb_range_limit_excess(pages, stride)) { flush_tlb_mm(mm); return; } @@ -547,7 +539,7 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 - if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { + if (__flush_tlb_range_limit_excess(pages, stride)) { flush_tlb_all(); return; } --=20 2.50.0.727.gbf7dc18ff4-goog From nobody Tue Oct 7 07:11:38 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0C3C02F2359 for ; Fri, 11 Jul 2025 16:18:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250686; cv=none; b=WLtMV39q18/cstCVw7kf+QQmcHvmqXedvjY8P3p5AG4Le4DqTataOUA3+SYKfUWCg22GqDKzzMoX0UoHJCaIFT/SHZ8AnBUFgHEucrR64p8pv8Gqvh6j21hyq87oB7/T2zClaPoyNmnak86VfWEH1nJMffDyZWFAc0QHPm0hULc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752250686; c=relaxed/simple; bh=dlETiSCP0bE/7Eo8LNeRjL0vHF3yV9EuAqXo0Nii9Qo=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=lOGgQeNN3c7Q1Vhn4VW0WK7zZVT4vMhDQcvpJz8u8Y2z7JK15XDiBL5dTTea505XI/ZkGJqrEZpGfNmfX4UdU1VzzjRBJADpOyPxoL3du7h+Ne6/W9KjB8lreMQpcjXmOiNtvrFX7hvn5Ppz3HXyMxa+eMKM+i8BWY7PN4ZE/8g= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=gUUS7ykY; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="gUUS7ykY" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A8A8AC4CEEF; Fri, 11 Jul 2025 16:18:03 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1752250685; bh=dlETiSCP0bE/7Eo8LNeRjL0vHF3yV9EuAqXo0Nii9Qo=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=gUUS7ykYexH4zJ5LeQzrknTtSpvDTGSrjRCCI3ivPvxM00SmGnWwhhvuHWCGpkXT9 gt0hJzVqbFvq5jJt7orVDYOtf5INnTQKdfDvPshlFZ98vgfC6TbjPzuB0PBlpiuEW5 vlJseJ8qKce72EIX9QuzcGs2zn0QybPYFhidA1fNQAyDJYWNym6hZQwpRwfoNBddyg FCkxwyVGD6yhLGX5Cctw1oizVqPJ+36kchYXqKd/Ou4NQafNI6z1Y4Dxu43XQtq88a JGgYX9NbtwPXMaRInCfT0//2bLgSpQIrzjVCvPMFJwDAUz8DwVfaagxMbG6vJoUO0p 6VNKKoKwp5yfQ== From: Will Deacon To: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org, Will Deacon , Ard Biesheuvel , Catalin Marinas , Ryan Roberts , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier Subject: [PATCH 10/10] arm64: mm: Re-implement the __flush_tlb_range_op macro in C Date: Fri, 11 Jul 2025 17:17:32 +0100 Message-Id: <20250711161732.384-11-will@kernel.org> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250711161732.384-1-will@kernel.org> References: <20250711161732.384-1-will@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The __flush_tlb_range_op() macro is horrible and has been a previous source of bugs thanks to multiple expansions of its arguments (see commit f7edb07ad7c6 ("Fix mmu notifiers for range-based invalidates")). Rewrite the thing in C. Suggested-by: Linus Torvalds Signed-off-by: Will Deacon Reviewed-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 63 +++++++++++++++++-------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 2541863721af..ee69efdc12ab 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -376,12 +376,12 @@ static inline void arch_tlbbatch_flush(struct arch_tl= bflush_unmap_batch *batch) /* * __flush_tlb_range_op - Perform TLBI operation upon a range * - * @op: TLBI instruction that operates on a range (has 'r' prefix) + * @op: TLBI instruction that operates on a range * @start: The start address of the range * @pages: Range as the number of pages from 'start' * @stride: Flush granularity * @asid: The ASID of the task (0 for IPA instructions) - * @tlb_level: Translation Table level hint, if known + * @level: Translation Table level hint, if known * @lpa2: If 'true', the lpa2 scheme is used as set out below * * When the CPU does not support TLB range operations, flush the TLB @@ -439,33 +439,38 @@ static __always_inline void __tlbi_range(const enum t= lbi_op op, u64 addr, #undef ___GEN_TLBI_OP_CASE #undef __GEN_TLBI_OP_CASE =20 -#define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, lpa2) \ -do { \ - typeof(start) __flush_start =3D start; \ - typeof(pages) __flush_pages =3D pages; \ - int num =3D 0; \ - int scale =3D 3; \ - \ - while (__flush_pages > 0) { \ - if (!system_supports_tlb_range() || \ - __flush_pages =3D=3D 1 || \ - (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ - __tlbi_level_asid(op, __flush_start, tlb_level, asid); \ - __flush_start +=3D stride; \ - __flush_pages -=3D stride >> PAGE_SHIFT; \ - continue; \ - } \ - \ - num =3D __TLBI_RANGE_NUM(__flush_pages, scale); \ - if (num >=3D 0) { \ - __tlbi_range(op, __flush_start, asid, scale, num, tlb_level, lpa2); \ - __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ - __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ - } \ - scale--; \ - } \ -} while (0) +static __always_inline void __flush_tlb_range_op(const enum tlbi_op op, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + u64 addr =3D start, end =3D start + pages * PAGE_SIZE; + int scale =3D 3; + + while (addr !=3D end) { + int num; + + pages =3D (end - addr) >> PAGE_SHIFT; + + if (!system_supports_tlb_range() || pages =3D=3D 1) + goto invalidate_one; + + if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) + goto invalidate_one; + + num =3D __TLBI_RANGE_NUM(pages, scale); + if (num >=3D 0) { + __tlbi_range(op, addr, asid, scale, num, level, lpa2); + addr +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + } + + scale--; + continue; +invalidate_one: + __tlbi_level_asid(op, addr, level, asid); + addr +=3D stride; + } +} =20 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); --=20 2.50.0.727.gbf7dc18ff4-goog