On Tue, Feb 25, 2025 at 10:00:35PM -0500, Rik van Riel wrote:
> Add support for broadcast TLB invalidation using AMD's INVLPGB instruction.
One more patch ontop from Tom.
Now lemme test this a bit again...
From: Tom Lendacky <thomas.lendacky@amd.com>
Date: Tue, 4 Mar 2025 12:59:56 +0100
Subject: [PATCH] x86/mm: Always set the ASID valid bit for the INVLPGB instruction
When executing the INVLPGB instruction on a bare-metal host or hypervisor, if
the ASID valid bit is not set, the instruction will flush the TLB entries that
match the specified criteria for any ASID, not just the those of the host. If
virtual machines are running on the system, this may result in inadvertent
flushes of guest TLB entries.
When executing the INVLPGB instruction in a guest and the INVLPGB instruction is
not intercepted by the hypervisor, the hardware will replace the requested ASID
with the guest ASID and set the ASID valid bit before doing the broadcast
invalidation. Thus a guest is only able to flush its own TLB entries.
So to limit the host TLB flushing reach, always set the ASID valid bit using an
ASID value of 0 (which represents the host/hypervisor). This will will result in
the desired effect in both host and guest.
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
---
arch/x86/include/asm/tlb.h | 58 +++++++++++++++++++++-----------------
1 file changed, 32 insertions(+), 26 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index e8561a846754..56fe331fb797 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -33,6 +33,27 @@ enum addr_stride {
PMD_STRIDE = 1
};
+/*
+ * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
+ * of the three. For example:
+ * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
+ * - FLAG_PCID: invalidate all TLB entries matching the PCID
+ *
+ * The first is used to invalidate (kernel) mappings at a particular
+ * address across all processes.
+ *
+ * The latter invalidates all TLB entries matching a PCID.
+ */
+#define INVLPGB_FLAG_VA BIT(0)
+#define INVLPGB_FLAG_PCID BIT(1)
+#define INVLPGB_FLAG_ASID BIT(2)
+#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3)
+#define INVLPGB_FLAG_FINAL_ONLY BIT(4)
+#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5)
+
+/* The implied mode when all bits are clear: */
+#define INVLPGB_MODE_ALL_NONGLOBALS 0UL
+
#ifdef CONFIG_BROADCAST_TLB_FLUSH
/*
* INVLPGB does broadcast TLB invalidation across all the CPUs in the system.
@@ -40,14 +61,20 @@ enum addr_stride {
* The INVLPGB instruction is weakly ordered, and a batch of invalidations can
* be done in a parallel fashion.
*
- * The instruction takes the number of extra pages to invalidate, beyond
- * the first page, while __invlpgb gets the more human readable number of
- * pages to invalidate.
+ * The instruction takes the number of extra pages to invalidate, beyond the
+ * first page, while __invlpgb gets the more human readable number of pages to
+ * invalidate.
*
* The bits in rax[0:2] determine respectively which components of the address
* (VA, PCID, ASID) get compared when flushing. If neither bits are set, *any*
* address in the specified range matches.
*
+ * Since it is desired to only flush TLB entries for the ASID that is executing
+ * the instruction (a host/hypervisor or a guest), the ASID valid bit should
+ * always be set. On a host/hypervisor, the hardware will use the ASID value
+ * specified in EDX[15:0] (which should be 0). On a guest, the hardware will
+ * use the actual ASID value of the guest.
+ *
* TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from
* this CPU have completed.
*/
@@ -55,9 +82,9 @@ static inline void __invlpgb(unsigned long asid, unsigned long pcid,
unsigned long addr, u16 nr_pages,
enum addr_stride stride, u8 flags)
{
- u32 edx = (pcid << 16) | asid;
+ u64 rax = addr | flags | INVLPGB_FLAG_ASID;
u32 ecx = (stride << 31) | (nr_pages - 1);
- u64 rax = addr | flags;
+ u32 edx = (pcid << 16) | asid;
/* The low bits in rax are for flags. Verify addr is clean. */
VM_WARN_ON_ONCE(addr & ~PAGE_MASK);
@@ -87,27 +114,6 @@ static inline void __invlpgb(unsigned long asid, unsigned long pcid,
static inline void __tlbsync(void) { }
#endif
-/*
- * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
- * of the three. For example:
- * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
- * - FLAG_PCID: invalidate all TLB entries matching the PCID
- *
- * The first is used to invalidate (kernel) mappings at a particular
- * address across all processes.
- *
- * The latter invalidates all TLB entries matching a PCID.
- */
-#define INVLPGB_FLAG_VA BIT(0)
-#define INVLPGB_FLAG_PCID BIT(1)
-#define INVLPGB_FLAG_ASID BIT(2)
-#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3)
-#define INVLPGB_FLAG_FINAL_ONLY BIT(4)
-#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5)
-
-/* The implied mode when all bits are clear: */
-#define INVLPGB_MODE_ALL_NONGLOBALS 0UL
-
static inline void __invlpgb_flush_user_nr_nosync(unsigned long pcid,
unsigned long addr,
u16 nr, bool stride)
--
2.43.0
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteOn Tue, Mar 04, 2025 at 01:04:49PM +0100, Borislav Petkov wrote:
> @@ -55,9 +82,9 @@ static inline void __invlpgb(unsigned long asid, unsigned long pcid,
> unsigned long addr, u16 nr_pages,
> enum addr_stride stride, u8 flags)
> {
> - u32 edx = (pcid << 16) | asid;
> + u64 rax = addr | flags | INVLPGB_FLAG_ASID;
So I did it this way instead of how you sent it to me privately and I think
doing it this way is equivalent.
However, my Zen3 doesn't like it somehow.
If I zap your patch, it boots fine.
So we need to stare at this some more first...
Thx.
[ 13.834550] (sd-mkdcre[570]: Changing mount flags /dev/shm (MS_RDONLY|MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_REMOUNT|MS_NOSYMFOLLOW|MS_BIND "")...
[ 13.834605] (sd-mkdcre[570]: Moving mount /dev/shm → /run/credentials/systemd-tmpfiles-setup-dev-early.service (MS_MOVE "")...
[ 13.835200] EXT4-fs (sda3): re-mounted 69500787-6b3d-4b03-96d6-6bb20668e924 r/w. Quota mode: none.
[ 13.867434] (journald)[561]: systemd-journald.service: Enabled MemoryDenyWriteExecute= with PR_SET_MDWE
[ 13.868764] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 8-.... } 8 jiffies s: 129 root: 0x1/.
[ 13.868779] rcu: blocking rcu_node structures (internal RCU debug): l=1:0-15:0x100/.
[ 13.868789] Sending NMI from CPU 19 to CPUs 8:
[ 13.868797] NMI backtrace for cpu 8
[ 13.868799] CPU: 8 UID: 0 PID: 566 Comm: mount Not tainted 6.14.0-rc3+ #1
[ 13.868802] Hardware name: Supermicro Super Server/H12SSL-i, BIOS 2.5 09/08/2022
[ 13.868804] RIP: 0010:delay_halt_mwaitx+0x38/0x40
[ 13.868809] Code: 31 d2 48 89 d1 48 05 00 60 00 00 0f 01 fa b8 ff ff ff ff b9 02 00 00 00 48 39 c6 48 0f 46 c6 48 89 c3 b8 f0 00 00 00 0f 01 fb <5b> e9 cd ea 23 ff 66 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90
[ 13.868810] RSP: 0018:ffffc90003b67ae0 EFLAGS: 00000097
[ 13.868813] RAX: 00000000000000f0 RBX: 0000000000000b9b RCX: 0000000000000002
[ 13.868814] RDX: 0000000000000000 RSI: 0000000000000b9b RDI: 00000036410a0588
[ 13.868816] RBP: 00000036410a0588 R08: 000000000000005b R09: 0000000000000006
[ 13.868817] R10: 0000000000000000 R11: 0000000000000006 R12: 0000000000000020
[ 13.868818] R13: ffffffff83fb5900 R14: 0000000000000000 R15: 0000000000000001
[ 13.868819] FS: 00007f04ffb9c800(0000) GS:ffff889002e00000(0000) knlGS:0000000000000000
[ 13.868821] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 13.868822] CR2: 00007f92418c8350 CR3: 000000012bbe4004 CR4: 0000000000770ef0
[ 13.868823] PKRU: 55555554
[ 13.868824] Call Trace:
[ 13.868826] <NMI>
[ 13.868828] ? nmi_cpu_backtrace.cold+0x16/0x6b
[ 13.868832] ? nmi_cpu_backtrace_handler+0xd/0x20
[ 13.868835] ? nmi_handle+0xcd/0x240
[ 13.868838] ? delay_halt_mwaitx+0x38/0x40
[ 13.868841] ? default_do_nmi+0x6b/0x180
[ 13.868844] ? exc_nmi+0x12e/0x1c0
[ 13.868847] ? end_repeat_nmi+0xf/0x53
[ 13.868854] ? delay_halt_mwaitx+0x38/0x40
[ 13.868857] ? delay_halt_mwaitx+0x38/0x40
[ 13.868860] ? delay_halt_mwaitx+0x38/0x40
[ 13.868863] </NMI>
[ 13.868864] <TASK>
[ 13.868865] delay_halt+0x3b/0x60
[ 13.868868] wait_for_lsr+0x3f/0xa0
[ 13.868873] serial8250_console_write+0x270/0x670
[ 13.868877] ? srso_alias_return_thunk+0x5/0xfbef5
[ 13.868879] ? lock_release+0x183/0x2d0
[ 13.868885] ? console_flush_all+0x381/0x6b0
[ 13.868888] console_flush_all+0x3ad/0x6b0
[ 13.868891] ? console_flush_all+0x381/0x6b0
[ 13.868893] ? console_flush_all+0x39/0x6b0
[ 13.868900] console_unlock+0x157/0x240
[ 13.868904] vprintk_emit+0x304/0x550
[ 13.868910] _printk+0x58/0x80
[ 13.868914] ? ___ratelimit+0x85/0x100
[ 13.868919] __ext4_msg.cold+0x7f/0x84
[ 13.868926] ? srso_alias_return_thunk+0x5/0xfbef5
[ 13.868928] ? ext4_register_li_request+0x1c9/0x2a0
[ 13.868933] ext4_reconfigure+0x503/0xc40
[ 13.868939] ? shrink_dentry_list+0x17/0x320
[ 13.868942] ? shrink_dentry_list+0x89/0x320
[ 13.868944] ? srso_alias_return_thunk+0x5/0xfbef5
[ 13.868946] ? shrink_dentry_list+0x16c/0x320
[ 13.868953] reconfigure_super+0xc5/0x210
[ 13.868958] __do_sys_fsconfig+0x607/0x750
[ 13.868967] do_syscall_64+0x5d/0x100
[ 13.868970] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 13.868973] RIP: 0033:0x7f04ffdb481a
[ 13.868976] Code: 73 01 c3 48 8b 0d 06 56 0d 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 49 89 ca b8 af 01 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d d6 55 0d 00 f7 d8 64 89 01 48
[ 13.868977] RSP: 002b:00007ffe3e177008 EFLAGS: 00000246 ORIG_RAX: 00000000000001af
[ 13.868980] RAX: ffffffffffffffda RBX: 000055ab49f74a50 RCX: 00007f04ffdb481a
[ 13.868981] RDX: 0000000000000000 RSI: 0000000000000007 RDI: 0000000000000004
[ 13.868982] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000001
[ 13.868983] R10: 0000000000000000 R11: 0000000000000246 R12: 000055ab49f75ee0
[ 13.868984] R13: 00007f04fff3b5e0 R14: 00007f04fff3d244 R15: 0000000000000065
[ 13.868993] </TASK>
[ 13.880199] (tmpfiles)[564]: (sd-mkdcreds) succeeded.
[ 13.936943] (journald)[561]: Restricting namespace to: n/a.
[ 13.937918] (tmpfiles)[564]: systemd-tmpfiles-setup-dev-early.service: Executing: systemd-tmpfiles --prefix=/dev --create --boot --graceful
[ 14.025641] (journald)[561]: systemd-journald.service: Executing: /usr/lib/systemd/systemd-journald
[ 14.137455] ACPI: bus type drm_connector registered
[ 14.306514] (udevadm)[569]: Found cgroup2 on /sys/fs/cgroup/, full unified hierarchy
[ 14.357107] (udevadm)[569]: Found cgroup2 on /sys/fs/cgroup/, full unified hierarch
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 634ab76159a842f890743fb435f74c228e06bb04
Gitweb: https://git.kernel.org/tip/634ab76159a842f890743fb435f74c228e06bb04
Author: Tom Lendacky <thomas.lendacky@amd.com>
AuthorDate: Tue, 04 Mar 2025 12:59:56 +01:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Wed, 19 Mar 2025 11:12:29 +01:00
x86/mm: Always set the ASID valid bit for the INVLPGB instruction
When executing the INVLPGB instruction on a bare-metal host or hypervisor, if
the ASID valid bit is not set, the instruction will flush the TLB entries that
match the specified criteria for any ASID, not just the those of the host. If
virtual machines are running on the system, this may result in inadvertent
flushes of guest TLB entries.
When executing the INVLPGB instruction in a guest and the INVLPGB instruction is
not intercepted by the hypervisor, the hardware will replace the requested ASID
with the guest ASID and set the ASID valid bit before doing the broadcast
invalidation. Thus a guest is only able to flush its own TLB entries.
So to limit the host TLB flushing reach, always set the ASID valid bit using an
ASID value of 0 (which represents the host/hypervisor). This will will result in
the desired effect in both host and guest.
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20250304120449.GHZ8bsYYyEBOKQIxBm@fat_crate.local
---
arch/x86/include/asm/tlb.h | 58 ++++++++++++++++++++-----------------
1 file changed, 32 insertions(+), 26 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 31f6db4..866ea78 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -33,6 +33,27 @@ enum addr_stride {
PMD_STRIDE = 1
};
+/*
+ * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
+ * of the three. For example:
+ * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
+ * - FLAG_PCID: invalidate all TLB entries matching the PCID
+ *
+ * The first is used to invalidate (kernel) mappings at a particular
+ * address across all processes.
+ *
+ * The latter invalidates all TLB entries matching a PCID.
+ */
+#define INVLPGB_FLAG_VA BIT(0)
+#define INVLPGB_FLAG_PCID BIT(1)
+#define INVLPGB_FLAG_ASID BIT(2)
+#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3)
+#define INVLPGB_FLAG_FINAL_ONLY BIT(4)
+#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5)
+
+/* The implied mode when all bits are clear: */
+#define INVLPGB_MODE_ALL_NONGLOBALS 0UL
+
#ifdef CONFIG_BROADCAST_TLB_FLUSH
/*
* INVLPGB does broadcast TLB invalidation across all the CPUs in the system.
@@ -40,14 +61,20 @@ enum addr_stride {
* The INVLPGB instruction is weakly ordered, and a batch of invalidations can
* be done in a parallel fashion.
*
- * The instruction takes the number of extra pages to invalidate, beyond
- * the first page, while __invlpgb gets the more human readable number of
- * pages to invalidate.
+ * The instruction takes the number of extra pages to invalidate, beyond the
+ * first page, while __invlpgb gets the more human readable number of pages to
+ * invalidate.
*
* The bits in rax[0:2] determine respectively which components of the address
* (VA, PCID, ASID) get compared when flushing. If neither bits are set, *any*
* address in the specified range matches.
*
+ * Since it is desired to only flush TLB entries for the ASID that is executing
+ * the instruction (a host/hypervisor or a guest), the ASID valid bit should
+ * always be set. On a host/hypervisor, the hardware will use the ASID value
+ * specified in EDX[15:0] (which should be 0). On a guest, the hardware will
+ * use the actual ASID value of the guest.
+ *
* TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from
* this CPU have completed.
*/
@@ -55,9 +82,9 @@ static inline void __invlpgb(unsigned long asid, unsigned long pcid,
unsigned long addr, u16 nr_pages,
enum addr_stride stride, u8 flags)
{
- u32 edx = (pcid << 16) | asid;
+ u64 rax = addr | flags | INVLPGB_FLAG_ASID;
u32 ecx = (stride << 31) | (nr_pages - 1);
- u64 rax = addr | flags;
+ u32 edx = (pcid << 16) | asid;
/* The low bits in rax are for flags. Verify addr is clean. */
VM_WARN_ON_ONCE(addr & ~PAGE_MASK);
@@ -93,27 +120,6 @@ static inline void __invlpgb_all(unsigned long asid, unsigned long pcid, u8 flag
static inline void __tlbsync(void) { }
#endif
-/*
- * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
- * of the three. For example:
- * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
- * - FLAG_PCID: invalidate all TLB entries matching the PCID
- *
- * The first is used to invalidate (kernel) mappings at a particular
- * address across all processes.
- *
- * The latter invalidates all TLB entries matching a PCID.
- */
-#define INVLPGB_FLAG_VA BIT(0)
-#define INVLPGB_FLAG_PCID BIT(1)
-#define INVLPGB_FLAG_ASID BIT(2)
-#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3)
-#define INVLPGB_FLAG_FINAL_ONLY BIT(4)
-#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5)
-
-/* The implied mode when all bits are clear: */
-#define INVLPGB_MODE_ALL_NONGLOBALS 0UL
-
static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
unsigned long addr,
u16 nr, bool stride)The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 0896acd80782ec49c6d36e576fcd53786f0a2bfb
Gitweb: https://git.kernel.org/tip/0896acd80782ec49c6d36e576fcd53786f0a2bfb
Author: Tom Lendacky <thomas.lendacky@amd.com>
AuthorDate: Tue, 04 Mar 2025 12:59:56 +01:00
Committer: Borislav Petkov (AMD) <bp@alien8.de>
CommitterDate: Wed, 05 Mar 2025 22:18:16 +01:00
x86/mm: Always set the ASID valid bit for the INVLPGB instruction
When executing the INVLPGB instruction on a bare-metal host or hypervisor, if
the ASID valid bit is not set, the instruction will flush the TLB entries that
match the specified criteria for any ASID, not just the those of the host. If
virtual machines are running on the system, this may result in inadvertent
flushes of guest TLB entries.
When executing the INVLPGB instruction in a guest and the INVLPGB instruction is
not intercepted by the hypervisor, the hardware will replace the requested ASID
with the guest ASID and set the ASID valid bit before doing the broadcast
invalidation. Thus a guest is only able to flush its own TLB entries.
So to limit the host TLB flushing reach, always set the ASID valid bit using an
ASID value of 0 (which represents the host/hypervisor). This will will result in
the desired effect in both host and guest.
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20250304120449.GHZ8bsYYyEBOKQIxBm@fat_crate.local
---
arch/x86/include/asm/tlb.h | 58 ++++++++++++++++++++-----------------
1 file changed, 32 insertions(+), 26 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 31f6db4..866ea78 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -33,6 +33,27 @@ enum addr_stride {
PMD_STRIDE = 1
};
+/*
+ * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
+ * of the three. For example:
+ * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
+ * - FLAG_PCID: invalidate all TLB entries matching the PCID
+ *
+ * The first is used to invalidate (kernel) mappings at a particular
+ * address across all processes.
+ *
+ * The latter invalidates all TLB entries matching a PCID.
+ */
+#define INVLPGB_FLAG_VA BIT(0)
+#define INVLPGB_FLAG_PCID BIT(1)
+#define INVLPGB_FLAG_ASID BIT(2)
+#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3)
+#define INVLPGB_FLAG_FINAL_ONLY BIT(4)
+#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5)
+
+/* The implied mode when all bits are clear: */
+#define INVLPGB_MODE_ALL_NONGLOBALS 0UL
+
#ifdef CONFIG_BROADCAST_TLB_FLUSH
/*
* INVLPGB does broadcast TLB invalidation across all the CPUs in the system.
@@ -40,14 +61,20 @@ enum addr_stride {
* The INVLPGB instruction is weakly ordered, and a batch of invalidations can
* be done in a parallel fashion.
*
- * The instruction takes the number of extra pages to invalidate, beyond
- * the first page, while __invlpgb gets the more human readable number of
- * pages to invalidate.
+ * The instruction takes the number of extra pages to invalidate, beyond the
+ * first page, while __invlpgb gets the more human readable number of pages to
+ * invalidate.
*
* The bits in rax[0:2] determine respectively which components of the address
* (VA, PCID, ASID) get compared when flushing. If neither bits are set, *any*
* address in the specified range matches.
*
+ * Since it is desired to only flush TLB entries for the ASID that is executing
+ * the instruction (a host/hypervisor or a guest), the ASID valid bit should
+ * always be set. On a host/hypervisor, the hardware will use the ASID value
+ * specified in EDX[15:0] (which should be 0). On a guest, the hardware will
+ * use the actual ASID value of the guest.
+ *
* TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from
* this CPU have completed.
*/
@@ -55,9 +82,9 @@ static inline void __invlpgb(unsigned long asid, unsigned long pcid,
unsigned long addr, u16 nr_pages,
enum addr_stride stride, u8 flags)
{
- u32 edx = (pcid << 16) | asid;
+ u64 rax = addr | flags | INVLPGB_FLAG_ASID;
u32 ecx = (stride << 31) | (nr_pages - 1);
- u64 rax = addr | flags;
+ u32 edx = (pcid << 16) | asid;
/* The low bits in rax are for flags. Verify addr is clean. */
VM_WARN_ON_ONCE(addr & ~PAGE_MASK);
@@ -93,27 +120,6 @@ static inline void __invlpgb_all(unsigned long asid, unsigned long pcid, u8 flag
static inline void __tlbsync(void) { }
#endif
-/*
- * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
- * of the three. For example:
- * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
- * - FLAG_PCID: invalidate all TLB entries matching the PCID
- *
- * The first is used to invalidate (kernel) mappings at a particular
- * address across all processes.
- *
- * The latter invalidates all TLB entries matching a PCID.
- */
-#define INVLPGB_FLAG_VA BIT(0)
-#define INVLPGB_FLAG_PCID BIT(1)
-#define INVLPGB_FLAG_ASID BIT(2)
-#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3)
-#define INVLPGB_FLAG_FINAL_ONLY BIT(4)
-#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5)
-
-/* The implied mode when all bits are clear: */
-#define INVLPGB_MODE_ALL_NONGLOBALS 0UL
-
static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
unsigned long addr,
u16 nr, bool stride)© 2016 - 2025 Red Hat, Inc.