drivers/iommu/intel/iommu.h | 10 + drivers/iommu/intel/dmar.c | 4 +- drivers/iommu/intel/iommu.c | 95 +++++- drivers/iommu/intel/perf.c | 2 +- drivers/iommu/intel/svm.c | 39 +-- Documentation/ABI/testing/debugfs-intel-iommu | 276 ++++++++++++++++++ drivers/iommu/intel/Kconfig | 11 - 7 files changed, 391 insertions(+), 46 deletions(-) create mode 100644 Documentation/ABI/testing/debugfs-intel-iommu
Hi Joerg,
The following changes have been queued for v6.9:
- Add rbtree to track iommu probed devices
- Add Intel IOMMU debugfs document
- Cleanup and refactoring
All patches are based on v6.8-rc6. The series is also available at:
https://github.com/LuBaolu/intel-iommu/commits/vtd-update-for-v6.9
Unfortunately, there is a merge conflict with your next branch, in
drivers/iommu/intel/svm.c. I resolved this conflict as below.
diff --cc drivers/iommu/intel/svm.c
index b644d57da841,bdf3584ca0af..c1bed89b1026
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@@ -698,14 -693,20 +688,18 @@@ bad_req
* If prq is to be handled outside iommu driver via receiver of
* the fault notifiers, we skip the page response here.
*/
- if (!pdev)
+ mutex_lock(&iommu->iopf_lock);
+ dev = device_rbtree_find(iommu, req->rid);
+ if (!dev) {
+ mutex_unlock(&iommu->iopf_lock);
goto bad_req;
+ }
- intel_svm_prq_report(iommu, &pdev->dev, req);
- trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1,
- if (intel_svm_prq_report(iommu, dev, req))
- handle_bad_prq_event(iommu, req, QI_RESP_INVALID);
- else
- trace_prq_report(iommu, dev, req->qw_0, req->qw_1,
- req->priv_data[0], req->priv_data[1],
- iommu->prq_seq_number++);
++ intel_svm_prq_report(iommu, dev, req);
++ trace_prq_report(iommu, dev, req->qw_0, req->qw_1,
+ req->priv_data[0], req->priv_data[1],
+ iommu->prq_seq_number++);
- pci_dev_put(pdev);
+ mutex_unlock(&iommu->iopf_lock);
prq_advance:
head = (head + sizeof(*req)) & PRQ_RING_MASK;
}
@@@ -781,17 -794,10 +775,8 @@@ void intel_svm_page_response(struct dev
qi_submit_sync(iommu, &desc, 1, 0);
}
-out:
- return ret;
}
- static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
- struct device *dev, ioasid_t pasid)
- {
- struct device_domain_info *info = dev_iommu_priv_get(dev);
- struct intel_iommu *iommu = info->iommu;
-
- return intel_svm_bind_mm(iommu, dev, domain, pasid);
- }
-
static void intel_svm_domain_free(struct iommu_domain *domain)
{
kfree(to_dmar_domain(domain));
With above in mind, please consider them for v6.9.
Best regards,
Baolu
Erick Archer (1):
iommu/vt-d: Use kcalloc() instead of kzalloc()
Jingqi Liu (1):
iommu/vt-d: Add the document for Intel IOMMU debugfs
Lu Baolu (3):
iommu/vt-d: Remove INTEL_IOMMU_BROKEN_GFX_WA
iommu/vt-d: Use rbtree to track iommu probed devices
iommu/vt-d: Use device rbtree in iopf reporting path
Tina Zhang (3):
iommu/vt-d: Remove treatment for revoking PASIDs with pending page
faults
iommu/vt-d: Remove initialization for dynamically heap-allocated
rcu_head
iommu/vt-d: Merge intel_svm_bind_mm() into its caller
drivers/iommu/intel/iommu.h | 10 +
drivers/iommu/intel/dmar.c | 4 +-
drivers/iommu/intel/iommu.c | 95 +++++-
drivers/iommu/intel/perf.c | 2 +-
drivers/iommu/intel/svm.c | 39 +--
Documentation/ABI/testing/debugfs-intel-iommu | 276 ++++++++++++++++++
drivers/iommu/intel/Kconfig | 11 -
7 files changed, 391 insertions(+), 46 deletions(-)
create mode 100644 Documentation/ABI/testing/debugfs-intel-iommu
--
2.34.1On Tue, Feb 27, 2024 at 10:14:33AM +0800, Lu Baolu wrote: > Hi Joerg, > > The following changes have been queued for v6.9: > > - Add rbtree to track iommu probed devices > - Add Intel IOMMU debugfs document > - Cleanup and refactoring Applied, thanks Baolu.
Commit 62edf5dc4a524 ("intel-iommu: Restore DMAR_BROKEN_GFX_WA option for
broken graphics drivers") was introduced 24 years ago as a temporary
workaround for graphics drivers that used physical addresses for DMA and
avoided DMA APIs. This workaround was disabled by default.
As 24 years have passed, it is expected that graphics driver developers
have migrated their drivers to use kernel DMA APIs. Therefore, this
workaround is no longer required and could been removed.
The Intel iommu driver also provides a "igfx_off" option to turn off
the DMA translation for the graphic dedicated IOMMU. Hence, there is
really no good reason to keep this config option.
Suggested-by: Kevin Tian <kevin.tian@intel.com>
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Link: https://lore.kernel.org/r/20240130060823.57990-1-baolu.lu@linux.intel.com
---
drivers/iommu/intel/iommu.c | 4 ----
drivers/iommu/intel/Kconfig | 11 -----------
2 files changed, 15 deletions(-)
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 11652e0bcab3..cfbe7c8e74fb 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -2712,10 +2712,6 @@ static int __init init_dmars(void)
iommu_set_root_entry(iommu);
}
-#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
- dmar_map_gfx = 0;
-#endif
-
if (!dmar_map_gfx)
iommu_identity_mapping |= IDENTMAP_GFX;
diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig
index 012cd2541a68..d2d34eb28d94 100644
--- a/drivers/iommu/intel/Kconfig
+++ b/drivers/iommu/intel/Kconfig
@@ -64,17 +64,6 @@ config INTEL_IOMMU_DEFAULT_ON
one is found. If this option is not selected, DMAR support can
be enabled by passing intel_iommu=on to the kernel.
-config INTEL_IOMMU_BROKEN_GFX_WA
- bool "Workaround broken graphics drivers (going away soon)"
- depends on BROKEN && X86
- help
- Current Graphics drivers tend to use physical address
- for DMA and avoid using DMA APIs. Setting this config
- option permits the IOMMU driver to set a unity map for
- all the OS-visible memory. Hence the driver can continue
- to use physical addresses for DMA, at least until this
- option is removed in the 2.6.32 kernel.
-
config INTEL_IOMMU_FLOPPY_WA
def_bool y
depends on X86
--
2.34.1From: Erick Archer <erick.archer@gmx.com>
This is an effort to get rid of all multiplications from allocation
functions in order to prevent integer overflows [1].
Here the multiplication is obviously safe because DMAR_LATENCY_NUM
is the number of latency types defined in the "latency_type" enum.
enum latency_type {
DMAR_LATENCY_INV_IOTLB = 0,
DMAR_LATENCY_INV_DEVTLB,
DMAR_LATENCY_INV_IEC,
DMAR_LATENCY_PRQ,
DMAR_LATENCY_NUM
};
However, using kcalloc() is more appropriate [2] and improves
readability. This patch has no effect on runtime behavior.
Link: https://github.com/KSPP/linux/issues/162 [1]
Link: https://www.kernel.org/doc/html/next/process/deprecated.html#open-coded-arithmetic-in-allocator-arguments [2]
Signed-off-by: Erick Archer <erick.archer@gmx.com>
Reviewed-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Gustavo A. R. Silva <gustavoars@kernel.org>
Link: https://lore.kernel.org/r/20240211175143.9229-1-erick.archer@gmx.com
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
drivers/iommu/intel/perf.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c
index 94ee70ac38e3..adc4de6bbd88 100644
--- a/drivers/iommu/intel/perf.c
+++ b/drivers/iommu/intel/perf.c
@@ -33,7 +33,7 @@ int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type)
spin_lock_irqsave(&latency_lock, flags);
if (!iommu->perf_statistic) {
- iommu->perf_statistic = kzalloc(sizeof(*lstat) * DMAR_LATENCY_NUM,
+ iommu->perf_statistic = kcalloc(DMAR_LATENCY_NUM, sizeof(*lstat),
GFP_ATOMIC);
if (!iommu->perf_statistic) {
ret = -ENOMEM;
--
2.34.1From: Jingqi Liu <Jingqi.liu@intel.com>
This document guides users to dump the Intel IOMMU internals by debugfs.
Signed-off-by: Jingqi Liu <Jingqi.liu@intel.com>
Link: https://lore.kernel.org/r/20240207090742.23857-1-Jingqi.liu@intel.com
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
Documentation/ABI/testing/debugfs-intel-iommu | 276 ++++++++++++++++++
1 file changed, 276 insertions(+)
create mode 100644 Documentation/ABI/testing/debugfs-intel-iommu
diff --git a/Documentation/ABI/testing/debugfs-intel-iommu b/Documentation/ABI/testing/debugfs-intel-iommu
new file mode 100644
index 000000000000..2ab8464504a9
--- /dev/null
+++ b/Documentation/ABI/testing/debugfs-intel-iommu
@@ -0,0 +1,276 @@
+What: /sys/kernel/debug/iommu/intel/iommu_regset
+Date: December 2023
+Contact: Jingqi Liu <Jingqi.liu@intel.com>
+Description:
+ This file dumps all the register contents for each IOMMU device.
+
+ Example in Kabylake:
+
+ ::
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/iommu_regset
+
+ IOMMU: dmar0 Register Base Address: 26be37000
+
+ Name Offset Contents
+ VER 0x00 0x0000000000000010
+ GCMD 0x18 0x0000000000000000
+ GSTS 0x1c 0x00000000c7000000
+ FSTS 0x34 0x0000000000000000
+ FECTL 0x38 0x0000000000000000
+
+ [...]
+
+ IOMMU: dmar1 Register Base Address: fed90000
+
+ Name Offset Contents
+ VER 0x00 0x0000000000000010
+ GCMD 0x18 0x0000000000000000
+ GSTS 0x1c 0x00000000c7000000
+ FSTS 0x34 0x0000000000000000
+ FECTL 0x38 0x0000000000000000
+
+ [...]
+
+ IOMMU: dmar2 Register Base Address: fed91000
+
+ Name Offset Contents
+ VER 0x00 0x0000000000000010
+ GCMD 0x18 0x0000000000000000
+ GSTS 0x1c 0x00000000c7000000
+ FSTS 0x34 0x0000000000000000
+ FECTL 0x38 0x0000000000000000
+
+ [...]
+
+What: /sys/kernel/debug/iommu/intel/ir_translation_struct
+Date: December 2023
+Contact: Jingqi Liu <Jingqi.liu@intel.com>
+Description:
+ This file dumps the table entries for Interrupt
+ remapping and Interrupt posting.
+
+ Example in Kabylake:
+
+ ::
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/ir_translation_struct
+
+ Remapped Interrupt supported on IOMMU: dmar0
+ IR table address:100900000
+
+ Entry SrcID DstID Vct IRTE_high IRTE_low
+ 0 00:0a.0 00000080 24 0000000000040050 000000800024000d
+ 1 00:0a.0 00000001 ef 0000000000040050 0000000100ef000d
+
+ Remapped Interrupt supported on IOMMU: dmar1
+ IR table address:100300000
+ Entry SrcID DstID Vct IRTE_high IRTE_low
+ 0 00:02.0 00000002 26 0000000000040010 000000020026000d
+
+ [...]
+
+ ****
+
+ Posted Interrupt supported on IOMMU: dmar0
+ IR table address:100900000
+ Entry SrcID PDA_high PDA_low Vct IRTE_high IRTE_low
+
+What: /sys/kernel/debug/iommu/intel/dmar_translation_struct
+Date: December 2023
+Contact: Jingqi Liu <Jingqi.liu@intel.com>
+Description:
+ This file dumps Intel IOMMU DMA remapping tables, such
+ as root table, context table, PASID directory and PASID
+ table entries in debugfs. For legacy mode, it doesn't
+ support PASID, and hence PASID field is defaulted to
+ '-1' and other PASID related fields are invalid.
+
+ Example in Kabylake:
+
+ ::
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/dmar_translation_struct
+
+ IOMMU dmar1: Root Table Address: 0x103027000
+ B.D.F Root_entry
+ 00:02.0 0x0000000000000000:0x000000010303e001
+
+ Context_entry
+ 0x0000000000000102:0x000000010303f005
+
+ PASID PASID_table_entry
+ -1 0x0000000000000000:0x0000000000000000:0x0000000000000000
+
+ IOMMU dmar0: Root Table Address: 0x103028000
+ B.D.F Root_entry
+ 00:0a.0 0x0000000000000000:0x00000001038a7001
+
+ Context_entry
+ 0x0000000000000000:0x0000000103220e7d
+
+ PASID PASID_table_entry
+ 0 0x0000000000000000:0x0000000000800002:0x00000001038a5089
+
+ [...]
+
+What: /sys/kernel/debug/iommu/intel/invalidation_queue
+Date: December 2023
+Contact: Jingqi Liu <Jingqi.liu@intel.com>
+Description:
+ This file exports invalidation queue internals of each
+ IOMMU device.
+
+ Example in Kabylake:
+
+ ::
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/invalidation_queue
+
+ Invalidation queue on IOMMU: dmar0
+ Base: 0x10022e000 Head: 20 Tail: 20
+ Index qw0 qw1 qw2
+ 0 0000000000000014 0000000000000000 0000000000000000
+ 1 0000000200000025 0000000100059c04 0000000000000000
+ 2 0000000000000014 0000000000000000 0000000000000000
+
+ qw3 status
+ 0000000000000000 0000000000000000
+ 0000000000000000 0000000000000000
+ 0000000000000000 0000000000000000
+
+ [...]
+
+ Invalidation queue on IOMMU: dmar1
+ Base: 0x10026e000 Head: 32 Tail: 32
+ Index qw0 qw1 status
+ 0 0000000000000004 0000000000000000 0000000000000000
+ 1 0000000200000025 0000000100059804 0000000000000000
+ 2 0000000000000011 0000000000000000 0000000000000000
+
+ [...]
+
+What: /sys/kernel/debug/iommu/intel/dmar_perf_latency
+Date: December 2023
+Contact: Jingqi Liu <Jingqi.liu@intel.com>
+Description:
+ This file is used to control and show counts of
+ execution time ranges for various types per DMAR.
+
+ Firstly, write a value to
+ /sys/kernel/debug/iommu/intel/dmar_perf_latency
+ to enable sampling.
+
+ The possible values are as follows:
+
+ * 0 - disable sampling all latency data
+
+ * 1 - enable sampling IOTLB invalidation latency data
+
+ * 2 - enable sampling devTLB invalidation latency data
+
+ * 3 - enable sampling intr entry cache invalidation latency data
+
+ Next, read /sys/kernel/debug/iommu/intel/dmar_perf_latency gives
+ a snapshot of sampling result of all enabled monitors.
+
+ Examples in Kabylake:
+
+ ::
+
+ 1) Disable sampling all latency data:
+
+ $ sudo echo 0 > /sys/kernel/debug/iommu/intel/dmar_perf_latency
+
+ 2) Enable sampling IOTLB invalidation latency data
+
+ $ sudo echo 1 > /sys/kernel/debug/iommu/intel/dmar_perf_latency
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/dmar_perf_latency
+
+ IOMMU: dmar0 Register Base Address: 26be37000
+ <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms
+ inv_iotlb 0 0 0 0 0
+
+ 1ms-10ms >=10ms min(us) max(us) average(us)
+ inv_iotlb 0 0 0 0 0
+
+ [...]
+
+ IOMMU: dmar2 Register Base Address: fed91000
+ <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms
+ inv_iotlb 0 0 18 0 0
+
+ 1ms-10ms >=10ms min(us) max(us) average(us)
+ inv_iotlb 0 0 2 2 2
+
+ 3) Enable sampling devTLB invalidation latency data
+
+ $ sudo echo 2 > /sys/kernel/debug/iommu/intel/dmar_perf_latency
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/dmar_perf_latency
+
+ IOMMU: dmar0 Register Base Address: 26be37000
+ <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms
+ inv_devtlb 0 0 0 0 0
+
+ >=10ms min(us) max(us) average(us)
+ inv_devtlb 0 0 0 0
+
+ [...]
+
+What: /sys/kernel/debug/iommu/intel/<bdf>/domain_translation_struct
+Date: December 2023
+Contact: Jingqi Liu <Jingqi.liu@intel.com>
+Description:
+ This file dumps a specified page table of Intel IOMMU
+ in legacy mode or scalable mode.
+
+ For a device that only supports legacy mode, dump its
+ page table by the debugfs file in the debugfs device
+ directory. e.g.
+ /sys/kernel/debug/iommu/intel/0000:00:02.0/domain_translation_struct.
+
+ For a device that supports scalable mode, dump the
+ page table of specified pasid by the debugfs file in
+ the debugfs pasid directory. e.g.
+ /sys/kernel/debug/iommu/intel/0000:00:02.0/1/domain_translation_struct.
+
+ Examples in Kabylake:
+
+ ::
+
+ 1) Dump the page table of device "0000:00:02.0" that only supports legacy mode.
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/0000:00:02.0/domain_translation_struct
+
+ Device 0000:00:02.0 @0x1017f8000
+ IOVA_PFN PML5E PML4E
+ 0x000000008d800 | 0x0000000000000000 0x00000001017f9003
+ 0x000000008d801 | 0x0000000000000000 0x00000001017f9003
+ 0x000000008d802 | 0x0000000000000000 0x00000001017f9003
+
+ PDPE PDE PTE
+ 0x00000001017fa003 0x00000001017fb003 0x000000008d800003
+ 0x00000001017fa003 0x00000001017fb003 0x000000008d801003
+ 0x00000001017fa003 0x00000001017fb003 0x000000008d802003
+
+ [...]
+
+ 2) Dump the page table of device "0000:00:0a.0" with PASID "1" that
+ supports scalable mode.
+
+ $ sudo cat /sys/kernel/debug/iommu/intel/0000:00:0a.0/1/domain_translation_struct
+
+ Device 0000:00:0a.0 with pasid 1 @0x10c112000
+ IOVA_PFN PML5E PML4E
+ 0x0000000000000 | 0x0000000000000000 0x000000010df93003
+ 0x0000000000001 | 0x0000000000000000 0x000000010df93003
+ 0x0000000000002 | 0x0000000000000000 0x000000010df93003
+
+ PDPE PDE PTE
+ 0x0000000106ae6003 0x0000000104b38003 0x0000000147c00803
+ 0x0000000106ae6003 0x0000000104b38003 0x0000000147c01803
+ 0x0000000106ae6003 0x0000000104b38003 0x0000000147c02803
+
+ [...]
--
2.34.1From: Tina Zhang <tina.zhang@intel.com>
Commit 2f26e0a9c986 ("iommu/vt-d: Add basic SVM PASID support") added a
special treatment to mandate that no page faults may be outstanding for
the PASID after intel_svm_unbind_mm() is called, as the PASID will be
released and reused after unbind.
This is unnecessary anymore as no outstanding page faults have been
ensured in the driver's remove_dev_pasid path:
- Tear down the pasid entry, which guarantees that new page faults for
the PASID will be rejected by the iommu hardware.
- All outstanding page faults have been responded to.
- All hardware pending faults are drained in intel_drain_pasid_prq().
Remove this unnecessary code.
Signed-off-by: Tina Zhang <tina.zhang@intel.com>
Link: https://lore.kernel.org/r/20240219125723.1645703-2-tina.zhang@intel.com
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
drivers/iommu/intel/svm.c | 7 -------
1 file changed, 7 deletions(-)
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index 40edd282903f..a815362c8e60 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -408,13 +408,6 @@ void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid)
if (svm->notifier.ops)
mmu_notifier_unregister(&svm->notifier, mm);
pasid_private_remove(svm->pasid);
- /*
- * We mandate that no page faults may be outstanding
- * for the PASID when intel_svm_unbind_mm() is called.
- * If that is not obeyed, subtle errors will happen.
- * Let's make them less subtle...
- */
- memset(svm, 0x6b, sizeof(*svm));
kfree(svm);
}
}
--
2.34.1From: Tina Zhang <tina.zhang@intel.com>
The rcu_head structures allocated dynamically in the heap don't need any
initialization. Therefore, remove the init_rcu_head().
Signed-off-by: Tina Zhang <tina.zhang@intel.com>
Link: https://lore.kernel.org/r/20240219125723.1645703-3-tina.zhang@intel.com
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
drivers/iommu/intel/svm.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index a815362c8e60..a92a9e2239e2 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -360,7 +360,6 @@ static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev,
sdev->iommu = iommu;
sdev->did = FLPT_DEFAULT_DID;
sdev->sid = PCI_DEVID(info->bus, info->devfn);
- init_rcu_head(&sdev->rcu);
if (info->ats_enabled) {
sdev->qdep = info->ats_qdep;
if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
--
2.34.1From: Tina Zhang <tina.zhang@intel.com>
intel_svm_set_dev_pasid() is the only caller of intel_svm_bind_mm().
Merge them and remove intel_svm_bind_mm(). No functional change
intended.
Signed-off-by: Tina Zhang <tina.zhang@intel.com>
Link: https://lore.kernel.org/r/20240219125723.1645703-4-tina.zhang@intel.com
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
drivers/iommu/intel/svm.c | 14 +++-----------
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index a92a9e2239e2..1dd56d4eb88c 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -315,10 +315,11 @@ static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,
return 0;
}
-static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev,
- struct iommu_domain *domain, ioasid_t pasid)
+static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
+ struct device *dev, ioasid_t pasid)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
+ struct intel_iommu *iommu = info->iommu;
struct mm_struct *mm = domain->mm;
struct intel_svm_dev *sdev;
struct intel_svm *svm;
@@ -796,15 +797,6 @@ int intel_svm_page_response(struct device *dev,
return ret;
}
-static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
- struct device *dev, ioasid_t pasid)
-{
- struct device_domain_info *info = dev_iommu_priv_get(dev);
- struct intel_iommu *iommu = info->iommu;
-
- return intel_svm_bind_mm(iommu, dev, domain, pasid);
-}
-
static void intel_svm_domain_free(struct iommu_domain *domain)
{
kfree(to_dmar_domain(domain));
--
2.34.1Use a red-black tree(rbtree) to track devices probed by the driver's
probe_device callback. These devices need to be looked up quickly by
a source ID when the hardware reports a fault, either recoverable or
unrecoverable.
Fault reporting paths are critical. Searching a list in this scenario
is inefficient, with an algorithm complexity of O(n). An rbtree is a
self-balancing binary search tree, offering an average search time
complexity of O(log(n)). This significant performance improvement
makes rbtrees a better choice.
Furthermore, rbtrees are implemented on a per-iommu basis, eliminating
the need for global searches and further enhancing efficiency in
critical fault paths. The rbtree is protected by a spin lock with
interrupts disabled to ensure thread-safe access even within interrupt
contexts.
Co-developed-by: Huang Jiaqing <jiaqing.huang@intel.com>
Signed-off-by: Huang Jiaqing <jiaqing.huang@intel.com>
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20240220065939.121116-2-baolu.lu@linux.intel.com
---
drivers/iommu/intel/iommu.h | 8 ++++
drivers/iommu/intel/dmar.c | 3 +-
drivers/iommu/intel/iommu.c | 88 ++++++++++++++++++++++++++++++++++++-
3 files changed, 96 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 4145c04cb1c6..df00240ebe90 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -722,6 +722,11 @@ struct intel_iommu {
struct q_inval *qi; /* Queued invalidation info */
u32 iommu_state[MAX_SR_DMAR_REGS]; /* Store iommu states between suspend and resume.*/
+ /* rb tree for all probed devices */
+ struct rb_root device_rbtree;
+ /* protect the device_rbtree */
+ spinlock_t device_rbtree_lock;
+
#ifdef CONFIG_IRQ_REMAP
struct ir_table *ir_table; /* Interrupt remapping info */
struct irq_domain *ir_domain;
@@ -755,6 +760,8 @@ struct device_domain_info {
struct intel_iommu *iommu; /* IOMMU used by this device */
struct dmar_domain *domain; /* pointer to domain */
struct pasid_table *pasid_table; /* pasid table */
+ /* device tracking node(lookup by PCI RID) */
+ struct rb_node node;
#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
struct dentry *debugfs_dentry; /* pointer to device directory dentry */
#endif
@@ -1081,6 +1088,7 @@ void free_pgtable_page(void *vaddr);
void iommu_flush_write_buffer(struct intel_iommu *iommu);
struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent,
const struct iommu_user_data *user_data);
+struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid);
#ifdef CONFIG_INTEL_IOMMU_SVM
void intel_svm_check(struct intel_iommu *iommu);
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index 23cb80d62a9a..f9b63c2875f7 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -1095,7 +1095,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
iommu->agaw = agaw;
iommu->msagaw = msagaw;
iommu->segment = drhd->segment;
-
+ iommu->device_rbtree = RB_ROOT;
+ spin_lock_init(&iommu->device_rbtree_lock);
iommu->node = NUMA_NO_NODE;
ver = readl(iommu->reg + DMAR_VER_REG);
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index cfbe7c8e74fb..5568f17d867f 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -97,6 +97,81 @@ static phys_addr_t root_entry_uctp(struct root_entry *re)
return re->hi & VTD_PAGE_MASK;
}
+static int device_rid_cmp_key(const void *key, const struct rb_node *node)
+{
+ struct device_domain_info *info =
+ rb_entry(node, struct device_domain_info, node);
+ const u16 *rid_lhs = key;
+
+ if (*rid_lhs < PCI_DEVID(info->bus, info->devfn))
+ return -1;
+
+ if (*rid_lhs > PCI_DEVID(info->bus, info->devfn))
+ return 1;
+
+ return 0;
+}
+
+static int device_rid_cmp(struct rb_node *lhs, const struct rb_node *rhs)
+{
+ struct device_domain_info *info =
+ rb_entry(lhs, struct device_domain_info, node);
+ u16 key = PCI_DEVID(info->bus, info->devfn);
+
+ return device_rid_cmp_key(&key, rhs);
+}
+
+/*
+ * Looks up an IOMMU-probed device using its source ID.
+ *
+ * Returns the pointer to the device if there is a match. Otherwise,
+ * returns NULL.
+ *
+ * Note that this helper doesn't guarantee that the device won't be
+ * released by the iommu subsystem after being returned. The caller
+ * should use its own synchronization mechanism to avoid the device
+ * being released during its use if its possibly the case.
+ */
+struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid)
+{
+ struct device_domain_info *info = NULL;
+ struct rb_node *node;
+ unsigned long flags;
+
+ spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
+ node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);
+ if (node)
+ info = rb_entry(node, struct device_domain_info, node);
+ spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
+
+ return info ? info->dev : NULL;
+}
+
+static int device_rbtree_insert(struct intel_iommu *iommu,
+ struct device_domain_info *info)
+{
+ struct rb_node *curr;
+ unsigned long flags;
+
+ spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
+ curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);
+ spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
+ if (WARN_ON(curr))
+ return -EEXIST;
+
+ return 0;
+}
+
+static void device_rbtree_remove(struct device_domain_info *info)
+{
+ struct intel_iommu *iommu = info->iommu;
+ unsigned long flags;
+
+ spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
+ rb_erase(&info->node, &iommu->device_rbtree);
+ spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
+}
+
/*
* This domain is a statically identity mapping domain.
* 1. This domain creats a static 1:1 mapping to all usable memory.
@@ -4326,25 +4401,34 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
}
dev_iommu_priv_set(dev, info);
+ ret = device_rbtree_insert(iommu, info);
+ if (ret)
+ goto free;
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
ret = intel_pasid_alloc_table(dev);
if (ret) {
dev_err(dev, "PASID table allocation failed\n");
- kfree(info);
- return ERR_PTR(ret);
+ goto clear_rbtree;
}
}
intel_iommu_debugfs_create_dev(info);
return &iommu->iommu;
+clear_rbtree:
+ device_rbtree_remove(info);
+free:
+ kfree(info);
+
+ return ERR_PTR(ret);
}
static void intel_iommu_release_device(struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
+ device_rbtree_remove(info);
dmar_remove_one_dev_info(dev);
intel_pasid_free_table(dev);
intel_iommu_debugfs_remove_dev(info);
--
2.34.1The existing I/O page fault handler currently locates the PCI device by
calling pci_get_domain_bus_and_slot(). This function searches the list
of all PCI devices until the desired device is found. To improve lookup
efficiency, replace it with device_rbtree_find() to search the device
within the probed device rbtree.
The I/O page fault is initiated by the device, which does not have any
synchronization mechanism with the software to ensure that the device
stays in the probed device tree. Theoretically, a device could be released
by the IOMMU subsystem after device_rbtree_find() and before
iopf_get_dev_fault_param(), which would cause a use-after-free problem.
Add a mutex to synchronize the I/O page fault reporting path and the IOMMU
release device path. This lock doesn't introduce any performance overhead,
as the conflict between I/O page fault reporting and device releasing is
very rare.
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20240220065939.121116-3-baolu.lu@linux.intel.com
---
drivers/iommu/intel/iommu.h | 2 ++
drivers/iommu/intel/dmar.c | 1 +
drivers/iommu/intel/iommu.c | 3 +++
drivers/iommu/intel/svm.c | 17 +++++++++--------
4 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index df00240ebe90..cd267ba64eda 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -719,6 +719,8 @@ struct intel_iommu {
#endif
struct iopf_queue *iopf_queue;
unsigned char iopfq_name[16];
+ /* Synchronization between fault report and iommu device release. */
+ struct mutex iopf_lock;
struct q_inval *qi; /* Queued invalidation info */
u32 iommu_state[MAX_SR_DMAR_REGS]; /* Store iommu states between suspend and resume.*/
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index f9b63c2875f7..d14797aabb7a 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -1097,6 +1097,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
iommu->segment = drhd->segment;
iommu->device_rbtree = RB_ROOT;
spin_lock_init(&iommu->device_rbtree_lock);
+ mutex_init(&iommu->iopf_lock);
iommu->node = NUMA_NO_NODE;
ver = readl(iommu->reg + DMAR_VER_REG);
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 5568f17d867f..eaa648c6c389 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -4427,8 +4427,11 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
static void intel_iommu_release_device(struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
+ struct intel_iommu *iommu = info->iommu;
+ mutex_lock(&iommu->iopf_lock);
device_rbtree_remove(info);
+ mutex_unlock(&iommu->iopf_lock);
dmar_remove_one_dev_info(dev);
intel_pasid_free_table(dev);
intel_iommu_debugfs_remove_dev(info);
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index 1dd56d4eb88c..bdf3584ca0af 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -643,7 +643,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
struct intel_iommu *iommu = d;
struct page_req_dsc *req;
int head, tail, handled;
- struct pci_dev *pdev;
+ struct device *dev;
u64 address;
/*
@@ -689,23 +689,24 @@ static irqreturn_t prq_event_thread(int irq, void *d)
if (unlikely(req->lpig && !req->rd_req && !req->wr_req))
goto prq_advance;
- pdev = pci_get_domain_bus_and_slot(iommu->segment,
- PCI_BUS_NUM(req->rid),
- req->rid & 0xff);
/*
* If prq is to be handled outside iommu driver via receiver of
* the fault notifiers, we skip the page response here.
*/
- if (!pdev)
+ mutex_lock(&iommu->iopf_lock);
+ dev = device_rbtree_find(iommu, req->rid);
+ if (!dev) {
+ mutex_unlock(&iommu->iopf_lock);
goto bad_req;
+ }
- if (intel_svm_prq_report(iommu, &pdev->dev, req))
+ if (intel_svm_prq_report(iommu, dev, req))
handle_bad_prq_event(iommu, req, QI_RESP_INVALID);
else
- trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1,
+ trace_prq_report(iommu, dev, req->qw_0, req->qw_1,
req->priv_data[0], req->priv_data[1],
iommu->prq_seq_number++);
- pci_dev_put(pdev);
+ mutex_unlock(&iommu->iopf_lock);
prq_advance:
head = (head + sizeof(*req)) & PRQ_RING_MASK;
}
--
2.34.1© 2016 - 2026 Red Hat, Inc.