Add iopf-capable hw page table attach/detach/replace helpers. The pointer
to iommufd_device is stored in the domain attachment handle, so that it
can be echo'ed back in the iopf_group.
The iopf-capable hw page tables can only be attached to devices that
support the IOMMU_DEV_FEAT_IOPF feature. On the first attachment of an
iopf-capable hw_pagetable to the device, the IOPF feature is enabled on
the device. Similarly, after the last iopf-capable hwpt is detached from
the device, the IOPF feature is disabled on the device.
The current implementation allows a replacement between iopf-capable and
non-iopf-capable hw page tables. This matches the nested translation use
case, where a parent domain is attached by default and can then be
replaced with a nested user domain with iopf support.
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
drivers/iommu/iommufd/iommufd_private.h | 41 +++++
drivers/iommu/iommufd/device.c | 7 +-
drivers/iommu/iommufd/fault.c | 190 ++++++++++++++++++++++++
3 files changed, 235 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index c8a4519f1405..aa4c26c87cb9 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -11,6 +11,7 @@
#include <linux/iommu.h>
#include <linux/iova_bitmap.h>
#include <uapi/linux/iommufd.h>
+#include "../iommu-priv.h"
struct iommu_domain;
struct iommu_group;
@@ -293,6 +294,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt,
struct iommufd_hw_pagetable {
struct iommufd_object obj;
struct iommu_domain *domain;
+ struct iommufd_fault *fault;
};
struct iommufd_hwpt_paging {
@@ -396,6 +398,9 @@ struct iommufd_device {
/* always the physical device */
struct device *dev;
bool enforce_cache_coherency;
+ /* protect iopf_enabled counter */
+ struct mutex iopf_lock;
+ unsigned int iopf_enabled;
};
static inline struct iommufd_device *
@@ -456,6 +461,42 @@ struct iommufd_attach_handle {
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd);
void iommufd_fault_destroy(struct iommufd_object *obj);
+int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev);
+void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev);
+int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
+ struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_hw_pagetable *old);
+
+static inline int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev)
+{
+ if (hwpt->fault)
+ return iommufd_fault_domain_attach_dev(hwpt, idev);
+
+ return iommu_attach_group(hwpt->domain, idev->igroup->group);
+}
+
+static inline void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev)
+{
+ if (hwpt->fault)
+ iommufd_fault_domain_detach_dev(hwpt, idev);
+
+ iommu_detach_group(hwpt->domain, idev->igroup->group);
+}
+
+static inline int iommufd_hwpt_replace_device(struct iommufd_device *idev,
+ struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_hw_pagetable *old)
+{
+ if (old->fault || hwpt->fault)
+ return iommufd_fault_domain_replace_dev(idev, hwpt, old);
+
+ return iommu_group_replace_domain(idev->igroup->group, hwpt->domain);
+}
+
#ifdef CONFIG_IOMMUFD_TEST
int iommufd_test(struct iommufd_ucmd *ucmd);
void iommufd_selftest_destroy(struct iommufd_object *obj);
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index 873630c111c1..9a7ec5997c61 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -215,6 +215,7 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
refcount_inc(&idev->obj.users);
/* igroup refcount moves into iommufd_device */
idev->igroup = igroup;
+ mutex_init(&idev->iopf_lock);
/*
* If the caller fails after this success it must call
@@ -376,7 +377,7 @@ int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
* attachment.
*/
if (list_empty(&idev->igroup->device_list)) {
- rc = iommu_attach_group(hwpt->domain, idev->igroup->group);
+ rc = iommufd_hwpt_attach_device(hwpt, idev);
if (rc)
goto err_unresv;
idev->igroup->hwpt = hwpt;
@@ -402,7 +403,7 @@ iommufd_hw_pagetable_detach(struct iommufd_device *idev)
mutex_lock(&idev->igroup->lock);
list_del(&idev->group_item);
if (list_empty(&idev->igroup->device_list)) {
- iommu_detach_group(hwpt->domain, idev->igroup->group);
+ iommufd_hwpt_detach_device(hwpt, idev);
idev->igroup->hwpt = NULL;
}
if (hwpt_is_paging(hwpt))
@@ -497,7 +498,7 @@ iommufd_device_do_replace(struct iommufd_device *idev,
goto err_unlock;
}
- rc = iommu_group_replace_domain(igroup->group, hwpt->domain);
+ rc = iommufd_hwpt_replace_device(idev, hwpt, old_hwpt);
if (rc)
goto err_unresv;
diff --git a/drivers/iommu/iommufd/fault.c b/drivers/iommu/iommufd/fault.c
index 68ff94671d48..4934ae572638 100644
--- a/drivers/iommu/iommufd/fault.c
+++ b/drivers/iommu/iommufd/fault.c
@@ -8,6 +8,7 @@
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/iommufd.h>
+#include <linux/pci.h>
#include <linux/poll.h>
#include <linux/anon_inodes.h>
#include <uapi/linux/iommufd.h>
@@ -15,6 +16,195 @@
#include "../iommu-priv.h"
#include "iommufd_private.h"
+static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
+{
+ struct device *dev = idev->dev;
+ int ret;
+
+ /*
+ * Once we turn on PCI/PRI support for VF, the response failure code
+ * should not be forwarded to the hardware due to PRI being a shared
+ * resource between PF and VFs. There is no coordination for this
+ * shared capability. This waits for a vPRI reset to recover.
+ */
+ if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
+ return -EINVAL;
+
+ mutex_lock(&idev->iopf_lock);
+ /* Device iopf has already been on. */
+ if (++idev->iopf_enabled > 1) {
+ mutex_unlock(&idev->iopf_lock);
+ return 0;
+ }
+
+ ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
+ if (ret)
+ --idev->iopf_enabled;
+ mutex_unlock(&idev->iopf_lock);
+
+ return ret;
+}
+
+static void iommufd_fault_iopf_disable(struct iommufd_device *idev)
+{
+ mutex_lock(&idev->iopf_lock);
+ if (!WARN_ON(idev->iopf_enabled == 0)) {
+ if (--idev->iopf_enabled == 0)
+ iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
+ }
+ mutex_unlock(&idev->iopf_lock);
+}
+
+static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev)
+{
+ struct iommufd_attach_handle *handle;
+ int ret;
+
+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
+ if (!handle)
+ return -ENOMEM;
+
+ handle->idev = idev;
+ ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
+ &handle->handle);
+ if (ret)
+ kfree(handle);
+
+ return ret;
+}
+
+int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev)
+{
+ int ret;
+
+ if (!hwpt->fault)
+ return -EINVAL;
+
+ ret = iommufd_fault_iopf_enable(idev);
+ if (ret)
+ return ret;
+
+ ret = __fault_domain_attach_dev(hwpt, idev);
+ if (ret)
+ iommufd_fault_iopf_disable(idev);
+
+ return ret;
+}
+
+static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_attach_handle *handle)
+{
+ struct iommufd_fault *fault = hwpt->fault;
+ struct iopf_group *group, *next;
+ unsigned long index;
+
+ if (!fault)
+ return;
+
+ mutex_lock(&fault->mutex);
+ list_for_each_entry_safe(group, next, &fault->deliver, node) {
+ if (group->attach_handle != &handle->handle)
+ continue;
+ list_del(&group->node);
+ iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
+ iopf_free_group(group);
+ }
+
+ xa_for_each(&fault->response, index, group) {
+ if (group->attach_handle != &handle->handle)
+ continue;
+ xa_erase(&fault->response, index);
+ iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
+ iopf_free_group(group);
+ }
+ mutex_unlock(&fault->mutex);
+}
+
+static struct iommufd_attach_handle *
+iommufd_device_get_attach_handle(struct iommufd_device *idev)
+{
+ struct iommu_attach_handle *handle;
+
+ handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
+ if (!handle)
+ return NULL;
+
+ return to_iommufd_handle(handle);
+}
+
+void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_device *idev)
+{
+ struct iommufd_attach_handle *handle;
+
+ handle = iommufd_device_get_attach_handle(idev);
+ iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
+ iommufd_auto_response_faults(hwpt, handle);
+ iommufd_fault_iopf_disable(idev);
+ kfree(handle);
+}
+
+static int __fault_domain_replace_dev(struct iommufd_device *idev,
+ struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_hw_pagetable *old)
+{
+ struct iommufd_attach_handle *handle, *curr = NULL;
+ int ret;
+
+ if (old->fault)
+ curr = iommufd_device_get_attach_handle(idev);
+
+ if (hwpt->fault) {
+ handle = kzalloc(sizeof(*handle), GFP_KERNEL);
+ if (!handle)
+ return -ENOMEM;
+
+ handle->handle.domain = hwpt->domain;
+ handle->idev = idev;
+ ret = iommu_replace_group_handle(idev->igroup->group,
+ hwpt->domain, &handle->handle);
+ } else {
+ ret = iommu_replace_group_handle(idev->igroup->group,
+ hwpt->domain, NULL);
+ }
+
+ if (!ret && curr) {
+ iommufd_auto_response_faults(old, curr);
+ kfree(curr);
+ }
+
+ return ret;
+}
+
+int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
+ struct iommufd_hw_pagetable *hwpt,
+ struct iommufd_hw_pagetable *old)
+{
+ bool iopf_off = !hwpt->fault && old->fault;
+ bool iopf_on = hwpt->fault && !old->fault;
+ int ret;
+
+ if (iopf_on) {
+ ret = iommufd_fault_iopf_enable(idev);
+ if (ret)
+ return ret;
+ }
+
+ ret = __fault_domain_replace_dev(idev, hwpt, old);
+ if (ret) {
+ if (iopf_on)
+ iommufd_fault_iopf_disable(idev);
+ return ret;
+ }
+
+ if (iopf_off)
+ iommufd_fault_iopf_disable(idev);
+
+ return 0;
+}
+
void iommufd_fault_destroy(struct iommufd_object *obj)
{
struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
--
2.34.1On Sun, Jun 16, 2024 at 02:11:52PM +0800, Lu Baolu wrote:
> +static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
> +{
> + struct device *dev = idev->dev;
> + int ret;
> +
> + /*
> + * Once we turn on PCI/PRI support for VF, the response failure code
> + * should not be forwarded to the hardware due to PRI being a shared
> + * resource between PF and VFs. There is no coordination for this
> + * shared capability. This waits for a vPRI reset to recover.
> + */
> + if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
> + return -EINVAL;
I don't quite get this remark, isn't not supporting PRI on VFs kind of
useless? What is the story here?
Jason
On 2024/6/29 5:17, Jason Gunthorpe wrote:
> On Sun, Jun 16, 2024 at 02:11:52PM +0800, Lu Baolu wrote:
>> +static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
>> +{
>> + struct device *dev = idev->dev;
>> + int ret;
>> +
>> + /*
>> + * Once we turn on PCI/PRI support for VF, the response failure code
>> + * should not be forwarded to the hardware due to PRI being a shared
>> + * resource between PF and VFs. There is no coordination for this
>> + * shared capability. This waits for a vPRI reset to recover.
>> + */
>> + if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
>> + return -EINVAL;
> I don't quite get this remark, isn't not supporting PRI on VFs kind of
> useless? What is the story here?
This remark is trying to explain why attaching an iopf-capable hwpt to a
VF is not supported for now. The PCI sepc (section 10.4.2.1) states that
a response failure will disable the PRI on the function. But for PF/VF
case, the PRI is a shared resource, therefore a response failure on a VF
might cause iopf on other VFs to malfunction. So, we start from simple
by not allowing it.
Best regards,
baolu
On Mon, Jul 01, 2024 at 01:55:12PM +0800, Baolu Lu wrote:
> On 2024/6/29 5:17, Jason Gunthorpe wrote:
> > On Sun, Jun 16, 2024 at 02:11:52PM +0800, Lu Baolu wrote:
> > > +static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
> > > +{
> > > + struct device *dev = idev->dev;
> > > + int ret;
> > > +
> > > + /*
> > > + * Once we turn on PCI/PRI support for VF, the response failure code
> > > + * should not be forwarded to the hardware due to PRI being a shared
> > > + * resource between PF and VFs. There is no coordination for this
> > > + * shared capability. This waits for a vPRI reset to recover.
> > > + */
> > > + if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
> > > + return -EINVAL;
> > I don't quite get this remark, isn't not supporting PRI on VFs kind of
> > useless? What is the story here?
>
> This remark is trying to explain why attaching an iopf-capable hwpt to a
> VF is not supported for now. The PCI sepc (section 10.4.2.1) states that
> a response failure will disable the PRI on the function. But for PF/VF
> case, the PRI is a shared resource, therefore a response failure on a VF
> might cause iopf on other VFs to malfunction. So, we start from simple
> by not allowing it.
You are talking about IOMMU_PAGE_RESP_FAILURE ?
But this is bad already, something like SVA could trigger
IOMMU_PAGE_RESP_FAILURE on a VF without iommufd today. Due to memory
allocation failure in iommu_report_device_fault()
And then we pass in code from userspace and blindly cast it to
enum iommu_page_response_code ?
Probably we should just only support IOMMU_PAGE_RESP_SUCCESS/INVALID
from userspace and block FAILURE entirely. Probably the VMM should
emulate FAILURE by disabling PRI on by changing to a non PRI domain.
And this subtle uABI leak needs a fix:
iopf_group_response(group, response.code);
response.code and enum iommu_page_response_code are different
enums, and there is no range check. Need a static assert at least and
a range check. Send a followup patch please
Jason
On 2024/7/10 1:36, Jason Gunthorpe wrote:
> On Mon, Jul 01, 2024 at 01:55:12PM +0800, Baolu Lu wrote:
>> On 2024/6/29 5:17, Jason Gunthorpe wrote:
>>> On Sun, Jun 16, 2024 at 02:11:52PM +0800, Lu Baolu wrote:
>>>> +static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
>>>> +{
>>>> + struct device *dev = idev->dev;
>>>> + int ret;
>>>> +
>>>> + /*
>>>> + * Once we turn on PCI/PRI support for VF, the response failure code
>>>> + * should not be forwarded to the hardware due to PRI being a shared
>>>> + * resource between PF and VFs. There is no coordination for this
>>>> + * shared capability. This waits for a vPRI reset to recover.
>>>> + */
>>>> + if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
>>>> + return -EINVAL;
>>> I don't quite get this remark, isn't not supporting PRI on VFs kind of
>>> useless? What is the story here?
>> This remark is trying to explain why attaching an iopf-capable hwpt to a
>> VF is not supported for now. The PCI sepc (section 10.4.2.1) states that
>> a response failure will disable the PRI on the function. But for PF/VF
>> case, the PRI is a shared resource, therefore a response failure on a VF
>> might cause iopf on other VFs to malfunction. So, we start from simple
>> by not allowing it.
> You are talking about IOMMU_PAGE_RESP_FAILURE ?
>
> But this is bad already, something like SVA could trigger
> IOMMU_PAGE_RESP_FAILURE on a VF without iommufd today. Due to memory
> allocation failure in iommu_report_device_fault()
>
> And then we pass in code from userspace and blindly cast it to
> enum iommu_page_response_code ?
>
> Probably we should just only support IOMMU_PAGE_RESP_SUCCESS/INVALID
> from userspace and block FAILURE entirely. Probably the VMM should
> emulate FAILURE by disabling PRI on by changing to a non PRI domain.
>
> And this subtle uABI leak needs a fix:
>
> iopf_group_response(group, response.code);
>
> response.code and enum iommu_page_response_code are different
> enums, and there is no range check. Need a static assert at least and
> a range check. Send a followup patch please
Yes, sure.
Thanks,
baolu
> From: Jason Gunthorpe <jgg@ziepe.ca>
> Sent: Wednesday, July 10, 2024 1:37 AM
>
> On Mon, Jul 01, 2024 at 01:55:12PM +0800, Baolu Lu wrote:
> > On 2024/6/29 5:17, Jason Gunthorpe wrote:
> > > On Sun, Jun 16, 2024 at 02:11:52PM +0800, Lu Baolu wrote:
> > > > +static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
> > > > +{
> > > > + struct device *dev = idev->dev;
> > > > + int ret;
> > > > +
> > > > + /*
> > > > + * Once we turn on PCI/PRI support for VF, the response failure code
> > > > + * should not be forwarded to the hardware due to PRI being a
> shared
> > > > + * resource between PF and VFs. There is no coordination for this
> > > > + * shared capability. This waits for a vPRI reset to recover.
> > > > + */
> > > > + if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
> > > > + return -EINVAL;
> > > I don't quite get this remark, isn't not supporting PRI on VFs kind of
> > > useless? What is the story here?
> >
> > This remark is trying to explain why attaching an iopf-capable hwpt to a
> > VF is not supported for now. The PCI sepc (section 10.4.2.1) states that
> > a response failure will disable the PRI on the function. But for PF/VF
> > case, the PRI is a shared resource, therefore a response failure on a VF
> > might cause iopf on other VFs to malfunction. So, we start from simple
> > by not allowing it.
>
> You are talking about IOMMU_PAGE_RESP_FAILURE ?
>
> But this is bad already, something like SVA could trigger
> IOMMU_PAGE_RESP_FAILURE on a VF without iommufd today. Due to
> memory
> allocation failure in iommu_report_device_fault()
>
> And then we pass in code from userspace and blindly cast it to
> enum iommu_page_response_code ?
>
> Probably we should just only support
> IOMMU_PAGE_RESP_SUCCESS/INVALID
> from userspace and block FAILURE entirely. Probably the VMM should
> emulate FAILURE by disabling PRI on by changing to a non PRI domain.
>
Agree. The definition of response failure is same as disabling the PRI
interface and requires re-enablement of PRI to recover. Using domain
switch can naturally leverage future fix on coordinating PRI enable/disable
between PF/VF.
© 2016 - 2025 Red Hat, Inc.