xen/drivers/passthrough/arm/ipmmu-vmsa.c | 2 +- xen/drivers/passthrough/arm/smmu-v3.c | 57 +++++++++++++++++++++++- xen/drivers/passthrough/arm/smmu.c | 4 +- xen/include/xen/device_tree.h | 5 ++- 4 files changed, 62 insertions(+), 6 deletions(-)
Allow for removing devices from SMMUv3. arm_smmu_deassign_dev handles
most of the work by disabling ATS and zeroing STEs. Additionally, unset
the dt_device_is_protected flag and free no longer needed smmu_master.
Rework dt_device_set_protected to accept a boolean parameter, update
callsites.
Tested on QEMU with SRIOV series[1] by repeatedly enabling/disabling
VFs.
[1]: https://patchew.org/Xen/cover.1772806036.git.mykyta._5Fpoturai@epam.com/
Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>
---
V1-V2:
* check for phantom functions
* simplify pci/dt device split
* improve error handling
* don't try to free master for unprotected devices
* rework dt_device_set_protected
---
xen/drivers/passthrough/arm/ipmmu-vmsa.c | 2 +-
xen/drivers/passthrough/arm/smmu-v3.c | 57 +++++++++++++++++++++++-
xen/drivers/passthrough/arm/smmu.c | 4 +-
xen/include/xen/device_tree.h | 5 ++-
4 files changed, 62 insertions(+), 6 deletions(-)
diff --git a/xen/drivers/passthrough/arm/ipmmu-vmsa.c b/xen/drivers/passthrough/arm/ipmmu-vmsa.c
index fa9ab9cb13..0648f9b407 100644
--- a/xen/drivers/passthrough/arm/ipmmu-vmsa.c
+++ b/xen/drivers/passthrough/arm/ipmmu-vmsa.c
@@ -1367,7 +1367,7 @@ static int ipmmu_add_device(u8 devfn, struct device *dev)
}
/* Let Xen know that the master device is protected by an IOMMU. */
- dt_device_set_protected(dev_to_dt(dev));
+ dt_device_set_protected(dev_to_dt(dev), true);
}
#ifdef CONFIG_HAS_PCI
if ( dev_is_pci(dev) )
diff --git a/xen/drivers/passthrough/arm/smmu-v3.c b/xen/drivers/passthrough/arm/smmu-v3.c
index bf153227db..8e080cd7d0 100644
--- a/xen/drivers/passthrough/arm/smmu-v3.c
+++ b/xen/drivers/passthrough/arm/smmu-v3.c
@@ -1493,6 +1493,60 @@ static int arm_smmu_assign_dev(struct domain *d, u8 devfn, struct device *dev,
static int arm_smmu_deassign_dev(struct domain *d, uint8_t devfn,
struct device *dev);
+static int arm_smmu_remove_device(u8 devfn, struct device *dev)
+{
+ struct arm_smmu_master *master;
+ struct iommu_fwspec *fwspec;
+ struct domain *d = NULL;
+
+ fwspec = dev_iommu_fwspec_get(dev);
+ if ( !fwspec )
+ return -ENODEV;
+
+ master = dev_iommu_priv_get(dev);
+ if ( !master )
+ return -ENODEV;
+
+ if ( IS_ENABLED(CONFIG_HAS_PCI) && dev_is_pci(dev) )
+ {
+ struct pci_dev *pdev = dev_to_pci(dev);
+
+ /* Ignore calls for phantom functions */
+ if ( devfn != pdev->devfn )
+ return 0;
+
+ d = pdev->domain;
+ }
+ else
+ {
+ if ( !dt_device_is_protected(dev_to_dt(dev)) )
+ {
+ dev_err(dev, "Not added to SMMUv3\n");
+ return -ENODEV;
+ }
+
+ dt_device_set_protected(dev_to_dt(dev), false);
+ if ( master->domain && master->domain->d )
+ d = master->domain->d;
+ }
+
+ if ( d )
+ {
+ int ret = arm_smmu_deassign_dev(d, devfn, dev);
+ /* This should never fail because we already checked the domain */
+ ASSERT(!ret);
+ }
+
+ arm_smmu_disable_pasid(master);
+
+ dev_info(dev, "Removed master device (SMMUv3 %s StreamIds %u)\n",
+ dev_name(fwspec->iommu_dev), fwspec->num_ids);
+
+ xfree(master);
+ dev_iommu_priv_set(dev, NULL);
+ return 0;
+}
+
static int arm_smmu_add_device(u8 devfn, struct device *dev)
{
int i, ret;
@@ -1571,7 +1625,7 @@ static int arm_smmu_add_device(u8 devfn, struct device *dev)
}
/* Let Xen know that the master device is protected by an IOMMU. */
- dt_device_set_protected(dev_to_dt(dev));
+ dt_device_set_protected(dev_to_dt(dev), true);
}
dev_info(dev, "Added master device (SMMUv3 %s StreamIds %u)\n",
@@ -2867,6 +2921,7 @@ static const struct iommu_ops arm_smmu_iommu_ops = {
.unmap_page = arm_iommu_unmap_page,
.dt_xlate = arm_smmu_dt_xlate,
.add_device = arm_smmu_add_device,
+ .remove_device = arm_smmu_remove_device,
};
static __init int arm_smmu_dt_init(struct dt_device_node *dev,
diff --git a/xen/drivers/passthrough/arm/smmu.c b/xen/drivers/passthrough/arm/smmu.c
index d63c901551..4d2f71f152 100644
--- a/xen/drivers/passthrough/arm/smmu.c
+++ b/xen/drivers/passthrough/arm/smmu.c
@@ -825,7 +825,7 @@ static int arm_smmu_dt_add_device_legacy(struct arm_smmu_device *smmu,
if ( !dev_is_pci(dev) )
{
/* Xen: Let Xen know that the device is protected by an SMMU */
- dt_device_set_protected(dev_node);
+ dt_device_set_protected(dev_node, true);
}
for (i = 0; i < fwspec->num_ids; ++i) {
@@ -862,7 +862,7 @@ static int arm_smmu_dt_remove_device_legacy(struct arm_smmu_device *smmu,
if ( !dev_is_pci(dev) )
/* Protected by dt_host_lock and dtdevs_lock as caller holds these locks. */
- dev_node->is_protected = false;
+ dt_device_set_protected(dev_node, false);
kfree(master);
return 0;
diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
index 06d7643622..76ae1e674a 100644
--- a/xen/include/xen/device_tree.h
+++ b/xen/include/xen/device_tree.h
@@ -300,9 +300,10 @@ static inline domid_t dt_device_used_by(const struct dt_device_node *device)
return device->used_by;
}
-static inline void dt_device_set_protected(struct dt_device_node *device)
+static inline void dt_device_set_protected(struct dt_device_node *device,
+ bool protected)
{
- device->is_protected = true;
+ device->is_protected = protected;
}
static inline bool dt_device_is_protected(const struct dt_device_node *device)
--
2.51.2Hi Mykyta,
> On 14 Apr 2026, at 14:15, Mykyta Poturai <Mykyta_Poturai@epam.com> wrote:
>
> Allow for removing devices from SMMUv3. arm_smmu_deassign_dev handles
> most of the work by disabling ATS and zeroing STEs. Additionally, unset
> the dt_device_is_protected flag and free no longer needed smmu_master.
>
> Rework dt_device_set_protected to accept a boolean parameter, update
> callsites.
>
Should ...
> Tested on QEMU with SRIOV series[1] by repeatedly enabling/disabling
> VFs.
>
> [1]: https://patchew.org/Xen/cover.1772806036.git.mykyta._5Fpoturai@epam.com/
this be omitted from commit message?
>
> Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>
> ---
> V1-V2:
> * check for phantom functions
> * simplify pci/dt device split
> * improve error handling
> * don't try to free master for unprotected devices
> * rework dt_device_set_protected
> ---
> xen/drivers/passthrough/arm/ipmmu-vmsa.c | 2 +-
> xen/drivers/passthrough/arm/smmu-v3.c | 57 +++++++++++++++++++++++-
> xen/drivers/passthrough/arm/smmu.c | 4 +-
> xen/include/xen/device_tree.h | 5 ++-
> 4 files changed, 62 insertions(+), 6 deletions(-)
>
> diff --git a/xen/drivers/passthrough/arm/ipmmu-vmsa.c b/xen/drivers/passthrough/arm/ipmmu-vmsa.c
> index fa9ab9cb13..0648f9b407 100644
> --- a/xen/drivers/passthrough/arm/ipmmu-vmsa.c
> +++ b/xen/drivers/passthrough/arm/ipmmu-vmsa.c
> @@ -1367,7 +1367,7 @@ static int ipmmu_add_device(u8 devfn, struct device *dev)
> }
>
> /* Let Xen know that the master device is protected by an IOMMU. */
> - dt_device_set_protected(dev_to_dt(dev));
> + dt_device_set_protected(dev_to_dt(dev), true);
> }
> #ifdef CONFIG_HAS_PCI
> if ( dev_is_pci(dev) )
> diff --git a/xen/drivers/passthrough/arm/smmu-v3.c b/xen/drivers/passthrough/arm/smmu-v3.c
> index bf153227db..8e080cd7d0 100644
> --- a/xen/drivers/passthrough/arm/smmu-v3.c
> +++ b/xen/drivers/passthrough/arm/smmu-v3.c
> @@ -1493,6 +1493,60 @@ static int arm_smmu_assign_dev(struct domain *d, u8 devfn, struct device *dev,
> static int arm_smmu_deassign_dev(struct domain *d, uint8_t devfn,
> struct device *dev);
>
> +static int arm_smmu_remove_device(u8 devfn, struct device *dev)
> +{
> + struct arm_smmu_master *master;
> + struct iommu_fwspec *fwspec;
I’m looking into the arm_smmu_remove_device, at some point there we allocate
the iommu_fwspec, but we are not using iommu_fwspec_free() here, I’ve tried
to look around and I’m not able to see what frees that structure apart form
iommu_remove_dt_device().
But pci_remove_device() -> iommu_remove_device() can call this function as well, I’m not sure I’ve understood
correctly the framework here so maybe someone with more PCI experience can help
@Stewart Hildebrand ?
> + struct domain *d = NULL;
> +
> + fwspec = dev_iommu_fwspec_get(dev);
> + if ( !fwspec )
> + return -ENODEV;
> +
> + master = dev_iommu_priv_get(dev);
> + if ( !master )
> + return -ENODEV;
> +
> + if ( IS_ENABLED(CONFIG_HAS_PCI) && dev_is_pci(dev) )
> + {
> + struct pci_dev *pdev = dev_to_pci(dev);
> +
> + /* Ignore calls for phantom functions */
> + if ( devfn != pdev->devfn )
> + return 0;
> +
> + d = pdev->domain;
> + }
> + else
> + {
> + if ( !dt_device_is_protected(dev_to_dt(dev)) )
> + {
> + dev_err(dev, "Not added to SMMUv3\n");
> + return -ENODEV;
> + }
> +
> + dt_device_set_protected(dev_to_dt(dev), false);
> + if ( master->domain && master->domain->d )
> + d = master->domain->d;
> + }
> +
> + if ( d )
> + {
> + int ret = arm_smmu_deassign_dev(d, devfn, dev);
> + /* This should never fail because we already checked the domain */
> + ASSERT(!ret);
> + }
> +
> + arm_smmu_disable_pasid(master);
> +
> + dev_info(dev, "Removed master device (SMMUv3 %s StreamIds %u)\n",
> + dev_name(fwspec->iommu_dev), fwspec->num_ids);
> +
> + xfree(master);
> + dev_iommu_priv_set(dev, NULL);
> + return 0;
> +}
> +
Cheers,
Luca
> On 15 Apr 2026, at 14:54, Luca Fancellu <Luca.Fancellu@arm.com> wrote:
>
> Hi Mykyta,
>
>> On 14 Apr 2026, at 14:15, Mykyta Poturai <Mykyta_Poturai@epam.com> wrote:
>>
>> Allow for removing devices from SMMUv3. arm_smmu_deassign_dev handles
>> most of the work by disabling ATS and zeroing STEs. Additionally, unset
>> the dt_device_is_protected flag and free no longer needed smmu_master.
>>
>> Rework dt_device_set_protected to accept a boolean parameter, update
>> callsites.
>>
>
> Should ...
>
>> Tested on QEMU with SRIOV series[1] by repeatedly enabling/disabling
>> VFs.
>>
>> [1]: https://patchew.org/Xen/cover.1772806036.git.mykyta._5Fpoturai@epam.com/
>
> this be omitted from commit message?
>
>>
>> Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>
>> ---
>> V1-V2:
>> * check for phantom functions
>> * simplify pci/dt device split
>> * improve error handling
>> * don't try to free master for unprotected devices
>> * rework dt_device_set_protected
>> ---
>> xen/drivers/passthrough/arm/ipmmu-vmsa.c | 2 +-
>> xen/drivers/passthrough/arm/smmu-v3.c | 57 +++++++++++++++++++++++-
>> xen/drivers/passthrough/arm/smmu.c | 4 +-
>> xen/include/xen/device_tree.h | 5 ++-
>> 4 files changed, 62 insertions(+), 6 deletions(-)
>>
>> diff --git a/xen/drivers/passthrough/arm/ipmmu-vmsa.c b/xen/drivers/passthrough/arm/ipmmu-vmsa.c
>> index fa9ab9cb13..0648f9b407 100644
>> --- a/xen/drivers/passthrough/arm/ipmmu-vmsa.c
>> +++ b/xen/drivers/passthrough/arm/ipmmu-vmsa.c
>> @@ -1367,7 +1367,7 @@ static int ipmmu_add_device(u8 devfn, struct device *dev)
>> }
>>
>> /* Let Xen know that the master device is protected by an IOMMU. */
>> - dt_device_set_protected(dev_to_dt(dev));
>> + dt_device_set_protected(dev_to_dt(dev), true);
>> }
>> #ifdef CONFIG_HAS_PCI
>> if ( dev_is_pci(dev) )
>> diff --git a/xen/drivers/passthrough/arm/smmu-v3.c b/xen/drivers/passthrough/arm/smmu-v3.c
>> index bf153227db..8e080cd7d0 100644
>> --- a/xen/drivers/passthrough/arm/smmu-v3.c
>> +++ b/xen/drivers/passthrough/arm/smmu-v3.c
>> @@ -1493,6 +1493,60 @@ static int arm_smmu_assign_dev(struct domain *d, u8 devfn, struct device *dev,
>> static int arm_smmu_deassign_dev(struct domain *d, uint8_t devfn,
>> struct device *dev);
>>
>> +static int arm_smmu_remove_device(u8 devfn, struct device *dev)
>> +{
>> + struct arm_smmu_master *master;
>> + struct iommu_fwspec *fwspec;
>
> I’m looking into the arm_smmu_remove_device, at some point there we allocate
Apologies, I meant arm_smmu_add_device here
> the iommu_fwspec, but we are not using iommu_fwspec_free() here, I’ve tried
> to look around and I’m not able to see what frees that structure apart form
> iommu_remove_dt_device().
> But pci_remove_device() -> iommu_remove_device() can call this function as well, I’m not sure I’ve understood
> correctly the framework here so maybe someone with more PCI experience can help
> @Stewart Hildebrand ?
>
>
>> + struct domain *d = NULL;
>> +
>> + fwspec = dev_iommu_fwspec_get(dev);
>> + if ( !fwspec )
>> + return -ENODEV;
>> +
>> + master = dev_iommu_priv_get(dev);
>> + if ( !master )
>> + return -ENODEV;
>> +
>> + if ( IS_ENABLED(CONFIG_HAS_PCI) && dev_is_pci(dev) )
>> + {
>> + struct pci_dev *pdev = dev_to_pci(dev);
>> +
>> + /* Ignore calls for phantom functions */
>> + if ( devfn != pdev->devfn )
>> + return 0;
>> +
>> + d = pdev->domain;
>> + }
>> + else
>> + {
>> + if ( !dt_device_is_protected(dev_to_dt(dev)) )
>> + {
>> + dev_err(dev, "Not added to SMMUv3\n");
>> + return -ENODEV;
>> + }
>> +
>> + dt_device_set_protected(dev_to_dt(dev), false);
>> + if ( master->domain && master->domain->d )
>> + d = master->domain->d;
>> + }
>> +
>> + if ( d )
>> + {
>> + int ret = arm_smmu_deassign_dev(d, devfn, dev);
>> + /* This should never fail because we already checked the domain */
>> + ASSERT(!ret);
>> + }
>> +
>> + arm_smmu_disable_pasid(master);
>> +
>> + dev_info(dev, "Removed master device (SMMUv3 %s StreamIds %u)\n",
>> + dev_name(fwspec->iommu_dev), fwspec->num_ids);
>> +
>> + xfree(master);
>> + dev_iommu_priv_set(dev, NULL);
>> + return 0;
>> +}
>> +
>
> Cheers,
> Luca
>
© 2016 - 2026 Red Hat, Inc.