In a non-driver context, it is crucial to ensure the consistency of a
device's iommu ops. Otherwise, it may result in a situation where a
device is released but it's iommu ops are still used.

Put the ops->release_device and __iommu_group_remove_device() in a some
group->mutext critical region, so that, as long as group->mutex is held
and the device is in its group's device list, its iommu ops are always
consistent.

Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
---
 drivers/iommu/iommu.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 6247883991e2..093692308b80 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -101,6 +101,10 @@ static int iommu_create_device_direct_mappings(struct iommu_group *group,
 static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
 static ssize_t iommu_group_store_type(struct iommu_group *group,
 				      const char *buf, size_t count);
+static struct group_device *
+__iommu_group_remove_device(struct iommu_group *group, struct device *dev);
+static void __iommu_group_release_device(struct iommu_group *group,
+					 struct group_device *grp_dev);
 
 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store)		\
 struct iommu_group_attribute iommu_group_attr_##_name =		\
@@ -466,18 +470,25 @@ int iommu_probe_device(struct device *dev)
 
 void iommu_release_device(struct device *dev)
 {
+	struct iommu_group *group = dev->iommu_group;
+	struct group_device *device;
 	const struct iommu_ops *ops;
 
-	if (!dev->iommu)
+	if (!dev->iommu || !group)
 		return;
 
 	iommu_device_unlink(dev->iommu->iommu_dev, dev);
 
+	mutex_lock(&group->mutex);
 	ops = dev_iommu_ops(dev);
 	if (ops->release_device)
 		ops->release_device(dev);
+	device = __iommu_group_remove_device(group, dev);
+	mutex_unlock(&group->mutex);
+
+	if (device)
+		__iommu_group_release_device(group, device);
 
-	iommu_group_remove_device(dev);
 	module_put(ops->owner);
 	dev_iommu_free(dev);
 }
-- 
2.34.1

On Fri, Feb 17, 2023 at 05:47:33PM +0800, Lu Baolu wrote:
> In a non-driver context, it is crucial to ensure the consistency of a
> device's iommu ops. Otherwise, it may result in a situation where a
> device is released but it's iommu ops are still used.
> 
> Put the ops->release_device and __iommu_group_remove_device() in a some
> group->mutext critical region, so that, as long as group->mutex is held
> and the device is in its group's device list, its iommu ops are always
> consistent.
> 
> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
> ---
>  drivers/iommu/iommu.c | 15 +++++++++++++--
>  1 file changed, 13 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> index 6247883991e2..093692308b80 100644
> --- a/drivers/iommu/iommu.c
> +++ b/drivers/iommu/iommu.c
> @@ -101,6 +101,10 @@ static int iommu_create_device_direct_mappings(struct iommu_group *group,
>  static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
>  static ssize_t iommu_group_store_type(struct iommu_group *group,
>  				      const char *buf, size_t count);
> +static struct group_device *
> +__iommu_group_remove_device(struct iommu_group *group, struct device *dev);
> +static void __iommu_group_release_device(struct iommu_group *group,
> +					 struct group_device *grp_dev);

Seems like a hunk is missing from this patch?

Jason

On 2/17/23 11:40 PM, Jason Gunthorpe wrote:
> On Fri, Feb 17, 2023 at 05:47:33PM +0800, Lu Baolu wrote:
>> In a non-driver context, it is crucial to ensure the consistency of a
>> device's iommu ops. Otherwise, it may result in a situation where a
>> device is released but it's iommu ops are still used.
>>
>> Put the ops->release_device and __iommu_group_remove_device() in a some
>> group->mutext critical region, so that, as long as group->mutex is held
>> and the device is in its group's device list, its iommu ops are always
>> consistent.
>>
>> Signed-off-by: Lu Baolu<baolu.lu@linux.intel.com>
>> ---
>>   drivers/iommu/iommu.c | 15 +++++++++++++--
>>   1 file changed, 13 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
>> index 6247883991e2..093692308b80 100644
>> --- a/drivers/iommu/iommu.c
>> +++ b/drivers/iommu/iommu.c
>> @@ -101,6 +101,10 @@ static int iommu_create_device_direct_mappings(struct iommu_group *group,
>>   static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
>>   static ssize_t iommu_group_store_type(struct iommu_group *group,
>>   				      const char *buf, size_t count);
>> +static struct group_device *
>> +__iommu_group_remove_device(struct iommu_group *group, struct device *dev);
>> +static void __iommu_group_release_device(struct iommu_group *group,
>> +					 struct group_device *grp_dev);
> Seems like a hunk is missing from this patch?

Did you mean below block of change? If so, I will add it in the next
version.

+
+	/*
+	 * If the group has become empty then ownership must have been released,
+	 * and the current domain must be set back to NULL or the default
+	 * domain.
+	 */
+	if (list_empty(&group->devices))
+		WARN_ON(group->owner_cnt ||
+			group->domain != group->default_domain);
+
+	/*
+	 * release_device() must stop using any attached domain on the device.
+	 * If there are still other devices in the group they are not effected
+	 * by this callback.
+	 *
+	 * The IOMMU driver must set the device to either an identity or
+	 * blocking translation and stop using any domain pointer, as it is
+	 * going to be freed.
+	 */
  	if (ops->release_device)
  		ops->release_device(dev);
+	mutex_unlock(&group->mutex);

By the way, can I add you signed-off-by when I use the code you posted
in the discussion thread?

Best regards,
baolu

On Sat, Feb 18, 2023 at 03:29:12PM +0800, Baolu Lu wrote:
> On 2/17/23 11:40 PM, Jason Gunthorpe wrote:
> > On Fri, Feb 17, 2023 at 05:47:33PM +0800, Lu Baolu wrote:
> > > In a non-driver context, it is crucial to ensure the consistency of a
> > > device's iommu ops. Otherwise, it may result in a situation where a
> > > device is released but it's iommu ops are still used.
> > > 
> > > Put the ops->release_device and __iommu_group_remove_device() in a some
> > > group->mutext critical region, so that, as long as group->mutex is held
> > > and the device is in its group's device list, its iommu ops are always
> > > consistent.
> > > 
> > > Signed-off-by: Lu Baolu<baolu.lu@linux.intel.com>
> > > ---
> > >   drivers/iommu/iommu.c | 15 +++++++++++++--
> > >   1 file changed, 13 insertions(+), 2 deletions(-)
> > > 
> > > diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> > > index 6247883991e2..093692308b80 100644
> > > --- a/drivers/iommu/iommu.c
> > > +++ b/drivers/iommu/iommu.c
> > > @@ -101,6 +101,10 @@ static int iommu_create_device_direct_mappings(struct iommu_group *group,
> > >   static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
> > >   static ssize_t iommu_group_store_type(struct iommu_group *group,
> > >   				      const char *buf, size_t count);
> > > +static struct group_device *
> > > +__iommu_group_remove_device(struct iommu_group *group, struct device *dev);
> > > +static void __iommu_group_release_device(struct iommu_group *group,
> > > +					 struct group_device *grp_dev);
> > Seems like a hunk is missing from this patch?
> 
> Did you mean below block of change? If so, I will add it in the next
> version.

I  mean this changed the protoype but I didn't see a change in the
actual funtion?

> By the way, can I add you signed-off-by when I use the code you posted
> in the discussion thread?

Yes

Jason