VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device

VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
address space specified by IOAS id.

VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
from the iommufd address space

Signed-off-by: Cindy Lu <lulu@redhat.com>
---
 drivers/vhost/vdpa.c       | 171 +++++++++++++++++++++++++++++++++++++
 include/uapi/linux/vhost.h |  66 ++++++++++++++
 2 files changed, 237 insertions(+)

diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 78379ffd2336..dfaddd833364 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -18,6 +18,7 @@
 #include <linux/mm.h>
 #include <linux/slab.h>
 #include <linux/iommu.h>
+#include <linux/iommufd.h>
 #include <linux/uuid.h>
 #include <linux/vdpa.h>
 #include <linux/nospec.h>
@@ -25,6 +26,8 @@
 
 #include "vhost.h"
 
+MODULE_IMPORT_NS(IOMMUFD);
+
 enum {
 	VHOST_VDPA_BACKEND_FEATURES =
 	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
@@ -69,6 +72,15 @@ static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
 				   struct vhost_iotlb *iotlb, u64 start,
 				   u64 last, u32 asid);
 
+void vhost_vdpa_lockdep_assert_held(struct vdpa_device *vdpa)
+{
+	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
+
+	if (WARN_ON(!v))
+		return;
+	lockdep_assert_held(&v->vdev.mutex);
+}
+
 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
 {
 	struct vhost_vdpa_as *as = container_of(iotlb, struct
@@ -551,6 +563,149 @@ static long vhost_vdpa_suspend(struct vhost_vdpa *v)
 
 	return ops->suspend(vdpa);
 }
+static long vhost_vdpa_iommufd_set_device(struct vhost_vdpa *v,
+					  void __user *argp)
+{
+	struct device *dma_dev = vdpa_get_dma_dev(v->vdpa);
+	struct vhost_vdpa_set_iommufd set_iommufd;
+	struct vdpa_device *vdpa = v->vdpa;
+	struct iommufd_ctx *ictx;
+	unsigned long minsz;
+	u32 ioas_id, dev_id;
+	struct fd f;
+	long r = 0;
+
+	minsz = offsetofend(struct vhost_vdpa_set_iommufd, iommufd_ioasid);
+	if (copy_from_user(&set_iommufd, argp, minsz))
+		return -EFAULT;
+
+	/* Unset IOMMUFD */
+	if (set_iommufd.iommufd < 0) {
+		if (!vdpa->iommufd_ictx || !vdpa->iommufd_device)
+			return -EINVAL;
+		if (atomic_read(&vdpa->iommufd_users)) {
+			atomic_dec(&vdpa->iommufd_users);
+			return 0;
+		}
+		vdpa_iommufd_unbind(v->vdpa);
+		vdpa->iommufd_device = NULL;
+		vdpa->iommufd_ictx = NULL;
+		return iommu_attach_device(v->domain, dma_dev);
+	}
+
+	/* For same device but different groups, ++refcount only */
+	if (vdpa->iommufd_device)
+		goto out_inc;
+
+	r = -EBADF;
+	f = fdget(set_iommufd.iommufd);
+	if (!f.file)
+		goto out;
+
+	r = -EINVAL;
+	ictx = iommufd_ctx_from_file(f.file);
+	if (IS_ERR(ictx))
+		goto out_fdput;
+
+	if (v->domain) {
+		iommu_device_unuse_default_domain(dma_dev);
+		iommu_detach_device(v->domain, dma_dev);
+	}
+
+	ioas_id = set_iommufd.iommufd_ioasid;
+	r = vdpa_iommufd_bind(vdpa, ictx, &ioas_id, &dev_id);
+	if (r)
+		goto out_reattach;
+
+	set_iommufd.out_dev_id = dev_id;
+	r = copy_to_user(argp + minsz, &set_iommufd.out_dev_id,
+			 sizeof(set_iommufd.out_dev_id)) ?
+		    -EFAULT :
+		    0;
+	if (r)
+		goto out_device_unbind;
+
+	vdpa->iommufd_ictx = ictx;
+
+out_inc:
+	atomic_inc(&vdpa->iommufd_users);
+
+	goto out_fdput;
+
+out_device_unbind:
+
+	vdpa_iommufd_unbind(vdpa);
+out_reattach:
+	iommu_device_use_default_domain(dma_dev);
+	iommu_attach_device(v->domain, dma_dev);
+	iommufd_ctx_put(ictx);
+out_fdput:
+	fdput(f);
+out:
+	return r;
+}
+int vhost_vdpa_iommufd_ioas_attach(struct vhost_vdpa *v, void __user *arg)
+{
+	struct vdpa_device_attach_iommufd_as attach;
+	unsigned long minsz;
+	int ret;
+
+	minsz = offsetofend(struct vdpa_device_attach_iommufd_as, ioas_id);
+
+	if (copy_from_user(&attach, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	if (attach.argsz < minsz || attach.flags)
+		return -EINVAL;
+
+	if (!v->vdpa->config->bind_iommufd)
+		return -ENODEV;
+
+	if (!v->vdpa->iommufd_ictx) {
+		ret = -EINVAL;
+		return ret;
+	}
+
+	ret = v->vdpa->config->attach_ioas(v->vdpa, &attach.ioas_id);
+
+	if (ret)
+		return ret;
+
+	ret = copy_to_user(
+		      (void __user *)arg +
+			      offsetofend(struct vdpa_device_attach_iommufd_as,
+					  flags),
+		      &attach.ioas_id, sizeof(attach.ioas_id)) ?
+		      -EFAULT :
+		      0;
+
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int vhost_vdpa_iommufd_ioas_detach(struct vhost_vdpa *v, void __user *arg)
+{
+	struct vdpa_device_detach_iommufd_as detach;
+	unsigned long minsz;
+
+	minsz = offsetofend(struct vdpa_device_detach_iommufd_as, flags);
+
+	if (copy_from_user(&detach, (void __user *)arg, minsz))
+		return -EFAULT;
+
+	if (detach.argsz < minsz || detach.flags)
+		return -EINVAL;
+
+	if (!v->vdpa->config->bind_iommufd)
+		return -ENODEV;
+
+	if (v->vdpa->iommufd_ictx) {
+		return -EINVAL;
+	}
+	return v->vdpa->config->detach_ioas(v->vdpa);
+}
 
 /* After a successful return of this ioctl the device resumes processing
  * virtqueue descriptors. The device becomes fully operational the same way it
@@ -744,6 +899,18 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 	case VHOST_SET_LOG_FD:
 		r = -ENOIOCTLCMD;
 		break;
+	case VHOST_VDPA_SET_IOMMU_FD:
+
+		r = vhost_vdpa_iommufd_set_device(v, argp);
+		break;
+	case VDPA_DEVICE_ATTACH_IOMMUFD_AS:
+		r = vhost_vdpa_iommufd_ioas_attach(v, (void __user *)arg);
+		break;
+
+	case VDPA_DEVICE_DETACH_IOMMUFD_AS:
+		r = vhost_vdpa_iommufd_ioas_detach(v, (void __user *)arg);
+		break;
+
 	case VHOST_VDPA_SET_CONFIG_CALL:
 		r = vhost_vdpa_set_config_call(v, argp);
 		break;
@@ -896,6 +1063,10 @@ static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 	} else if (ops->set_map) {
 		if (!v->in_batch)
 			r = ops->set_map(vdpa, asid, iotlb);
+	} else if (!vdpa->iommufd_ictx) {
+		/* Legacy iommu domain pathway without IOMMUFD */
+		r = iommu_map(v->domain, iova, pa, size,
+			      perm_to_iommu_flags(perm), GFP_KERNEL);
 	} else {
 		r = iommu_map(v->domain, iova, pa, size,
 			      perm_to_iommu_flags(perm), GFP_KERNEL);
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index f5c48b61ab62..07e1b2c443ca 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -219,4 +219,70 @@
  */
 #define VHOST_VDPA_RESUME		_IO(VHOST_VIRTIO, 0x7E)
 
+/* vhost_vdpa_set_iommufd
+ * Input parameters:
+ * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
+ * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
+ * Output parameters:
+ * @out_dev_id: device identifier
+ */
+struct vhost_vdpa_set_iommufd {
+	__s32 iommufd;
+	__u32 iommufd_ioasid;
+	__u32 out_dev_id;
+};
+
+#define VHOST_VDPA_SET_IOMMU_FD \
+	_IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
+
+/*
+ * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
+ * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
+ *
+ * Attach a vdpa device to an iommufd address space specified by IOAS
+ * id.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
+ *
+ * @argsz:	user filled size of this data.
+ * @flags:	must be 0.
+ * @ioas_id:	Input the target id which can represent an ioas
+ *		allocated via iommufd subsystem.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_attach_iommufd_as {
+	__u32 argsz;
+	__u32 flags;
+	__u32 ioas_id;
+};
+
+#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
+	_IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
+
+/*
+ * VDPA_DEVICE_DETACH_IOMMUFD_AS
+ *
+ * Detach a vdpa device from the iommufd address space it has been
+ * attached to. After it, device should be in a blocking DMA state.
+ *
+ * Available only after a device has been bound to iommufd via
+ * VHOST_VDPA_SET_IOMMU_FD
+ *
+ * @argsz:	user filled size of this data.
+ * @flags:	must be 0.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vdpa_device_detach_iommufd_as {
+	__u32 argsz;
+	__u32 flags;
+};
+
+#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
+	_IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
+
 #endif
-- 
2.34.3

On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---

[...]

> diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> index f5c48b61ab62..07e1b2c443ca 100644
> --- a/include/uapi/linux/vhost.h
> +++ b/include/uapi/linux/vhost.h
> @@ -219,4 +219,70 @@
>   */
>  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
>
> +/* vhost_vdpa_set_iommufd
> + * Input parameters:
> + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> + * Output parameters:
> + * @out_dev_id: device identifier
> + */
> +struct vhost_vdpa_set_iommufd {
> +       __s32 iommufd;
> +       __u32 iommufd_ioasid;
> +       __u32 out_dev_id;
> +};
> +
> +#define VHOST_VDPA_SET_IOMMU_FD \
> +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> +
> +/*
> + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> + *
> + * Attach a vdpa device to an iommufd address space specified by IOAS
> + * id.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> + *
> + * @argsz:     user filled size of this data.
> + * @flags:     must be 0.
> + * @ioas_id:   Input the target id which can represent an ioas
> + *             allocated via iommufd subsystem.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_attach_iommufd_as {
> +       __u32 argsz;
> +       __u32 flags;
> +       __u32 ioas_id;
> +};

I think we need to map ioas to vDPA AS, so there should be an ASID
from the view of vDPA?

Thanks

> +
> +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> +
> +/*
> + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> + *
> + * Detach a vdpa device from the iommufd address space it has been
> + * attached to. After it, device should be in a blocking DMA state.
> + *
> + * Available only after a device has been bound to iommufd via
> + * VHOST_VDPA_SET_IOMMU_FD
> + *
> + * @argsz:     user filled size of this data.
> + * @flags:     must be 0.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vdpa_device_detach_iommufd_as {
> +       __u32 argsz;
> +       __u32 flags;
> +};
> +
> +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> +
>  #endif
> --
> 2.34.3
>

On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
>
> On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> >
> > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> >
> > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > address space specified by IOAS id.
> >
> > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > from the iommufd address space
> >
> > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > ---
>
> [...]
>
> > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > index f5c48b61ab62..07e1b2c443ca 100644
> > --- a/include/uapi/linux/vhost.h
> > +++ b/include/uapi/linux/vhost.h
> > @@ -219,4 +219,70 @@
> >   */
> >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> >
> > +/* vhost_vdpa_set_iommufd
> > + * Input parameters:
> > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > + * Output parameters:
> > + * @out_dev_id: device identifier
> > + */
> > +struct vhost_vdpa_set_iommufd {
> > +       __s32 iommufd;
> > +       __u32 iommufd_ioasid;
> > +       __u32 out_dev_id;
> > +};
> > +
> > +#define VHOST_VDPA_SET_IOMMU_FD \
> > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > +
> > +/*
> > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > + *
> > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > + * id.
> > + *
> > + * Available only after a device has been bound to iommufd via
> > + * VHOST_VDPA_SET_IOMMU_FD
> > + *
> > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > + *
> > + * @argsz:     user filled size of this data.
> > + * @flags:     must be 0.
> > + * @ioas_id:   Input the target id which can represent an ioas
> > + *             allocated via iommufd subsystem.
> > + *
> > + * Return: 0 on success, -errno on failure.
> > + */
> > +struct vdpa_device_attach_iommufd_as {
> > +       __u32 argsz;
> > +       __u32 flags;
> > +       __u32 ioas_id;
> > +};
>
> I think we need to map ioas to vDPA AS, so there should be an ASID
> from the view of vDPA?
>
> Thanks
>
The qemu will have a structure save and  maintain this information,So
I didn't add this
 in kernel，we can add this but maybe only for check?
this in
Thanks
Cindy
> > +
> > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > +
> > +/*
> > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > + *
> > + * Detach a vdpa device from the iommufd address space it has been
> > + * attached to. After it, device should be in a blocking DMA state.
> > + *
> > + * Available only after a device has been bound to iommufd via
> > + * VHOST_VDPA_SET_IOMMU_FD
> > + *
> > + * @argsz:     user filled size of this data.
> > + * @flags:     must be 0.
> > + *
> > + * Return: 0 on success, -errno on failure.
> > + */
> > +struct vdpa_device_detach_iommufd_as {
> > +       __u32 argsz;
> > +       __u32 flags;
> > +};
> > +
> > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > +
> >  #endif
> > --
> > 2.34.3
> >
>

On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
>
> On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > >
> > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > address space specified by IOAS id.
> > >
> > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > from the iommufd address space
> > >
> > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > ---
> >
> > [...]
> >
> > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > index f5c48b61ab62..07e1b2c443ca 100644
> > > --- a/include/uapi/linux/vhost.h
> > > +++ b/include/uapi/linux/vhost.h
> > > @@ -219,4 +219,70 @@
> > >   */
> > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > >
> > > +/* vhost_vdpa_set_iommufd
> > > + * Input parameters:
> > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > + * Output parameters:
> > > + * @out_dev_id: device identifier
> > > + */
> > > +struct vhost_vdpa_set_iommufd {
> > > +       __s32 iommufd;
> > > +       __u32 iommufd_ioasid;
> > > +       __u32 out_dev_id;
> > > +};
> > > +
> > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > +
> > > +/*
> > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > + *
> > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > + * id.
> > > + *
> > > + * Available only after a device has been bound to iommufd via
> > > + * VHOST_VDPA_SET_IOMMU_FD
> > > + *
> > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > + *
> > > + * @argsz:     user filled size of this data.
> > > + * @flags:     must be 0.
> > > + * @ioas_id:   Input the target id which can represent an ioas
> > > + *             allocated via iommufd subsystem.
> > > + *
> > > + * Return: 0 on success, -errno on failure.
> > > + */
> > > +struct vdpa_device_attach_iommufd_as {
> > > +       __u32 argsz;
> > > +       __u32 flags;
> > > +       __u32 ioas_id;
> > > +};
> >
> > I think we need to map ioas to vDPA AS, so there should be an ASID
> > from the view of vDPA?
> >
> > Thanks
> >
> The qemu will have a structure save and  maintain this information,So
> I didn't add this
>  in kernel，we can add this but maybe only for check?

I meant for example, a simulator has two AS. How can we attach an ioas
to a specific AS with the above uAPI?

Thanks

> this in
> Thanks
> Cindy
> > > +
> > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > +
> > > +/*
> > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > + *
> > > + * Detach a vdpa device from the iommufd address space it has been
> > > + * attached to. After it, device should be in a blocking DMA state.
> > > + *
> > > + * Available only after a device has been bound to iommufd via
> > > + * VHOST_VDPA_SET_IOMMU_FD
> > > + *
> > > + * @argsz:     user filled size of this data.
> > > + * @flags:     must be 0.
> > > + *
> > > + * Return: 0 on success, -errno on failure.
> > > + */
> > > +struct vdpa_device_detach_iommufd_as {
> > > +       __u32 argsz;
> > > +       __u32 flags;
> > > +};
> > > +
> > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > +
> > >  #endif
> > > --
> > > 2.34.3
> > >
> >
>

On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
>
> On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> >
> > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > >
> > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > >
> > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > address space specified by IOAS id.
> > > >
> > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > from the iommufd address space
> > > >
> > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > ---
> > >
> > > [...]
> > >
> > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > --- a/include/uapi/linux/vhost.h
> > > > +++ b/include/uapi/linux/vhost.h
> > > > @@ -219,4 +219,70 @@
> > > >   */
> > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > >
> > > > +/* vhost_vdpa_set_iommufd
> > > > + * Input parameters:
> > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > + * Output parameters:
> > > > + * @out_dev_id: device identifier
> > > > + */
> > > > +struct vhost_vdpa_set_iommufd {
> > > > +       __s32 iommufd;
> > > > +       __u32 iommufd_ioasid;
> > > > +       __u32 out_dev_id;
> > > > +};
> > > > +
> > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > +
> > > > +/*
> > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > + *
> > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > + * id.
> > > > + *
> > > > + * Available only after a device has been bound to iommufd via
> > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > + *
> > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > + *
> > > > + * @argsz:     user filled size of this data.
> > > > + * @flags:     must be 0.
> > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > + *             allocated via iommufd subsystem.
> > > > + *
> > > > + * Return: 0 on success, -errno on failure.
> > > > + */
> > > > +struct vdpa_device_attach_iommufd_as {
> > > > +       __u32 argsz;
> > > > +       __u32 flags;
> > > > +       __u32 ioas_id;
> > > > +};
> > >
> > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > from the view of vDPA?
> > >
> > > Thanks
> > >
> > The qemu will have a structure save and  maintain this information,So
> > I didn't add this
> >  in kernel，we can add this but maybe only for check?
>
> I meant for example, a simulator has two AS. How can we attach an ioas
> to a specific AS with the above uAPI?
>
> Thank>
this   __u32 ioas_id here is alloc from the iommufd system. maybe I
need to change to new name iommuds_asid to
make this more clear
the process in qemu is

1) qemu want to use AS 0 (for example)
2) checking the existing asid. the asid 0 not used before
3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
qemu will save this relation 3<-->0 in the driver.
4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
to the kernel
5） while map the memory， qemu will use ASID 3 to map /umap
and use ASID 0 for legacy mode map/umap

So kernel here will not maintain the ioas_id from iommufd，
and this also make the code strange since there will 2 different asid
for the same AS, maybe we can save these information in the kernel
Thanks
cindy
> > Thanks
> > Cindy
> > > > +
> > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > +
> > > > +/*
> > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > + *
> > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > + *
> > > > + * Available only after a device has been bound to iommufd via
> > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > + *
> > > > + * @argsz:     user filled size of this data.
> > > > + * @flags:     must be 0.
> > > > + *
> > > > + * Return: 0 on success, -errno on failure.
> > > > + */
> > > > +struct vdpa_device_detach_iommufd_as {
> > > > +       __u32 argsz;
> > > > +       __u32 flags;
> > > > +};
> > > > +
> > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > +
> > > >  #endif
> > > > --
> > > > 2.34.3
> > > >
> > >
> >
>

On Wed, Nov 8, 2023 at 2:39 PM Cindy Lu <lulu@redhat.com> wrote:
>
> On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > > >
> > > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > > >
> > > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > > address space specified by IOAS id.
> > > > >
> > > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > > from the iommufd address space
> > > > >
> > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > ---
> > > >
> > > > [...]
> > > >
> > > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > > --- a/include/uapi/linux/vhost.h
> > > > > +++ b/include/uapi/linux/vhost.h
> > > > > @@ -219,4 +219,70 @@
> > > > >   */
> > > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > > >
> > > > > +/* vhost_vdpa_set_iommufd
> > > > > + * Input parameters:
> > > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > > + * Output parameters:
> > > > > + * @out_dev_id: device identifier
> > > > > + */
> > > > > +struct vhost_vdpa_set_iommufd {
> > > > > +       __s32 iommufd;
> > > > > +       __u32 iommufd_ioasid;
> > > > > +       __u32 out_dev_id;
> > > > > +};
> > > > > +
> > > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > > +
> > > > > +/*
> > > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > > + *
> > > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > > + * id.
> > > > > + *
> > > > > + * Available only after a device has been bound to iommufd via
> > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > + *
> > > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > > + *
> > > > > + * @argsz:     user filled size of this data.
> > > > > + * @flags:     must be 0.
> > > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > > + *             allocated via iommufd subsystem.
> > > > > + *
> > > > > + * Return: 0 on success, -errno on failure.
> > > > > + */
> > > > > +struct vdpa_device_attach_iommufd_as {
> > > > > +       __u32 argsz;
> > > > > +       __u32 flags;
> > > > > +       __u32 ioas_id;
> > > > > +};
> > > >
> > > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > > from the view of vDPA?
> > > >
> > > > Thanks
> > > >
> > > The qemu will have a structure save and  maintain this information,So
> > > I didn't add this
> > >  in kernel，we can add this but maybe only for check?
> >
> > I meant for example, a simulator has two AS. How can we attach an ioas
> > to a specific AS with the above uAPI?
> >
> > Thank>
> this   __u32 ioas_id here is alloc from the iommufd system. maybe I
> need to change to new name iommuds_asid to
> make this more clear
> the process in qemu is
>
> 1) qemu want to use AS 0 (for example)
> 2) checking the existing asid. the asid 0 not used before
> 3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
> qemu will save this relation 3<-->0 in the driver.
> 4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
> to the kernel

So if we want to map IOMMUFD AS 3 to VDPA AS 0, how can it be done?

For example I didn't see a vDPA AS parameter in the above uAPI.

vhost_vdpa_set_iommufd has iommufd_ioasid which is obviously not the vDPA AS.

And ioas_id of vdpa_device_attach_iommufd_as (as you explained above)
is not vDPA AS.

Thanks


> 5） while map the memory， qemu will use ASID 3 to map /umap
> and use ASID 0 for legacy mode map/umap
>
> So kernel here will not maintain the ioas_id from iommufd，
> and this also make the code strange since there will 2 different asid
> for the same AS, maybe we can save these information in the kernel
> Thanks
> cindy
> > > Thanks
> > > Cindy
> > > > > +
> > > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > > +
> > > > > +/*
> > > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > > + *
> > > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > > + *
> > > > > + * Available only after a device has been bound to iommufd via
> > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > + *
> > > > > + * @argsz:     user filled size of this data.
> > > > > + * @flags:     must be 0.
> > > > > + *
> > > > > + * Return: 0 on success, -errno on failure.
> > > > > + */
> > > > > +struct vdpa_device_detach_iommufd_as {
> > > > > +       __u32 argsz;
> > > > > +       __u32 flags;
> > > > > +};
> > > > > +
> > > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > > +
> > > > >  #endif
> > > > > --
> > > > > 2.34.3
> > > > >
> > > >
> > >
> >
>

On Wed, Nov 8, 2023 at 3:09 PM Jason Wang <jasowang@redhat.com> wrote:
>
> On Wed, Nov 8, 2023 at 2:39 PM Cindy Lu <lulu@redhat.com> wrote:
> >
> > On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> > > >
> > > > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > > > >
> > > > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > > > >
> > > > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > > > >
> > > > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > > > address space specified by IOAS id.
> > > > > >
> > > > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > > > from the iommufd address space
> > > > > >
> > > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > > ---
> > > > >
> > > > > [...]
> > > > >
> > > > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > > > --- a/include/uapi/linux/vhost.h
> > > > > > +++ b/include/uapi/linux/vhost.h
> > > > > > @@ -219,4 +219,70 @@
> > > > > >   */
> > > > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > > > >
> > > > > > +/* vhost_vdpa_set_iommufd
> > > > > > + * Input parameters:
> > > > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > > > + * Output parameters:
> > > > > > + * @out_dev_id: device identifier
> > > > > > + */
> > > > > > +struct vhost_vdpa_set_iommufd {
> > > > > > +       __s32 iommufd;
> > > > > > +       __u32 iommufd_ioasid;
> > > > > > +       __u32 out_dev_id;
> > > > > > +};
> > > > > > +
> > > > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > > > +
> > > > > > +/*
> > > > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > > > + *
> > > > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > > > + * id.
> > > > > > + *
> > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > + *
> > > > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > > > + *
> > > > > > + * @argsz:     user filled size of this data.
> > > > > > + * @flags:     must be 0.
> > > > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > > > + *             allocated via iommufd subsystem.
> > > > > > + *
> > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > + */
> > > > > > +struct vdpa_device_attach_iommufd_as {
> > > > > > +       __u32 argsz;
> > > > > > +       __u32 flags;
> > > > > > +       __u32 ioas_id;
> > > > > > +};
> > > > >
> > > > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > > > from the view of vDPA?
> > > > >
> > > > > Thanks
> > > > >
> > > > The qemu will have a structure save and  maintain this information,So
> > > > I didn't add this
> > > >  in kernel，we can add this but maybe only for check?
> > >
> > > I meant for example, a simulator has two AS. How can we attach an ioas
> > > to a specific AS with the above uAPI?
> > >
> > > Thank>
> > this   __u32 ioas_id here is alloc from the iommufd system. maybe I
> > need to change to new name iommuds_asid to
> > make this more clear
> > the process in qemu is
> >
> > 1) qemu want to use AS 0 (for example)
> > 2) checking the existing asid. the asid 0 not used before
> > 3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
> > qemu will save this relation 3<-->0 in the driver.
> > 4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
> > to the kernel
>
> So if we want to map IOMMUFD AS 3 to VDPA AS 0, how can it be done?
>
> For example I didn't see a vDPA AS parameter in the above uAPI.
>
> vhost_vdpa_set_iommufd has iommufd_ioasid which is obviously not the vDPA AS.
>
> And ioas_id of vdpa_device_attach_iommufd_as (as you explained above)
> is not vDPA AS.

For example, the simulator/mlx5e has two ASes. It needs to know the
mapping between vDPA AS and iommufd AS. Otherwise the translation will
be problematic.

Thanks

>
> Thanks
>
>
> > 5） while map the memory， qemu will use ASID 3 to map /umap
> > and use ASID 0 for legacy mode map/umap
> >
> > So kernel here will not maintain the ioas_id from iommufd，
> > and this also make the code strange since there will 2 different asid
> > for the same AS, maybe we can save these information in the kernel
> > Thanks
> > cindy
> > > > Thanks
> > > > Cindy
> > > > > > +
> > > > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > > > +
> > > > > > +/*
> > > > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > > > + *
> > > > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > > > + *
> > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > + *
> > > > > > + * @argsz:     user filled size of this data.
> > > > > > + * @flags:     must be 0.
> > > > > > + *
> > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > + */
> > > > > > +struct vdpa_device_detach_iommufd_as {
> > > > > > +       __u32 argsz;
> > > > > > +       __u32 flags;
> > > > > > +};
> > > > > > +
> > > > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > > > +
> > > > > >  #endif
> > > > > > --
> > > > > > 2.34.3
> > > > > >
> > > > >
> > > >
> > >
> >

Jason Wang <jasowang@redhat.com> 于2023年11月10日周五 10:32写道：
>
> On Wed, Nov 8, 2023 at 3:09 PM Jason Wang <jasowang@redhat.com> wrote:
> >
> > On Wed, Nov 8, 2023 at 2:39 PM Cindy Lu <lulu@redhat.com> wrote:
> > >
> > > On Wed, Nov 8, 2023 at 11:03 AM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > > On Tue, Nov 7, 2023 at 2:57 PM Cindy Lu <lulu@redhat.com> wrote:
> > > > >
> > > > > On Mon, Nov 6, 2023 at 3:30 PM Jason Wang <jasowang@redhat.com> wrote:
> > > > > >
> > > > > > On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
> > > > > > >
> > > > > > > VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
> > > > > > >
> > > > > > > VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> > > > > > > address space specified by IOAS id.
> > > > > > >
> > > > > > > VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> > > > > > > from the iommufd address space
> > > > > > >
> > > > > > > Signed-off-by: Cindy Lu <lulu@redhat.com>
> > > > > > > ---
> > > > > >
> > > > > > [...]
> > > > > >
> > > > > > > diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
> > > > > > > index f5c48b61ab62..07e1b2c443ca 100644
> > > > > > > --- a/include/uapi/linux/vhost.h
> > > > > > > +++ b/include/uapi/linux/vhost.h
> > > > > > > @@ -219,4 +219,70 @@
> > > > > > >   */
> > > > > > >  #define VHOST_VDPA_RESUME              _IO(VHOST_VIRTIO, 0x7E)
> > > > > > >
> > > > > > > +/* vhost_vdpa_set_iommufd
> > > > > > > + * Input parameters:
> > > > > > > + * @iommufd: file descriptor from /dev/iommu; pass -1 to unset
> > > > > > > + * @iommufd_ioasid: IOAS identifier returned from ioctl(IOMMU_IOAS_ALLOC)
> > > > > > > + * Output parameters:
> > > > > > > + * @out_dev_id: device identifier
> > > > > > > + */
> > > > > > > +struct vhost_vdpa_set_iommufd {
> > > > > > > +       __s32 iommufd;
> > > > > > > +       __u32 iommufd_ioasid;
> > > > > > > +       __u32 out_dev_id;
> > > > > > > +};
> > > > > > > +
> > > > > > > +#define VHOST_VDPA_SET_IOMMU_FD \
> > > > > > > +       _IOW(VHOST_VIRTIO, 0x7F, struct vhost_vdpa_set_iommufd)
> > > > > > > +
> > > > > > > +/*
> > > > > > > + * VDPA_DEVICE_ATTACH_IOMMUFD_AS -
> > > > > > > + * _IOW(VHOST_VIRTIO, 0x7f, struct vdpa_device_attach_iommufd_as)
> > > > > > > + *
> > > > > > > + * Attach a vdpa device to an iommufd address space specified by IOAS
> > > > > > > + * id.
> > > > > > > + *
> > > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > > + *
> > > > > > > + * Undo by VDPA_DEVICE_DETACH_IOMMUFD_AS or device fd close.
> > > > > > > + *
> > > > > > > + * @argsz:     user filled size of this data.
> > > > > > > + * @flags:     must be 0.
> > > > > > > + * @ioas_id:   Input the target id which can represent an ioas
> > > > > > > + *             allocated via iommufd subsystem.
> > > > > > > + *
> > > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > > + */
> > > > > > > +struct vdpa_device_attach_iommufd_as {
> > > > > > > +       __u32 argsz;
> > > > > > > +       __u32 flags;
> > > > > > > +       __u32 ioas_id;
> > > > > > > +};
> > > > > >
> > > > > > I think we need to map ioas to vDPA AS, so there should be an ASID
> > > > > > from the view of vDPA?
> > > > > >
> > > > > > Thanks
> > > > > >
> > > > > The qemu will have a structure save and  maintain this information,So
> > > > > I didn't add this
> > > > >  in kernel，we can add this but maybe only for check?
> > > >
> > > > I meant for example, a simulator has two AS. How can we attach an ioas
> > > > to a specific AS with the above uAPI?
> > > >
> > > > Thank>
> > > this   __u32 ioas_id here is alloc from the iommufd system. maybe I
> > > need to change to new name iommuds_asid to
> > > make this more clear
> > > the process in qemu is
> > >
> > > 1) qemu want to use AS 0 (for example)
> > > 2) checking the existing asid. the asid 0 not used before
> > > 3 )alloc new asid from iommufd system, get new ioas_id (maybe 3 for example)
> > > qemu will save this relation 3<-->0 in the driver.
> > > 4) setting the ioctl VDPA_DEVICE_ATTACH_IOMMUFD_AS to attach new ASID
> > > to the kernel
> >
> > So if we want to map IOMMUFD AS 3 to VDPA AS 0, how can it be done?
> >
> > For example I didn't see a vDPA AS parameter in the above uAPI.
> >
> > vhost_vdpa_set_iommufd has iommufd_ioasid which is obviously not the vDPA AS.
> >
> > And ioas_id of vdpa_device_attach_iommufd_as (as you explained above)
> > is not vDPA AS.
>
> For example, the simulator/mlx5e has two ASes. It needs to know the
> mapping between vDPA AS and iommufd AS. Otherwise the translation will
> be problematic.
>
> Thanks
>
Got it, thanks Jason. I will re-write this part
Thanks
Cindy
> >
> > Thanks
> >
> >
> > > 5） while map the memory， qemu will use ASID 3 to map /umap
> > > and use ASID 0 for legacy mode map/umap
> > >
> > > So kernel here will not maintain the ioas_id from iommufd，
> > > and this also make the code strange since there will 2 different asid
> > > for the same AS, maybe we can save these information in the kernel
> > > Thanks
> > > cindy
> > > > > Thanks
> > > > > Cindy
> > > > > > > +
> > > > > > > +#define VDPA_DEVICE_ATTACH_IOMMUFD_AS \
> > > > > > > +       _IOW(VHOST_VIRTIO, 0x82, struct vdpa_device_attach_iommufd_as)
> > > > > > > +
> > > > > > > +/*
> > > > > > > + * VDPA_DEVICE_DETACH_IOMMUFD_AS
> > > > > > > + *
> > > > > > > + * Detach a vdpa device from the iommufd address space it has been
> > > > > > > + * attached to. After it, device should be in a blocking DMA state.
> > > > > > > + *
> > > > > > > + * Available only after a device has been bound to iommufd via
> > > > > > > + * VHOST_VDPA_SET_IOMMU_FD
> > > > > > > + *
> > > > > > > + * @argsz:     user filled size of this data.
> > > > > > > + * @flags:     must be 0.
> > > > > > > + *
> > > > > > > + * Return: 0 on success, -errno on failure.
> > > > > > > + */
> > > > > > > +struct vdpa_device_detach_iommufd_as {
> > > > > > > +       __u32 argsz;
> > > > > > > +       __u32 flags;
> > > > > > > +};
> > > > > > > +
> > > > > > > +#define VDPA_DEVICE_DETACH_IOMMUFD_AS \
> > > > > > > +       _IOW(VHOST_VIRTIO, 0x83, struct vdpa_device_detach_iommufd_as)
> > > > > > > +
> > > > > > >  #endif
> > > > > > > --
> > > > > > > 2.34.3
> > > > > > >
> > > > > >
> > > > >
> > > >
> > >
>

On Sat, Nov 4, 2023 at 1:17 AM Cindy Lu <lulu@redhat.com> wrote:
>
> VHOST_VDPA_SET_IOMMU_FD: bind the device to iommufd device
>
> VDPA_DEVICE_ATTACH_IOMMUFD_AS: Attach a vdpa device to an iommufd
> address space specified by IOAS id.
>
> VDPA_DEVICE_DETACH_IOMMUFD_AS: Detach a vdpa device
> from the iommufd address space
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>

As discussed in the previous version, any reason/advantages of this
compared to just having a single  VDPA_DEVICE_ATTACH_IOMMUFD_AS?

Thanks