[libvirt PATCH v2 5/5] qemu: Implement support for vDPA block devices

Jonathon Jongsma posted 5 patches 10 months, 1 week ago
[libvirt PATCH v2 5/5] qemu: Implement support for vDPA block devices
Posted by Jonathon Jongsma 10 months, 1 week ago
Requires recent qemu with support for the virtio-blk-vhost-vdpa device
and the ability to pass a /dev/fdset/N path for the vdpa path (8.1.0)

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1900770
Signed-off-by: Jonathon Jongsma <jjongsma@redhat.com>
---
 src/qemu/qemu_block.c                         | 20 ++++++++--
 src/qemu/qemu_process.c                       | 34 +++++++++++++++++
 src/qemu/qemu_validate.c                      | 32 ++++++++++++++--
 .../disk-vhostvdpa.x86_64-latest.args         | 37 +++++++++++++++++++
 tests/qemuxml2argvdata/disk-vhostvdpa.xml     | 21 +++++++++++
 tests/qemuxml2argvtest.c                      | 34 +++++++++++++++++
 tests/testutilsqemu.c                         | 11 ++++++
 tests/testutilsqemu.h                         |  2 +
 8 files changed, 185 insertions(+), 6 deletions(-)
 create mode 100644 tests/qemuxml2argvdata/disk-vhostvdpa.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/disk-vhostvdpa.xml

diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
index 0b4c2dbcf4..d31bbde0f4 100644
--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
@@ -778,6 +778,20 @@ qemuBlockStorageSourceGetNVMeProps(virStorageSource *src)
 }
 
 
+static virJSONValue *
+qemuBlockStorageSourceGetVhostVdpaProps(virStorageSource *src)
+{
+    virJSONValue *ret = NULL;
+    qemuDomainStorageSourcePrivate *srcpriv = QEMU_DOMAIN_STORAGE_SOURCE_PRIVATE(src);
+
+    ignore_value(virJSONValueObjectAdd(&ret,
+                                       "s:driver", "virtio-blk-vhost-vdpa",
+                                       "s:path", qemuFDPassGetPath(srcpriv->fdpass),
+                                       NULL));
+    return ret;
+}
+
+
 static int
 qemuBlockStorageSourceGetBlockdevGetCacheProps(virStorageSource *src,
                                                virJSONValue *props)
@@ -874,9 +888,9 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
         break;
 
     case VIR_STORAGE_TYPE_VHOST_VDPA:
-        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                       _("vhostvdpa disk type not yet supported"));
-        return NULL;
+        if (!(fileprops = qemuBlockStorageSourceGetVhostVdpaProps(src)))
+            return NULL;
+        break;
 
     case VIR_STORAGE_TYPE_VHOST_USER:
         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 7a1cdb0302..42837c4a8a 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -6820,6 +6820,28 @@ qemuProcessPrepareLaunchSecurityGuestInput(virDomainObj *vm)
 }
 
 
+static int
+qemuProcessPrepareHostStorageSourceVDPA(virStorageSource *src,
+                                        qemuDomainObjPrivate *priv)
+{
+    qemuDomainStorageSourcePrivate *srcpriv = NULL;
+    virStorageType actualType = virStorageSourceGetActualType(src);
+    int vdpafd = -1;
+
+    if (actualType != VIR_STORAGE_TYPE_VHOST_VDPA)
+        return 0;
+
+    if ((vdpafd = qemuVDPAConnect(src->vdpadev)) < 0)
+        return -1;
+
+    srcpriv = qemuDomainStorageSourcePrivateFetch(src);
+
+    srcpriv->fdpass = qemuFDPassNew(src->nodestorage, priv);
+    qemuFDPassAddFD(srcpriv->fdpass, &vdpafd, "-vdpa");
+    return 0;
+}
+
+
 static int
 qemuProcessPrepareHostStorage(virQEMUDriver *driver,
                               virDomainObj *vm,
@@ -6856,6 +6878,18 @@ qemuProcessPrepareHostStorage(virQEMUDriver *driver,
         return -1;
     }
 
+    /* connect to any necessary vdpa block devices */
+    for (i = vm->def->ndisks; i > 0; i--) {
+        size_t idx = i - 1;
+        virDomainDiskDef *disk = vm->def->disks[idx];
+        virStorageSource *src;
+
+        for (src = disk->src; virStorageSourceIsBacking(src); src = src->backingStore) {
+            if (qemuProcessPrepareHostStorageSourceVDPA(src, vm->privateData) < 0)
+                return -1;
+        }
+    }
+
     return 0;
 }
 
diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c
index 165ab3a66a..5bae56b00f 100644
--- a/src/qemu/qemu_validate.c
+++ b/src/qemu/qemu_validate.c
@@ -3175,13 +3175,39 @@ qemuValidateDomainDeviceDefDisk(const virDomainDiskDef *disk,
     }
 
     if (disk->src->type == VIR_STORAGE_TYPE_VHOST_USER) {
+        const char *vhosttype = virStorageTypeToString(disk->src->type);
+
         if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VHOST_USER_BLK)) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                           _("vhostuser disk is not supported with this QEMU binary"));
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("%1$s disk is not supported with this QEMU binary"),
+                           vhosttype);
+            return -1;
+        }
+
+        if (qemuValidateDomainDefVhostUserRequireSharedMemory(def, vhosttype) < 0)
+            return -1;
+    }
+
+    if (disk->src->type == VIR_STORAGE_TYPE_VHOST_VDPA) {
+        const char *vhosttype = virStorageTypeToString(disk->src->type);
+
+        if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VIRTIO_BLK_VHOST_VDPA)) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("%1$s disk is not supported with this QEMU binary"),
+                           vhosttype);
             return -1;
         }
 
-        if (qemuValidateDomainDefVhostUserRequireSharedMemory(def, "vhostuser") < 0) {
+        if (qemuValidateDomainDefVhostUserRequireSharedMemory(def, vhosttype) < 0)
+            return -1;
+
+        if (disk->cachemode != VIR_DOMAIN_DISK_CACHE_DIRECTSYNC &&
+            disk->cachemode != VIR_DOMAIN_DISK_CACHE_DISABLE) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("disk type '%1$s' requires cache mode '%2$s' or '%3$s'"),
+                           virStorageTypeToString(disk->src->type),
+                           virDomainDiskCacheTypeToString(VIR_DOMAIN_DISK_CACHE_DIRECTSYNC),
+                           virDomainDiskCacheTypeToString(VIR_DOMAIN_DISK_CACHE_DISABLE));
             return -1;
         }
     }
diff --git a/tests/qemuxml2argvdata/disk-vhostvdpa.x86_64-latest.args b/tests/qemuxml2argvdata/disk-vhostvdpa.x86_64-latest.args
new file mode 100644
index 0000000000..b987455ee4
--- /dev/null
+++ b/tests/qemuxml2argvdata/disk-vhostvdpa.x86_64-latest.args
@@ -0,0 +1,37 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1 \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1/.local/share \
+XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1/.cache \
+XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1/.config \
+/usr/bin/qemu-system-x86_64 \
+-name guest=QEMUGuest1,debug-threads=on \
+-S \
+-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain--1-QEMUGuest1/master-key.aes"}' \
+-machine pc,usb=off,dump-guest-core=off,memory-backend=pc.ram,acpi=off \
+-accel tcg \
+-cpu qemu64 \
+-m size=219136k \
+-object '{"qom-type":"memory-backend-file","id":"pc.ram","mem-path":"/var/lib/libvirt/qemu/ram/-1-QEMUGuest1/pc.ram","share":true,"x-use-canonical-path-for-ramblock-id":false,"size":224395264}' \
+-overcommit mem-lock=off \
+-smp 1,sockets=1,cores=1,threads=1 \
+-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,fd=1729,server=on,wait=off \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-boot strict=on \
+-device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-add-fd set=0,fd=801,opaque=libvirt-1-storage-vdpa \
+-blockdev '{"driver":"virtio-blk-vhost-vdpa","path":"/dev/fdset/0","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
+-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \
+-device '{"driver":"virtio-blk-pci","bus":"pci.0","addr":"0x2","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \
+-audiodev '{"id":"audio1","driver":"none"}' \
+-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.0","addr":"0x3"}' \
+-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
+-msg timestamp=on
diff --git a/tests/qemuxml2argvdata/disk-vhostvdpa.xml b/tests/qemuxml2argvdata/disk-vhostvdpa.xml
new file mode 100644
index 0000000000..0ac3899a34
--- /dev/null
+++ b/tests/qemuxml2argvdata/disk-vhostvdpa.xml
@@ -0,0 +1,21 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <memoryBacking>
+    <access mode='shared'/>
+  </memoryBacking>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc'>hvm</type>
+  </os>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <disk type='vhostvdpa' device='disk'>
+      <driver name='qemu' type='raw' cache='none'/>
+      <source dev='/dev/vhost-vdpa-0'/>
+      <target dev='vda' bus='virtio'/>
+    </disk>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index 2d06e2a0d1..0dd10f1fab 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -304,6 +304,7 @@ testCompareXMLToArgvCreateArgs(virQEMUDriver *drv,
 
     for (i = 0; i < vm->def->ndisks; i++) {
         virDomainDiskDef *disk = vm->def->disks[i];
+        virStorageSource *src;
 
         /* host cdrom requires special treatment in qemu, mock it */
         if (disk->device == VIR_DOMAIN_DISK_DEVICE_CDROM &&
@@ -311,6 +312,37 @@ testCompareXMLToArgvCreateArgs(virQEMUDriver *drv,
             virStorageSourceIsBlockLocal(disk->src) &&
             STREQ(disk->src->path, "/dev/cdrom"))
             disk->src->hostcdrom = true;
+
+        if (info->args.vdpafds) {
+            for (src = disk->src; virStorageSourceIsBacking(src); src = src->backingStore) {
+                gpointer value;
+
+                if (src->type != VIR_STORAGE_TYPE_VHOST_VDPA)
+                    continue;
+
+                if ((value = g_hash_table_lookup(info->args.vdpafds, src->vdpadev))) {
+                    int fd = GPOINTER_TO_INT(value);
+                    qemuDomainStorageSourcePrivate *srcpriv;
+                    VIR_AUTOCLOSE fakefd = open("/dev/zero", O_RDWR);
+
+                    if (fcntl(fd, F_GETFD) != -1) {
+                        fprintf(stderr, "fd '%d' is already in use\n", fd);
+                        abort();
+                    }
+
+                    if (dup2(fakefd, fd) < 0) {
+                        fprintf(stderr, "failed to duplicate fake fd: %s",
+                                g_strerror(errno));
+                        abort();
+                    }
+
+                    srcpriv = qemuDomainStorageSourcePrivateFetch(src);
+
+                    srcpriv->fdpass = qemuFDPassNew(src->nodestorage, priv);
+                    qemuFDPassAddFD(srcpriv->fdpass, &fd, "-vdpa");
+                }
+            }
+        }
     }
 
     if (vm->def->vsock) {
@@ -1129,6 +1161,8 @@ mymain(void)
     DO_TEST_CAPS_VER("disk-vhostuser-numa", "4.2.0");
     DO_TEST_CAPS_LATEST("disk-vhostuser-numa");
     DO_TEST_CAPS_LATEST("disk-vhostuser");
+    DO_TEST_CAPS_ARCH_LATEST_FULL("disk-vhostvdpa", "x86_64",
+                                  ARG_VDPA_FD, "/dev/vhost-vdpa-0", 801);
     DO_TEST_CAPS_LATEST_PARSE_ERROR("disk-device-lun-type-invalid");
     DO_TEST_CAPS_LATEST_PARSE_ERROR("disk-attaching-partition-nosupport");
     DO_TEST_CAPS_LATEST("disk-usb-device");
diff --git a/tests/testutilsqemu.c b/tests/testutilsqemu.c
index bb3e1b52c5..7f8bf4c01c 100644
--- a/tests/testutilsqemu.c
+++ b/tests/testutilsqemu.c
@@ -709,6 +709,17 @@ testQemuInfoSetArgs(struct testQemuInfo *info,
             break;
         }
 
+        case ARG_VDPA_FD: {
+            const char *vdpadev = va_arg(argptr, char *);
+            int vdpafd = va_arg(argptr, unsigned int);
+
+            if (!info->args.vdpafds)
+                info->args.vdpafds = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+
+            g_hash_table_insert(info->args.vdpafds, g_strdup(vdpadev), GINT_TO_POINTER(vdpafd));
+            break;
+        }
+
         case ARG_END:
         default:
             info->args.invalidarg = true;
diff --git a/tests/testutilsqemu.h b/tests/testutilsqemu.h
index 2d7e2f7779..8398627f71 100644
--- a/tests/testutilsqemu.h
+++ b/tests/testutilsqemu.h
@@ -49,6 +49,7 @@ typedef enum {
     ARG_CAPS_VARIANT,
     ARG_CAPS_HOST_CPU_MODEL,
     ARG_FD_GROUP, /* name, nfds, fd[0], ... fd[n-1] */
+    ARG_VDPA_FD, /* vdpadev, fd */
     ARG_END,
 } testQemuInfoArgName;
 
@@ -85,6 +86,7 @@ struct testQemuArgs {
     qemuTestCPUDef capsHostCPUModel;
     int gic;
     GHashTable *fds;
+    GHashTable *vdpafds;
     bool invalidarg;
 };
 
-- 
2.41.0