Currently, memory device (def->mems) part of cmd line is
generated before any controller. In majority of cases it doesn't
matter because neither of memory devices live on a bus that's
created by an exposed controller (e.g. there's no DIMM
controller, at least not exposed). Except for virtio-mem and
virtio-pmem, which do have a PCI address. And if it so happens
that the device goes onto non-default bus (pci.0) starting such
guest fails, because the controller that creates the desired bus
wasn't processed yet. QEMU processes arguments in order.

For instance, if virtio-mem has address with bus='0x01' QEMU
refuses to start with the following message:

  Bus 'pci.1' not found

Similarly for virtio-pmem. I've successfully tested migration and
changing the order does not affect migration stream.

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2047271
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
---
 src/qemu/qemu_command.c                                   | 6 +++---
 tests/qemuxml2argvdata/hugepages-memaccess.args           | 4 ++--
 tests/qemuxml2argvdata/hugepages-memaccess2.args          | 4 ++--
 tests/qemuxml2argvdata/hugepages-numa-default-dimm.args   | 4 ++--
 .../qemuxml2argvdata/hugepages-nvdimm.x86_64-latest.args  | 4 ++--
 .../qemuxml2argvdata/memfd-memory-numa.x86_64-latest.args | 4 ++--
 tests/qemuxml2argvdata/memory-hotplug-dimm-addr.args      | 8 ++++----
 tests/qemuxml2argvdata/memory-hotplug-dimm.args           | 8 ++++----
 .../memory-hotplug-nvdimm-access.x86_64-latest.args       | 4 ++--
 .../memory-hotplug-nvdimm-align.x86_64-5.2.0.args         | 4 ++--
 .../memory-hotplug-nvdimm-align.x86_64-latest.args        | 4 ++--
 .../memory-hotplug-nvdimm-label.x86_64-5.2.0.args         | 4 ++--
 .../memory-hotplug-nvdimm-label.x86_64-latest.args        | 4 ++--
 .../memory-hotplug-nvdimm-pmem.x86_64-5.2.0.args          | 4 ++--
 .../memory-hotplug-nvdimm-pmem.x86_64-latest.args         | 4 ++--
 .../memory-hotplug-nvdimm-ppc64-abi-update.args           | 4 ++--
 tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64.args   | 4 ++--
 .../memory-hotplug-nvdimm-readonly.x86_64-5.2.0.args      | 4 ++--
 .../memory-hotplug-nvdimm-readonly.x86_64-latest.args     | 4 ++--
 .../memory-hotplug-nvdimm.x86_64-latest.args              | 4 ++--
 .../memory-hotplug-ppc64-nonuma-abi-update.args           | 8 ++++----
 tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma.args   | 8 ++++----
 .../memory-hotplug-virtio-mem.x86_64-latest.args          | 8 ++++----
 .../memory-hotplug-virtio-pmem.x86_64-5.2.0.args          | 4 ++--
 .../memory-hotplug-virtio-pmem.x86_64-latest.args         | 4 ++--
 tests/qemuxml2argvdata/pages-dimm-discard.args            | 8 ++++----
 26 files changed, 65 insertions(+), 65 deletions(-)

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index fc778901d1..e12512a78c 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -10579,9 +10579,6 @@ qemuBuildCommandLine(virQEMUDriver *driver,
         qemuBuildNumaCommandLine(cfg, def, cmd, priv) < 0)
         return NULL;
 
-    if (qemuBuildMemoryDeviceCommandLine(cmd, cfg, def, priv) < 0)
-        return NULL;
-
     virUUIDFormat(def->uuid, uuid);
     virCommandAddArgList(cmd, "-uuid", uuid, NULL);
 
@@ -10631,6 +10628,9 @@ qemuBuildCommandLine(virQEMUDriver *driver,
     if (qemuBuildControllersCommandLine(cmd, def, qemuCaps) < 0)
         return NULL;
 
+    if (qemuBuildMemoryDeviceCommandLine(cmd, cfg, def, priv) < 0)
+        return NULL;
+
     if (qemuBuildHubCommandLine(cmd, def, qemuCaps) < 0)
         return NULL;
 
diff --git a/tests/qemuxml2argvdata/hugepages-memaccess.args b/tests/qemuxml2argvdata/hugepages-memaccess.args
index b89f791697..a369c7f6da 100644
--- a/tests/qemuxml2argvdata/hugepages-memaccess.args
+++ b/tests/qemuxml2argvdata/hugepages-memaccess.args
@@ -24,8 +24,6 @@ QEMU_AUDIO_DRV=none \
 -numa node,nodeid=2,cpus=2,memdev=ram-node2 \
 -object memory-backend-file,id=ram-node3,mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,share=off,prealloc=on,size=1073741824,host-nodes=3,policy=bind \
 -numa node,nodeid=3,cpus=3,memdev=ram-node3 \
--object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,share=on,prealloc=on,size=536870912,host-nodes=0-3,policy=bind \
--device pc-dimm,node=1,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -37,6 +35,8 @@ QEMU_AUDIO_DRV=none \
 -no-acpi \
 -boot strict=on \
 -usb \
+-object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,share=on,prealloc=on,size=536870912,host-nodes=0-3,policy=bind \
+-device pc-dimm,node=1,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
 -drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-ide0-0-0 \
 -device ide-hd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0,bootindex=1 \
 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \
diff --git a/tests/qemuxml2argvdata/hugepages-memaccess2.args b/tests/qemuxml2argvdata/hugepages-memaccess2.args
index 0c296797e9..434ebdaa62 100644
--- a/tests/qemuxml2argvdata/hugepages-memaccess2.args
+++ b/tests/qemuxml2argvdata/hugepages-memaccess2.args
@@ -24,8 +24,6 @@ QEMU_AUDIO_DRV=none \
 -numa node,nodeid=2,cpus=2,memdev=ram-node2 \
 -object memory-backend-file,id=ram-node3,mem-path=/var/lib/libvirt/qemu/ram/-1-QEMUGuest1/ram-node3,share=off,size=1073741824,host-nodes=3,policy=bind \
 -numa node,nodeid=3,cpus=3,memdev=ram-node3 \
--object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,share=on,prealloc=on,size=536870912,host-nodes=0-3,policy=bind \
--device pc-dimm,node=1,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -37,6 +35,8 @@ QEMU_AUDIO_DRV=none \
 -no-acpi \
 -boot strict=on \
 -usb \
+-object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,share=on,prealloc=on,size=536870912,host-nodes=0-3,policy=bind \
+-device pc-dimm,node=1,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
 -drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-ide0-0-0 \
 -device ide-hd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0,bootindex=1 \
 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \
diff --git a/tests/qemuxml2argvdata/hugepages-numa-default-dimm.args b/tests/qemuxml2argvdata/hugepages-numa-default-dimm.args
index f560cabf8e..57bb70346d 100644
--- a/tests/qemuxml2argvdata/hugepages-numa-default-dimm.args
+++ b/tests/qemuxml2argvdata/hugepages-numa-default-dimm.args
@@ -19,8 +19,6 @@ QEMU_AUDIO_DRV=none \
 -mem-prealloc \
 -mem-path /dev/hugepages2M/libvirt/qemu/-1-fedora \
 -numa node,nodeid=0,cpus=0-1,mem=1024 \
--object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages1G/libvirt/qemu/-1-fedora,size=1073741824,host-nodes=1-3,policy=bind \
--device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
 -uuid 63840878-0deb-4095-97e6-fc444d9bc9fa \
 -display none \
 -no-user-config \
@@ -32,4 +30,6 @@ QEMU_AUDIO_DRV=none \
 -no-acpi \
 -boot strict=on \
 -usb \
+-object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages1G/libvirt/qemu/-1-fedora,size=1073741824,host-nodes=1-3,policy=bind \
+-device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
 -msg timestamp=on
diff --git a/tests/qemuxml2argvdata/hugepages-nvdimm.x86_64-latest.args b/tests/qemuxml2argvdata/hugepages-nvdimm.x86_64-latest.args
index 72eae74869..ac719b8c9c 100644
--- a/tests/qemuxml2argvdata/hugepages-nvdimm.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/hugepages-nvdimm.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-file","id":"ram-node0","mem-path":"/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1","share":true,"prealloc":true,"size":1073741824}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":true,"prealloc":true,"size":536870912}' \
--device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":true,"prealloc":true,"size":536870912}' \
+-device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -audiodev '{"id":"audio1","driver":"none"}' \
 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
 -msg timestamp=on
diff --git a/tests/qemuxml2argvdata/memfd-memory-numa.x86_64-latest.args b/tests/qemuxml2argvdata/memfd-memory-numa.x86_64-latest.args
index 9d1f9831d9..6f5a983d3a 100644
--- a/tests/qemuxml2argvdata/memfd-memory-numa.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memfd-memory-numa.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-instance-00000092/.config \
 -smp 8,sockets=1,dies=1,cores=8,threads=1 \
 -object '{"qom-type":"memory-backend-memfd","id":"ram-node0","hugetlb":true,"hugetlbsize":2097152,"share":true,"prealloc":true,"size":15032385536,"host-nodes":[3],"policy":"preferred"}' \
 -numa node,nodeid=0,cpus=0-7,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":true,"prealloc":true,"size":536870912,"host-nodes":[3],"policy":"preferred"}' \
--device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid 126f2720-6f8e-45ab-a886-ec9277079a67 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-instance-00000092/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":true,"prealloc":true,"size":536870912,"host-nodes":[3],"policy":"preferred"}' \
+-device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -audiodev '{"id":"audio1","driver":"none"}' \
 -device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.0","addr":"0x3"}' \
 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-dimm-addr.args b/tests/qemuxml2argvdata/memory-hotplug-dimm-addr.args
index 907072d55d..b36117f1e8 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-dimm-addr.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-dimm-addr.args
@@ -17,10 +17,6 @@ QEMU_AUDIO_DRV=none \
 -realtime mlock=off \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=214 \
--object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,prealloc=on,size=536870912,host-nodes=1-3,policy=bind \
--device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
--object memory-backend-ram,id=memdimm2,size=536870912 \
--device pc-dimm,node=0,memdev=memdimm2,id=dimm2,slot=2 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -32,6 +28,10 @@ QEMU_AUDIO_DRV=none \
 -no-acpi \
 -boot strict=on \
 -usb \
+-object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,prealloc=on,size=536870912,host-nodes=1-3,policy=bind \
+-device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
+-object memory-backend-ram,id=memdimm2,size=536870912 \
+-device pc-dimm,node=0,memdev=memdimm2,id=dimm2,slot=2 \
 -drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-ide0-0-0 \
 -device ide-hd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0,bootindex=1 \
 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x2 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-dimm.args b/tests/qemuxml2argvdata/memory-hotplug-dimm.args
index 5d87f4a3ef..72c2803c5e 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-dimm.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-dimm.args
@@ -17,10 +17,6 @@ QEMU_AUDIO_DRV=none \
 -realtime mlock=off \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=214 \
--object memory-backend-ram,id=memdimm0,size=536870912 \
--device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
--object memory-backend-file,id=memdimm1,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,prealloc=on,size=536870912,host-nodes=1-3,policy=bind \
--device pc-dimm,node=0,memdev=memdimm1,id=dimm1,slot=1 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -32,6 +28,10 @@ QEMU_AUDIO_DRV=none \
 -no-acpi \
 -boot strict=on \
 -usb \
+-object memory-backend-ram,id=memdimm0,size=536870912 \
+-device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
+-object memory-backend-file,id=memdimm1,mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,prealloc=on,size=536870912,host-nodes=1-3,policy=bind \
+-device pc-dimm,node=0,memdev=memdimm1,id=dimm1,slot=1 \
 -drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-ide0-0-0 \
 -device ide-hd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0,bootindex=1 \
 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x2 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-access.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-access.x86_64-latest.args
index 1d4fee4b6e..5967b3566e 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-access.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-access.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":224395264}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912}' \
--device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912}' \
+-device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-5.2.0.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-5.2.0.args
index 6c28c86004..0b29a6fded 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-5.2.0.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-5.2.0.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object memory-backend-ram,id=ram-node0,size=224395264 \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,prealloc=on,size=536870912,align=2097152 \
--device nvdimm,node=0,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,prealloc=on,size=536870912,align=2097152 \
+-device nvdimm,node=0,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device ide-hd,bus=ide.0,unit=0,drive=libvirt-1-format,id=ide0-0-0,bootindex=1 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-latest.args
index 711864037b..dd6884ca14 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-align.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":224395264}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912,"align":2097152}' \
--device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912,"align":2097152}' \
+-device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-5.2.0.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-5.2.0.args
index c45b401af5..7c67161b78 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-5.2.0.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-5.2.0.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object memory-backend-ram,id=ram-node0,size=224395264 \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,prealloc=on,size=536870912 \
--device nvdimm,node=0,label-size=131072,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,prealloc=on,size=536870912 \
+-device nvdimm,node=0,label-size=131072,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device ide-hd,bus=ide.0,unit=0,drive=libvirt-1-format,id=ide0-0-0,bootindex=1 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-latest.args
index ca4bd6c406..2a1ae03004 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-label.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":224395264}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912}' \
--device '{"driver":"nvdimm","node":0,"label-size":131072,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912}' \
+-device '{"driver":"nvdimm","node":0,"label-size":131072,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-5.2.0.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-5.2.0.args
index 3f35d4dca2..c4dae0cfd1 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-5.2.0.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-5.2.0.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object memory-backend-ram,id=ram-node0,size=224395264 \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,size=536870912,pmem=on \
--device nvdimm,node=0,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,size=536870912,pmem=on \
+-device nvdimm,node=0,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device ide-hd,bus=ide.0,unit=0,drive=libvirt-1-format,id=ide0-0-0,bootindex=1 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-latest.args
index 68174e96b5..a0d30a15e9 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-pmem.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":224395264}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"size":536870912,"pmem":true}' \
--device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"size":536870912,"pmem":true}' \
+-device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64-abi-update.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64-abi-update.args
index ebdb0429d0..55b16c9dc2 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64-abi-update.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64-abi-update.args
@@ -17,8 +17,6 @@ QEMU_AUDIO_DRV=none \
 -realtime mlock=off \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=1024 \
--object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,prealloc=on,size=537001984 \
--device nvdimm,node=0,label-size=131072,uuid=49545eb3-75e1-2d0a-acdd-f0294406c99e,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -28,4 +26,6 @@ QEMU_AUDIO_DRV=none \
 -rtc base=utc \
 -no-shutdown \
 -boot strict=on \
+-object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,prealloc=on,size=537001984 \
+-device nvdimm,node=0,label-size=131072,uuid=49545eb3-75e1-2d0a-acdd-f0294406c99e,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -msg timestamp=on
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64.args
index ebdb0429d0..55b16c9dc2 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-ppc64.args
@@ -17,8 +17,6 @@ QEMU_AUDIO_DRV=none \
 -realtime mlock=off \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=1024 \
--object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,prealloc=on,size=537001984 \
--device nvdimm,node=0,label-size=131072,uuid=49545eb3-75e1-2d0a-acdd-f0294406c99e,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -28,4 +26,6 @@ QEMU_AUDIO_DRV=none \
 -rtc base=utc \
 -no-shutdown \
 -boot strict=on \
+-object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,prealloc=on,size=537001984 \
+-device nvdimm,node=0,label-size=131072,uuid=49545eb3-75e1-2d0a-acdd-f0294406c99e,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -msg timestamp=on
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-5.2.0.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-5.2.0.args
index bca2f286ba..b1873c100f 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-5.2.0.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-5.2.0.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object memory-backend-ram,id=ram-node0,size=224395264 \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,prealloc=on,size=536870912 \
--device nvdimm,node=0,unarmed=on,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-object memory-backend-file,id=memnvdimm0,mem-path=/tmp/nvdimm,share=off,prealloc=on,size=536870912 \
+-device nvdimm,node=0,unarmed=on,memdev=memnvdimm0,id=nvdimm0,slot=0 \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device ide-hd,bus=ide.0,unit=0,drive=libvirt-1-format,id=ide0-0-0,bootindex=1 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-latest.args
index a1709c918c..9ad3f239da 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm-readonly.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":224395264}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912}' \
--device '{"driver":"nvdimm","node":0,"unarmed":true,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","share":false,"prealloc":true,"size":536870912}' \
+-device '{"driver":"nvdimm","node":0,"unarmed":true,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-nvdimm.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-nvdimm.x86_64-latest.args
index 1dd3f9f1a4..4e8eea2bcf 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-nvdimm.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-nvdimm.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":1073741824}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","prealloc":true,"size":536870912}' \
--device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memnvdimm0","mem-path":"/tmp/nvdimm","prealloc":true,"size":536870912}' \
+-device '{"driver":"nvdimm","node":0,"memdev":"memnvdimm0","id":"nvdimm0","slot":0}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma-abi-update.args b/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma-abi-update.args
index 5380c9e805..e8a6451841 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma-abi-update.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma-abi-update.args
@@ -16,10 +16,6 @@ QEMU_AUDIO_DRV=none \
 -m size=1048576k,slots=16,maxmem=4194304k \
 -realtime mlock=off \
 -smp 1,sockets=1,cores=1,threads=1 \
--object memory-backend-ram,id=memdimm0,size=536870912 \
--device pc-dimm,memdev=memdimm0,id=dimm0,slot=0 \
--object memory-backend-ram,id=memdimm1,size=536870912 \
--device pc-dimm,memdev=memdimm1,id=dimm1,slot=1 \
 -uuid 49545eb3-75e1-2d0a-acdd-f0294406c99e \
 -display none \
 -no-user-config \
@@ -29,4 +25,8 @@ QEMU_AUDIO_DRV=none \
 -rtc base=utc \
 -no-shutdown \
 -boot strict=on \
+-object memory-backend-ram,id=memdimm0,size=536870912 \
+-device pc-dimm,memdev=memdimm0,id=dimm0,slot=0 \
+-object memory-backend-ram,id=memdimm1,size=536870912 \
+-device pc-dimm,memdev=memdimm1,id=dimm1,slot=1 \
 -msg timestamp=on
diff --git a/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma.args b/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma.args
index acff36eb78..181409cde0 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-ppc64-nonuma.args
@@ -16,10 +16,6 @@ QEMU_AUDIO_DRV=none \
 -m size=1310720k,slots=16,maxmem=4194304k \
 -realtime mlock=off \
 -smp 1,sockets=1,cores=1,threads=1 \
--object memory-backend-ram,id=memdimm0,size=536870912 \
--device pc-dimm,memdev=memdimm0,id=dimm0,slot=0 \
--object memory-backend-ram,id=memdimm1,size=536870912 \
--device pc-dimm,memdev=memdimm1,id=dimm1,slot=1 \
 -uuid 49545eb3-75e1-2d0a-acdd-f0294406c99e \
 -display none \
 -no-user-config \
@@ -29,4 +25,8 @@ QEMU_AUDIO_DRV=none \
 -rtc base=utc \
 -no-shutdown \
 -boot strict=on \
+-object memory-backend-ram,id=memdimm0,size=536870912 \
+-device pc-dimm,memdev=memdimm0,id=dimm0,slot=0 \
+-object memory-backend-ram,id=memdimm1,size=536870912 \
+-device pc-dimm,memdev=memdimm1,id=dimm1,slot=1 \
 -msg timestamp=on
diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
index 77dbc0c89c..dba2452ccf 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
@@ -18,10 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":2145386496}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-ram","id":"memvirtiomem0","reserve":false,"size":1073741824}' \
--device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":536870912,"memdev":"memvirtiomem0","id":"virtiomem0","bus":"pci.0","addr":"0x2"}' \
--object '{"qom-type":"memory-backend-file","id":"memvirtiomem1","mem-path":"/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1","reserve":false,"size":2147483648,"host-nodes":[1,2,3],"policy":"bind"}' \
--device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":1073741824,"memdev":"memvirtiomem1","prealloc":true,"id":"virtiomem1","bus":"pci.0","addr":"0x3"}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -33,6 +29,10 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-ram","id":"memvirtiomem0","reserve":false,"size":1073741824}' \
+-device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":536870912,"memdev":"memvirtiomem0","id":"virtiomem0","bus":"pci.0","addr":"0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memvirtiomem1","mem-path":"/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1","reserve":false,"size":2147483648,"host-nodes":[1,2,3],"policy":"bind"}' \
+-device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":1073741824,"memdev":"memvirtiomem1","prealloc":true,"id":"virtiomem1","bus":"pci.0","addr":"0x3"}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args
index 17fd98fb88..8fa678c209 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object memory-backend-ram,id=ram-node0,size=2145386496 \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object memory-backend-file,id=memvirtiopmem0,mem-path=/tmp/virtio_pmem,share=on,size=536870912 \
--device virtio-pmem-pci,memdev=memvirtiopmem0,id=virtiopmem0,bus=pci.0,addr=0x5 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-object memory-backend-file,id=memvirtiopmem0,mem-path=/tmp/virtio_pmem,share=on,size=536870912 \
+-device virtio-pmem-pci,memdev=memvirtiopmem0,id=virtiopmem0,bus=pci.0,addr=0x5 \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device ide-hd,bus=ide.0,unit=0,drive=libvirt-1-format,id=ide0-0-0,bootindex=1 \
diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args
index fdbdfa00c3..25b521978f 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args
@@ -18,8 +18,6 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -smp 2,sockets=2,dies=1,cores=1,threads=1 \
 -object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":2145386496}' \
 -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \
--object '{"qom-type":"memory-backend-file","id":"memvirtiopmem0","mem-path":"/tmp/virtio_pmem","share":true,"size":536870912}' \
--device '{"driver":"virtio-pmem-pci","memdev":"memvirtiopmem0","id":"virtiopmem0","bus":"pci.0","addr":"0x5"}' \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -display none \
 -no-user-config \
@@ -31,6 +29,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-acpi \
 -boot strict=on \
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
+-object '{"qom-type":"memory-backend-file","id":"memvirtiopmem0","mem-path":"/tmp/virtio_pmem","share":true,"size":536870912}' \
+-device '{"driver":"virtio-pmem-pci","memdev":"memvirtiopmem0","id":"virtiopmem0","bus":"pci.0","addr":"0x5"}' \
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \
diff --git a/tests/qemuxml2argvdata/pages-dimm-discard.args b/tests/qemuxml2argvdata/pages-dimm-discard.args
index 2ebe9c1350..e63c908549 100644
--- a/tests/qemuxml2argvdata/pages-dimm-discard.args
+++ b/tests/qemuxml2argvdata/pages-dimm-discard.args
@@ -17,10 +17,6 @@ QEMU_AUDIO_DRV=none \
 -realtime mlock=off \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=1024 \
--object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages1G/libvirt/qemu/-1-fedora,prealloc=on,size=1073741824,host-nodes=1-3,policy=bind \
--device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
--object memory-backend-file,id=memdimm1,mem-path=/var/lib/libvirt/qemu/ram/-1-fedora/dimm1,discard-data=on,share=off,size=536870912 \
--device pc-dimm,node=0,memdev=memdimm1,id=dimm1,slot=1 \
 -uuid 63840878-0deb-4095-97e6-fc444d9bc9fa \
 -display none \
 -no-user-config \
@@ -32,4 +28,8 @@ QEMU_AUDIO_DRV=none \
 -no-acpi \
 -boot strict=on \
 -usb \
+-object memory-backend-file,id=memdimm0,mem-path=/dev/hugepages1G/libvirt/qemu/-1-fedora,prealloc=on,size=1073741824,host-nodes=1-3,policy=bind \
+-device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
+-object memory-backend-file,id=memdimm1,mem-path=/var/lib/libvirt/qemu/ram/-1-fedora/dimm1,discard-data=on,share=off,size=536870912 \
+-device pc-dimm,node=0,memdev=memdimm1,id=dimm1,slot=1 \
 -msg timestamp=on
-- 
2.34.1

On Thu, Jan 27, 2022 at 02:47:43PM +0100, Michal Privoznik wrote:
> Currently, memory device (def->mems) part of cmd line is
> generated before any controller. In majority of cases it doesn't
> matter because neither of memory devices live on a bus that's
> created by an exposed controller (e.g. there's no DIMM
> controller, at least not exposed). Except for virtio-mem and
> virtio-pmem, which do have a PCI address. And if it so happens
> that the device goes onto non-default bus (pci.0) starting such
> guest fails, because the controller that creates the desired bus
> wasn't processed yet. QEMU processes arguments in order.
>
> For instance, if virtio-mem has address with bus='0x01' QEMU
> refuses to start with the following message:
>
>   Bus 'pci.1' not found
>
> Similarly for virtio-pmem. I've successfully tested migration and
> changing the order does not affect migration stream.

Maybe we should add a test case where the memory device is not on the
root bus? We can't catch the QEMU error of course, but that would at
least serve as some sort of implicit documentation of the fact that
we expect that scenario to work.

For this patch,

  Reviewed-by: Andrea Bolognani <abologna@redhat.com>

-- 
Andrea Bolognani / Red Hat / Virtualization

On 2/2/22 13:44, Andrea Bolognani wrote:
> On Thu, Jan 27, 2022 at 02:47:43PM +0100, Michal Privoznik wrote:
>> Currently, memory device (def->mems) part of cmd line is
>> generated before any controller. In majority of cases it doesn't
>> matter because neither of memory devices live on a bus that's
>> created by an exposed controller (e.g. there's no DIMM
>> controller, at least not exposed). Except for virtio-mem and
>> virtio-pmem, which do have a PCI address. And if it so happens
>> that the device goes onto non-default bus (pci.0) starting such
>> guest fails, because the controller that creates the desired bus
>> wasn't processed yet. QEMU processes arguments in order.
>>
>> For instance, if virtio-mem has address with bus='0x01' QEMU
>> refuses to start with the following message:
>>
>>   Bus 'pci.1' not found
>>
>> Similarly for virtio-pmem. I've successfully tested migration and
>> changing the order does not affect migration stream.
> 
> Maybe we should add a test case where the memory device is not on the
> root bus? We can't catch the QEMU error of course, but that would at
> least serve as some sort of implicit documentation of the fact that
> we expect that scenario to work.

Sure, I can do that. I'm not that convinced on its value, but I can
alter an existing test case.

> 
> For this patch,
> 
>   Reviewed-by: Andrea Bolognani <abologna@redhat.com>
> 

Pushed, thanks.

Michal

On Wed, Feb 02, 2022 at 02:26:23PM +0100, Michal Prívozník wrote:
> On 2/2/22 13:44, Andrea Bolognani wrote:
> > Maybe we should add a test case where the memory device is not on the
> > root bus? We can't catch the QEMU error of course, but that would at
> > least serve as some sort of implicit documentation of the fact that
> > we expect that scenario to work.
>
> Sure, I can do that. I'm not that convinced on its value, but I can
> alter an existing test case.

If you don't think it's going to be useful, then just don't do it :)

-- 
Andrea Bolognani / Red Hat / Virtualization

On 2/2/22 14:34, Andrea Bolognani wrote:
> On Wed, Feb 02, 2022 at 02:26:23PM +0100, Michal Prívozník wrote:
>> On 2/2/22 13:44, Andrea Bolognani wrote:
>>> Maybe we should add a test case where the memory device is not on the
>>> root bus? We can't catch the QEMU error of course, but that would at
>>> least serve as some sort of implicit documentation of the fact that
>>> we expect that scenario to work.
>>
>> Sure, I can do that. I'm not that convinced on its value, but I can
>> alter an existing test case.
> 
> If you don't think it's going to be useful, then just don't do it :)
> 

Yeah, thing is, this bug depends on how QEMU behaves (namely order in
which it parses arguments). Libvirt produced "correct" output (in sense
that devices that need to be there are there). So unless we are starting
QEMU we won't notice the QEMU behavior.

Think of this in a different way, if QEMU started parsing arguments in
different order (highly improbable, but let's assume that for a while).
 Even if I added test case as you suggest, our test suite would not
notice anything different, and yet - users would be unable to start
their guests.

But maybe I'm missing something and we might get something valuable from
such test? What was your thinking?

Michal

On Wed, Feb 02, 2022 at 03:11:03PM +0100, Michal Prívozník wrote:
> On 2/2/22 14:34, Andrea Bolognani wrote:
> > On Wed, Feb 02, 2022 at 02:26:23PM +0100, Michal Prívozník wrote:
> >> On 2/2/22 13:44, Andrea Bolognani wrote:
> >>> Maybe we should add a test case where the memory device is not on the
> >>> root bus? We can't catch the QEMU error of course, but that would at
> >>> least serve as some sort of implicit documentation of the fact that
> >>> we expect that scenario to work.
> >>
> >> Sure, I can do that. I'm not that convinced on its value, but I can
> >> alter an existing test case.
> >
> > If you don't think it's going to be useful, then just don't do it :)
>
> Yeah, thing is, this bug depends on how QEMU behaves (namely order in
> which it parses arguments). Libvirt produced "correct" output (in sense
> that devices that need to be there are there). So unless we are starting
> QEMU we won't notice the QEMU behavior.
>
> Think of this in a different way, if QEMU started parsing arguments in
> different order (highly improbable, but let's assume that for a while).
>  Even if I added test case as you suggest, our test suite would not
> notice anything different, and yet - users would be unable to start
> their guests.
>
> But maybe I'm missing something and we might get something valuable from
> such test? What was your thinking?

I was imagining some change happening in the future under the
assumption that memory devices will only ever be plugged into
pci(e).0: the current test suite would not catch that, but if we had
at least one case in which a memory device is plugged into a
different bus then we'd have a chance of noticing.

Maybe that's a bit far-fetched, but even though the potential gain
would be small the effort needed to obtain it would also be small
(something along the lines of the diff below), so overall it seems
like a decent deal to me :)

I agree completely with you that we can't have a proper test for the
issue that your patch fixed without actually running QEMU.



diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
index dba2452ccf..5aa8110aeb 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args
@@ -28,11 +28,12 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
 -no-shutdown \
 -no-acpi \
 -boot strict=on \
+-device '{"driver":"pci-bridge","chassis_nr":1,"id":"pci.1","bus":"pci.0","addr":"0x3"}'
\
 -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}'
\
 -object '{"qom-type":"memory-backend-ram","id":"memvirtiomem0","reserve":false,"size":1073741824}'
\
 -device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":536870912,"memdev":"memvirtiomem0","id":"virtiomem0","bus":"pci.0","addr":"0x2"}'
\
 -object '{"qom-type":"memory-backend-file","id":"memvirtiomem1","mem-path":"/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1","reserve":false,"size":2147483648,"host-nodes":[1,2,3],"policy":"bind"}'
\
--device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":1073741824,"memdev":"memvirtiomem1","prealloc":true,"id":"virtiomem1","bus":"pci.0","addr":"0x3"}'
\
+-device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":1073741824,"memdev":"memvirtiomem1","prealloc":true,"id":"virtiomem1","bus":"pci.1","addr":"0x1"}'
\
 -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}'
\
 -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}'
\
 -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}'
\
diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml
b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml
index ea9f5e8765..73036d8602 100644
--- a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml
+++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml
@@ -35,6 +35,11 @@
       <address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x2'/>
     </controller>
     <controller type='pci' index='0' model='pci-root'/>
+    <controller type='pci' index='1' model='pci-bridge'>
+      <model name='pci-bridge'/>
+      <target chassisNr='1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x0'/>
+    </controller>
     <input type='mouse' bus='ps2'/>
     <input type='keyboard' bus='ps2'/>
     <audio id='1' type='none'/>
@@ -61,7 +66,7 @@
         <block unit='KiB'>2048</block>
         <requested unit='KiB'>1048576</requested>
       </target>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x0'/>
+      <address type='pci' domain='0x0000' bus='0x01' slot='0x01'
function='0x0'/>
     </memory>
   </devices>
 </domain>
-- 
Andrea Bolognani / Red Hat / Virtualization

On Wed, Feb 02, 2022 at 06:57:09AM -0800, Andrea Bolognani wrote:
> I was imagining some change happening in the future under the
> assumption that memory devices will only ever be plugged into
> pci(e).0: the current test suite would not catch that, but if we had
> at least one case in which a memory device is plugged into a
> different bus then we'd have a chance of noticing.
>
> Maybe that's a bit far-fetched, but even though the potential gain
> would be small the effort needed to obtain it would also be small
> (something along the lines of the diff below), so overall it seems
> like a decent deal to me :)

I've actually gone ahead and posted that as a proper patch:

  https://listman.redhat.com/archives/libvir-list/2022-February/msg00097.html

-- 
Andrea Bolognani / Red Hat / Virtualization