[libvirt PATCH v5 00/32] Use nbdkit for http/ftp/ssh network drives in libvirt

Jonathon Jongsma posted 32 patches 1 year, 1 month ago
Patches applied successfully (tree, apply log)
git fetch https://github.com/patchew-project/libvirt tags/patchew/20230214170819.3143132-1-jjongsma@redhat.com
There is a newer version of this series
build-aux/syntax-check.mk                     |    2 +-
docs/formatdomain.rst                         |   41 +-
meson.build                                   |   14 +
meson_options.txt                             |    1 +
po/POTFILES                                   |    2 +
src/conf/domain_conf.c                        |   32 +
src/conf/schemas/domaincommon.rng             |   53 +
src/conf/storage_source_conf.c                |    3 +
src/conf/storage_source_conf.h                |    6 +-
src/libvirt_private.syms                      |    1 +
src/qemu/meson.build                          |    3 +
src/qemu/qemu_block.c                         |  162 +-
src/qemu/qemu_conf.c                          |   22 +
src/qemu/qemu_conf.h                          |    6 +
src/qemu/qemu_domain.c                        |  419 ++----
src/qemu/qemu_domain.h                        |   39 +-
src/qemu/qemu_driver.c                        |    3 +
src/qemu/qemu_extdevice.c                     |   56 +
src/qemu/qemu_hotplug.c                       |    7 +
src/qemu/qemu_logcontext.c                    |  329 +++++
src/qemu/qemu_logcontext.h                    |   41 +
src/qemu/qemu_nbdkit.c                        | 1298 +++++++++++++++++
src/qemu/qemu_nbdkit.h                        |  113 ++
src/qemu/qemu_nbdkitpriv.h                    |   31 +
src/qemu/qemu_process.c                       |  114 +-
src/util/vircommand.c                         |   19 +-
src/util/vircommand.h                         |    8 +
src/util/vircommandpriv.h                     |    4 +
src/util/virfilecache.c                       |   14 +-
src/util/virfilecache.h                       |    2 +-
tests/meson.build                             |    1 +
tests/qemublocktest.c                         |    2 +-
...w2-invalid.json => network-ssh-qcow2.json} |    0
...cow2-invalid.xml => network-ssh-qcow2.xml} |    0
.../disk-cdrom-network.args.disk0             |    6 +
.../disk-cdrom-network.args.disk1             |    8 +
.../disk-cdrom-network.args.disk1.pipe.778    |    1 +
.../disk-cdrom-network.args.disk2             |    8 +
.../disk-cdrom-network.args.disk2.pipe.780    |    1 +
.../disk-network-http.args.disk0              |    6 +
.../disk-network-http.args.disk1              |    5 +
.../disk-network-http.args.disk2              |    6 +
.../disk-network-http.args.disk2.pipe.778     |    1 +
.../disk-network-http.args.disk3              |    7 +
.../disk-network-http.args.disk3.pipe.780     |    1 +
...work-source-curl-nbdkit-backing.args.disk0 |    7 +
...ce-curl-nbdkit-backing.args.disk0.pipe.778 |    1 +
.../disk-network-source-curl.args.disk0       |    7 +
...sk-network-source-curl.args.disk0.pipe.778 |    1 +
.../disk-network-source-curl.args.disk1       |    9 +
...sk-network-source-curl.args.disk1.pipe.780 |    1 +
...sk-network-source-curl.args.disk1.pipe.782 |    1 +
.../disk-network-source-curl.args.disk2       |    7 +
...sk-network-source-curl.args.disk2.pipe.782 |    1 +
...sk-network-source-curl.args.disk2.pipe.784 |    1 +
.../disk-network-source-curl.args.disk3       |    6 +
.../disk-network-source-curl.args.disk4       |    6 +
.../disk-network-ssh-key.args.disk0           |   10 +
.../disk-network-ssh-password.args.disk0      |    9 +
...k-network-ssh-password.args.disk0.pipe.778 |    1 +
.../disk-network-ssh.args.disk0               |    7 +
.../disk-network-ssh.args.disk1               |    8 +
.../disk-network-ssh.args.disk1.pipe.778      |    1 +
.../disk-network-ssh.args.disk2               |    9 +
tests/qemunbdkittest.c                        |  302 ++++
tests/qemustatusxml2xmldata/modern-in.xml     |    4 +
...sk-cdrom-network-nbdkit.x86_64-latest.args |   42 +
.../disk-cdrom-network-nbdkit.xml             |    1 +
...isk-network-http-nbdkit.x86_64-latest.args |   45 +
.../disk-network-http-nbdkit.xml              |    1 +
...rce-curl-nbdkit-backing.x86_64-latest.args |   38 +
...isk-network-source-curl-nbdkit-backing.xml |   45 +
...work-source-curl-nbdkit.x86_64-latest.args |   50 +
.../disk-network-source-curl-nbdkit.xml       |    1 +
...isk-network-source-curl.x86_64-latest.args |   54 +
.../disk-network-source-curl.xml              |   74 +
.../qemuxml2argvdata/disk-network-ssh-key.xml |   33 +
...disk-network-ssh-nbdkit.x86_64-latest.args |   36 +
.../disk-network-ssh-nbdkit.xml               |    1 +
...sk-network-ssh-password.x86_64-latest.args |   36 +
.../disk-network-ssh-password.xml             |   35 +
.../disk-network-ssh.x86_64-latest.args       |   36 +
tests/qemuxml2argvdata/disk-network-ssh.xml   |   32 +
tests/qemuxml2argvtest.c                      |   19 +
tests/testutilsqemu.c                         |   27 +
tests/testutilsqemu.h                         |    5 +
86 files changed, 3433 insertions(+), 475 deletions(-)
create mode 100644 src/qemu/qemu_logcontext.c
create mode 100644 src/qemu/qemu_logcontext.h
create mode 100644 src/qemu/qemu_nbdkit.c
create mode 100644 src/qemu/qemu_nbdkit.h
create mode 100644 src/qemu/qemu_nbdkitpriv.h
rename tests/qemublocktestdata/imagecreate/{network-ssh-qcow2-invalid.json => network-ssh-qcow2.json} (100%)
rename tests/qemublocktestdata/imagecreate/{network-ssh-qcow2-invalid.xml => network-ssh-qcow2.xml} (100%)
create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk1
create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk1.pipe.778
create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk2
create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk2.pipe.780
create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk1
create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk2
create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk2.pipe.778
create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk3
create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk3.pipe.780
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl-nbdkit-backing.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl-nbdkit-backing.args.disk0.pipe.778
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk0.pipe.778
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk1
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk1.pipe.780
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk1.pipe.782
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk2
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk2.pipe.782
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk2.pipe.784
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk3
create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk4
create mode 100644 tests/qemunbdkitdata/disk-network-ssh-key.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-network-ssh-password.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-network-ssh-password.args.disk0.pipe.778
create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk0
create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk1
create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk1.pipe.778
create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk2
create mode 100644 tests/qemunbdkittest.c
create mode 100644 tests/qemuxml2argvdata/disk-cdrom-network-nbdkit.x86_64-latest.args
create mode 120000 tests/qemuxml2argvdata/disk-cdrom-network-nbdkit.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-http-nbdkit.x86_64-latest.args
create mode 120000 tests/qemuxml2argvdata/disk-network-http-nbdkit.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit-backing.x86_64-latest.args
create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit-backing.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit.x86_64-latest.args
create mode 120000 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl.x86_64-latest.args
create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-key.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-nbdkit.x86_64-latest.args
create mode 120000 tests/qemuxml2argvdata/disk-network-ssh-nbdkit.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-password.x86_64-latest.args
create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-password.xml
create mode 100644 tests/qemuxml2argvdata/disk-network-ssh.x86_64-latest.args
create mode 100644 tests/qemuxml2argvdata/disk-network-ssh.xml
[libvirt PATCH v5 00/32] Use nbdkit for http/ftp/ssh network drives in libvirt
Posted by Jonathon Jongsma 1 year, 1 month ago
This is the fifth version of this patch series. See
https://bugzilla.redhat.com/show_bug.cgi?id=2016527 for more information about
the goal, but the summary is that RHEL does not want to ship the qemu storage
plugins for curl and ssh.  Handling them outside of the qemu process provides
several advantages such as reduced attack surface and stability.

See previous series for more info:
https://listman.redhat.com/archives/libvir-list/2022-October/235052.html

Note that gitlab CI will not work for this series without changes to the ci
definitions due to the addition of libnbd dependency. It also will require
changes to selinux policy to enable nbdkit to execute in the proper context.

Changes in v5:
 - securely erase secrets, including in virCommand send buffers
 - don't kill a running guest when restarting libvirt if we can't properly
   initialize nbdkit state
 - make sure to restart nbdkit when libvirt restarts if nbdkit has died in the
   meantime
 - simplify ownership of event data used for monitoring and restart of nbdkit
   process
 - other smaller changes suggested by Peter in v4

Jonathon Jongsma (32):
  schema: allow 'ssh' as a protocol for network disks
  qemu: Add functions for determining nbdkit availability
  qemu: expand nbdkit capabilities
  util: Allow virFileCache data to be any GObject
  qemu: implement basic virFileCache for nbdkit caps
  qemu: implement persistent file cache for nbdkit caps
  qemu: use file cache for nbdkit caps
  qemu: Add qemuNbdkitProcess
  qemu: query nbdkit module dir from binary
  qemu: add functions to start and stop nbdkit
  qemu: remove unused 'mode' param from qemuDomainLogContextNew()
  Generalize qemuDomainLogContextNew()
  qemu: Extract qemuDomainLogContext into a new file
  qemu: move qemuProcessReadLog() to qemuLogContext
  qemu: log error output from nbdkit
  tests: add ability to test various nbdkit capabilities
  qemu: split qemuDomainSecretStorageSourcePrepare
  qemu: include nbdkit state in private xml
  util: secure erase virCommand send buffers
  qemu: pass sensitive data to nbdkit via pipe
  qemu: use nbdkit to serve network disks if available
  util: make virCommandSetSendBuffer testable
  tests: add tests for nbdkit invocation
  qemu: add test for authenticating a https network disk
  qemu: Monitor nbdkit process for exit
  qemu: try to connect to nbdkit early to detect errors
  schema: add password configuration for ssh disk
  qemu: implement password auth for ssh disks with nbdkit
  schema: add configuration for host verification of ssh disks
  qemu: implement knownHosts for ssh disks with nbdkit
  schema: add keyfile configuration for ssh disks
  qemu: implement keyfile auth for ssh disk with nbdkit

 build-aux/syntax-check.mk                     |    2 +-
 docs/formatdomain.rst                         |   41 +-
 meson.build                                   |   14 +
 meson_options.txt                             |    1 +
 po/POTFILES                                   |    2 +
 src/conf/domain_conf.c                        |   32 +
 src/conf/schemas/domaincommon.rng             |   53 +
 src/conf/storage_source_conf.c                |    3 +
 src/conf/storage_source_conf.h                |    6 +-
 src/libvirt_private.syms                      |    1 +
 src/qemu/meson.build                          |    3 +
 src/qemu/qemu_block.c                         |  162 +-
 src/qemu/qemu_conf.c                          |   22 +
 src/qemu/qemu_conf.h                          |    6 +
 src/qemu/qemu_domain.c                        |  419 ++----
 src/qemu/qemu_domain.h                        |   39 +-
 src/qemu/qemu_driver.c                        |    3 +
 src/qemu/qemu_extdevice.c                     |   56 +
 src/qemu/qemu_hotplug.c                       |    7 +
 src/qemu/qemu_logcontext.c                    |  329 +++++
 src/qemu/qemu_logcontext.h                    |   41 +
 src/qemu/qemu_nbdkit.c                        | 1298 +++++++++++++++++
 src/qemu/qemu_nbdkit.h                        |  113 ++
 src/qemu/qemu_nbdkitpriv.h                    |   31 +
 src/qemu/qemu_process.c                       |  114 +-
 src/util/vircommand.c                         |   19 +-
 src/util/vircommand.h                         |    8 +
 src/util/vircommandpriv.h                     |    4 +
 src/util/virfilecache.c                       |   14 +-
 src/util/virfilecache.h                       |    2 +-
 tests/meson.build                             |    1 +
 tests/qemublocktest.c                         |    2 +-
 ...w2-invalid.json => network-ssh-qcow2.json} |    0
 ...cow2-invalid.xml => network-ssh-qcow2.xml} |    0
 .../disk-cdrom-network.args.disk0             |    6 +
 .../disk-cdrom-network.args.disk1             |    8 +
 .../disk-cdrom-network.args.disk1.pipe.778    |    1 +
 .../disk-cdrom-network.args.disk2             |    8 +
 .../disk-cdrom-network.args.disk2.pipe.780    |    1 +
 .../disk-network-http.args.disk0              |    6 +
 .../disk-network-http.args.disk1              |    5 +
 .../disk-network-http.args.disk2              |    6 +
 .../disk-network-http.args.disk2.pipe.778     |    1 +
 .../disk-network-http.args.disk3              |    7 +
 .../disk-network-http.args.disk3.pipe.780     |    1 +
 ...work-source-curl-nbdkit-backing.args.disk0 |    7 +
 ...ce-curl-nbdkit-backing.args.disk0.pipe.778 |    1 +
 .../disk-network-source-curl.args.disk0       |    7 +
 ...sk-network-source-curl.args.disk0.pipe.778 |    1 +
 .../disk-network-source-curl.args.disk1       |    9 +
 ...sk-network-source-curl.args.disk1.pipe.780 |    1 +
 ...sk-network-source-curl.args.disk1.pipe.782 |    1 +
 .../disk-network-source-curl.args.disk2       |    7 +
 ...sk-network-source-curl.args.disk2.pipe.782 |    1 +
 ...sk-network-source-curl.args.disk2.pipe.784 |    1 +
 .../disk-network-source-curl.args.disk3       |    6 +
 .../disk-network-source-curl.args.disk4       |    6 +
 .../disk-network-ssh-key.args.disk0           |   10 +
 .../disk-network-ssh-password.args.disk0      |    9 +
 ...k-network-ssh-password.args.disk0.pipe.778 |    1 +
 .../disk-network-ssh.args.disk0               |    7 +
 .../disk-network-ssh.args.disk1               |    8 +
 .../disk-network-ssh.args.disk1.pipe.778      |    1 +
 .../disk-network-ssh.args.disk2               |    9 +
 tests/qemunbdkittest.c                        |  302 ++++
 tests/qemustatusxml2xmldata/modern-in.xml     |    4 +
 ...sk-cdrom-network-nbdkit.x86_64-latest.args |   42 +
 .../disk-cdrom-network-nbdkit.xml             |    1 +
 ...isk-network-http-nbdkit.x86_64-latest.args |   45 +
 .../disk-network-http-nbdkit.xml              |    1 +
 ...rce-curl-nbdkit-backing.x86_64-latest.args |   38 +
 ...isk-network-source-curl-nbdkit-backing.xml |   45 +
 ...work-source-curl-nbdkit.x86_64-latest.args |   50 +
 .../disk-network-source-curl-nbdkit.xml       |    1 +
 ...isk-network-source-curl.x86_64-latest.args |   54 +
 .../disk-network-source-curl.xml              |   74 +
 .../qemuxml2argvdata/disk-network-ssh-key.xml |   33 +
 ...disk-network-ssh-nbdkit.x86_64-latest.args |   36 +
 .../disk-network-ssh-nbdkit.xml               |    1 +
 ...sk-network-ssh-password.x86_64-latest.args |   36 +
 .../disk-network-ssh-password.xml             |   35 +
 .../disk-network-ssh.x86_64-latest.args       |   36 +
 tests/qemuxml2argvdata/disk-network-ssh.xml   |   32 +
 tests/qemuxml2argvtest.c                      |   19 +
 tests/testutilsqemu.c                         |   27 +
 tests/testutilsqemu.h                         |    5 +
 86 files changed, 3433 insertions(+), 475 deletions(-)
 create mode 100644 src/qemu/qemu_logcontext.c
 create mode 100644 src/qemu/qemu_logcontext.h
 create mode 100644 src/qemu/qemu_nbdkit.c
 create mode 100644 src/qemu/qemu_nbdkit.h
 create mode 100644 src/qemu/qemu_nbdkitpriv.h
 rename tests/qemublocktestdata/imagecreate/{network-ssh-qcow2-invalid.json => network-ssh-qcow2.json} (100%)
 rename tests/qemublocktestdata/imagecreate/{network-ssh-qcow2-invalid.xml => network-ssh-qcow2.xml} (100%)
 create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk1
 create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk1.pipe.778
 create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk2
 create mode 100644 tests/qemunbdkitdata/disk-cdrom-network.args.disk2.pipe.780
 create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk1
 create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk2
 create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk2.pipe.778
 create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk3
 create mode 100644 tests/qemunbdkitdata/disk-network-http.args.disk3.pipe.780
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl-nbdkit-backing.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl-nbdkit-backing.args.disk0.pipe.778
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk0.pipe.778
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk1
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk1.pipe.780
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk1.pipe.782
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk2
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk2.pipe.782
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk2.pipe.784
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk3
 create mode 100644 tests/qemunbdkitdata/disk-network-source-curl.args.disk4
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh-key.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh-password.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh-password.args.disk0.pipe.778
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk0
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk1
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk1.pipe.778
 create mode 100644 tests/qemunbdkitdata/disk-network-ssh.args.disk2
 create mode 100644 tests/qemunbdkittest.c
 create mode 100644 tests/qemuxml2argvdata/disk-cdrom-network-nbdkit.x86_64-latest.args
 create mode 120000 tests/qemuxml2argvdata/disk-cdrom-network-nbdkit.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-http-nbdkit.x86_64-latest.args
 create mode 120000 tests/qemuxml2argvdata/disk-network-http-nbdkit.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit-backing.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit-backing.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit.x86_64-latest.args
 create mode 120000 tests/qemuxml2argvdata/disk-network-source-curl-nbdkit.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/disk-network-source-curl.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-key.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-nbdkit.x86_64-latest.args
 create mode 120000 tests/qemuxml2argvdata/disk-network-ssh-nbdkit.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-password.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/disk-network-ssh-password.xml
 create mode 100644 tests/qemuxml2argvdata/disk-network-ssh.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/disk-network-ssh.xml

-- 
2.39.1