Currently, nbdkit support will automatically be enabled as long as
the pidfd_open(2) syscall is available. Optionally, libnbd is used
to generate more user-friendly error messages.

In theory this is all good, since use of nbdkit is supposed to be
transparent to the user. In practice, however, there is a problem:
if support for it is enabled at build time and the necessary
runtime components are installed, nbdkit will always be preferred,
with no way for the user to opt out.

This will arguably be fine in the long run, but right now none of
the platforms that we target ships with a SELinux policy that
allows libvirt to launch nbdkit, and the AppArmor policy that we
maintain ourselves hasn't been updated either.

So, in practice, as of today having nbdkit installed on the host
makes network disks completely unusable unless you're willing to
compromise the overall security of the system by disabling
SELinux/AppArmor.

In order to make the transition smoother, provide a convenient
way for users and distro packagers to disable nbdkit support at
compile time until SELinux and AppArmor are ready.

In the process, detection is completely overhauled. libnbd is
made mandatory when nbdkit support is enabled, since availability
across operating systems is comparable and offering users the
option to make error messages worse doesn't make a lot of sense;
we also make sure that an explicit request from the user to
enable/disable nbdkit support is either complied with, or results
in a build failure when that's not possible. Last but not least,
we avoid linking against libnbd when nbdkit support is disabled.

At the RPM level, we disable the feature when building against
RHEL 8, which doesn't have pidfd_open(2), and also allow it to
be disabled at build time the same as other optional features,
that is, by passing "--define '_without_nbdkit 1'" to rpmbuild.
Finally, if nbdkit support has been disabled, installing libvirt
will no longer drag it in as a (weak) dependency.

Signed-off-by: Andrea Bolognani <abologna@redhat.com>
---
 libvirt.spec.in        | 23 ++++++++++++++++++++---
 meson.build            | 29 +++++++++++++++++++++--------
 meson_options.txt      |  2 +-
 src/qemu/qemu_nbdkit.c |  6 +++---
 4 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/libvirt.spec.in b/libvirt.spec.in
index f3d21ccc8f..2ea465348c 100644
--- a/libvirt.spec.in
+++ b/libvirt.spec.in
@@ -95,6 +95,7 @@
 %define with_fuse             0
 %define with_sanlock          0
 %define with_numad            0
+%define with_nbdkit           0
 %define with_firewalld_zone   0
 %define with_netcf            0
 %define with_libssh2          0
@@ -173,6 +174,13 @@
     %endif
 %endif
 
+# nbdkit support requires pidfd_open(2), which is not in RHEL 8
+%if %{with_qemu}
+    %if 0%{?fedora} || 0%{?rhel} >= 9
+        %define with_nbdkit 0%{!?_without_nbdkit:1}
+    %endif
+%endif
+
 %ifarch %{arches_dmidecode}
     %define with_dmidecode 0%{!?_without_dmidecode:1}
 %endif
@@ -312,6 +320,9 @@ BuildRequires: util-linux
 BuildRequires: libacl-devel
 # From QEMU RPMs, used by virstoragetest
 BuildRequires: /usr/bin/qemu-img
+%endif
+# nbdkit support requires libnbd
+%if %{with_nbdkit}
 BuildRequires: libnbd-devel
 %endif
 # For LVM drivers
@@ -769,9 +780,11 @@ Requires: numad
 Recommends: passt
 Recommends: passt-selinux
     %endif
+    %if %{with_nbdkit}
 Recommends: nbdkit
 Recommends: nbdkit-curl-plugin
 Recommends: nbdkit-ssh-plugin
+    %endif
 
 %description daemon-driver-qemu
 The qemu driver plugin for the libvirtd daemon, providing
@@ -1078,10 +1091,8 @@ exit 1
 
 %if %{with_qemu}
     %define arg_qemu -Ddriver_qemu=enabled
-    %define arg_libnbd -Dlibnbd=enabled
 %else
     %define arg_qemu -Ddriver_qemu=disabled
-    %define arg_libnbd -Dlibnbd=disabled
 %endif
 
 %if %{with_openvz}
@@ -1158,6 +1169,12 @@ exit 1
     %define arg_numad -Dnumad=disabled
 %endif
 
+%if %{with_nbdkit}
+    %define arg_nbdkit -Dnbdkit=enabled
+%else
+    %define arg_nbdkit -Dnbdkit=disabled
+%endif
+
 %if %{with_fuse}
     %define arg_fuse -Dfuse=enabled
 %else
@@ -1270,7 +1287,7 @@ export SOURCE_DATE_EPOCH=$(stat --printf='%Y' %{_specdir}/libvirt.spec)
            -Dyajl=enabled \
            %{?arg_sanlock} \
            -Dlibpcap=enabled \
-           %{?arg_libnbd} \
+           %{?arg_nbdkit} \
            -Dlibnl=enabled \
            -Daudit=enabled \
            -Ddtrace=enabled \
diff --git a/meson.build b/meson.build
index 6fa1f74670..de23fbda1e 100644
--- a/meson.build
+++ b/meson.build
@@ -1011,10 +1011,27 @@ endif
 libiscsi_version = '1.18.0'
 libiscsi_dep = dependency('libiscsi', version: '>=' + libiscsi_version, required: get_option('libiscsi'))
 
-libnbd_version = '1.0'
-libnbd_dep = dependency('libnbd', version: '>=' + libnbd_version, required: get_option('libnbd'))
-if libnbd_dep.found()
-  conf.set('WITH_LIBNBD', 1)
+if not get_option('nbdkit').disabled()
+  libnbd_version = '1.0'
+  libnbd_dep = dependency('libnbd', version: '>=' + libnbd_version, required: false)
+
+  nbdkit_requested = get_option('nbdkit').enabled()
+  nbdkit_syscall_ok = conf.has('WITH_DECL_SYS_PIDFD_OPEN')
+  nbdkit_libnbd_ok = libnbd_dep.found()
+
+  if not nbdkit_syscall_ok and nbdkit_requested
+    error('nbdkit support requires pidfd_open(2)')
+  endif
+  if not nbdkit_libnbd_ok and nbdkit_requested
+    error('nbdkit support requires libnbd')
+  endif
+
+  if nbdkit_syscall_ok and nbdkit_libnbd_ok
+    conf.set('WITH_NBDKIT', 1)
+  endif
+endif
+if not conf.has('WITH_NBDKIT')
+  libnbd_dep = dependency('', required: false)
 endif
 
 libnl_version = '3.0'
@@ -2024,10 +2041,6 @@ endif
 
 conf.set_quoted('TLS_PRIORITY', get_option('tls_priority'))
 
-if conf.has('WITH_DECL_SYS_PIDFD_OPEN')
-  conf.set('WITH_NBDKIT', 1)
-endif
-
 # Various definitions
 
 # Python3 < 3.7 treats the C locale as 7-bit only. We must force env vars so
diff --git a/meson_options.txt b/meson_options.txt
index ba6e49afc5..7c428a9eb0 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -25,7 +25,6 @@ option('curl', type: 'feature', value: 'auto', description: 'curl support')
 option('fuse', type: 'feature', value: 'auto', description: 'fuse support')
 option('glusterfs', type: 'feature', value: 'auto', description: 'glusterfs support')
 option('libiscsi', type: 'feature', value: 'auto', description: 'libiscsi support')
-option('libnbd', type: 'feature', value: 'auto', description: 'libnbd support')
 option('libnl', type: 'feature', value: 'auto', description: 'libnl support')
 option('libpcap', type: 'feature', value: 'auto', description: 'libpcap support')
 option('libssh', type: 'feature', value: 'auto', description: 'libssh support')
@@ -105,6 +104,7 @@ option('loader_nvram', type: 'string', value: '', description: 'Pass list of pai
 option('login_shell', type: 'feature', value: 'auto', description: 'build virt-login-shell')
 option('nss', type: 'feature', value: 'auto', description: 'enable Name Service Switch plugin for resolving guest IP addresses')
 option('numad', type: 'feature', value: 'auto', description: 'use numad to manage CPU placement dynamically')
+option('nbdkit', type: 'feature', value: 'auto', description: 'use nbdkit to access network disks')
 option('pm_utils', type: 'feature', value: 'auto', description: 'use pm-utils for power management')
 option('sysctl_config', type: 'feature', value: 'auto', description: 'Whether to install sysctl configs')
 option('tls_priority', type: 'string', value: 'NORMAL', description: 'set the default TLS session priority string')
diff --git a/src/qemu/qemu_nbdkit.c b/src/qemu/qemu_nbdkit.c
index 17819ca992..3ad63cfaa0 100644
--- a/src/qemu/qemu_nbdkit.c
+++ b/src/qemu/qemu_nbdkit.c
@@ -19,7 +19,7 @@
 
 #include <config.h>
 #include <glib.h>
-#if WITH_LIBNBD
+#if WITH_NBDKIT
 # include <libnbd.h>
 #endif
 #include <sys/syscall.h>
@@ -1159,7 +1159,7 @@ qemuNbdkitProcessStart(qemuNbdkitProcess *proc,
     g_autofree char *basename = g_strdup_printf("%s-nbdkit-%i", vm->def->name, proc->source->id);
     int logfd = -1;
     g_autoptr(qemuLogContext) logContext = NULL;
-#if WITH_LIBNBD
+#if WITH_NBDKIT
     struct nbd_handle *nbd = NULL;
 #endif
 
@@ -1214,7 +1214,7 @@ qemuNbdkitProcessStart(qemuNbdkitProcess *proc,
 
     while (virTimeBackOffWait(&timebackoff)) {
         if (virFileExists(proc->socketfile)) {
-#if WITH_LIBNBD
+#if WITH_NBDKIT
             /* if the disk source was misconfigured, nbdkit will not produce an error
              * until somebody connects to the socket and tries to access the nbd
              * export. This results in poor user experience because the only error we
-- 
2.41.0