Support for nested kvm is handled via a kernel module configuration
adjustment which if done after libvirtd is started and/or the last
QEMU capabilities adjustment can result in the inability to start a
guest and use nested kvm until the capabilities cache is invalidated.

Thus, let's fetch and save the setting during initialization and then
when the capabilities are checked for various host related adjustments
that could affect whether the capabilities cache is updated add a check
whether the nested value was set for either kvm_intel or kvm_amd to
force a refetch of the capabilities.

Signed-off-by: John Ferlan <jferlan@redhat.com>
---
 src/qemu/qemu_capabilities.c | 43 ++++++++++++++++++++++++++++++++++++
 src/qemu/qemu_capspriv.h     |  2 ++
 tests/qemucapabilitiestest.c |  3 +++
 3 files changed, 48 insertions(+)

diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
index 2ca5af3297..ebe0c0c7df 100644
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@@ -40,6 +40,7 @@
 #include "virnodesuspend.h"
 #include "virnuma.h"
 #include "virhostcpu.h"
+#include "virkmod.h"
 #include "qemu_monitor.h"
 #include "virstring.h"
 #include "qemu_hostdev.h"
@@ -551,6 +552,7 @@ struct _virQEMUCaps {
     virObject parent;
 
     bool usedQMP;
+    bool isNested;
 
     char *binary;
     time_t ctime;
@@ -1512,6 +1514,7 @@ virQEMUCapsPtr virQEMUCapsNewCopy(virQEMUCapsPtr qemuCaps)
         return NULL;
 
     ret->usedQMP = qemuCaps->usedQMP;
+    ret->isNested = qemuCaps->isNested;
 
     if (VIR_STRDUP(ret->binary, qemuCaps->binary) < 0)
         goto error;
@@ -3567,6 +3570,9 @@ virQEMUCapsLoadCache(virArch hostArch,
     virQEMUCapsInitHostCPUModel(qemuCaps, hostArch, VIR_DOMAIN_VIRT_KVM);
     virQEMUCapsInitHostCPUModel(qemuCaps, hostArch, VIR_DOMAIN_VIRT_QEMU);
 
+    qemuCaps->isNested = virXPathBoolean("count(./isNested) > 0",
+                                         ctxt) > 0;
+
     ret = 0;
  cleanup:
     VIR_FREE(str);
@@ -3786,6 +3792,9 @@ virQEMUCapsFormatCache(virQEMUCapsPtr qemuCaps)
     if (qemuCaps->sevCapabilities)
         virQEMUCapsFormatSEVInfo(qemuCaps, &buf);
 
+    if (qemuCaps->isNested)
+        virBufferAddLit(&buf, "<isNested/>\n");
+
     virBufferAdjustIndent(&buf, -2);
     virBufferAddLit(&buf, "</qemuCaps>\n");
 
@@ -3826,6 +3835,28 @@ virQEMUCapsSaveFile(void *data,
 }
 
 
+static bool
+virQEMUCapsIsNested(void)
+{
+    VIR_AUTOFREE(char *) kConfig = NULL;
+
+    if ((kConfig = virKModConfig()) &&
+        (strstr(kConfig, "kvm_intel nested=1") ||
+         strstr(kConfig, "kvm_amd nested=1")))
+        return true;
+    return false;
+}
+
+
+void
+virQEMUCapsClearIsNested(virQEMUCapsPtr qemuCaps)
+{
+    /* For qemucapabilitiestest to avoid printing the </isNested> on
+     * hosts with nested set in the kernel */
+    qemuCaps->isNested = false;
+}
+
+
 static bool
 virQEMUCapsIsValid(void *data,
                    void *privData)
@@ -3834,6 +3865,7 @@ virQEMUCapsIsValid(void *data,
     virQEMUCapsCachePrivPtr priv = privData;
     bool kvmUsable;
     struct stat sb;
+    bool isNested;
 
     if (!qemuCaps->binary)
         return true;
@@ -3866,6 +3898,15 @@ virQEMUCapsIsValid(void *data,
         return false;
     }
 
+    /* Check if someone changed the nested={0|1} value for the kernel from
+     * the previous time we checked. If so, then refresh the capabilities. */
+    isNested = virQEMUCapsIsNested();
+    if (isNested != qemuCaps->isNested) {
+        VIR_WARN("changed kernel nested kvm value was %d", qemuCaps->isNested);
+        qemuCaps->isNested = isNested;
+        return false;
+    }
+
     if (!virQEMUCapsGuestIsNative(priv->hostArch, qemuCaps->arch)) {
         VIR_DEBUG("Guest arch (%s) is not native to host arch (%s), "
                   "skipping KVM-related checks",
@@ -4452,6 +4493,8 @@ virQEMUCapsInitQMP(virQEMUCapsPtr qemuCaps,
     if (virQEMUCapsInitQMPMonitor(qemuCaps, cmd->mon) < 0)
         goto cleanup;
 
+    qemuCaps->isNested = virQEMUCapsIsNested();
+
     if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_KVM)) {
         virQEMUCapsInitQMPCommandAbort(cmd);
         if ((rc = virQEMUCapsInitQMPCommandRun(cmd, true)) != 0) {
diff --git a/src/qemu/qemu_capspriv.h b/src/qemu/qemu_capspriv.h
index 8d1a40fe74..b5d6aae2e5 100644
--- a/src/qemu/qemu_capspriv.h
+++ b/src/qemu/qemu_capspriv.h
@@ -48,6 +48,8 @@ int
 virQEMUCapsInitQMPMonitor(virQEMUCapsPtr qemuCaps,
                           qemuMonitorPtr mon);
 
+void virQEMUCapsClearIsNested(virQEMUCapsPtr qemuCaps);
+
 int
 virQEMUCapsInitQMPMonitorTCG(virQEMUCapsPtr qemuCaps,
                              qemuMonitorPtr mon);
diff --git a/tests/qemucapabilitiestest.c b/tests/qemucapabilitiestest.c
index 8fe5a55e1d..703fb6a125 100644
--- a/tests/qemucapabilitiestest.c
+++ b/tests/qemucapabilitiestest.c
@@ -63,6 +63,9 @@ testQemuCaps(const void *opaque)
                                   qemuMonitorTestGetMonitor(mon)) < 0)
         goto cleanup;
 
+    /* Don't apply what the host has... force clear for testing purposes */
+    virQEMUCapsClearIsNested(capsActual);
+
     if (virQEMUCapsGet(capsActual, QEMU_CAPS_KVM)) {
         qemuMonitorResetCommandID(qemuMonitorTestGetMonitor(mon));
         if (virQEMUCapsInitQMPMonitorTCG(capsActual,
-- 
2.17.2

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

On Tue, Nov 13, 2018 at 09:21 PM +0100, John Ferlan <jferlan@redhat.com> wrote:
> Support for nested kvm is handled via a kernel module configuration
> adjustment which if done after libvirtd is started and/or the last
> QEMU capabilities adjustment can result in the inability to start a
> guest and use nested kvm until the capabilities cache is invalidated.
>
> Thus, let's fetch and save the setting during initialization and then
> when the capabilities are checked for various host related adjustments
> that could affect whether the capabilities cache is updated add a check
> whether the nested value was set for either kvm_intel or kvm_amd to
> force a refetch of the capabilities.
>
> Signed-off-by: John Ferlan <jferlan@redhat.com>
> ---
>  src/qemu/qemu_capabilities.c | 43 ++++++++++++++++++++++++++++++++++++
>  src/qemu/qemu_capspriv.h     |  2 ++
>  tests/qemucapabilitiestest.c |  3 +++
>  3 files changed, 48 insertions(+)
>
> diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
> index 2ca5af3297..ebe0c0c7df 100644
> --- a/src/qemu/qemu_capabilities.c
> +++ b/src/qemu/qemu_capabilities.c
> @@ -40,6 +40,7 @@
>  #include "virnodesuspend.h"
>  #include "virnuma.h"
>  #include "virhostcpu.h"
> +#include "virkmod.h"
>  #include "qemu_monitor.h"
>  #include "virstring.h"
>  #include "qemu_hostdev.h"
> @@ -551,6 +552,7 @@ struct _virQEMUCaps {
>      virObject parent;
>
>      bool usedQMP;
> +    bool isNested;
>
>      char *binary;
>      time_t ctime;
> @@ -1512,6 +1514,7 @@ virQEMUCapsPtr virQEMUCapsNewCopy(virQEMUCapsPtr qemuCaps)
>          return NULL;
>
>      ret->usedQMP = qemuCaps->usedQMP;
> +    ret->isNested = qemuCaps->isNested;
>
>      if (VIR_STRDUP(ret->binary, qemuCaps->binary) < 0)
>          goto error;
> @@ -3567,6 +3570,9 @@ virQEMUCapsLoadCache(virArch hostArch,
>      virQEMUCapsInitHostCPUModel(qemuCaps, hostArch, VIR_DOMAIN_VIRT_KVM);
>      virQEMUCapsInitHostCPUModel(qemuCaps, hostArch, VIR_DOMAIN_VIRT_QEMU);
>
> +    qemuCaps->isNested = virXPathBoolean("count(./isNested) > 0",
> +                                         ctxt) > 0;
> +
>      ret = 0;
>   cleanup:
>      VIR_FREE(str);
> @@ -3786,6 +3792,9 @@ virQEMUCapsFormatCache(virQEMUCapsPtr qemuCaps)
>      if (qemuCaps->sevCapabilities)
>          virQEMUCapsFormatSEVInfo(qemuCaps, &buf);
>
> +    if (qemuCaps->isNested)
> +        virBufferAddLit(&buf, "<isNested/>\n");
> +
>      virBufferAdjustIndent(&buf, -2);
>      virBufferAddLit(&buf, "</qemuCaps>\n");
>
> @@ -3826,6 +3835,28 @@ virQEMUCapsSaveFile(void *data,
>  }
>
>
> +static bool
> +virQEMUCapsIsNested(void)

Not sure if “isNested” is the best wording… Since we’re talking about
nested KVM support here.

> +{
> +    VIR_AUTOFREE(char *) kConfig = NULL;
> +
> +    if ((kConfig = virKModConfig()) &&
> +        (strstr(kConfig, "kvm_intel nested=1") ||
> +         strstr(kConfig, "kvm_amd nested=1")))

Please add a check for "kvm nested=1" here since this is used by s390x :)

> +        return true;
> +    return false;
> +}
> +
> +
> +void
> +virQEMUCapsClearIsNested(virQEMUCapsPtr qemuCaps)
> +{
> +    /* For qemucapabilitiestest to avoid printing the </isNested> on
> +     * hosts with nested set in the kernel */
> +    qemuCaps->isNested = false;
> +}
> +
> +
>  static bool
>  virQEMUCapsIsValid(void *data,
>                     void *privData)
> @@ -3834,6 +3865,7 @@ virQEMUCapsIsValid(void *data,
>      virQEMUCapsCachePrivPtr priv = privData;
>      bool kvmUsable;
>      struct stat sb;
> +    bool isNested;
>
>      if (!qemuCaps->binary)
>          return true;
> @@ -3866,6 +3898,15 @@ virQEMUCapsIsValid(void *data,
>          return false;
>      }
>
> +    /* Check if someone changed the nested={0|1} value for the kernel from
> +     * the previous time we checked. If so, then refresh the capabilities. */
> +    isNested = virQEMUCapsIsNested();
> +    if (isNested != qemuCaps->isNested) {
> +        VIR_WARN("changed kernel nested kvm value was %d", qemuCaps->isNested);
> +        qemuCaps->isNested = isNested;
> +        return false;
> +    }
> +
>      if (!virQEMUCapsGuestIsNative(priv->hostArch, qemuCaps->arch)) {
>          VIR_DEBUG("Guest arch (%s) is not native to host arch (%s), "
>                    "skipping KVM-related checks",
> @@ -4452,6 +4493,8 @@ virQEMUCapsInitQMP(virQEMUCapsPtr qemuCaps,
>      if (virQEMUCapsInitQMPMonitor(qemuCaps, cmd->mon) < 0)
>          goto cleanup;
>
> +    qemuCaps->isNested = virQEMUCapsIsNested();
> +
>      if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_KVM)) {
>          virQEMUCapsInitQMPCommandAbort(cmd);
>          if ((rc = virQEMUCapsInitQMPCommandRun(cmd, true)) != 0) {
> diff --git a/src/qemu/qemu_capspriv.h b/src/qemu/qemu_capspriv.h
> index 8d1a40fe74..b5d6aae2e5 100644
> --- a/src/qemu/qemu_capspriv.h
> +++ b/src/qemu/qemu_capspriv.h
> @@ -48,6 +48,8 @@ int
>  virQEMUCapsInitQMPMonitor(virQEMUCapsPtr qemuCaps,
>                            qemuMonitorPtr mon);
>
> +void virQEMUCapsClearIsNested(virQEMUCapsPtr qemuCaps);
> +
>  int
>  virQEMUCapsInitQMPMonitorTCG(virQEMUCapsPtr qemuCaps,
>                               qemuMonitorPtr mon);
> diff --git a/tests/qemucapabilitiestest.c b/tests/qemucapabilitiestest.c
> index 8fe5a55e1d..703fb6a125 100644
> --- a/tests/qemucapabilitiestest.c
> +++ b/tests/qemucapabilitiestest.c
> @@ -63,6 +63,9 @@ testQemuCaps(const void *opaque)
>                                    qemuMonitorTestGetMonitor(mon)) < 0)
>          goto cleanup;
>
> +    /* Don't apply what the host has... force clear for testing purposes */
> +    virQEMUCapsClearIsNested(capsActual);
> +
>      if (virQEMUCapsGet(capsActual, QEMU_CAPS_KVM)) {
>          qemuMonitorResetCommandID(qemuMonitorTestGetMonitor(mon));
>          if (virQEMUCapsInitQMPMonitorTCG(capsActual,
> --
> 2.17.2
>
> --
> libvir-list mailing list
> libvir-list@redhat.com
> https://www.redhat.com/mailman/listinfo/libvir-list
>
--
Kind regards / Beste Grüße
   Marc Hartmayer

IBM Deutschland Research & Development GmbH
Vorsitzende des Aufsichtsrats: Martina Koederitz
Geschäftsführung: Dirk Wittkopp
Sitz der Gesellschaft: Böblingen
Registergericht: Amtsgericht Stuttgart, HRB 243294


--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list