SHMEM_MAP creates a separate RAM MemoryRegion for every mapping. It can
therefore make a vhost-user memory table exceed the number of slots
negotiated with the backend.

Reject a mapping before changing the memory topology if KVM or a vhost
backend has no unreserved slot left. Also validate each memory table
against the negotiated vhost-user limit as a final defense for mappings
that become visible in a later memory transaction.

Fixes: b52e1896e764 ("vhost-user: Add VirtIO Shared Memory map request")
Signed-off-by: Feifan Qian <bea1e@proton.me>
---
Tested with an ASan/UBSan x86_64 build and a vhost-user backend that
filled the negotiated 512-slot memory table.  The next SHMEM_MAP request
was rejected and QEMU remained alive, with no sanitizer report.
qtest-x86_64/qos-test also passed all 134 subtests.

 hw/virtio/vhost-user.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 2881cec72d..3673653f67 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -10,6 +10,7 @@

 #include "qemu/osdep.h"
 #include "qapi/error.h"
+#include "hw/mem/memory-device.h"
 #include "hw/virtio/virtio-dmabuf.h"
 #include "hw/virtio/virtio-qmp.h"
 #include "hw/virtio/vhost.h"
@@ -1126,6 +1127,13 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,
             dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
     int ret;

+    if (mem->nregions > u->user->memory_slots) {
+        error_report("vhost-user memory table has %u regions, "
+                     "but the backend supports only %d",
+                     mem->nregions, u->user->memory_slots);
+        return -ENOSPC;
+    }
+
     if (do_postcopy) {
         /*
          * Postcopy has enough differences that it's best done in it's own
@@ -1940,6 +1948,7 @@ vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
     VhostUserMMap *vu_mmap = &payload->mmap;
     VirtioSharedMemoryMapping *existing;
     Error *local_err = NULL;
+    unsigned int reserved_memslots;
     int ret = 0;

     if (fd < 0) {
@@ -1980,6 +1989,15 @@ vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
         }
     }

+    reserved_memslots = memory_devices_get_reserved_memslots();
+    if ((kvm_enabled() &&
+         kvm_get_free_memslots() <= reserved_memslots) ||
+        vhost_get_free_memslots() <= reserved_memslots) {
+        error_report("No free memory slots for shared memory mapping");
+        ret = -ENOSPC;
+        goto send_reply;
+    }
+
     memory_region_transaction_begin();

     /* Create VirtioSharedMemoryMapping object */
--
2.43.0

Hi Feifan,

On Thu, Aug 6, 2026 at 4:14 PM Feifan Qian <bea1e@proton.me> wrote:
>
> SHMEM_MAP creates a separate RAM MemoryRegion for every mapping. It can
> therefore make a vhost-user memory table exceed the number of slots
> negotiated with the backend.
>
> Reject a mapping before changing the memory topology if KVM or a vhost
> backend has no unreserved slot left. Also validate each memory table
> against the negotiated vhost-user limit as a final defense for mappings
> that become visible in a later memory transaction.
>
> Fixes: b52e1896e764 ("vhost-user: Add VirtIO Shared Memory map request")
> Signed-off-by: Feifan Qian <bea1e@proton.me>
> ---
> Tested with an ASan/UBSan x86_64 build and a vhost-user backend that
> filled the negotiated 512-slot memory table.  The next SHMEM_MAP request
> was rejected and QEMU remained alive, with no sanitizer report.
> qtest-x86_64/qos-test also passed all 134 subtests.
>
>  hw/virtio/vhost-user.c | 18 ++++++++++++++++++
>  1 file changed, 18 insertions(+)
>
> diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
> index 2881cec72d..3673653f67 100644
> --- a/hw/virtio/vhost-user.c
> +++ b/hw/virtio/vhost-user.c
> @@ -10,6 +10,7 @@
>
>  #include "qemu/osdep.h"
>  #include "qapi/error.h"
> +#include "hw/mem/memory-device.h"
>  #include "hw/virtio/virtio-dmabuf.h"
>  #include "hw/virtio/virtio-qmp.h"
>  #include "hw/virtio/vhost.h"
> @@ -1126,6 +1127,13 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,
>              dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
>      int ret;
>
> +    if (mem->nregions > u->user->memory_slots) {
> +        error_report("vhost-user memory table has %u regions, "
> +                     "but the backend supports only %d",
> +                     mem->nregions, u->user->memory_slots);
> +        return -ENOSPC;
> +    }
> +
>      if (do_postcopy) {
>          /*
>           * Postcopy has enough differences that it's best done in it's own
> @@ -1940,6 +1948,7 @@ vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
>      VhostUserMMap *vu_mmap = &payload->mmap;
>      VirtioSharedMemoryMapping *existing;
>      Error *local_err = NULL;
> +    unsigned int reserved_memslots;
>      int ret = 0;
>
>      if (fd < 0) {
> @@ -1980,6 +1989,15 @@ vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
>          }
>      }
>
> +    reserved_memslots = memory_devices_get_reserved_memslots();
> +    if ((kvm_enabled() &&
> +         kvm_get_free_memslots() <= reserved_memslots) ||
> +        vhost_get_free_memslots() <= reserved_memslots) {

This is correct with current baseline, but after this[1] series lands
all SHMEM mapping will be filtered by vhost. So this could check
kvm_get_free_memslots() only and avoid reserved_memslots altogether.

Perhaps you could base this series on Dorinda's work above instead?
Actually, the check above is also partially redundant for SHMEM_MAP
regions given the filter in Dorinda's patch. But it's probably still a
valid safety net for ADD_MEM_REG.

[1] https://lore.kernel.org/all/20260717134920.265128-2-dbassey@redhat.com/
[2] https://www.qemu.org/docs/master/devel/submitting-a-patch.html#base-patches-against-current-git-master

BR,
Albert

> +        error_report("No free memory slots for shared memory mapping");
> +        ret = -ENOSPC;
> +        goto send_reply;
> +    }
> +
>      memory_region_transaction_begin();
>
>      /* Create VirtioSharedMemoryMapping object */
> --
> 2.43.0
>