contrib/vhost-user-gpu/vhost-user-gpu.c | 25 ++++++++++----- contrib/vhost-user-gpu/virgl.c | 6 +++- contrib/vhost-user-gpu/vugbm.c | 5 ++- hw/display/virtio-gpu-rutabaga.c | 14 +++++++-- hw/display/virtio-gpu-udmabuf.c | 7 +++-- hw/display/virtio-gpu-virgl.c | 6 +++- hw/display/virtio-gpu.c | 42 ++++++++++++++++++------- 7 files changed, 79 insertions(+), 26 deletions(-)
From: Marc-André Lureau <marcandre.lureau@redhat.com>
Use g_try_malloc/g_try_new0 for guest-controlled allocation, so failure
returns an error to the guest rather than crashing the host (glib
behaviour).
Resolves: https://gitlab.com/qemu-project/qemu/-/work_items/3898
Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
---
v5:
- also fail gracefully in virtio_gpu_create_udmabuf()
---
contrib/vhost-user-gpu/vhost-user-gpu.c | 25 ++++++++++-----
contrib/vhost-user-gpu/virgl.c | 6 +++-
contrib/vhost-user-gpu/vugbm.c | 5 ++-
hw/display/virtio-gpu-rutabaga.c | 14 +++++++--
hw/display/virtio-gpu-udmabuf.c | 7 +++--
hw/display/virtio-gpu-virgl.c | 6 +++-
hw/display/virtio-gpu.c | 42 ++++++++++++++++++-------
7 files changed, 79 insertions(+), 26 deletions(-)
diff --git a/contrib/vhost-user-gpu/vhost-user-gpu.c b/contrib/vhost-user-gpu/vhost-user-gpu.c
index 786488150932..933bdbb671c0 100644
--- a/contrib/vhost-user-gpu/vhost-user-gpu.c
+++ b/contrib/vhost-user-gpu/vhost-user-gpu.c
@@ -487,7 +487,7 @@ vg_create_mapping_iov(VuGpu *g,
struct virtio_gpu_ctrl_command *cmd,
struct iovec **iov)
{
- struct virtio_gpu_mem_entry *ents;
+ g_autofree struct virtio_gpu_mem_entry *ents = NULL;
size_t esize, s;
int i;
@@ -498,17 +498,22 @@ vg_create_mapping_iov(VuGpu *g,
}
esize = sizeof(*ents) * ab->nr_entries;
- ents = g_malloc(esize);
+ ents = g_try_malloc(esize);
+ if (!ents && esize) {
+ return -1;
+ }
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
sizeof(*ab), ents, esize);
if (s != esize) {
g_critical("%s: command data size incorrect %zu vs %zu",
__func__, s, esize);
- g_free(ents);
return -1;
}
- *iov = g_new0(struct iovec, ab->nr_entries);
+ *iov = g_try_new0(struct iovec, ab->nr_entries);
+ if (!*iov && ab->nr_entries) {
+ return -1;
+ }
for (i = 0; i < ab->nr_entries; i++) {
uint64_t len = ents[i].length;
(*iov)[i].iov_len = ents[i].length;
@@ -517,12 +522,10 @@ vg_create_mapping_iov(VuGpu *g,
g_critical("%s: resource %d element %d",
__func__, ab->resource_id, i);
g_free(*iov);
- g_free(ents);
*iov = NULL;
return -1;
}
}
- g_free(ents);
return 0;
}
@@ -828,8 +831,14 @@ vg_resource_flush(VuGpu *g,
PIXMAN_FORMAT_BPP(pixman_image_get_format(res->image)) / 8;
size_t size = width * height * bpp;
- void *p = g_malloc(VHOST_USER_GPU_HDR_SIZE +
- sizeof(VhostUserGpuUpdate) + size);
+ void *p = g_try_malloc(VHOST_USER_GPU_HDR_SIZE +
+ sizeof(VhostUserGpuUpdate) + size);
+ if (!p) {
+ pixman_region_fini(®ion);
+ pixman_region_fini(&finalregion);
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+ break;
+ }
VhostUserGpuMsg *msg = p;
msg->request = VHOST_USER_GPU_UPDATE;
msg->size = sizeof(VhostUserGpuUpdate) + size;
diff --git a/contrib/vhost-user-gpu/virgl.c b/contrib/vhost-user-gpu/virgl.c
index 550fd03bf5c4..20bae57d0fe4 100644
--- a/contrib/vhost-user-gpu/virgl.c
+++ b/contrib/vhost-user-gpu/virgl.c
@@ -209,7 +209,11 @@ virgl_cmd_submit_3d(VuGpu *g,
return;
}
- buf = g_malloc(cs.size);
+ buf = g_try_malloc(cs.size);
+ if (!buf && cs.size) {
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+ return;
+ }
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
sizeof(cs), buf, cs.size);
if (s != cs.size) {
diff --git a/contrib/vhost-user-gpu/vugbm.c b/contrib/vhost-user-gpu/vugbm.c
index 710d54529779..e2d8385fd857 100644
--- a/contrib/vhost-user-gpu/vugbm.c
+++ b/contrib/vhost-user-gpu/vugbm.c
@@ -13,7 +13,10 @@
static bool
mem_alloc_bo(struct vugbm_buffer *buf)
{
- buf->mmap = g_malloc((uint64_t)buf->width * buf->height * 4);
+ buf->mmap = g_try_malloc((uint64_t)buf->width * buf->height * 4);
+ if (!buf->mmap && buf->width && buf->height) {
+ return false;
+ }
buf->stride = buf->width * 4;
return true;
}
diff --git a/hw/display/virtio-gpu-rutabaga.c b/hw/display/virtio-gpu-rutabaga.c
index a054f8117f14..041216a10d04 100644
--- a/hw/display/virtio-gpu-rutabaga.c
+++ b/hw/display/virtio-gpu-rutabaga.c
@@ -366,10 +366,20 @@ rutabaga_cmd_submit_3d(VirtIOGPU *g,
return;
}
- buf = g_new0(uint8_t, cs.size);
+ buf = g_try_new0(uint8_t, cs.size);
+ if (!buf && cs.size) {
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+ return;
+ }
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
sizeof(cs), buf, cs.size);
- CHECK(s == cs.size, cmd);
+ if (s != cs.size) {
+ qemu_log_mask(LOG_GUEST_ERROR,
+ "%s: size mismatch (%zu/%u)\n",
+ __func__, s, cs.size);
+ cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+ return;
+ }
rutabaga_cmd.ctx_id = cs.hdr.ctx_id;
rutabaga_cmd.cmd = buf;
diff --git a/hw/display/virtio-gpu-udmabuf.c b/hw/display/virtio-gpu-udmabuf.c
index 5f08c855dde1..816f52a51457 100644
--- a/hw/display/virtio-gpu-udmabuf.c
+++ b/hw/display/virtio-gpu-udmabuf.c
@@ -39,8 +39,11 @@ static void virtio_gpu_create_udmabuf(struct virtio_gpu_simple_resource *res)
return;
}
- list = g_malloc0(sizeof(struct udmabuf_create_list) +
- sizeof(struct udmabuf_create_item) * res->iov_cnt);
+ list = g_try_malloc0(sizeof(struct udmabuf_create_list) +
+ sizeof(struct udmabuf_create_item) * res->iov_cnt);
+ if (!list) {
+ return;
+ }
for (i = 0; i < res->iov_cnt; i++) {
rcu_read_lock();
diff --git a/hw/display/virtio-gpu-virgl.c b/hw/display/virtio-gpu-virgl.c
index 6e298f997d66..9bda572426b2 100644
--- a/hw/display/virtio-gpu-virgl.c
+++ b/hw/display/virtio-gpu-virgl.c
@@ -620,7 +620,11 @@ static void virgl_cmd_submit_3d(VirtIOGPU *g,
return;
}
- buf = g_malloc(cs.size);
+ buf = g_try_malloc(cs.size);
+ if (!buf && cs.size) {
+ cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+ return;
+ }
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
sizeof(cs), buf, cs.size);
if (s != cs.size) {
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
index fbb6fec7a0ad..9eb010082d0d 100644
--- a/hw/display/virtio-gpu.c
+++ b/hw/display/virtio-gpu.c
@@ -892,7 +892,10 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
}
esize = sizeof(*ents) * nr_entries;
- ents = g_malloc(esize);
+ ents = g_try_malloc(esize);
+ if (!ents && esize) {
+ return -1;
+ }
s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
offset, ents, esize);
if (s != esize) {
@@ -913,6 +916,7 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
hwaddr len;
void *map;
+ /* TODO: a common DMA map SG helper */
do {
len = l;
map = dma_memory_map(VIRTIO_DEVICE(g)->dma_as, a, &len,
@@ -921,20 +925,27 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
if (!map) {
qemu_log_mask(LOG_GUEST_ERROR, "%s: failed to map MMIO memory for"
" element %d\n", __func__, e);
- virtio_gpu_cleanup_mapping_iov(g, *iov, v);
- g_free(ents);
- *iov = NULL;
- if (addr) {
- g_free(*addr);
- *addr = NULL;
- }
- return -1;
+ goto err;
}
if (!(v % 16)) {
- *iov = g_renew(struct iovec, *iov, v + 16);
+ struct iovec *new_iov;
+ new_iov = g_try_renew(struct iovec, *iov, v + 16);
+ if (!new_iov) {
+ dma_memory_unmap(VIRTIO_DEVICE(g)->dma_as, map, len,
+ DMA_DIRECTION_TO_DEVICE, len);
+ goto err;
+ }
+ *iov = new_iov;
if (addr) {
- *addr = g_renew(uint64_t, *addr, v + 16);
+ uint64_t *new_addr;
+ new_addr = g_try_renew(uint64_t, *addr, v + 16);
+ if (!new_addr) {
+ dma_memory_unmap(VIRTIO_DEVICE(g)->dma_as, map, len,
+ DMA_DIRECTION_TO_DEVICE, len);
+ goto err;
+ }
+ *addr = new_addr;
}
}
(*iov)[v].iov_base = map;
@@ -952,6 +963,15 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
g_free(ents);
return 0;
+
+err:
+ virtio_gpu_cleanup_mapping_iov(g, *iov, v);
+ *iov = NULL;
+ if (addr) {
+ g_clear_pointer(addr, g_free);
+ }
+ g_free(ents);
+ return -1;
}
void virtio_gpu_cleanup_mapping_iov(VirtIOGPU *g,
--
2.55.0
On 2026/08/05 22:01, marcandre.lureau@redhat.com wrote:
> From: Marc-André Lureau <marcandre.lureau@redhat.com>
>
> Use g_try_malloc/g_try_new0 for guest-controlled allocation, so failure
> returns an error to the guest rather than crashing the host (glib
> behaviour).
>
> Resolves: https://gitlab.com/qemu-project/qemu/-/work_items/3898
> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
Reviewed-by: Akihiko Odaki <odaki@rsg.ci.i.u-tokyo.ac.jp>
Regards,
Akihiko Odaki
> ---
> v5:
> - also fail gracefully in virtio_gpu_create_udmabuf()
> ---
> contrib/vhost-user-gpu/vhost-user-gpu.c | 25 ++++++++++-----
> contrib/vhost-user-gpu/virgl.c | 6 +++-
> contrib/vhost-user-gpu/vugbm.c | 5 ++-
> hw/display/virtio-gpu-rutabaga.c | 14 +++++++--
> hw/display/virtio-gpu-udmabuf.c | 7 +++--
> hw/display/virtio-gpu-virgl.c | 6 +++-
> hw/display/virtio-gpu.c | 42 ++++++++++++++++++-------
> 7 files changed, 79 insertions(+), 26 deletions(-)
>
> diff --git a/contrib/vhost-user-gpu/vhost-user-gpu.c b/contrib/vhost-user-gpu/vhost-user-gpu.c
> index 786488150932..933bdbb671c0 100644
> --- a/contrib/vhost-user-gpu/vhost-user-gpu.c
> +++ b/contrib/vhost-user-gpu/vhost-user-gpu.c
> @@ -487,7 +487,7 @@ vg_create_mapping_iov(VuGpu *g,
> struct virtio_gpu_ctrl_command *cmd,
> struct iovec **iov)
> {
> - struct virtio_gpu_mem_entry *ents;
> + g_autofree struct virtio_gpu_mem_entry *ents = NULL;
> size_t esize, s;
> int i;
>
> @@ -498,17 +498,22 @@ vg_create_mapping_iov(VuGpu *g,
> }
>
> esize = sizeof(*ents) * ab->nr_entries;
> - ents = g_malloc(esize);
> + ents = g_try_malloc(esize);
> + if (!ents && esize) {
> + return -1;
> + }
> s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
> sizeof(*ab), ents, esize);
> if (s != esize) {
> g_critical("%s: command data size incorrect %zu vs %zu",
> __func__, s, esize);
> - g_free(ents);
> return -1;
> }
>
> - *iov = g_new0(struct iovec, ab->nr_entries);
> + *iov = g_try_new0(struct iovec, ab->nr_entries);
> + if (!*iov && ab->nr_entries) {
> + return -1;
> + }
> for (i = 0; i < ab->nr_entries; i++) {
> uint64_t len = ents[i].length;
> (*iov)[i].iov_len = ents[i].length;
> @@ -517,12 +522,10 @@ vg_create_mapping_iov(VuGpu *g,
> g_critical("%s: resource %d element %d",
> __func__, ab->resource_id, i);
> g_free(*iov);
> - g_free(ents);
> *iov = NULL;
> return -1;
> }
> }
> - g_free(ents);
> return 0;
> }
>
> @@ -828,8 +831,14 @@ vg_resource_flush(VuGpu *g,
> PIXMAN_FORMAT_BPP(pixman_image_get_format(res->image)) / 8;
> size_t size = width * height * bpp;
>
> - void *p = g_malloc(VHOST_USER_GPU_HDR_SIZE +
> - sizeof(VhostUserGpuUpdate) + size);
> + void *p = g_try_malloc(VHOST_USER_GPU_HDR_SIZE +
> + sizeof(VhostUserGpuUpdate) + size);
> + if (!p) {
> + pixman_region_fini(®ion);
> + pixman_region_fini(&finalregion);
> + cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
> + break;
> + }
> VhostUserGpuMsg *msg = p;
> msg->request = VHOST_USER_GPU_UPDATE;
> msg->size = sizeof(VhostUserGpuUpdate) + size;
> diff --git a/contrib/vhost-user-gpu/virgl.c b/contrib/vhost-user-gpu/virgl.c
> index 550fd03bf5c4..20bae57d0fe4 100644
> --- a/contrib/vhost-user-gpu/virgl.c
> +++ b/contrib/vhost-user-gpu/virgl.c
> @@ -209,7 +209,11 @@ virgl_cmd_submit_3d(VuGpu *g,
> return;
> }
>
> - buf = g_malloc(cs.size);
> + buf = g_try_malloc(cs.size);
> + if (!buf && cs.size) {
> + cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
> + return;
> + }
> s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
> sizeof(cs), buf, cs.size);
> if (s != cs.size) {
> diff --git a/contrib/vhost-user-gpu/vugbm.c b/contrib/vhost-user-gpu/vugbm.c
> index 710d54529779..e2d8385fd857 100644
> --- a/contrib/vhost-user-gpu/vugbm.c
> +++ b/contrib/vhost-user-gpu/vugbm.c
> @@ -13,7 +13,10 @@
> static bool
> mem_alloc_bo(struct vugbm_buffer *buf)
> {
> - buf->mmap = g_malloc((uint64_t)buf->width * buf->height * 4);
> + buf->mmap = g_try_malloc((uint64_t)buf->width * buf->height * 4);
> + if (!buf->mmap && buf->width && buf->height) {
> + return false;
> + }
> buf->stride = buf->width * 4;
> return true;
> }
> diff --git a/hw/display/virtio-gpu-rutabaga.c b/hw/display/virtio-gpu-rutabaga.c
> index a054f8117f14..041216a10d04 100644
> --- a/hw/display/virtio-gpu-rutabaga.c
> +++ b/hw/display/virtio-gpu-rutabaga.c
> @@ -366,10 +366,20 @@ rutabaga_cmd_submit_3d(VirtIOGPU *g,
> return;
> }
>
> - buf = g_new0(uint8_t, cs.size);
> + buf = g_try_new0(uint8_t, cs.size);
> + if (!buf && cs.size) {
> + cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
> + return;
> + }
> s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
> sizeof(cs), buf, cs.size);
> - CHECK(s == cs.size, cmd);
> + if (s != cs.size) {
> + qemu_log_mask(LOG_GUEST_ERROR,
> + "%s: size mismatch (%zu/%u)\n",
> + __func__, s, cs.size);
> + cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
> + return;
> + }
>
> rutabaga_cmd.ctx_id = cs.hdr.ctx_id;
> rutabaga_cmd.cmd = buf;
> diff --git a/hw/display/virtio-gpu-udmabuf.c b/hw/display/virtio-gpu-udmabuf.c
> index 5f08c855dde1..816f52a51457 100644
> --- a/hw/display/virtio-gpu-udmabuf.c
> +++ b/hw/display/virtio-gpu-udmabuf.c
> @@ -39,8 +39,11 @@ static void virtio_gpu_create_udmabuf(struct virtio_gpu_simple_resource *res)
> return;
> }
>
> - list = g_malloc0(sizeof(struct udmabuf_create_list) +
> - sizeof(struct udmabuf_create_item) * res->iov_cnt);
> + list = g_try_malloc0(sizeof(struct udmabuf_create_list) +
> + sizeof(struct udmabuf_create_item) * res->iov_cnt);
> + if (!list) {
> + return;
> + }
>
> for (i = 0; i < res->iov_cnt; i++) {
> rcu_read_lock();
> diff --git a/hw/display/virtio-gpu-virgl.c b/hw/display/virtio-gpu-virgl.c
> index 6e298f997d66..9bda572426b2 100644
> --- a/hw/display/virtio-gpu-virgl.c
> +++ b/hw/display/virtio-gpu-virgl.c
> @@ -620,7 +620,11 @@ static void virgl_cmd_submit_3d(VirtIOGPU *g,
> return;
> }
>
> - buf = g_malloc(cs.size);
> + buf = g_try_malloc(cs.size);
> + if (!buf && cs.size) {
> + cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
> + return;
> + }
> s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
> sizeof(cs), buf, cs.size);
> if (s != cs.size) {
> diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
> index fbb6fec7a0ad..9eb010082d0d 100644
> --- a/hw/display/virtio-gpu.c
> +++ b/hw/display/virtio-gpu.c
> @@ -892,7 +892,10 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
> }
>
> esize = sizeof(*ents) * nr_entries;
> - ents = g_malloc(esize);
> + ents = g_try_malloc(esize);
> + if (!ents && esize) {
> + return -1;
> + }
> s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
> offset, ents, esize);
> if (s != esize) {
> @@ -913,6 +916,7 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
> hwaddr len;
> void *map;
>
> + /* TODO: a common DMA map SG helper */
> do {
> len = l;
> map = dma_memory_map(VIRTIO_DEVICE(g)->dma_as, a, &len,
> @@ -921,20 +925,27 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
> if (!map) {
> qemu_log_mask(LOG_GUEST_ERROR, "%s: failed to map MMIO memory for"
> " element %d\n", __func__, e);
> - virtio_gpu_cleanup_mapping_iov(g, *iov, v);
> - g_free(ents);
> - *iov = NULL;
> - if (addr) {
> - g_free(*addr);
> - *addr = NULL;
> - }
> - return -1;
> + goto err;
> }
>
> if (!(v % 16)) {
> - *iov = g_renew(struct iovec, *iov, v + 16);
> + struct iovec *new_iov;
> + new_iov = g_try_renew(struct iovec, *iov, v + 16);
> + if (!new_iov) {
> + dma_memory_unmap(VIRTIO_DEVICE(g)->dma_as, map, len,
> + DMA_DIRECTION_TO_DEVICE, len);
> + goto err;
> + }
> + *iov = new_iov;
> if (addr) {
> - *addr = g_renew(uint64_t, *addr, v + 16);
> + uint64_t *new_addr;
> + new_addr = g_try_renew(uint64_t, *addr, v + 16);
> + if (!new_addr) {
> + dma_memory_unmap(VIRTIO_DEVICE(g)->dma_as, map, len,
> + DMA_DIRECTION_TO_DEVICE, len);
> + goto err;
> + }
> + *addr = new_addr;
> }
> }
> (*iov)[v].iov_base = map;
> @@ -952,6 +963,15 @@ int virtio_gpu_create_mapping_iov(VirtIOGPU *g,
>
> g_free(ents);
> return 0;
> +
> +err:
> + virtio_gpu_cleanup_mapping_iov(g, *iov, v);
> + *iov = NULL;
> + if (addr) {
> + g_clear_pointer(addr, g_free);
> + }
> + g_free(ents);
> + return -1;
> }
>
> void virtio_gpu_cleanup_mapping_iov(VirtIOGPU *g,
© 2016 - 2026 Red Hat, Inc.