virtio-blk and virtio-scsi devices need a way to specify the mapping between IOThreads and virtqueues. At the moment all virtqueues are assigned to a single IOThread or the main loop. This single thread can be a CPU bottleneck, so it is necessary to allow finer-grained assignment to spread the load. With this series applied, "pidstat -t 1" shows that guests with -smp 2 or higher are able to exploit multiple IOThreads. This series introduces command-line syntax for the new iothread-vq-mapping property is as follows: --device '{"driver":"virtio-blk-pci","iothread-vq-mapping":[{"iothread":"iothread0","vqs":[0,1,2]},...]},...' IOThreads are specified by name and virtqueues are specified by 0-based index. It will be common to simply assign virtqueues round-robin across a set of IOThreads. A convenient syntax that does not require specifying individual virtqueue indices is available: --device '{"driver":"virtio-blk-pci","iothread-vq-mapping":[{"iothread":"iothread0"},{"iothread":"iothread1"},...]},...' There is no way to reassign virtqueues at runtime and I expect that to be a very rare requirement. Note that JSON --device syntax is required for the iothread-vq-mapping parameter because it's non-scalar. Based-on: 20230912231037.826804-1-stefanha@redhat.com ("[PATCH v3 0/5] block-backend: process I/O in the current AioContext") Stefan Hajnoczi (2): qdev: add IOThreadVirtQueueMappingList property type virtio-blk: add iothread-vq-mapping parameter qapi/virtio.json | 30 +++++ hw/block/dataplane/virtio-blk.h | 3 + include/hw/qdev-properties-system.h | 4 + include/hw/virtio/virtio-blk.h | 2 + hw/block/dataplane/virtio-blk.c | 163 +++++++++++++++++++++------- hw/block/virtio-blk.c | 92 ++++++++++++++-- hw/core/qdev-properties-system.c | 47 ++++++++ 7 files changed, 287 insertions(+), 54 deletions(-) -- 2.41.0 virtio-blk and virtio-scsi devices will need a way to specify the mapping between IOThreads and virtqueues. At the moment all virtqueues are assigned to a single IOThread or the main loop. This single thread can be a CPU bottleneck, so it is necessary to allow finer-grained assignment to spread the load. Introduce DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST() so devices can take a parameter that maps virtqueues to IOThreads. The command-line syntax for this new property is as follows: --device '{"driver":"foo","iothread-vq-mapping":[{"iothread":"iothread0","vqs":[0,1,2]},...]}' IOThreads are specified by name and virtqueues are specified by 0-based index. It will be common to simply assign virtqueues round-robin across a set of IOThreads. A convenient syntax that does not require specifying individual virtqueue indices is available: --device '{"driver":"foo","iothread-vq-mapping":[{"iothread":"iothread0"},{"iothread":"iothread1"},...]}' Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- qapi/virtio.json | 30 ++++++++++++++++++ include/hw/qdev-properties-system.h | 4 +++ hw/core/qdev-properties-system.c | 47 +++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/qapi/virtio.json b/qapi/virtio.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/virtio.json +++ b/qapi/virtio.json @@ -XXX,XX +XXX,XX @@ 'data': { 'path': 'str', 'queue': 'uint16', '*index': 'uint16' }, 'returns': 'VirtioQueueElement', 'features': [ 'unstable' ] } + +## +# @IOThreadVirtQueueMapping: +# +# Describes the subset of virtqueues assigned to an IOThread. +# +# @iothread: the id of IOThread object +# @vqs: an optional array of virtqueue indices that will be handled by this +# IOThread. When absent, virtqueues are assigned round-robin across all +# IOThreadVirtQueueMappings provided. Either all +# IOThreadVirtQueueMappings must have @vqs or none of them must have it. +# +# Since: 8.2 +# +## + +{ 'struct': 'IOThreadVirtQueueMapping', + 'data': { 'iothread': 'str', '*vqs': ['uint16'] } } + +## +# @IOThreadVirtQueueMappings: +# +# IOThreadVirtQueueMapping list. This struct is not actually used but the +# IOThreadVirtQueueMappingList type it generates is! +# +# Since: 8.2 +## + +{ 'struct': 'IOThreadVirtQueueMappings', + 'data': { 'mappings': ['IOThreadVirtQueueMapping'] } } diff --git a/include/hw/qdev-properties-system.h b/include/hw/qdev-properties-system.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/qdev-properties-system.h +++ b/include/hw/qdev-properties-system.h @@ -XXX,XX +XXX,XX @@ extern const PropertyInfo qdev_prop_audiodev; extern const PropertyInfo qdev_prop_off_auto_pcibar; extern const PropertyInfo qdev_prop_pcie_link_speed; extern const PropertyInfo qdev_prop_pcie_link_width; +extern const PropertyInfo qdev_prop_iothread_vq_mapping_list; #define DEFINE_PROP_PCI_DEVFN(_n, _s, _f, _d) \ DEFINE_PROP_SIGNED(_n, _s, _f, _d, qdev_prop_pci_devfn, int32_t) @@ -XXX,XX +XXX,XX @@ extern const PropertyInfo qdev_prop_pcie_link_width; #define DEFINE_PROP_UUID_NODEFAULT(_name, _state, _field) \ DEFINE_PROP(_name, _state, _field, qdev_prop_uuid, QemuUUID) +#define DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST(_name, _state, _field) \ + DEFINE_PROP(_name, _state, _field, qdev_prop_iothread_vq_mapping_list, \ + IOThreadVirtQueueMappingList *) #endif diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/qdev-properties-system.c +++ b/hw/core/qdev-properties-system.c @@ -XXX,XX +XXX,XX @@ #include "qapi/qapi-types-block.h" #include "qapi/qapi-types-machine.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qapi-visit-virtio.h" #include "qapi/qmp/qerror.h" #include "qemu/ctype.h" #include "qemu/cutils.h" @@ -XXX,XX +XXX,XX @@ const PropertyInfo qdev_prop_uuid = { .set = set_uuid, .set_default_value = set_default_uuid_auto, }; + +/* --- IOThreadVirtQueueMappingList --- */ + +static void get_iothread_vq_mapping_list(Object *obj, Visitor *v, + const char *name, void *opaque, Error **errp) +{ + IOThreadVirtQueueMappingList **prop_ptr = + object_field_prop_ptr(obj, opaque); + IOThreadVirtQueueMappingList *list = *prop_ptr; + + visit_type_IOThreadVirtQueueMappingList(v, name, &list, errp); +} + +static void set_iothread_vq_mapping_list(Object *obj, Visitor *v, + const char *name, void *opaque, Error **errp) +{ + IOThreadVirtQueueMappingList **prop_ptr = + object_field_prop_ptr(obj, opaque); + IOThreadVirtQueueMappingList *list; + + if (!visit_type_IOThreadVirtQueueMappingList(v, name, &list, errp)) { + return; + } + + qapi_free_IOThreadVirtQueueMappingList(*prop_ptr); + *prop_ptr = list; +} + +static void release_iothread_vq_mapping_list(Object *obj, + const char *name, void *opaque) +{ + IOThreadVirtQueueMappingList **prop_ptr = + object_field_prop_ptr(obj, opaque); + + qapi_free_IOThreadVirtQueueMappingList(*prop_ptr); + *prop_ptr = NULL; +} + +const PropertyInfo qdev_prop_iothread_vq_mapping_list = { + .name = "IOThreadVirtQueueMappingList", + .description = "IOThread virtqueue mapping list [{\"iothread\":\"<id>\", " + "\"vqs\":[1,2,3,...]},...]", + .get = get_iothread_vq_mapping_list, + .set = set_iothread_vq_mapping_list, + .release = release_iothread_vq_mapping_list, +}; -- 2.41.0 Add the iothread-vq-mapping parameter to assign virtqueues to IOThreads. Store the vq:AioContext mapping in the new struct VirtIOBlockDataPlane->vq_aio_context[] field and refactor the code to use the per-vq AioContext instead of the BlockDriverState's AioContext. Reimplement --device virtio-blk-pci,iothread= and non-IOThread mode by assigning all virtqueues to the IOThread and main loop's AioContext in vq_aio_context[], respectively. The comment in struct VirtIOBlockDataPlane about EventNotifiers is stale. Remove it. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/block/dataplane/virtio-blk.h | 3 + include/hw/virtio/virtio-blk.h | 2 + hw/block/dataplane/virtio-blk.c | 163 ++++++++++++++++++++++++-------- hw/block/virtio-blk.c | 92 +++++++++++++++--- 4 files changed, 206 insertions(+), 54 deletions(-) diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h index XXXXXXX..XXXXXXX 100644 --- a/hw/block/dataplane/virtio-blk.h +++ b/hw/block/dataplane/virtio-blk.h @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq); int virtio_blk_data_plane_start(VirtIODevice *vdev); void virtio_blk_data_plane_stop(VirtIODevice *vdev); +void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s); +void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s); + #endif /* HW_DATAPLANE_VIRTIO_BLK_H */ diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/virtio/virtio-blk.h +++ b/include/hw/virtio/virtio-blk.h @@ -XXX,XX +XXX,XX @@ #include "sysemu/block-backend.h" #include "sysemu/block-ram-registrar.h" #include "qom/object.h" +#include "qapi/qapi-types-virtio.h" #define TYPE_VIRTIO_BLK "virtio-blk-device" OBJECT_DECLARE_SIMPLE_TYPE(VirtIOBlock, VIRTIO_BLK) @@ -XXX,XX +XXX,XX @@ struct VirtIOBlkConf { BlockConf conf; IOThread *iothread; + IOThreadVirtQueueMappingList *iothread_vq_mapping_list; char *serial; uint32_t request_merging; uint16_t num_queues; diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/dataplane/virtio-blk.c +++ b/hw/block/dataplane/virtio-blk.c @@ -XXX,XX +XXX,XX @@ struct VirtIOBlockDataPlane { VirtIOBlkConf *conf; VirtIODevice *vdev; - /* Note that these EventNotifiers are assigned by value. This is - * fine as long as you do not call event_notifier_cleanup on them - * (because you don't own the file descriptor or handle; you just - * use it). + /* + * The AioContext for each virtqueue. The BlockDriverState will use the + * first element as its AioContext. */ - IOThread *iothread; - AioContext *ctx; + AioContext **vq_aio_context; }; /* Raise an interrupt to signal guest, if necessary */ @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) virtio_notify_irqfd(s->vdev, vq); } +/* Generate vq:AioContext mappings from a validated iothread-vq-mapping list */ +static void +apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list, + AioContext **vq_aio_context, uint16_t num_queues) +{ + IOThreadVirtQueueMappingList *node; + size_t num_iothreads = 0; + size_t cur_iothread = 0; + + for (node = iothread_vq_mapping_list; node; node = node->next) { + num_iothreads++; + } + + for (node = iothread_vq_mapping_list; node; node = node->next) { + IOThread *iothread = iothread_by_id(node->value->iothread); + AioContext *ctx = iothread_get_aio_context(iothread); + + /* Released in virtio_blk_data_plane_destroy() */ + object_ref(OBJECT(iothread)); + + if (node->value->vqs) { + uint16List *vq; + + /* Explicit vq:IOThread assignment */ + for (vq = node->value->vqs; vq; vq = vq->next) { + vq_aio_context[vq->value] = ctx; + } + } else { + /* Round-robin vq:IOThread assignment */ + for (unsigned i = cur_iothread; i < num_queues; + i += num_iothreads) { + vq_aio_context[i] = ctx; + } + } + + cur_iothread++; + } +} + /* Context: QEMU global mutex held */ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, VirtIOBlockDataPlane **dataplane, @@ -XXX,XX +XXX,XX @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, *dataplane = NULL; - if (conf->iothread) { + if (conf->iothread || conf->iothread_vq_mapping_list) { if (!k->set_guest_notifiers || !k->ioeventfd_assign) { error_setg(errp, "device is incompatible with iothread " @@ -XXX,XX +XXX,XX @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, s = g_new0(VirtIOBlockDataPlane, 1); s->vdev = vdev; s->conf = conf; + s->vq_aio_context = g_new(AioContext *, conf->num_queues); - if (conf->iothread) { - s->iothread = conf->iothread; - object_ref(OBJECT(s->iothread)); - s->ctx = iothread_get_aio_context(s->iothread); + if (conf->iothread_vq_mapping_list) { + apply_vq_mapping(conf->iothread_vq_mapping_list, s->vq_aio_context, + conf->num_queues); + } else if (conf->iothread) { + AioContext *ctx = iothread_get_aio_context(conf->iothread); + for (unsigned i = 0; i < conf->num_queues; i++) { + s->vq_aio_context[i] = ctx; + } + + /* Released in virtio_blk_data_plane_destroy() */ + object_ref(OBJECT(conf->iothread)); } else { - s->ctx = qemu_get_aio_context(); + AioContext *ctx = qemu_get_aio_context(); + for (unsigned i = 0; i < conf->num_queues; i++) { + s->vq_aio_context[i] = ctx; + } } *dataplane = s; @@ -XXX,XX +XXX,XX @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) { VirtIOBlock *vblk; + VirtIOBlkConf *conf = s->conf; if (!s) { return; @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) vblk = VIRTIO_BLK(s->vdev); assert(!vblk->dataplane_started); - if (s->iothread) { - object_unref(OBJECT(s->iothread)); + + if (conf->iothread_vq_mapping_list) { + IOThreadVirtQueueMappingList *node; + + for (node = conf->iothread_vq_mapping_list; node; node = node->next) { + IOThread *iothread = iothread_by_id(node->value->iothread); + object_unref(OBJECT(iothread)); + } } + + if (conf->iothread) { + object_unref(OBJECT(conf->iothread)); + } + + g_free(s->vq_aio_context); g_free(s); } @@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) old_context = blk_get_aio_context(s->conf->conf.blk); aio_context_acquire(old_context); - r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); + r = blk_set_aio_context(s->conf->conf.blk, s->vq_aio_context[0], + &local_err); aio_context_release(old_context); if (r < 0) { error_report_err(local_err); goto fail_aio_context; } - /* Kick right away to begin processing requests already in vring */ - for (i = 0; i < nvqs; i++) { - VirtQueue *vq = virtio_get_queue(s->vdev, i); - - event_notifier_set(virtio_queue_get_host_notifier(vq)); - } - /* * These fields must be visible to the IOThread when it processes the * virtqueue, otherwise it will think dataplane has not started yet. @@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) /* Get this show started by hooking up our callbacks */ if (!blk_in_drain(s->conf->conf.blk)) { - aio_context_acquire(s->ctx); for (i = 0; i < nvqs; i++) { VirtQueue *vq = virtio_get_queue(s->vdev, i); + AioContext *ctx = s->vq_aio_context[i]; - virtio_queue_aio_attach_host_notifier(vq, s->ctx); + /* Kick right away to begin processing requests already in vring */ + event_notifier_set(virtio_queue_get_host_notifier(vq)); + + virtio_queue_aio_attach_host_notifier(vq, ctx); } - aio_context_release(s->ctx); } return 0; @@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) * * Context: BH in IOThread */ -static void virtio_blk_data_plane_stop_bh(void *opaque) +static void virtio_blk_data_plane_stop_vq_bh(void *opaque) { - VirtIOBlockDataPlane *s = opaque; - unsigned i; + VirtQueue *vq = opaque; + EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); - for (i = 0; i < s->conf->num_queues; i++) { - VirtQueue *vq = virtio_get_queue(s->vdev, i); - EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); + virtio_queue_aio_detach_host_notifier(vq, qemu_get_current_aio_context()); - virtio_queue_aio_detach_host_notifier(vq, s->ctx); - - /* - * Test and clear notifier after disabling event, in case poll callback - * didn't have time to run. - */ - virtio_queue_host_notifier_read(host_notifier); - } + /* + * Test and clear notifier after disabling event, in case poll callback + * didn't have time to run. + */ + virtio_queue_host_notifier_read(host_notifier); } /* Context: QEMU global mutex held */ @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) trace_virtio_blk_data_plane_stop(s); if (!blk_in_drain(s->conf->conf.blk)) { - aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); + for (i = 0; i < nvqs; i++) { + VirtQueue *vq = virtio_get_queue(s->vdev, i); + AioContext *ctx = s->vq_aio_context[i]; + + aio_context_acquire(ctx); + aio_wait_bh_oneshot(ctx, virtio_blk_data_plane_stop_vq_bh, vq); + aio_context_release(ctx); + } } /* @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) */ vblk->dataplane_started = false; - aio_context_acquire(s->ctx); + aio_context_acquire(s->vq_aio_context[0]); /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */ blk_drain(s->conf->conf.blk); @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) */ blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); - aio_context_release(s->ctx); + aio_context_release(s->vq_aio_context[0]); /* Clean up guest notifier (irq) */ k->set_guest_notifiers(qbus->parent, nvqs, false); s->stopping = false; } + +void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev); + + for (uint16_t i = 0; i < s->conf->num_queues; i++) { + VirtQueue *vq = virtio_get_queue(vdev, i); + virtio_queue_aio_detach_host_notifier(vq, s->vq_aio_context[i]); + } +} + +void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev); + + for (uint16_t i = 0; i < s->conf->num_queues; i++) { + VirtQueue *vq = virtio_get_queue(vdev, i); + virtio_queue_aio_attach_host_notifier(vq, s->vq_aio_context[i]); + } +} diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -XXX,XX +XXX,XX @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq) return; } } + virtio_blk_handle_vq(s, vq); } @@ -XXX,XX +XXX,XX @@ static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f, return 0; } +static bool +validate_iothread_vq_mapping_list(IOThreadVirtQueueMappingList *list, + uint16_t num_queues, Error **errp) +{ + g_autofree unsigned long *vqs = bitmap_new(num_queues); + g_autoptr(GHashTable) iothreads = + g_hash_table_new(g_str_hash, g_str_equal); + + for (IOThreadVirtQueueMappingList *node = list; node; node = node->next) { + const char *name = node->value->iothread; + uint16List *vq; + + if (!iothread_by_id(name)) { + error_setg(errp, "IOThread \"%s\" object does not exist", name); + return false; + } + + if (!g_hash_table_add(iothreads, (gpointer)name)) { + error_setg(errp, + "duplicate IOThread name \"%s\" in iothread-vq-mapping", + name); + return false; + } + + if (node != list) { + if (!!node->value->vqs != !!list->value->vqs) { + error_setg(errp, "either all items in iothread-vq-mapping " + "must have vqs or none of them must have it"); + return false; + } + } + + for (vq = node->value->vqs; vq; vq = vq->next) { + if (vq->value >= num_queues) { + error_setg(errp, "vq index %u for IOThread \"%s\" must be " + "less than num_queues %u in iothread-vq-mapping", + vq->value, name, num_queues); + return false; + } + + if (test_and_set_bit(vq->value, vqs)) { + error_setg(errp, "cannot assign vq %u to IOThread \"%s\" " + "because it is already assigned", vq->value, name); + return false; + } + } + } + + if (list->value->vqs) { + for (uint16_t i = 0; i < num_queues; i++) { + if (!test_bit(i, vqs)) { + error_setg(errp, + "missing vq %u IOThread assignment in iothread-vq-mapping", + i); + return false; + } + } + } + + return true; +} + static void virtio_resize_cb(void *opaque) { VirtIODevice *vdev = opaque; @@ -XXX,XX +XXX,XX @@ static void virtio_blk_resize(void *opaque) static void virtio_blk_drained_begin(void *opaque) { VirtIOBlock *s = opaque; - VirtIODevice *vdev = VIRTIO_DEVICE(opaque); - AioContext *ctx = blk_get_aio_context(s->conf.conf.blk); if (!s->dataplane || !s->dataplane_started) { return; } - for (uint16_t i = 0; i < s->conf.num_queues; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_detach_host_notifier(vq, ctx); - } + virtio_blk_data_plane_detach(s->dataplane); } /* Resume virtqueue ioeventfd processing after drain */ static void virtio_blk_drained_end(void *opaque) { VirtIOBlock *s = opaque; - VirtIODevice *vdev = VIRTIO_DEVICE(opaque); - AioContext *ctx = blk_get_aio_context(s->conf.conf.blk); if (!s->dataplane || !s->dataplane_started) { return; } - for (uint16_t i = 0; i < s->conf.num_queues; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_attach_host_notifier(vq, ctx); - } + virtio_blk_data_plane_attach(s->dataplane); } static const BlockDevOps virtio_block_ops = { @@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) return; } + if (conf->iothread_vq_mapping_list) { + if (conf->iothread) { + error_setg(errp, "iothread and iothread-vq-mapping properties " + "cannot be set at the same time"); + return; + } + + if (!validate_iothread_vq_mapping_list(conf->iothread_vq_mapping_list, + conf->num_queues, errp)) { + return; + } + } + s->config_size = virtio_get_config_size(&virtio_blk_cfg_size_params, s->host_features); virtio_init(vdev, VIRTIO_ID_BLOCK, s->config_size); @@ -XXX,XX +XXX,XX @@ static Property virtio_blk_properties[] = { DEFINE_PROP_BOOL("seg-max-adjust", VirtIOBlock, conf.seg_max_adjust, true), DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD, IOThread *), + DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST("iothread-vq-mapping", VirtIOBlock, + conf.iothread_vq_mapping_list), DEFINE_PROP_BIT64("discard", VirtIOBlock, host_features, VIRTIO_BLK_F_DISCARD, true), DEFINE_PROP_BOOL("report-discard-granularity", VirtIOBlock, -- 2.41.0 v4: - Use DummyVirtioForceArrays naming in QAPI schema [Markus] v3: - Rebased onto Kevin's block branch - Add StringOutputVisitor "<omitted>" patch to fix "info qtree" crash - Fix QAPI schema formatting [Markus] - Eliminate unnecessary local variable in get_iothread_vq_mapping_list() [Markus] virtio-blk and virtio-scsi devices need a way to specify the mapping between IOThreads and virtqueues. At the moment all virtqueues are assigned to a single IOThread or the main loop. This single thread can be a CPU bottleneck, so it is necessary to allow finer-grained assignment to spread the load. With this series applied, "pidstat -t 1" shows that guests with -smp 2 or higher are able to exploit multiple IOThreads. This series introduces command-line syntax for the new iothread-vq-mapping property is as follows: --device '{"driver":"virtio-blk-pci","iothread-vq-mapping":[{"iothread":"iothread0","vqs":[0,1,2]},...]},...' IOThreads are specified by name and virtqueues are specified by 0-based index. It will be common to simply assign virtqueues round-robin across a set of IOThreads. A convenient syntax that does not require specifying individual virtqueue indices is available: --device '{"driver":"virtio-blk-pci","iothread-vq-mapping":[{"iothread":"iothread0"},{"iothread":"iothread1"},...]},...' There is no way to reassign virtqueues at runtime and I expect that to be a very rare requirement. Note that JSON --device syntax is required for the iothread-vq-mapping parameter because it's non-scalar. Based-on: 81e69329d6a4018f4b37d15b6fc845fbe585a93b (https://repo.or.cz/qemu/kevin.git block) Stefan Hajnoczi (4): qdev-properties: alias all object class properties string-output-visitor: show structs as "<omitted>" qdev: add IOThreadVirtQueueMappingList property type virtio-blk: add iothread-vq-mapping parameter qapi/virtio.json | 29 +++++ hw/block/dataplane/virtio-blk.h | 3 + include/hw/qdev-properties-system.h | 5 + include/hw/virtio/virtio-blk.h | 2 + include/qapi/string-output-visitor.h | 6 +- hw/block/dataplane/virtio-blk.c | 155 ++++++++++++++++++++------- hw/block/virtio-blk.c | 92 +++++++++++++--- hw/core/qdev-properties-system.c | 46 ++++++++ hw/core/qdev-properties.c | 18 ++-- qapi/string-output-visitor.c | 16 +++ 10 files changed, 311 insertions(+), 61 deletions(-) -- 2.43.0 qdev_alias_all_properties() aliases a DeviceState's qdev properties onto an Object. This is used for VirtioPCIProxy types so that --device virtio-blk-pci has properties of its embedded --device virtio-blk-device object. Currently this function is implemented using qdev properties. Change the function to use QOM object class properties instead. This works because qdev properties create QOM object class properties, but it also catches any QOM object class-only properties that have no qdev properties. This change ensures that properties of devices are shown with --device foo,\? even if they are QOM object class properties. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/core/qdev-properties.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/hw/core/qdev-properties.c b/hw/core/qdev-properties.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/qdev-properties.c +++ b/hw/core/qdev-properties.c @@ -XXX,XX +XXX,XX @@ void device_class_set_props(DeviceClass *dc, Property *props) void qdev_alias_all_properties(DeviceState *target, Object *source) { ObjectClass *class; - Property *prop; + ObjectPropertyIterator iter; + ObjectProperty *prop; class = object_get_class(OBJECT(target)); - do { - DeviceClass *dc = DEVICE_CLASS(class); - for (prop = dc->props_; prop && prop->name; prop++) { - object_property_add_alias(source, prop->name, - OBJECT(target), prop->name); + object_class_property_iter_init(&iter, class); + while ((prop = object_property_iter_next(&iter))) { + if (object_property_find(source, prop->name)) { + continue; /* skip duplicate properties */ } - class = object_class_get_parent(class); - } while (class != object_class_by_name(TYPE_DEVICE)); + + object_property_add_alias(source, prop->name, + OBJECT(target), prop->name); + } } -- 2.43.0 StringOutputVisitor crashes when it visits a struct because ->start_struct() is NULL. Show "<omitted>" instead of crashing. This is necessary because the virtio-blk-pci iothread-vq-mapping parameter that I'd like to introduce soon is a list of IOThreadMapping structs. This patch is a quick fix to solve the crash, but the long-term solution is replacing StringOutputVisitor with something that can handle the full gamut of values in QEMU. Reviewed-by: Markus Armbruster <armbru@redhat.com> Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- include/qapi/string-output-visitor.h | 6 +++--- qapi/string-output-visitor.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/include/qapi/string-output-visitor.h b/include/qapi/string-output-visitor.h index XXXXXXX..XXXXXXX 100644 --- a/include/qapi/string-output-visitor.h +++ b/include/qapi/string-output-visitor.h @@ -XXX,XX +XXX,XX @@ typedef struct StringOutputVisitor StringOutputVisitor; * If everything else succeeds, pass @result to visit_complete() to * collect the result of the visit. * - * The string output visitor does not implement support for visiting - * QAPI structs, alternates, null, or arbitrary QTypes. It also - * requires a non-null list argument to visit_start_list(). + * The string output visitor does not implement support for alternates, null, + * or arbitrary QTypes. Struct fields are not shown. It also requires a + * non-null list argument to visit_start_list(). */ Visitor *string_output_visitor_new(bool human, char **result); diff --git a/qapi/string-output-visitor.c b/qapi/string-output-visitor.c index XXXXXXX..XXXXXXX 100644 --- a/qapi/string-output-visitor.c +++ b/qapi/string-output-visitor.c @@ -XXX,XX +XXX,XX @@ static bool print_type_null(Visitor *v, const char *name, QNull **obj, return true; } +static bool start_struct(Visitor *v, const char *name, void **obj, + size_t size, Error **errp) +{ + return true; +} + +static void end_struct(Visitor *v, void **obj) +{ + StringOutputVisitor *sov = to_sov(v); + + /* TODO actually print struct fields */ + string_output_set(sov, g_strdup("<omitted>")); +} + static bool start_list(Visitor *v, const char *name, GenericList **list, size_t size, Error **errp) @@ -XXX,XX +XXX,XX @@ Visitor *string_output_visitor_new(bool human, char **result) v->visitor.type_str = print_type_str; v->visitor.type_number = print_type_number; v->visitor.type_null = print_type_null; + v->visitor.start_struct = start_struct; + v->visitor.end_struct = end_struct; v->visitor.start_list = start_list; v->visitor.next_list = next_list; v->visitor.end_list = end_list; -- 2.43.0 virtio-blk and virtio-scsi devices will need a way to specify the mapping between IOThreads and virtqueues. At the moment all virtqueues are assigned to a single IOThread or the main loop. This single thread can be a CPU bottleneck, so it is necessary to allow finer-grained assignment to spread the load. Introduce DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST() so devices can take a parameter that maps virtqueues to IOThreads. The command-line syntax for this new property is as follows: --device '{"driver":"foo","iothread-vq-mapping":[{"iothread":"iothread0","vqs":[0,1,2]},...]}' IOThreads are specified by name and virtqueues are specified by 0-based index. It will be common to simply assign virtqueues round-robin across a set of IOThreads. A convenient syntax that does not require specifying individual virtqueue indices is available: --device '{"driver":"foo","iothread-vq-mapping":[{"iothread":"iothread0"},{"iothread":"iothread1"},...]}' Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- qapi/virtio.json | 29 ++++++++++++++++++ include/hw/qdev-properties-system.h | 5 ++++ hw/core/qdev-properties-system.c | 46 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/qapi/virtio.json b/qapi/virtio.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/virtio.json +++ b/qapi/virtio.json @@ -XXX,XX +XXX,XX @@ 'data': { 'path': 'str', 'queue': 'uint16', '*index': 'uint16' }, 'returns': 'VirtioQueueElement', 'features': [ 'unstable' ] } + +## +# @IOThreadVirtQueueMapping: +# +# Describes the subset of virtqueues assigned to an IOThread. +# +# @iothread: the id of IOThread object +# +# @vqs: an optional array of virtqueue indices that will be handled by this +# IOThread. When absent, virtqueues are assigned round-robin across all +# IOThreadVirtQueueMappings provided. Either all IOThreadVirtQueueMappings +# must have @vqs or none of them must have it. +# +# Since: 9.0 +## + +{ 'struct': 'IOThreadVirtQueueMapping', + 'data': { 'iothread': 'str', '*vqs': ['uint16'] } } + +## +# @DummyVirtioForceArrays: +# +# Not used by QMP; hack to let us use IOThreadVirtQueueMappingList internally +# +# Since: 9.0 +## + +{ 'struct': 'DummyVirtioForceArrays', + 'data': { 'unused-iothread-vq-mapping': ['IOThreadVirtQueueMapping'] } } diff --git a/include/hw/qdev-properties-system.h b/include/hw/qdev-properties-system.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/qdev-properties-system.h +++ b/include/hw/qdev-properties-system.h @@ -XXX,XX +XXX,XX @@ extern const PropertyInfo qdev_prop_off_auto_pcibar; extern const PropertyInfo qdev_prop_pcie_link_speed; extern const PropertyInfo qdev_prop_pcie_link_width; extern const PropertyInfo qdev_prop_cpus390entitlement; +extern const PropertyInfo qdev_prop_iothread_vq_mapping_list; #define DEFINE_PROP_PCI_DEVFN(_n, _s, _f, _d) \ DEFINE_PROP_SIGNED(_n, _s, _f, _d, qdev_prop_pci_devfn, int32_t) @@ -XXX,XX +XXX,XX @@ extern const PropertyInfo qdev_prop_cpus390entitlement; DEFINE_PROP_SIGNED(_n, _s, _f, _d, qdev_prop_cpus390entitlement, \ CpuS390Entitlement) +#define DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST(_name, _state, _field) \ + DEFINE_PROP(_name, _state, _field, qdev_prop_iothread_vq_mapping_list, \ + IOThreadVirtQueueMappingList *) + #endif diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/qdev-properties-system.c +++ b/hw/core/qdev-properties-system.c @@ -XXX,XX +XXX,XX @@ #include "qapi/qapi-types-block.h" #include "qapi/qapi-types-machine.h" #include "qapi/qapi-types-migration.h" +#include "qapi/qapi-visit-virtio.h" #include "qapi/qmp/qerror.h" #include "qemu/ctype.h" #include "qemu/cutils.h" @@ -XXX,XX +XXX,XX @@ const PropertyInfo qdev_prop_cpus390entitlement = { .set = qdev_propinfo_set_enum, .set_default_value = qdev_propinfo_set_default_value_enum, }; + +/* --- IOThreadVirtQueueMappingList --- */ + +static void get_iothread_vq_mapping_list(Object *obj, Visitor *v, + const char *name, void *opaque, Error **errp) +{ + IOThreadVirtQueueMappingList **prop_ptr = + object_field_prop_ptr(obj, opaque); + + visit_type_IOThreadVirtQueueMappingList(v, name, prop_ptr, errp); +} + +static void set_iothread_vq_mapping_list(Object *obj, Visitor *v, + const char *name, void *opaque, Error **errp) +{ + IOThreadVirtQueueMappingList **prop_ptr = + object_field_prop_ptr(obj, opaque); + IOThreadVirtQueueMappingList *list; + + if (!visit_type_IOThreadVirtQueueMappingList(v, name, &list, errp)) { + return; + } + + qapi_free_IOThreadVirtQueueMappingList(*prop_ptr); + *prop_ptr = list; +} + +static void release_iothread_vq_mapping_list(Object *obj, + const char *name, void *opaque) +{ + IOThreadVirtQueueMappingList **prop_ptr = + object_field_prop_ptr(obj, opaque); + + qapi_free_IOThreadVirtQueueMappingList(*prop_ptr); + *prop_ptr = NULL; +} + +const PropertyInfo qdev_prop_iothread_vq_mapping_list = { + .name = "IOThreadVirtQueueMappingList", + .description = "IOThread virtqueue mapping list [{\"iothread\":\"<id>\", " + "\"vqs\":[1,2,3,...]},...]", + .get = get_iothread_vq_mapping_list, + .set = set_iothread_vq_mapping_list, + .release = release_iothread_vq_mapping_list, +}; -- 2.43.0 Add the iothread-vq-mapping parameter to assign virtqueues to IOThreads. Store the vq:AioContext mapping in the new struct VirtIOBlockDataPlane->vq_aio_context[] field and refactor the code to use the per-vq AioContext instead of the BlockDriverState's AioContext. Reimplement --device virtio-blk-pci,iothread= and non-IOThread mode by assigning all virtqueues to the IOThread and main loop's AioContext in vq_aio_context[], respectively. The comment in struct VirtIOBlockDataPlane about EventNotifiers is stale. Remove it. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/block/dataplane/virtio-blk.h | 3 + include/hw/virtio/virtio-blk.h | 2 + hw/block/dataplane/virtio-blk.c | 155 ++++++++++++++++++++++++-------- hw/block/virtio-blk.c | 92 ++++++++++++++++--- 4 files changed, 202 insertions(+), 50 deletions(-) diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h index XXXXXXX..XXXXXXX 100644 --- a/hw/block/dataplane/virtio-blk.h +++ b/hw/block/dataplane/virtio-blk.h @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq); int virtio_blk_data_plane_start(VirtIODevice *vdev); void virtio_blk_data_plane_stop(VirtIODevice *vdev); +void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s); +void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s); + #endif /* HW_DATAPLANE_VIRTIO_BLK_H */ diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/virtio/virtio-blk.h +++ b/include/hw/virtio/virtio-blk.h @@ -XXX,XX +XXX,XX @@ #include "sysemu/block-backend.h" #include "sysemu/block-ram-registrar.h" #include "qom/object.h" +#include "qapi/qapi-types-virtio.h" #define TYPE_VIRTIO_BLK "virtio-blk-device" OBJECT_DECLARE_SIMPLE_TYPE(VirtIOBlock, VIRTIO_BLK) @@ -XXX,XX +XXX,XX @@ struct VirtIOBlkConf { BlockConf conf; IOThread *iothread; + IOThreadVirtQueueMappingList *iothread_vq_mapping_list; char *serial; uint32_t request_merging; uint16_t num_queues; diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/dataplane/virtio-blk.c +++ b/hw/block/dataplane/virtio-blk.c @@ -XXX,XX +XXX,XX @@ struct VirtIOBlockDataPlane { VirtIOBlkConf *conf; VirtIODevice *vdev; - /* Note that these EventNotifiers are assigned by value. This is - * fine as long as you do not call event_notifier_cleanup on them - * (because you don't own the file descriptor or handle; you just - * use it). + /* + * The AioContext for each virtqueue. The BlockDriverState will use the + * first element as its AioContext. */ - IOThread *iothread; - AioContext *ctx; + AioContext **vq_aio_context; }; /* Raise an interrupt to signal guest, if necessary */ @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) virtio_notify_irqfd(s->vdev, vq); } +/* Generate vq:AioContext mappings from a validated iothread-vq-mapping list */ +static void +apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list, + AioContext **vq_aio_context, uint16_t num_queues) +{ + IOThreadVirtQueueMappingList *node; + size_t num_iothreads = 0; + size_t cur_iothread = 0; + + for (node = iothread_vq_mapping_list; node; node = node->next) { + num_iothreads++; + } + + for (node = iothread_vq_mapping_list; node; node = node->next) { + IOThread *iothread = iothread_by_id(node->value->iothread); + AioContext *ctx = iothread_get_aio_context(iothread); + + /* Released in virtio_blk_data_plane_destroy() */ + object_ref(OBJECT(iothread)); + + if (node->value->vqs) { + uint16List *vq; + + /* Explicit vq:IOThread assignment */ + for (vq = node->value->vqs; vq; vq = vq->next) { + vq_aio_context[vq->value] = ctx; + } + } else { + /* Round-robin vq:IOThread assignment */ + for (unsigned i = cur_iothread; i < num_queues; + i += num_iothreads) { + vq_aio_context[i] = ctx; + } + } + + cur_iothread++; + } +} + /* Context: QEMU global mutex held */ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, VirtIOBlockDataPlane **dataplane, @@ -XXX,XX +XXX,XX @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, *dataplane = NULL; - if (conf->iothread) { + if (conf->iothread || conf->iothread_vq_mapping_list) { if (!k->set_guest_notifiers || !k->ioeventfd_assign) { error_setg(errp, "device is incompatible with iothread " @@ -XXX,XX +XXX,XX @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, s = g_new0(VirtIOBlockDataPlane, 1); s->vdev = vdev; s->conf = conf; + s->vq_aio_context = g_new(AioContext *, conf->num_queues); - if (conf->iothread) { - s->iothread = conf->iothread; - object_ref(OBJECT(s->iothread)); - s->ctx = iothread_get_aio_context(s->iothread); + if (conf->iothread_vq_mapping_list) { + apply_vq_mapping(conf->iothread_vq_mapping_list, s->vq_aio_context, + conf->num_queues); + } else if (conf->iothread) { + AioContext *ctx = iothread_get_aio_context(conf->iothread); + for (unsigned i = 0; i < conf->num_queues; i++) { + s->vq_aio_context[i] = ctx; + } + + /* Released in virtio_blk_data_plane_destroy() */ + object_ref(OBJECT(conf->iothread)); } else { - s->ctx = qemu_get_aio_context(); + AioContext *ctx = qemu_get_aio_context(); + for (unsigned i = 0; i < conf->num_queues; i++) { + s->vq_aio_context[i] = ctx; + } } *dataplane = s; @@ -XXX,XX +XXX,XX @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) { VirtIOBlock *vblk; + VirtIOBlkConf *conf = s->conf; if (!s) { return; @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) vblk = VIRTIO_BLK(s->vdev); assert(!vblk->dataplane_started); - if (s->iothread) { - object_unref(OBJECT(s->iothread)); + + if (conf->iothread_vq_mapping_list) { + IOThreadVirtQueueMappingList *node; + + for (node = conf->iothread_vq_mapping_list; node; node = node->next) { + IOThread *iothread = iothread_by_id(node->value->iothread); + object_unref(OBJECT(iothread)); + } } + + if (conf->iothread) { + object_unref(OBJECT(conf->iothread)); + } + + g_free(s->vq_aio_context); g_free(s); } @@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) trace_virtio_blk_data_plane_start(s); - r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); + r = blk_set_aio_context(s->conf->conf.blk, s->vq_aio_context[0], + &local_err); if (r < 0) { error_report_err(local_err); goto fail_aio_context; } - /* Kick right away to begin processing requests already in vring */ - for (i = 0; i < nvqs; i++) { - VirtQueue *vq = virtio_get_queue(s->vdev, i); - - event_notifier_set(virtio_queue_get_host_notifier(vq)); - } - /* * These fields must be visible to the IOThread when it processes the * virtqueue, otherwise it will think dataplane has not started yet. @@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) if (!blk_in_drain(s->conf->conf.blk)) { for (i = 0; i < nvqs; i++) { VirtQueue *vq = virtio_get_queue(s->vdev, i); + AioContext *ctx = s->vq_aio_context[i]; - virtio_queue_aio_attach_host_notifier(vq, s->ctx); + /* Kick right away to begin processing requests already in vring */ + event_notifier_set(virtio_queue_get_host_notifier(vq)); + + virtio_queue_aio_attach_host_notifier(vq, ctx); } } return 0; @@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) * * Context: BH in IOThread */ -static void virtio_blk_data_plane_stop_bh(void *opaque) +static void virtio_blk_data_plane_stop_vq_bh(void *opaque) { - VirtIOBlockDataPlane *s = opaque; - unsigned i; + VirtQueue *vq = opaque; + EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); - for (i = 0; i < s->conf->num_queues; i++) { - VirtQueue *vq = virtio_get_queue(s->vdev, i); - EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); + virtio_queue_aio_detach_host_notifier(vq, qemu_get_current_aio_context()); - virtio_queue_aio_detach_host_notifier(vq, s->ctx); - - /* - * Test and clear notifier after disabling event, in case poll callback - * didn't have time to run. - */ - virtio_queue_host_notifier_read(host_notifier); - } + /* + * Test and clear notifier after disabling event, in case poll callback + * didn't have time to run. + */ + virtio_queue_host_notifier_read(host_notifier); } /* Context: QEMU global mutex held */ @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) trace_virtio_blk_data_plane_stop(s); if (!blk_in_drain(s->conf->conf.blk)) { - aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); + for (i = 0; i < nvqs; i++) { + VirtQueue *vq = virtio_get_queue(s->vdev, i); + AioContext *ctx = s->vq_aio_context[i]; + + aio_wait_bh_oneshot(ctx, virtio_blk_data_plane_stop_vq_bh, vq); + } } /* @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) s->stopping = false; } + +void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev); + + for (uint16_t i = 0; i < s->conf->num_queues; i++) { + VirtQueue *vq = virtio_get_queue(vdev, i); + virtio_queue_aio_detach_host_notifier(vq, s->vq_aio_context[i]); + } +} + +void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev); + + for (uint16_t i = 0; i < s->conf->num_queues; i++) { + VirtQueue *vq = virtio_get_queue(vdev, i); + virtio_queue_aio_attach_host_notifier(vq, s->vq_aio_context[i]); + } +} diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -XXX,XX +XXX,XX @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq) return; } } + virtio_blk_handle_vq(s, vq); } @@ -XXX,XX +XXX,XX @@ static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f, return 0; } +static bool +validate_iothread_vq_mapping_list(IOThreadVirtQueueMappingList *list, + uint16_t num_queues, Error **errp) +{ + g_autofree unsigned long *vqs = bitmap_new(num_queues); + g_autoptr(GHashTable) iothreads = + g_hash_table_new(g_str_hash, g_str_equal); + + for (IOThreadVirtQueueMappingList *node = list; node; node = node->next) { + const char *name = node->value->iothread; + uint16List *vq; + + if (!iothread_by_id(name)) { + error_setg(errp, "IOThread \"%s\" object does not exist", name); + return false; + } + + if (!g_hash_table_add(iothreads, (gpointer)name)) { + error_setg(errp, + "duplicate IOThread name \"%s\" in iothread-vq-mapping", + name); + return false; + } + + if (node != list) { + if (!!node->value->vqs != !!list->value->vqs) { + error_setg(errp, "either all items in iothread-vq-mapping " + "must have vqs or none of them must have it"); + return false; + } + } + + for (vq = node->value->vqs; vq; vq = vq->next) { + if (vq->value >= num_queues) { + error_setg(errp, "vq index %u for IOThread \"%s\" must be " + "less than num_queues %u in iothread-vq-mapping", + vq->value, name, num_queues); + return false; + } + + if (test_and_set_bit(vq->value, vqs)) { + error_setg(errp, "cannot assign vq %u to IOThread \"%s\" " + "because it is already assigned", vq->value, name); + return false; + } + } + } + + if (list->value->vqs) { + for (uint16_t i = 0; i < num_queues; i++) { + if (!test_bit(i, vqs)) { + error_setg(errp, + "missing vq %u IOThread assignment in iothread-vq-mapping", + i); + return false; + } + } + } + + return true; +} + static void virtio_resize_cb(void *opaque) { VirtIODevice *vdev = opaque; @@ -XXX,XX +XXX,XX @@ static void virtio_blk_resize(void *opaque) static void virtio_blk_drained_begin(void *opaque) { VirtIOBlock *s = opaque; - VirtIODevice *vdev = VIRTIO_DEVICE(opaque); - AioContext *ctx = blk_get_aio_context(s->conf.conf.blk); if (!s->dataplane || !s->dataplane_started) { return; } - for (uint16_t i = 0; i < s->conf.num_queues; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_detach_host_notifier(vq, ctx); - } + virtio_blk_data_plane_detach(s->dataplane); } /* Resume virtqueue ioeventfd processing after drain */ static void virtio_blk_drained_end(void *opaque) { VirtIOBlock *s = opaque; - VirtIODevice *vdev = VIRTIO_DEVICE(opaque); - AioContext *ctx = blk_get_aio_context(s->conf.conf.blk); if (!s->dataplane || !s->dataplane_started) { return; } - for (uint16_t i = 0; i < s->conf.num_queues; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_attach_host_notifier(vq, ctx); - } + virtio_blk_data_plane_attach(s->dataplane); } static const BlockDevOps virtio_block_ops = { @@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) return; } + if (conf->iothread_vq_mapping_list) { + if (conf->iothread) { + error_setg(errp, "iothread and iothread-vq-mapping properties " + "cannot be set at the same time"); + return; + } + + if (!validate_iothread_vq_mapping_list(conf->iothread_vq_mapping_list, + conf->num_queues, errp)) { + return; + } + } + s->config_size = virtio_get_config_size(&virtio_blk_cfg_size_params, s->host_features); virtio_init(vdev, VIRTIO_ID_BLOCK, s->config_size); @@ -XXX,XX +XXX,XX @@ static Property virtio_blk_properties[] = { DEFINE_PROP_BOOL("seg-max-adjust", VirtIOBlock, conf.seg_max_adjust, true), DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD, IOThread *), + DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST("iothread-vq-mapping", VirtIOBlock, + conf.iothread_vq_mapping_list), DEFINE_PROP_BIT64("discard", VirtIOBlock, host_features, VIRTIO_BLK_F_DISCARD, true), DEFINE_PROP_BOOL("report-discard-granularity", VirtIOBlock, -- 2.43.0