From: Philippe Mathieu-Daudé <philmd@redhat.com>

Keep statistics of some hardware errors, and number of

aligned/unaligned I/O accesses.

QMP example booting a full RHEL 8.3 aarch64 guest:

{ "execute": "query-blockstats" }

{

"return": [

{

"device": "",

"node-name": "drive0",

"stats": {

"flush_total_time_ns": 6026948,

"wr_highest_offset": 3383991230464,

"wr_total_time_ns": 807450995,

"failed_wr_operations": 0,

"failed_rd_operations": 0,

"wr_merged": 3,

"wr_bytes": 50133504,

"failed_unmap_operations": 0,

"failed_flush_operations": 0,

"account_invalid": false,

"rd_total_time_ns": 1846979900,

"flush_operations": 130,

"wr_operations": 659,

"rd_merged": 1192,

"rd_bytes": 218244096,

"account_failed": false,

"idle_time_ns": 2678641497,

"rd_operations": 7406,

},

"driver-specific": {

"driver": "nvme",

"completion-errors": 0,

"unaligned-accesses": 2959,

"aligned-accesses": 4477

},

"qdev": "/machine/peripheral-anon/device[0]/virtio-backend"

}

]

}

Suggested-by: Stefan Hajnoczi <stefanha@gmail.com>

Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Acked-by: Markus Armbruster <armbru@redhat.com>

Message-id: 20201001162939.1567915-1-philmd@redhat.com

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

qapi/block-core.json | 24 +++++++++++++++++++++++-

block/nvme.c | 27 +++++++++++++++++++++++++++

2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json

index XXXXXXX..XXXXXXX 100644

--- a/qapi/block-core.json

+++ b/qapi/block-core.json

@@ -XXX,XX +XXX,XX @@

'discard-nb-failed': 'uint64',

'discard-bytes-ok': 'uint64' } }

+##

+# @BlockStatsSpecificNvme:

+#

+# NVMe driver statistics

+#

+# @completion-errors: The number of completion errors.

+#

+# @aligned-accesses: The number of aligned accesses performed by

+# the driver.

+#

+# @unaligned-accesses: The number of unaligned accesses performed by

+# the driver.

+#

+# Since: 5.2

+##

+{ 'struct': 'BlockStatsSpecificNvme',

+ 'data': {

+ 'completion-errors': 'uint64',

+ 'aligned-accesses': 'uint64',

+ 'unaligned-accesses': 'uint64' } }

+

##

# @BlockStatsSpecific:

#

@@ -XXX,XX +XXX,XX @@

'discriminator': 'driver',

'data': {

'file': 'BlockStatsSpecificFile',

- 'host_device': 'BlockStatsSpecificFile' } }

+ 'host_device': 'BlockStatsSpecificFile',

+ 'nvme': 'BlockStatsSpecificNvme' } }

##

# @BlockStats:

diff --git a/block/nvme.c b/block/nvme.c

index XXXXXXX..XXXXXXX 100644

--- a/block/nvme.c

+++ b/block/nvme.c

@@ -XXX,XX +XXX,XX @@ struct BDRVNVMeState {

/* PCI address (required for nvme_refresh_filename()) */

char *device;

+

+ struct {

+ uint64_t completion_errors;

+ uint64_t aligned_accesses;

+ uint64_t unaligned_accesses;

+ } stats;

};

#define NVME_BLOCK_OPT_DEVICE "device"

@@ -XXX,XX +XXX,XX @@ static bool nvme_process_completion(NVMeQueuePair *q)

break;

}

ret = nvme_translate_error(c);

+ if (ret) {

+ s->stats.completion_errors++;

+ }

q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;

if (!q->cq.head) {

q->cq_phase = !q->cq_phase;

@@ -XXX,XX +XXX,XX @@ static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,

assert(QEMU_IS_ALIGNED(bytes, s->page_size));

assert(bytes <= s->max_transfer);

if (nvme_qiov_aligned(bs, qiov)) {

+ s->stats.aligned_accesses++;

return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);

}

+ s->stats.unaligned_accesses++;

trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);

buf = qemu_try_memalign(s->page_size, bytes);

@@ -XXX,XX +XXX,XX @@ static void nvme_unregister_buf(BlockDriverState *bs, void *host)

qemu_vfio_dma_unmap(s->vfio, host);

}

+static BlockStatsSpecific *nvme_get_specific_stats(BlockDriverState *bs)

+{

+ BlockStatsSpecific *stats = g_new(BlockStatsSpecific, 1);

+ BDRVNVMeState *s = bs->opaque;

+

+ stats->driver = BLOCKDEV_DRIVER_NVME;

+ stats->u.nvme = (BlockStatsSpecificNvme) {

+ .completion_errors = s->stats.completion_errors,

+ .aligned_accesses = s->stats.aligned_accesses,

+ .unaligned_accesses = s->stats.unaligned_accesses,

+ };

+

+ return stats;

+}

+

static const char *const nvme_strong_runtime_opts[] = {

NVME_BLOCK_OPT_DEVICE,

NVME_BLOCK_OPT_NAMESPACE,

@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_nvme = {

.bdrv_refresh_filename = nvme_refresh_filename,

.bdrv_refresh_limits = nvme_refresh_limits,

.strong_runtime_opts = nvme_strong_runtime_opts,

+ .bdrv_get_specific_stats = nvme_get_specific_stats,

.bdrv_detach_aio_context = nvme_detach_aio_context,

.bdrv_attach_aio_context = nvme_attach_aio_context,

--

2.26.2

From: Coiby Xu <coiby.xu@gmail.com>

Allow vu_message_read to be replaced by one which will make use of the

QIOChannel functions. Thus reading vhost-user message won't stall the

guest. For slave channel, we still use the default vu_message_read.

Reviewed-by: Marc-André Lureau <marcandre.lureau@redhat.com>

Signed-off-by: Coiby Xu <coiby.xu@gmail.com>

Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>

Message-id: 20200918080912.321299-2-coiby.xu@gmail.com

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

contrib/libvhost-user/libvhost-user.h | 21 +++++++++++++++++++++

contrib/libvhost-user/libvhost-user-glib.c | 2 +-

contrib/libvhost-user/libvhost-user.c | 14 +++++++-------

tests/vhost-user-bridge.c | 2 ++

tools/virtiofsd/fuse_virtio.c | 4 ++--

5 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/contrib/libvhost-user/libvhost-user.h b/contrib/libvhost-user/libvhost-user.h

index XXXXXXX..XXXXXXX 100644

--- a/contrib/libvhost-user/libvhost-user.h

+++ b/contrib/libvhost-user/libvhost-user.h

@@ -XXX,XX +XXX,XX @@

*/

#define VHOST_USER_MAX_RAM_SLOTS 32

+#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)

+

typedef enum VhostSetConfigType {

VHOST_SET_CONFIG_TYPE_MASTER = 0,

VHOST_SET_CONFIG_TYPE_MIGRATION = 1,

@@ -XXX,XX +XXX,XX @@ typedef uint64_t (*vu_get_features_cb) (VuDev *dev);

typedef void (*vu_set_features_cb) (VuDev *dev, uint64_t features);

typedef int (*vu_process_msg_cb) (VuDev *dev, VhostUserMsg *vmsg,

int *do_reply);

+typedef bool (*vu_read_msg_cb) (VuDev *dev, int sock, VhostUserMsg *vmsg);

typedef void (*vu_queue_set_started_cb) (VuDev *dev, int qidx, bool started);

typedef bool (*vu_queue_is_processed_in_order_cb) (VuDev *dev, int qidx);

typedef int (*vu_get_config_cb) (VuDev *dev, uint8_t *config, uint32_t len);

@@ -XXX,XX +XXX,XX @@ struct VuDev {

bool broken;

uint16_t max_queues;

+ /* @read_msg: custom method to read vhost-user message

+ *

+ * Read data from vhost_user socket fd and fill up

+ * the passed VhostUserMsg *vmsg struct.

+ *

+ * If reading fails, it should close the received set of file

+ * descriptors as socket message's auxiliary data.

+ *

+ * For the details, please refer to vu_message_read in libvhost-user.c

+ * which will be used by default if not custom method is provided when

+ * calling vu_init

+ *

+ * Returns: true if vhost-user message successfully received,

+ * otherwise return false.

+ *

+ */

+ vu_read_msg_cb read_msg;

/* @set_watch: add or update the given fd to the watch set,

* call cb when condition is met */

vu_set_watch_cb set_watch;

@@ -XXX,XX +XXX,XX @@ bool vu_init(VuDev *dev,

uint16_t max_queues,

int socket,

vu_panic_cb panic,

+ vu_read_msg_cb read_msg,

vu_set_watch_cb set_watch,

vu_remove_watch_cb remove_watch,

const VuDevIface *iface);

diff --git a/contrib/libvhost-user/libvhost-user-glib.c b/contrib/libvhost-user/libvhost-user-glib.c

index XXXXXXX..XXXXXXX 100644

--- a/contrib/libvhost-user/libvhost-user-glib.c

+++ b/contrib/libvhost-user/libvhost-user-glib.c

@@ -XXX,XX +XXX,XX @@ vug_init(VugDev *dev, uint16_t max_queues, int socket,

g_assert(dev);

g_assert(iface);

- if (!vu_init(&dev->parent, max_queues, socket, panic, set_watch,

+ if (!vu_init(&dev->parent, max_queues, socket, panic, NULL, set_watch,

remove_watch, iface)) {

return false;

}

diff --git a/contrib/libvhost-user/libvhost-user.c b/contrib/libvhost-user/libvhost-user.c

index XXXXXXX..XXXXXXX 100644

--- a/contrib/libvhost-user/libvhost-user.c

+++ b/contrib/libvhost-user/libvhost-user.c

@@ -XXX,XX +XXX,XX @@

/* The version of inflight buffer */

#define INFLIGHT_VERSION 1

-#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)

-

/* The version of the protocol we support */

#define VHOST_USER_VERSION 1

#define LIBVHOST_USER_DEBUG 0

@@ -XXX,XX +XXX,XX @@ have_userfault(void)

}

static bool

-vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)

+vu_message_read_default(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)

{

char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = {};

struct iovec iov = {

@@ -XXX,XX +XXX,XX @@ vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)

goto out;

}

- if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) {

+ if (!vu_message_read_default(dev, dev->slave_fd, &msg_reply)) {

goto out;

}

@@ -XXX,XX +XXX,XX @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)

/* Wait for QEMU to confirm that it's registered the handler for the

* faults.

*/

- if (!vu_message_read(dev, dev->sock, vmsg) ||

+ if (!dev->read_msg(dev, dev->sock, vmsg) ||

vmsg->size != sizeof(vmsg->payload.u64) ||

vmsg->payload.u64 != 0) {

vu_panic(dev, "failed to receive valid ack for postcopy set-mem-table");

@@ -XXX,XX +XXX,XX @@ vu_dispatch(VuDev *dev)

int reply_requested;

bool need_reply, success = false;

- if (!vu_message_read(dev, dev->sock, &vmsg)) {

+ if (!dev->read_msg(dev, dev->sock, &vmsg)) {

goto end;

}

@@ -XXX,XX +XXX,XX @@ vu_init(VuDev *dev,

uint16_t max_queues,

int socket,

vu_panic_cb panic,

+ vu_read_msg_cb read_msg,

vu_set_watch_cb set_watch,

vu_remove_watch_cb remove_watch,

const VuDevIface *iface)

@@ -XXX,XX +XXX,XX @@ vu_init(VuDev *dev,

dev->sock = socket;

dev->panic = panic;

+ dev->read_msg = read_msg ? read_msg : vu_message_read_default;

dev->set_watch = set_watch;

dev->remove_watch = remove_watch;

dev->iface = iface;

@@ -XXX,XX +XXX,XX @@ static void _vu_queue_notify(VuDev *dev, VuVirtq *vq, bool sync)

vu_message_write(dev, dev->slave_fd, &vmsg);

if (ack) {

- vu_message_read(dev, dev->slave_fd, &vmsg);

+ vu_message_read_default(dev, dev->slave_fd, &vmsg);

}

return;

}

diff --git a/tests/vhost-user-bridge.c b/tests/vhost-user-bridge.c

index XXXXXXX..XXXXXXX 100644

--- a/tests/vhost-user-bridge.c

+++ b/tests/vhost-user-bridge.c

@@ -XXX,XX +XXX,XX @@ vubr_accept_cb(int sock, void *ctx)

VHOST_USER_BRIDGE_MAX_QUEUES,

conn_fd,

vubr_panic,

+ NULL,

vubr_set_watch,

vubr_remove_watch,

&vuiface)) {

@@ -XXX,XX +XXX,XX @@ vubr_new(const char *path, bool client)

VHOST_USER_BRIDGE_MAX_QUEUES,

dev->sock,

vubr_panic,

+ NULL,

vubr_set_watch,

vubr_remove_watch,

&vuiface)) {

diff --git a/tools/virtiofsd/fuse_virtio.c b/tools/virtiofsd/fuse_virtio.c

index XXXXXXX..XXXXXXX 100644

--- a/tools/virtiofsd/fuse_virtio.c

+++ b/tools/virtiofsd/fuse_virtio.c

@@ -XXX,XX +XXX,XX @@ int virtio_session_mount(struct fuse_session *se)

se->vu_socketfd = data_sock;

se->virtio_dev->se = se;

pthread_rwlock_init(&se->virtio_dev->vu_dispatch_rwlock, NULL);

- vu_init(&se->virtio_dev->dev, 2, se->vu_socketfd, fv_panic, fv_set_watch,

- fv_remove_watch, &fv_iface);

+ vu_init(&se->virtio_dev->dev, 2, se->vu_socketfd, fv_panic, NULL,

+ fv_set_watch, fv_remove_watch, &fv_iface);

return 0;

}

--

2.26.2

From: Coiby Xu <coiby.xu@gmail.com>

When the client is running in gdb and quit command is run in gdb,

QEMU will still dispatch the event which will cause segment fault in

the callback function.

Signed-off-by: Coiby Xu <coiby.xu@gmail.com>

Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Marc-André Lureau <marcandre.lureau@redhat.com>

Message-id: 20200918080912.321299-3-coiby.xu@gmail.com

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

contrib/libvhost-user/libvhost-user.c | 1 +

1 file changed, 1 insertion(+)

diff --git a/contrib/libvhost-user/libvhost-user.c b/contrib/libvhost-user/libvhost-user.c

index XXXXXXX..XXXXXXX 100644

--- a/contrib/libvhost-user/libvhost-user.c

+++ b/contrib/libvhost-user/libvhost-user.c

@@ -XXX,XX +XXX,XX @@ vu_deinit(VuDev *dev)

}

if (vq->kick_fd != -1) {

+ dev->remove_watch(dev, vq->kick_fd);

close(vq->kick_fd);

vq->kick_fd = -1;

}

--

2.26.2

From: Coiby Xu <coiby.xu@gmail.com>

Sharing QEMU devices via vhost-user protocol.

Only one vhost-user client can connect to the server one time.

Suggested-by: Kevin Wolf <kwolf@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Signed-off-by: Coiby Xu <coiby.xu@gmail.com>

Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Marc-André Lureau <marcandre.lureau@redhat.com>

Message-id: 20200918080912.321299-4-coiby.xu@gmail.com

[Fixed size_t %lu -> %zu format string compiler error.

--Stefan]

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

util/vhost-user-server.h | 65 ++++++

util/vhost-user-server.c | 428 +++++++++++++++++++++++++++++++++++++++

util/meson.build | 1 +

3 files changed, 494 insertions(+)

create mode 100644 util/vhost-user-server.h

create mode 100644 util/vhost-user-server.c

diff --git a/util/vhost-user-server.h b/util/vhost-user-server.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/util/vhost-user-server.h

@@ -XXX,XX +XXX,XX @@

+/*

+ * Sharing QEMU devices via vhost-user protocol

+ *

+ * Copyright (c) Coiby Xu <coiby.xu@gmail.com>.

+ * Copyright (c) 2020 Red Hat, Inc.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or

+ * later. See the COPYING file in the top-level directory.

+ */

+

+#ifndef VHOST_USER_SERVER_H

+#define VHOST_USER_SERVER_H

+

+#include "contrib/libvhost-user/libvhost-user.h"

+#include "io/channel-socket.h"

+#include "io/channel-file.h"

+#include "io/net-listener.h"

+#include "qemu/error-report.h"

+#include "qapi/error.h"

+#include "standard-headers/linux/virtio_blk.h"

+

+typedef struct VuFdWatch {

+ VuDev *vu_dev;

+ int fd; /*kick fd*/

+ void *pvt;

+ vu_watch_cb cb;

+ bool processing;

+ QTAILQ_ENTRY(VuFdWatch) next;

+} VuFdWatch;

+

+typedef struct VuServer VuServer;

+typedef void DevicePanicNotifierFn(VuServer *server);

+

+struct VuServer {

+ QIONetListener *listener;

+ AioContext *ctx;

+ DevicePanicNotifierFn *device_panic_notifier;

+ int max_queues;

+ const VuDevIface *vu_iface;

+ VuDev vu_dev;

+ QIOChannel *ioc; /* The I/O channel with the client */

+ QIOChannelSocket *sioc; /* The underlying data channel with the client */

+ /* IOChannel for fd provided via VHOST_USER_SET_SLAVE_REQ_FD */

+ QIOChannel *ioc_slave;

+ QIOChannelSocket *sioc_slave;

+ Coroutine *co_trip; /* coroutine for processing VhostUserMsg */

+ QTAILQ_HEAD(, VuFdWatch) vu_fd_watches;

+ /* restart coroutine co_trip if AIOContext is changed */

+ bool aio_context_changed;

+ bool processing_msg;

+};

+

+bool vhost_user_server_start(VuServer *server,

+ SocketAddress *unix_socket,

+ AioContext *ctx,

+ uint16_t max_queues,

+ DevicePanicNotifierFn *device_panic_notifier,

+ const VuDevIface *vu_iface,

+ Error **errp);

+

+void vhost_user_server_stop(VuServer *server);

+

+void vhost_user_server_set_aio_context(VuServer *server, AioContext *ctx);

+

+#endif /* VHOST_USER_SERVER_H */

diff --git a/util/vhost-user-server.c b/util/vhost-user-server.c

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/util/vhost-user-server.c

@@ -XXX,XX +XXX,XX @@

+/*

+ * Sharing QEMU devices via vhost-user protocol

+ *

+ * Copyright (c) Coiby Xu <coiby.xu@gmail.com>.

+ * Copyright (c) 2020 Red Hat, Inc.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or

+ * later. See the COPYING file in the top-level directory.

+ */

+#include "qemu/osdep.h"

+#include "qemu/main-loop.h"

+#include "vhost-user-server.h"

+

+static void vmsg_close_fds(VhostUserMsg *vmsg)

+{

+ int i;

+ for (i = 0; i < vmsg->fd_num; i++) {

+ close(vmsg->fds[i]);

+ }

+}

+

+static void vmsg_unblock_fds(VhostUserMsg *vmsg)

+{

+ int i;

+ for (i = 0; i < vmsg->fd_num; i++) {

+ qemu_set_nonblock(vmsg->fds[i]);

+ }

+}

+

+static void vu_accept(QIONetListener *listener, QIOChannelSocket *sioc,

+ gpointer opaque);

+

+static void close_client(VuServer *server)

+{

+ /*

+ * Before closing the client

+ *

+ * 1. Let vu_client_trip stop processing new vhost-user msg

+ *

+ * 2. remove kick_handler

+ *

+ * 3. wait for the kick handler to be finished

+ *

+ * 4. wait for the current vhost-user msg to be finished processing

+ */

+

+ QIOChannelSocket *sioc = server->sioc;

+ /* When this is set vu_client_trip will stop new processing vhost-user message */

+ server->sioc = NULL;

+

+ VuFdWatch *vu_fd_watch, *next;

+ QTAILQ_FOREACH_SAFE(vu_fd_watch, &server->vu_fd_watches, next, next) {

+ aio_set_fd_handler(server->ioc->ctx, vu_fd_watch->fd, true, NULL,

+ NULL, NULL, NULL);

+ }

+

+ while (!QTAILQ_EMPTY(&server->vu_fd_watches)) {

+ QTAILQ_FOREACH_SAFE(vu_fd_watch, &server->vu_fd_watches, next, next) {

+ if (!vu_fd_watch->processing) {

+ QTAILQ_REMOVE(&server->vu_fd_watches, vu_fd_watch, next);

+ g_free(vu_fd_watch);

+ }

+ }

+ }

+

+ while (server->processing_msg) {

+ if (server->ioc->read_coroutine) {

+ server->ioc->read_coroutine = NULL;

+ qio_channel_set_aio_fd_handler(server->ioc, server->ioc->ctx, NULL,

+ NULL, server->ioc);

+ server->processing_msg = false;

+ }

+ }

+

+ vu_deinit(&server->vu_dev);

+ object_unref(OBJECT(sioc));

+ object_unref(OBJECT(server->ioc));

+}

+

+static void panic_cb(VuDev *vu_dev, const char *buf)

+{

+ VuServer *server = container_of(vu_dev, VuServer, vu_dev);

+

+ /* avoid while loop in close_client */

+ server->processing_msg = false;

+

+ if (buf) {

+ error_report("vu_panic: %s", buf);

+ }

+

+ if (server->sioc) {

+ close_client(server);

+ }

+

+ if (server->device_panic_notifier) {

+ server->device_panic_notifier(server);

+ }

+

+ /*

+ * Set the callback function for network listener so another

+ * vhost-user client can connect to this server

+ */

+ qio_net_listener_set_client_func(server->listener,

+ vu_accept,

+ server,

+ NULL);

+}

+

+static bool coroutine_fn

+vu_message_read(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)

+{

+ struct iovec iov = {

+ .iov_base = (char *)vmsg,

+ .iov_len = VHOST_USER_HDR_SIZE,

+ };

+ int rc, read_bytes = 0;

+ Error *local_err = NULL;

+ /*

+ * Store fds/nfds returned from qio_channel_readv_full into

+ * temporary variables.

+ *

+ * VhostUserMsg is a packed structure, gcc will complain about passing

+ * pointer to a packed structure member if we pass &VhostUserMsg.fd_num

+ * and &VhostUserMsg.fds directly when calling qio_channel_readv_full,

+ * thus two temporary variables nfds and fds are used here.

+ */

+ size_t nfds = 0, nfds_t = 0;

+ const size_t max_fds = G_N_ELEMENTS(vmsg->fds);

+ int *fds_t = NULL;

+ VuServer *server = container_of(vu_dev, VuServer, vu_dev);

+ QIOChannel *ioc = server->ioc;

+

+ if (!ioc) {

+ error_report_err(local_err);

+ goto fail;

+ }

+

+ assert(qemu_in_coroutine());

+ do {

+ /*

+ * qio_channel_readv_full may have short reads, keeping calling it

+ * until getting VHOST_USER_HDR_SIZE or 0 bytes in total

+ */

+ rc = qio_channel_readv_full(ioc, &iov, 1, &fds_t, &nfds_t, &local_err);

+ if (rc < 0) {

+ if (rc == QIO_CHANNEL_ERR_BLOCK) {

+ qio_channel_yield(ioc, G_IO_IN);

+ continue;

+ } else {

+ error_report_err(local_err);

+ return false;

+ }

+ }

+ read_bytes += rc;

+ if (nfds_t > 0) {

+ if (nfds + nfds_t > max_fds) {

+ error_report("A maximum of %zu fds are allowed, "

+ "however got %zu fds now",

+ max_fds, nfds + nfds_t);

+ goto fail;

+ }

+ memcpy(vmsg->fds + nfds, fds_t,

+ nfds_t *sizeof(vmsg->fds[0]));

+ nfds += nfds_t;

+ g_free(fds_t);

+ }

+ if (read_bytes == VHOST_USER_HDR_SIZE || rc == 0) {

+ break;

+ }

+ iov.iov_base = (char *)vmsg + read_bytes;

+ iov.iov_len = VHOST_USER_HDR_SIZE - read_bytes;

+ } while (true);

+

+ vmsg->fd_num = nfds;

+ /* qio_channel_readv_full will make socket fds blocking, unblock them */

+ vmsg_unblock_fds(vmsg);

+ if (vmsg->size > sizeof(vmsg->payload)) {

+ error_report("Error: too big message request: %d, "

+ "size: vmsg->size: %u, "

+ "while sizeof(vmsg->payload) = %zu",

+ vmsg->request, vmsg->size, sizeof(vmsg->payload));

+ goto fail;

+ }

+

+ struct iovec iov_payload = {

+ .iov_base = (char *)&vmsg->payload,

+ .iov_len = vmsg->size,

+ };

+ if (vmsg->size) {

+ rc = qio_channel_readv_all_eof(ioc, &iov_payload, 1, &local_err);

+ if (rc == -1) {

+ error_report_err(local_err);

+ goto fail;

+ }

+ }

+

+ return true;

+

+fail:

+ vmsg_close_fds(vmsg);

+

+ return false;

+}

+

+

+static void vu_client_start(VuServer *server);

+static coroutine_fn void vu_client_trip(void *opaque)

+{

+ VuServer *server = opaque;

+

+ while (!server->aio_context_changed && server->sioc) {

+ server->processing_msg = true;

+ vu_dispatch(&server->vu_dev);

+ server->processing_msg = false;

+ }

+

+ if (server->aio_context_changed && server->sioc) {

+ server->aio_context_changed = false;

+ vu_client_start(server);

+ }

+}

+

+static void vu_client_start(VuServer *server)

+{

+ server->co_trip = qemu_coroutine_create(vu_client_trip, server);

+ aio_co_enter(server->ctx, server->co_trip);

+}

+

+/*

+ * a wrapper for vu_kick_cb

+ *

+ * since aio_dispatch can only pass one user data pointer to the

+ * callback function, pack VuDev and pvt into a struct. Then unpack it

+ * and pass them to vu_kick_cb

+ */

+static void kick_handler(void *opaque)

+{

+ VuFdWatch *vu_fd_watch = opaque;

+ vu_fd_watch->processing = true;

+ vu_fd_watch->cb(vu_fd_watch->vu_dev, 0, vu_fd_watch->pvt);

+ vu_fd_watch->processing = false;

+}

+

+

+static VuFdWatch *find_vu_fd_watch(VuServer *server, int fd)

+{

+

+ VuFdWatch *vu_fd_watch, *next;

+ QTAILQ_FOREACH_SAFE(vu_fd_watch, &server->vu_fd_watches, next, next) {

+ if (vu_fd_watch->fd == fd) {

+ return vu_fd_watch;

+ }

+ }

+ return NULL;

+}

+

+static void

+set_watch(VuDev *vu_dev, int fd, int vu_evt,

+ vu_watch_cb cb, void *pvt)

+{

+

+ VuServer *server = container_of(vu_dev, VuServer, vu_dev);

+ g_assert(vu_dev);

+ g_assert(fd >= 0);

+ g_assert(cb);

+

+ VuFdWatch *vu_fd_watch = find_vu_fd_watch(server, fd);

+

+ if (!vu_fd_watch) {

+ VuFdWatch *vu_fd_watch = g_new0(VuFdWatch, 1);

+

+ QTAILQ_INSERT_TAIL(&server->vu_fd_watches, vu_fd_watch, next);

+

+ vu_fd_watch->fd = fd;

+ vu_fd_watch->cb = cb;

+ qemu_set_nonblock(fd);

+ aio_set_fd_handler(server->ioc->ctx, fd, true, kick_handler,

+ NULL, NULL, vu_fd_watch);

+ vu_fd_watch->vu_dev = vu_dev;

+ vu_fd_watch->pvt = pvt;

+ }

+}

+

+

+static void remove_watch(VuDev *vu_dev, int fd)

+{

+ VuServer *server;

+ g_assert(vu_dev);

+ g_assert(fd >= 0);

+

+ server = container_of(vu_dev, VuServer, vu_dev);

+

+ VuFdWatch *vu_fd_watch = find_vu_fd_watch(server, fd);

+

+ if (!vu_fd_watch) {

+ return;

+ }

+ aio_set_fd_handler(server->ioc->ctx, fd, true, NULL, NULL, NULL, NULL);

+

+ QTAILQ_REMOVE(&server->vu_fd_watches, vu_fd_watch, next);

+ g_free(vu_fd_watch);

+}

+

+

+static void vu_accept(QIONetListener *listener, QIOChannelSocket *sioc,

+ gpointer opaque)

+{

+ VuServer *server = opaque;

+

+ if (server->sioc) {

+ warn_report("Only one vhost-user client is allowed to "

+ "connect the server one time");

+ return;

+ }

+

+ if (!vu_init(&server->vu_dev, server->max_queues, sioc->fd, panic_cb,

+ vu_message_read, set_watch, remove_watch, server->vu_iface)) {

+ error_report("Failed to initialize libvhost-user");

+ return;

+ }

+

+ /*

+ * Unset the callback function for network listener to make another

+ * vhost-user client keeping waiting until this client disconnects

+ */

+ qio_net_listener_set_client_func(server->listener,

+ NULL,

+ NULL,

+ NULL);

+ server->sioc = sioc;

+ /*

+ * Increase the object reference, so sioc will not freed by

+ * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))

+ */

+ object_ref(OBJECT(server->sioc));

+ qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");

+ server->ioc = QIO_CHANNEL(sioc);

+ object_ref(OBJECT(server->ioc));

+ qio_channel_attach_aio_context(server->ioc, server->ctx);

+ qio_channel_set_blocking(QIO_CHANNEL(server->sioc), false, NULL);

+ vu_client_start(server);

+}

+

+

+void vhost_user_server_stop(VuServer *server)

+{

+ if (server->sioc) {

+ close_client(server);

+ }

+

+ if (server->listener) {

+ qio_net_listener_disconnect(server->listener);

+ object_unref(OBJECT(server->listener));

+ }

+

+}

+

+void vhost_user_server_set_aio_context(VuServer *server, AioContext *ctx)

+{

+ VuFdWatch *vu_fd_watch, *next;

+ void *opaque = NULL;

+ IOHandler *io_read = NULL;

+ bool attach;

+

+ server->ctx = ctx ? ctx : qemu_get_aio_context();

+

+ if (!server->sioc) {

+ /* not yet serving any client*/

+ return;

+ }

+

+ if (ctx) {

+ qio_channel_attach_aio_context(server->ioc, ctx);

+ server->aio_context_changed = true;

+ io_read = kick_handler;

+ attach = true;

+ } else {

+ qio_channel_detach_aio_context(server->ioc);

+ /* server->ioc->ctx keeps the old AioConext */

+ ctx = server->ioc->ctx;

+ attach = false;

+ }

+

+ QTAILQ_FOREACH_SAFE(vu_fd_watch, &server->vu_fd_watches, next, next) {

+ if (vu_fd_watch->cb) {

+ opaque = attach ? vu_fd_watch : NULL;

+ aio_set_fd_handler(ctx, vu_fd_watch->fd, true,

+ io_read, NULL, NULL,

+ opaque);

+ }

+ }

+}

+

+

+bool vhost_user_server_start(VuServer *server,

+ SocketAddress *socket_addr,

+ AioContext *ctx,

+ uint16_t max_queues,

+ DevicePanicNotifierFn *device_panic_notifier,

+ const VuDevIface *vu_iface,

+ Error **errp)

+{

+ QIONetListener *listener = qio_net_listener_new();

+ if (qio_net_listener_open_sync(listener, socket_addr, 1,

+ errp) < 0) {

+ object_unref(OBJECT(listener));

+ return false;

+ }

+

+ /* zero out unspecified fileds */

+ *server = (VuServer) {

+ .listener = listener,

+ .vu_iface = vu_iface,

+ .max_queues = max_queues,

+ .ctx = ctx,

+ .device_panic_notifier = device_panic_notifier,

+ };

+

+ qio_net_listener_set_name(server->listener, "vhost-user-backend-listener");

+

+ qio_net_listener_set_client_func(server->listener,

+ vu_accept,

+ server,

+ NULL);

+

+ QTAILQ_INIT(&server->vu_fd_watches);

+ return true;

+}

diff --git a/util/meson.build b/util/meson.build

index XXXXXXX..XXXXXXX 100644

--- a/util/meson.build

+++ b/util/meson.build

@@ -XXX,XX +XXX,XX @@ if have_block

util_ss.add(files('main-loop.c'))

util_ss.add(files('nvdimm-utils.c'))

util_ss.add(files('qemu-coroutine.c', 'qemu-coroutine-lock.c', 'qemu-coroutine-io.c'))

+ util_ss.add(when: 'CONFIG_LINUX', if_true: files('vhost-user-server.c'))

util_ss.add(files('qemu-coroutine-sleep.c'))

util_ss.add(files('qemu-co-shared-resource.c'))

util_ss.add(files('thread-pool.c', 'qemu-timer.c'))

--

2.26.2

From: Coiby Xu <coiby.xu@gmail.com>

Move the constants from hw/core/qdev-properties.c to

util/block-helpers.h so that knowledge of the min/max values is

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Signed-off-by: Coiby Xu <coiby.xu@gmail.com>

Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Marc-André Lureau <marcandre.lureau@redhat.com>

Acked-by: Eduardo Habkost <ehabkost@redhat.com>

Message-id: 20200918080912.321299-5-coiby.xu@gmail.com

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

util/block-helpers.h | 19 +++++++++++++

hw/core/qdev-properties-system.c | 31 ++++-----------------

util/block-helpers.c | 46 ++++++++++++++++++++++++++++++++

util/meson.build | 1 +

4 files changed, 71 insertions(+), 26 deletions(-)

create mode 100644 util/block-helpers.h

create mode 100644 util/block-helpers.c

diff --git a/util/block-helpers.h b/util/block-helpers.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/util/block-helpers.h

@@ -XXX,XX +XXX,XX @@

+#ifndef BLOCK_HELPERS_H

+#define BLOCK_HELPERS_H

+

+#include "qemu/units.h"

+

+/* lower limit is sector size */

+#define MIN_BLOCK_SIZE INT64_C(512)

+#define MIN_BLOCK_SIZE_STR "512 B"

+/*

+ * upper limit is arbitrary, 2 MiB looks sufficient for all sensible uses, and

+ * matches qcow2 cluster size limit

+ */

+#define MAX_BLOCK_SIZE (2 * MiB)

+#define MAX_BLOCK_SIZE_STR "2 MiB"

+

+void check_block_size(const char *id, const char *name, int64_t value,

+ Error **errp);

+

+#endif /* BLOCK_HELPERS_H */

diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/core/qdev-properties-system.c

+++ b/hw/core/qdev-properties-system.c

@@ -XXX,XX +XXX,XX @@

#include "sysemu/blockdev.h"

#include "net/net.h"

#include "hw/pci/pci.h"

+#include "util/block-helpers.h"

static bool check_prop_still_unset(DeviceState *dev, const char *name,

const void *old_val, const char *new_val,

@@ -XXX,XX +XXX,XX @@ const PropertyInfo qdev_prop_losttickpolicy = {

/* --- blocksize --- */

-/* lower limit is sector size */

-#define MIN_BLOCK_SIZE 512

-#define MIN_BLOCK_SIZE_STR "512 B"

-/*

- * upper limit is arbitrary, 2 MiB looks sufficient for all sensible uses, and

- * matches qcow2 cluster size limit

- */

-#define MAX_BLOCK_SIZE (2 * MiB)

-#define MAX_BLOCK_SIZE_STR "2 MiB"

-

static void set_blocksize(Object *obj, Visitor *v, const char *name,

void *opaque, Error **errp)

{

@@ -XXX,XX +XXX,XX @@ static void set_blocksize(Object *obj, Visitor *v, const char *name,

Property *prop = opaque;

uint32_t *ptr = qdev_get_prop_ptr(dev, prop);

uint64_t value;

+ Error *local_err = NULL;

if (dev->realized) {

qdev_prop_set_after_realize(dev, name, errp);

@@ -XXX,XX +XXX,XX @@ static void set_blocksize(Object *obj, Visitor *v, const char *name,

if (!visit_type_size(v, name, &value, errp)) {

return;

}

- /* value of 0 means "unset" */

- if (value && (value < MIN_BLOCK_SIZE || value > MAX_BLOCK_SIZE)) {

- error_setg(errp,

- "Property %s.%s doesn't take value %" PRIu64

- " (minimum: " MIN_BLOCK_SIZE_STR

- ", maximum: " MAX_BLOCK_SIZE_STR ")",

- dev->id ? : "", name, value);

+ check_block_size(dev->id ? : "", name, value, &local_err);

+ if (local_err) {

+ error_propagate(errp, local_err);

return;

}

-

- /* We rely on power-of-2 blocksizes for bitmasks */

- if ((value & (value - 1)) != 0) {

- error_setg(errp,

- "Property %s.%s doesn't take value '%" PRId64 "', "

- "it's not a power of 2", dev->id ?: "", name, (int64_t)value);

- return;

- }

-

*ptr = value;

}

diff --git a/util/block-helpers.c b/util/block-helpers.c

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/util/block-helpers.c

@@ -XXX,XX +XXX,XX @@

+/*

+ * Block utility functions

+ *

+ * Copyright IBM, Corp. 2011

+ * Copyright (c) 2020 Coiby Xu <coiby.xu@gmail.com>

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

+ * See the COPYING file in the top-level directory.

+ */

+

+#include "qemu/osdep.h"

+#include "qapi/error.h"

+#include "qapi/qmp/qerror.h"

+#include "block-helpers.h"

+

+/**

+ * check_block_size:

+ * @id: The unique ID of the object

+ * @name: The name of the property being validated

+ * @value: The block size in bytes

+ * @errp: A pointer to an area to store an error

+ *

+ * This function checks that the block size meets the following conditions:

+ * 1. At least MIN_BLOCK_SIZE

+ * 2. No larger than MAX_BLOCK_SIZE

+ * 3. A power of 2

+ */

+void check_block_size(const char *id, const char *name, int64_t value,

+ Error **errp)

+{

+ /* value of 0 means "unset" */

+ if (value && (value < MIN_BLOCK_SIZE || value > MAX_BLOCK_SIZE)) {

+ error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,

+ id, name, value, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);

+ return;

+ }

+

+ /* We rely on power-of-2 blocksizes for bitmasks */

+ if ((value & (value - 1)) != 0) {

+ error_setg(errp,

+ "Property %s.%s doesn't take value '%" PRId64

+ "', it's not a power of 2",

+ id, name, value);

+ return;

+ }

+}

diff --git a/util/meson.build b/util/meson.build

index XXXXXXX..XXXXXXX 100644

--- a/util/meson.build

+++ b/util/meson.build

@@ -XXX,XX +XXX,XX @@ if have_block

util_ss.add(files('nvdimm-utils.c'))

util_ss.add(files('qemu-coroutine.c', 'qemu-coroutine-lock.c', 'qemu-coroutine-io.c'))

util_ss.add(when: 'CONFIG_LINUX', if_true: files('vhost-user-server.c'))

+ util_ss.add(files('block-helpers.c'))

util_ss.add(files('qemu-coroutine-sleep.c'))

util_ss.add(files('qemu-co-shared-resource.c'))

util_ss.add(files('thread-pool.c', 'qemu-timer.c'))

--

2.26.2

From: Coiby Xu <coiby.xu@gmail.com>

By making use of libvhost-user, block device drive can be shared to

the connected vhost-user client. Only one client can connect to the

server one time.

Since vhost-user-server needs a block drive to be created first, delay

the creation of this object.

Suggested-by: Kevin Wolf <kwolf@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Signed-off-by: Coiby Xu <coiby.xu@gmail.com>

Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Marc-André Lureau <marcandre.lureau@redhat.com>

Message-id: 20200918080912.321299-6-coiby.xu@gmail.com

[Shorten "vhost_user_blk_server" string to "vhost_user_blk" to avoid the

following compiler warning:

../block/export/vhost-user-blk-server.c:178:50: error: ‘%s’ directive output truncated writing 21 bytes into a region of size 20 [-Werror=format-truncation=]

and fix "Invalid size %ld ..." ssize_t format string arguments for

32-bit hosts.

--Stefan]

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

block/export/vhost-user-blk-server.h | 36 ++

block/export/vhost-user-blk-server.c | 661 +++++++++++++++++++++++++++

softmmu/vl.c | 4 +

block/meson.build | 1 +

4 files changed, 702 insertions(+)

create mode 100644 block/export/vhost-user-blk-server.h

create mode 100644 block/export/vhost-user-blk-server.c

diff --git a/block/export/vhost-user-blk-server.h b/block/export/vhost-user-blk-server.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/block/export/vhost-user-blk-server.h

@@ -XXX,XX +XXX,XX @@

+/*

+ * Sharing QEMU block devices via vhost-user protocal

+ *

+ * Copyright (c) Coiby Xu <coiby.xu@gmail.com>.

+ * Copyright (c) 2020 Red Hat, Inc.

+ *

+ * This work is licensed under the terms of the GNU GPL, version 2 or

+ * later. See the COPYING file in the top-level directory.

+ */

+

+#ifndef VHOST_USER_BLK_SERVER_H

+#define VHOST_USER_BLK_SERVER_H

+#include "util/vhost-user-server.h"

+

+typedef struct VuBlockDev VuBlockDev;

+#define TYPE_VHOST_USER_BLK_SERVER "vhost-user-blk-server"

+#define VHOST_USER_BLK_SERVER(obj) \

+ OBJECT_CHECK(VuBlockDev, obj, TYPE_VHOST_USER_BLK_SERVER)

+

+/* vhost user block device */

+struct VuBlockDev {

+ Object parent_obj;

+ char *node_name;

+ SocketAddress *addr;

+ AioContext *ctx;

+ VuServer vu_server;

+ bool running;

+ uint32_t blk_size;

+ BlockBackend *backend;

+ QIOChannelSocket *sioc;

+ QTAILQ_ENTRY(VuBlockDev) next;

+ struct virtio_blk_config blkcfg;

+ bool writable;

+};

+

+#endif /* VHOST_USER_BLK_SERVER_H */

diff --git a/block/export/vhost-user-blk-server.c b/block/export/vhost-user-blk-server.c

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/block/export/vhost-user-blk-server.c

@@ -XXX,XX +XXX,XX @@