The following changes since commit b4fbe1f65a4769c09e6bf2d79fc84360f840f40e:

Merge remote-tracking branch 'remotes/pmaydell/tags/pull-target-arm-20190129' into staging (2019-01-29 12:00:19 +0000)

https://git.xanclic.moe/XanClic/qemu.git tags/pull-block-2019-01-31

for you to fetch changes up to 908b30164bbffad7430d551b2a03a8fbcaa631ef:

iotests: Allow 147 to be run concurrently (2019-01-31 00:44:55 +0100)

Block patches:

- New debugging QMP command to explore block graphs

- Converted DPRINTF()s to trace events

- Fixed qemu-io's use of getopt() for systems with optreset

- Minor NVMe emulation fixes

- An iotest fix

Li Qiang (3):

nvme: use TYPE_NVME instead of constant string

nvme: ensure the num_queues is not zero

nvme: use pci_dev directly in nvme_realize

Max Reitz (3):

iotests.py: Add qemu_nbd_pipe()

iotests: Bind qemu-nbd to localhost in 147

iotests: Allow 147 to be run concurrently

Richard W.M. Jones (1):

qemu-io: Add generic function for reinitializing optind.

Vladimir Sementsov-Ogievskiy (2):

qapi: add x-debug-query-block-graph

scripts: add render_block_graph function for QEMUMachine

configure | 14 ++++

qapi/block-core.json | 108 ++++++++++++++++++++++++

include/block/block.h | 1 +

include/qemu/osdep.h | 16 ++++

include/sysemu/block-backend.h | 2 +

block.c | 148 +++++++++++++++++++++++++++++++++

block/block-backend.c | 5 ++

block/curl.c | 29 ++-----

block/file-posix.c | 25 ++----

block/sheepdog.c | 47 ++++-------

block/ssh.c | 46 ++++------

blockdev.c | 5 ++

hw/block/nvme.c | 15 ++--

qemu-img.c | 2 +-

qemu-io-cmds.c | 2 +-

block/trace-events | 47 +++++++++++

scripts/render_block_graph.py | 120 ++++++++++++++++++++++++++

tests/qemu-iotests/147 | 98 +++++++++++++++-------

tests/qemu-iotests/iotests.py | 14 ++++

19 files changed, 608 insertions(+), 136 deletions(-)

create mode 100755 scripts/render_block_graph.py

2.20.1

From: Laurent Vivier <lvivier@redhat.com>

Signed-off-by: Laurent Vivier <lvivier@redhat.com>

Reviewed-by: Richard W.M. Jones <rjones@redhat.com>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Message-id: 20181213162727.17438-2-lvivier@redhat.com

[mreitz: Fixed type of ssh_{read,write}_return's parameter to be ssize_t

instead of size_t]

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/ssh.c | 46 +++++++++++++++++-----------------------------

block/trace-events | 17 +++++++++++++++++

2 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/block/ssh.c b/block/ssh.c

--- a/block/ssh.c

+++ b/block/ssh.c

@@ -XXX,XX +XXX,XX @@

#include "qapi/qmp/qstring.h"

#include "qapi/qobject-input-visitor.h"

#include "qapi/qobject-output-visitor.h"

+#include "trace.h"

-/* DEBUG_SSH=1 enables the DPRINTF (debugging printf) statements in

- * this block driver code.

- *

* TRACE_LIBSSH2=<bitmask> enables tracing in libssh2 itself. Note

* that this requires that libssh2 was specially compiled with the

* `./configure --enable-debug' option, so most likely you will have

* to compile it yourself. The meaning of <bitmask> is described

* here: http://www.libssh2.org/libssh2_trace.html

*/

-#define DEBUG_SSH 0

#define TRACE_LIBSSH2 0 /* or try: LIBSSH2_TRACE_SFTP */

-#define DPRINTF(fmt, ...) \

- do { \

- if (DEBUG_SSH) { \

- fprintf(stderr, "ssh: %-15s " fmt "\n", \

- __func__, ##__VA_ARGS__); \

- } \

- } while (0)

-

typedef struct BDRVSSHState {

/* Coroutine. */

CoMutex lock;

@@ -XXX,XX +XXX,XX @@ static int check_host_key_knownhosts(BDRVSSHState *s,

switch (r) {

case LIBSSH2_KNOWNHOST_CHECK_MATCH:

/* OK */

- DPRINTF("host key OK: %s", found->key);

+ trace_ssh_check_host_key_knownhosts(found->key);

break;

case LIBSSH2_KNOWNHOST_CHECK_MISMATCH:

ret = -EINVAL;

@@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts,

/* Open the remote file. */

- DPRINTF("opening file %s flags=0x%x creat_mode=0%o",

- opts->path, ssh_flags, creat_mode);

+ trace_ssh_connect_to_ssh(opts->path, ssh_flags, creat_mode);

s->sftp_handle = libssh2_sftp_open(s->sftp, opts->path, ssh_flags,

creat_mode);

if (!s->sftp_handle) {

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn ssh_co_create_opts(const char *filename, QemuOpts *opts,

/* Get desired file size. */

ssh_opts->size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),

BDRV_SECTOR_SIZE);

- DPRINTF("total_size=%" PRIi64, ssh_opts->size);

+ trace_ssh_co_create_opts(ssh_opts->size);

uri_options = qdict_new();

ret = parse_uri(filename, uri_options, errp);

@@ -XXX,XX +XXX,XX @@ static void restart_coroutine(void *opaque)

BDRVSSHState *s = bs->opaque;

AioContext *ctx = bdrv_get_aio_context(bs);

- DPRINTF("co=%p", restart->co);

+ trace_ssh_restart_coroutine(restart->co);

aio_set_fd_handler(ctx, s->sock, false, NULL, NULL, NULL, NULL);

aio_co_wake(restart->co);

@@ -XXX,XX +XXX,XX @@ static coroutine_fn void co_yield(BDRVSSHState *s, BlockDriverState *bs)

wr_handler = restart_coroutine;

/* SFTP has a function `libssh2_sftp_seek64' which seeks to a position

@@ -XXX,XX +XXX,XX @@ static void ssh_seek(BDRVSSHState *s, int64_t offset, int flags)

bool force = (flags & SSH_SEEK_FORCE) != 0;

if (force || op_read != s->offset_op_read || offset != s->offset) {

- DPRINTF("seeking to offset=%" PRIi64, offset);

+ trace_ssh_seek(offset);

libssh2_sftp_seek64(s->sftp_handle, offset);

s->offset = offset;

s->offset_op_read = op_read;

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs,

char *buf, *end_of_vec;

struct iovec *i;

- DPRINTF("offset=%" PRIi64 " size=%zu", offset, size);

+ trace_ssh_read(offset, size);

ssh_seek(s, offset, SSH_SEEK_READ);

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs,

*/

for (got = 0; got < size; ) {

again:

- DPRINTF("sftp_read buf=%p size=%zu", buf, end_of_vec - buf);

+ trace_ssh_read_buf(buf, end_of_vec - buf);

r = libssh2_sftp_read(s->sftp_handle, buf, end_of_vec - buf);

- DPRINTF("sftp_read returned %zd", r);

+ trace_ssh_read_return(r);

if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) {

co_yield(s, bs);

@@ -XXX,XX +XXX,XX @@ static int ssh_write(BDRVSSHState *s, BlockDriverState *bs,

char *buf, *end_of_vec;

struct iovec *i;

- DPRINTF("offset=%" PRIi64 " size=%zu", offset, size);

+ trace_ssh_write(offset, size);

ssh_seek(s, offset, SSH_SEEK_WRITE);

@@ -XXX,XX +XXX,XX @@ static int ssh_write(BDRVSSHState *s, BlockDriverState *bs,

for (written = 0; written < size; ) {

again:

- DPRINTF("sftp_write buf=%p size=%zu", buf, end_of_vec - buf);

+ trace_ssh_write_buf(buf, end_of_vec - buf);

r = libssh2_sftp_write(s->sftp_handle, buf, end_of_vec - buf);

- DPRINTF("sftp_write returned %zd", r);

+ trace_ssh_write_return(r);

if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) {

co_yield(s, bs);

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_flush(BDRVSSHState *s, BlockDriverState *bs)

int r;

- DPRINTF("fsync");

+ trace_ssh_flush();

again:

r = libssh2_sftp_fsync(s->sftp_handle);

if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) {

@@ -XXX,XX +XXX,XX @@ static int64_t ssh_getlength(BlockDriverState *bs)

/* Note we cannot make a libssh2 call here. */

length = (int64_t) s->attrs.filesize;

- DPRINTF("length=%" PRIi64, length);

+ trace_ssh_getlength(length);

return length;

diff --git a/block/trace-events b/block/trace-events

index XXXXXXX..XXXXXXX 100644

--- a/block/trace-events

+++ b/block/trace-events

@@ -XXX,XX +XXX,XX @@ iscsi_xcopy(void *src_lun, uint64_t src_off, void *dst_lun, uint64_t dst_off, ui

# block/nbd-client.c

nbd_read_reply_entry_fail(int ret, const char *err) "ret = %d, err: %s"

nbd_co_request_fail(uint64_t from, uint32_t len, uint64_t handle, uint16_t flags, uint16_t type, const char *name, int ret, const char *err) "Request failed { .from = %" PRIu64", .len = %" PRIu32 ", .handle = %" PRIu64 ", .flags = 0x%" PRIx16 ", .type = %" PRIu16 " (%s) } ret = %d, err: %s"

+

+# block/ssh.c

+ssh_restart_coroutine(void *co) "co=%p"

+ssh_flush(void) "fsync"

+ssh_check_host_key_knownhosts(const char *key) "host key OK: %s"

+ssh_connect_to_ssh(char *path, int flags, int mode) "opening file %s flags=0x%x creat_mode=0%o"

+ssh_co_yield(int sock, void *rd_handler, void *wr_handler) "s->sock=%d rd_handler=%p wr_handler=%p"

+ssh_co_yield_back(int sock) "s->sock=%d - back"

+ssh_getlength(int64_t length) "length=%" PRIi64

+ssh_co_create_opts(uint64_t size) "total_size=%" PRIu64

+ssh_read(int64_t offset, size_t size) "offset=%" PRIi64 " size=%zu"

+ssh_read_buf(void *buf, size_t size) "sftp_read buf=%p size=%zu"

+ssh_read_return(ssize_t ret) "sftp_read returned %zd"

+ssh_write(int64_t offset, size_t size) "offset=%" PRIi64 " size=%zu"

+ssh_write_buf(void *buf, size_t size) "sftp_write buf=%p size=%zu"

+ssh_write_return(ssize_t ret) "sftp_write returned %zd"

+ssh_seek(int64_t offset) "seeking to offset=%" PRIi64

2.20.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Add a new command, returning block nodes (and their users) graph.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Message-id: 20181221170909.25584-2-vsementsov@virtuozzo.com

Signed-off-by: Max Reitz <mreitz@redhat.com>

qapi/block-core.json | 108 ++++++++++++++++++++++++

include/block/block.h | 1 +

include/sysemu/block-backend.h | 2 +

block.c | 148 +++++++++++++++++++++++++++++++++

block/block-backend.c | 5 ++

blockdev.c | 5 ++

6 files changed, 269 insertions(+)

diff --git a/qapi/block-core.json b/qapi/block-core.json

index XXXXXXX..XXXXXXX 100644

--- a/qapi/block-core.json

+++ b/qapi/block-core.json

@@ -XXX,XX +XXX,XX @@

{ 'command': 'query-named-block-nodes', 'returns': [ 'BlockDeviceInfo' ] }

+##

+# @XDbgBlockGraphNodeType:

+#

+# @block-backend: corresponds to BlockBackend

+#

+# @block-job: corresonds to BlockJob

+#

+# @block-driver: corresponds to BlockDriverState

+#

+# Since: 4.0

+##

+{ 'enum': 'XDbgBlockGraphNodeType',

+ 'data': [ 'block-backend', 'block-job', 'block-driver' ] }

+

+##

+# @XDbgBlockGraphNode:

+#

+# @id: Block graph node identifier. This @id is generated only for

+# x-debug-query-block-graph and does not relate to any other identifiers in

+# Qemu.

+#

+# @type: Type of graph node. Can be one of block-backend, block-job or

+# block-driver-state.

+#

+# @name: Human readable name of the node. Corresponds to node-name for

+# block-driver-state nodes; is not guaranteed to be unique in the whole

+# graph (with block-jobs and block-backends).

+#

+# Since: 4.0

+##

+{ 'struct': 'XDbgBlockGraphNode',

+ 'data': { 'id': 'uint64', 'type': 'XDbgBlockGraphNodeType', 'name': 'str' } }

+

+##

+# @BlockPermission:

+#

+# Enum of base block permissions.

+#

+# @consistent-read: A user that has the "permission" of consistent reads is

+# guaranteed that their view of the contents of the block

+# device is complete and self-consistent, representing the

+# contents of a disk at a specific point.

+# For most block devices (including their backing files) this

+# is true, but the property cannot be maintained in a few

+# situations like for intermediate nodes of a commit block

+# job.

+#

+# @write: This permission is required to change the visible disk contents.

+#

+# @write-unchanged: This permission (which is weaker than BLK_PERM_WRITE) is

+# both enough and required for writes to the block node when

+# the caller promises that the visible disk content doesn't

+# change.

+# As the BLK_PERM_WRITE permission is strictly stronger,

+# either is sufficient to perform an unchanging write.

+#

+# @resize: This permission is required to change the size of a block node.

+#

+# @graph-mod: This permission is required to change the node that this

+# BdrvChild points to.

+#

+# Since: 4.0

+##

+ { 'enum': 'BlockPermission',

+ 'data': [ 'consistent-read', 'write', 'write-unchanged', 'resize',

+ 'graph-mod' ] }

+##

+# @XDbgBlockGraphEdge:

+#

+# Block Graph edge description for x-debug-query-block-graph.

+#

+# @parent: parent id

+#

+# @child: child id

+#

+# @name: name of the relation (examples are 'file' and 'backing')

+#

+# @perm: granted permissions for the parent operating on the child

+#

+# @shared-perm: permissions that can still be granted to other users of the

+# child while it is still attached to this parent

+#

+# Since: 4.0

+##

+{ 'struct': 'XDbgBlockGraphEdge',

+ 'data': { 'parent': 'uint64', 'child': 'uint64',

+ 'name': 'str', 'perm': [ 'BlockPermission' ],

+ 'shared-perm': [ 'BlockPermission' ] } }

+

+##

+# @XDbgBlockGraph:

+#

+# Block Graph - list of nodes and list of edges.

+#

+# Since: 4.0

+##

+{ 'struct': 'XDbgBlockGraph',

+ 'data': { 'nodes': ['XDbgBlockGraphNode'], 'edges': ['XDbgBlockGraphEdge'] } }

+

+##

+# @x-debug-query-block-graph:

+#

+# Get the block graph.

+#

+# Since: 4.0

+##

+{ 'command': 'x-debug-query-block-graph', 'returns': 'XDbgBlockGraph' }

+

##

# @drive-mirror:

diff --git a/include/block/block.h b/include/block/block.h

index XXXXXXX..XXXXXXX 100644

--- a/include/block/block.h

+++ b/include/block/block.h

@@ -XXX,XX +XXX,XX @@ void bdrv_eject(BlockDriverState *bs, bool eject_flag);

const char *bdrv_get_format_name(BlockDriverState *bs);

BlockDriverState *bdrv_find_node(const char *node_name);

BlockDeviceInfoList *bdrv_named_nodes_list(Error **errp);

+XDbgBlockGraph *bdrv_get_xdbg_block_graph(Error **errp);

BlockDriverState *bdrv_lookup_bs(const char *device,

const char *node_name,

Error **errp);

diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h

index XXXXXXX..XXXXXXX 100644

--- a/include/sysemu/block-backend.h

+++ b/include/sysemu/block-backend.h

@@ -XXX,XX +XXX,XX @@ int coroutine_fn blk_co_copy_range(BlockBackend *blk_in, int64_t off_in,

int bytes, BdrvRequestFlags read_flags,

BdrvRequestFlags write_flags);

+const BdrvChild *blk_root(BlockBackend *blk);

+

#endif

diff --git a/block.c b/block.c

index XXXXXXX..XXXXXXX 100644

--- a/block.c

+++ b/block.c

@@ -XXX,XX +XXX,XX @@ BlockDeviceInfoList *bdrv_named_nodes_list(Error **errp)

return list;

+#define QAPI_LIST_ADD(list, element) do { \

+ typeof(list) _tmp = g_new(typeof(*(list)), 1); \

+ _tmp->value = (element); \

+ _tmp->next = (list); \

+ (list) = _tmp; \

+} while (0)

+

+typedef struct XDbgBlockGraphConstructor {

+ XDbgBlockGraph *graph;

+ GHashTable *graph_nodes;

+} XDbgBlockGraphConstructor;

+

+static XDbgBlockGraphConstructor *xdbg_graph_new(void)

+{

+ XDbgBlockGraphConstructor *gr = g_new(XDbgBlockGraphConstructor, 1);

+

+ gr->graph = g_new0(XDbgBlockGraph, 1);

+ gr->graph_nodes = g_hash_table_new(NULL, NULL);

+

+ return gr;

+}

+

+static XDbgBlockGraph *xdbg_graph_finalize(XDbgBlockGraphConstructor *gr)

+{

+ XDbgBlockGraph *graph = gr->graph;

+

+ g_hash_table_destroy(gr->graph_nodes);

+ g_free(gr);

+

+ return graph;

+}

+

+static uintptr_t xdbg_graph_node_num(XDbgBlockGraphConstructor *gr, void *node)

+{

+ uintptr_t ret = (uintptr_t)g_hash_table_lookup(gr->graph_nodes, node);

+

+ if (ret != 0) {

+ return ret;

+ /*

+ * Start counting from 1, not 0, because 0 interferes with not-found (NULL)

+ * answer of g_hash_table_lookup.

+ */

+ ret = g_hash_table_size(gr->graph_nodes) + 1;

+ g_hash_table_insert(gr->graph_nodes, node, (void *)ret);

+

+ return ret;

+}

+

+static void xdbg_graph_add_node(XDbgBlockGraphConstructor *gr, void *node,

+ XDbgBlockGraphNodeType type, const char *name)

+{

+ XDbgBlockGraphNode *n;

+

+ n = g_new0(XDbgBlockGraphNode, 1);

+

+ n->id = xdbg_graph_node_num(gr, node);

+ n->type = type;

+ n->name = g_strdup(name);

+

+ QAPI_LIST_ADD(gr->graph->nodes, n);

+}

+

+static void xdbg_graph_add_edge(XDbgBlockGraphConstructor *gr, void *parent,

+ const BdrvChild *child)

+{

+ typedef struct {

+ unsigned int flag;

+ BlockPermission num;

+ } PermissionMap;

+

+ static const PermissionMap permissions[] = {

+ { BLK_PERM_CONSISTENT_READ, BLOCK_PERMISSION_CONSISTENT_READ },

+ { BLK_PERM_WRITE, BLOCK_PERMISSION_WRITE },

+ { BLK_PERM_WRITE_UNCHANGED, BLOCK_PERMISSION_WRITE_UNCHANGED },

+ { BLK_PERM_RESIZE, BLOCK_PERMISSION_RESIZE },

+ { BLK_PERM_GRAPH_MOD, BLOCK_PERMISSION_GRAPH_MOD },

+ { 0, 0 }

+ };

+ const PermissionMap *p;

+ XDbgBlockGraphEdge *edge;

+

+ QEMU_BUILD_BUG_ON(1UL << (ARRAY_SIZE(permissions) - 1) != BLK_PERM_ALL + 1);

+

+ edge = g_new0(XDbgBlockGraphEdge, 1);

+

+ edge->parent = xdbg_graph_node_num(gr, parent);

+ edge->child = xdbg_graph_node_num(gr, child->bs);

+ edge->name = g_strdup(child->name);

+

+ for (p = permissions; p->flag; p++) {

+ if (p->flag & child->perm) {

+ QAPI_LIST_ADD(edge->perm, p->num);

+ }

+ if (p->flag & child->shared_perm) {

+ QAPI_LIST_ADD(edge->shared_perm, p->num);

+ }

+ QAPI_LIST_ADD(gr->graph->edges, edge);

+}

+

+

+XDbgBlockGraph *bdrv_get_xdbg_block_graph(Error **errp)

+{

+ BlockBackend *blk;

+ BlockJob *job;

+ BlockDriverState *bs;

+ BdrvChild *child;

+ XDbgBlockGraphConstructor *gr = xdbg_graph_new();

+

+ for (blk = blk_all_next(NULL); blk; blk = blk_all_next(blk)) {

+ char *allocated_name = NULL;

+ const char *name = blk_name(blk);

+

+ if (!*name) {

+ name = allocated_name = blk_get_attached_dev_id(blk);

+ }

+ xdbg_graph_add_node(gr, blk, X_DBG_BLOCK_GRAPH_NODE_TYPE_BLOCK_BACKEND,

+ name);

+ g_free(allocated_name);

+ if (blk_root(blk)) {

+ xdbg_graph_add_edge(gr, blk, blk_root(blk));

+ }

+ for (job = block_job_next(NULL); job; job = block_job_next(job)) {

+ GSList *el;

+

+ xdbg_graph_add_node(gr, job, X_DBG_BLOCK_GRAPH_NODE_TYPE_BLOCK_JOB,

+ job->job.id);

+ for (el = job->nodes; el; el = el->next) {

+ xdbg_graph_add_edge(gr, job, (BdrvChild *)el->data);

+ }

+ }

+

+ QTAILQ_FOREACH(bs, &graph_bdrv_states, node_list) {

+ xdbg_graph_add_node(gr, bs, X_DBG_BLOCK_GRAPH_NODE_TYPE_BLOCK_DRIVER,

+ bs->node_name);

+ QLIST_FOREACH(child, &bs->children, next) {

+ xdbg_graph_add_edge(gr, bs, child);

+ }

+ }

+

+ return xdbg_graph_finalize(gr);

+}

+

BlockDriverState *bdrv_lookup_bs(const char *device,

const char *node_name,

Error **errp)

diff --git a/block/block-backend.c b/block/block-backend.c

index XXXXXXX..XXXXXXX 100644

--- a/block/block-backend.c

+++ b/block/block-backend.c

@@ -XXX,XX +XXX,XX @@ int coroutine_fn blk_co_copy_range(BlockBackend *blk_in, int64_t off_in,

blk_out->root, off_out,

bytes, read_flags, write_flags);

}

+

+const BdrvChild *blk_root(BlockBackend *blk)

+{

+ return blk->root;

+}

diff --git a/blockdev.c b/blockdev.c

index XXXXXXX..XXXXXXX 100644

--- a/blockdev.c

+++ b/blockdev.c

@@ -XXX,XX +XXX,XX @@ BlockDeviceInfoList *qmp_query_named_block_nodes(Error **errp)

return bdrv_named_nodes_list(errp);

}

+XDbgBlockGraph *qmp_x_debug_query_block_graph(Error **errp)

+{

+ return bdrv_get_xdbg_block_graph(errp);

+}

+

BlockJob *do_blockdev_backup(BlockdevBackup *backup, JobTxn *txn,

Error **errp)

{

2.20.1