The following changes since commit 506e4a00de01e0b29fa83db5cbbc3d154253b4ea: Merge remote-tracking branch 'remotes/dgibson/tags/ppc-for-3.1-20180925' into staging (2018-09-25 13:30:45 +0100) are available in the Git repository at: https://git.xanclic.moe/XanClic/qemu.git tags/pull-block-2018-09-25 for you to fetch changes up to 9c76ff9c16be890e70fce30754b096ff9950d1ee: Merge remote-tracking branch 'kevin/tags/for-upstream' into block (2018-09-25 16:12:44 +0200) ---------------------------------------------------------------- Block layer patches: - Drain fixes - node-name parameters for block-commit - Refactor block jobs to use transactional callbacks for exiting ---------------------------------------------------------------- Alberto Garcia (2): block: Fix use after free error in bdrv_open_inherit() qemu-iotests: Test snapshot=on with nonexistent TMPDIR Fam Zheng (1): job: Fix nested aio_poll() hanging in job_txn_apply John Snow (16): block/commit: add block job creation flags block/mirror: add block job creation flags block/stream: add block job creation flags block/commit: refactor commit to use job callbacks block/mirror: don't install backing chain on abort block/mirror: conservative mirror_exit refactor block/stream: refactor stream to use job callbacks tests/blockjob: replace Blockjob with Job tests/test-blockjob: remove exit callback tests/test-blockjob-txn: move .exit to .clean jobs: remove .exit callback qapi/block-commit: expose new job properties qapi/block-mirror: expose new job properties qapi/block-stream: expose new job properties block/backup: qapi documentation fixup blockdev: document transactional shortcomings Kevin Wolf (21): commit: Add top-node/base-node options qemu-iotests: Test commit with top-node/base-node job: Fix missing locking due to mismerge blockjob: Wake up BDS when job becomes idle aio-wait: Increase num_waiters even in home thread test-bdrv-drain: Drain with block jobs in an I/O thread test-blockjob: Acquire AioContext around job_cancel_sync() job: Use AIO_WAIT_WHILE() in job_finish_sync() test-bdrv-drain: Test AIO_WAIT_WHILE() in completion callback block: Add missing locking in bdrv_co_drain_bh_cb() block-backend: Add .drained_poll callback block-backend: Fix potential double blk_delete() block-backend: Decrease in_flight only after callback blockjob: Lie better in child_job_drained_poll() block: Remove aio_poll() in bdrv_drain_poll variants test-bdrv-drain: Test nested poll in bdrv_drain_poll_top_level() job: Avoid deadlocks in job_completed_txn_abort() test-bdrv-drain: AIO_WAIT_WHILE() in job .commit/.abort test-bdrv-drain: Fix outdated comments block: Use a single global AioWait test-bdrv-drain: Test draining job source child and parent Max Reitz (1): Merge remote-tracking branch 'kevin/tags/for-upstream' into block Sergio Lopez (2): block/linux-aio: acquire AioContext before qemu_laio_process_completions util/async: use qemu_aio_coroutine_enter in co_schedule_bh_cb qapi/block-core.json | 104 ++++++++++++--- include/block/aio-wait.h | 28 ++-- include/block/block.h | 6 +- include/block/block_int.h | 18 ++- include/block/blockjob.h | 3 + include/qemu/coroutine.h | 5 + include/qemu/job.h | 23 ++-- block.c | 6 +- block/block-backend.c | 31 +++-- block/commit.c | 97 ++++++++------ block/io.c | 30 +++-- block/linux-aio.c | 2 +- block/mirror.c | 49 +++++-- block/stream.c | 28 ++-- blockdev.c | 84 ++++++++++-- blockjob.c | 9 +- hmp.c | 5 +- job.c | 144 +++++++++++---------- tests/test-bdrv-drain.c | 294 +++++++++++++++++++++++++++++++++++++++--- tests/test-blockjob-txn.c | 4 +- tests/test-blockjob.c | 120 ++++++++--------- util/aio-wait.c | 11 +- util/async.c | 2 +- util/qemu-coroutine.c | 5 + tests/qemu-iotests/040 | 52 +++++++- tests/qemu-iotests/040.out | 4 +- tests/qemu-iotests/051 | 3 + tests/qemu-iotests/051.out | 3 + tests/qemu-iotests/051.pc.out | 3 + 29 files changed, 856 insertions(+), 317 deletions(-) -- 2.17.1 From: John Snow <jsnow@redhat.com> Add support for taking and passing forward job creation flags. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Jeff Cody <jcody@redhat.com> Message-id: 20180906130225.5118-2-jsnow@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block_int.h | 5 ++++- block/commit.c | 5 +++-- blockdev.c | 7 ++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, * @bs: Active block device. * @top: Top block device to be committed. * @base: Block device that will be written into, and become the new top. + * @creation_flags: Flags that control the behavior of the Job lifetime. + * See @BlockJobCreateFlags * @speed: The maximum speed, in bytes per second, or 0 for unlimited. * @on_error: The action to take upon error. * @backing_file_str: String to use as the backing file in @top's overlay @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, * */ void commit_start(const char *job_id, BlockDriverState *bs, - BlockDriverState *base, BlockDriverState *top, int64_t speed, + BlockDriverState *base, BlockDriverState *top, + int creation_flags, int64_t speed, BlockdevOnError on_error, const char *backing_file_str, const char *filter_node_name, Error **errp); /** diff --git a/block/commit.c b/block/commit.c index XXXXXXX..XXXXXXX 100644 --- a/block/commit.c +++ b/block/commit.c @@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_commit_top = { }; void commit_start(const char *job_id, BlockDriverState *bs, - BlockDriverState *base, BlockDriverState *top, int64_t speed, + BlockDriverState *base, BlockDriverState *top, + int creation_flags, int64_t speed, BlockdevOnError on_error, const char *backing_file_str, const char *filter_node_name, Error **errp) { @@ -XXX,XX +XXX,XX @@ void commit_start(const char *job_id, BlockDriverState *bs, } s = block_job_create(job_id, &commit_job_driver, NULL, bs, 0, BLK_PERM_ALL, - speed, JOB_DEFAULT, NULL, NULL, errp); + speed, creation_flags, NULL, NULL, errp); if (!s) { return; } diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, * BlockdevOnError change for blkmirror makes it in */ BlockdevOnError on_error = BLOCKDEV_ON_ERROR_REPORT; + int job_flags = JOB_DEFAULT; if (!has_speed) { speed = 0; @@ -XXX,XX +XXX,XX @@ void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, goto out; } commit_active_start(has_job_id ? job_id : NULL, bs, base_bs, - JOB_DEFAULT, speed, on_error, + job_flags, speed, on_error, filter_node_name, NULL, NULL, false, &local_err); } else { BlockDriverState *overlay_bs = bdrv_find_overlay(bs, top_bs); if (bdrv_op_is_blocked(overlay_bs, BLOCK_OP_TYPE_COMMIT_TARGET, errp)) { goto out; } - commit_start(has_job_id ? job_id : NULL, bs, base_bs, top_bs, speed, - on_error, has_backing_file ? backing_file : NULL, + commit_start(has_job_id ? job_id : NULL, bs, base_bs, top_bs, job_flags, + speed, on_error, has_backing_file ? backing_file : NULL, filter_node_name, &local_err); } if (local_err != NULL) { -- 2.17.1 From: John Snow <jsnow@redhat.com> Add support for taking and passing forward job creation flags. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Jeff Cody <jcody@redhat.com> Message-id: 20180906130225.5118-3-jsnow@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block_int.h | 5 ++++- block/mirror.c | 5 +++-- blockdev.c | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ void commit_active_start(const char *job_id, BlockDriverState *bs, * @target: Block device to write to. * @replaces: Block graph node name to replace once the mirror is done. Can * only be used when full mirroring is selected. + * @creation_flags: Flags that control the behavior of the Job lifetime. + * See @BlockJobCreateFlags * @speed: The maximum speed, in bytes per second, or 0 for unlimited. * @granularity: The chosen granularity for the dirty bitmap. * @buf_size: The amount of data that can be in flight at one time. @@ -XXX,XX +XXX,XX @@ void commit_active_start(const char *job_id, BlockDriverState *bs, */ void mirror_start(const char *job_id, BlockDriverState *bs, BlockDriverState *target, const char *replaces, - int64_t speed, uint32_t granularity, int64_t buf_size, + int creation_flags, int64_t speed, + uint32_t granularity, int64_t buf_size, MirrorSyncMode mode, BlockMirrorBackingMode backing_mode, BlockdevOnError on_source_error, BlockdevOnError on_target_error, diff --git a/block/mirror.c b/block/mirror.c index XXXXXXX..XXXXXXX 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -XXX,XX +XXX,XX @@ fail: void mirror_start(const char *job_id, BlockDriverState *bs, BlockDriverState *target, const char *replaces, - int64_t speed, uint32_t granularity, int64_t buf_size, + int creation_flags, int64_t speed, + uint32_t granularity, int64_t buf_size, MirrorSyncMode mode, BlockMirrorBackingMode backing_mode, BlockdevOnError on_source_error, BlockdevOnError on_target_error, @@ -XXX,XX +XXX,XX @@ void mirror_start(const char *job_id, BlockDriverState *bs, } is_none_mode = mode == MIRROR_SYNC_MODE_NONE; base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL; - mirror_start_job(job_id, bs, JOB_DEFAULT, target, replaces, + mirror_start_job(job_id, bs, creation_flags, target, replaces, speed, granularity, buf_size, backing_mode, on_source_error, on_target_error, unmap, NULL, NULL, &mirror_job_driver, is_none_mode, base, false, diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs, bool has_copy_mode, MirrorCopyMode copy_mode, Error **errp) { + int job_flags = JOB_DEFAULT; if (!has_speed) { speed = 0; @@ -XXX,XX +XXX,XX @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs, * and will allow to check whether the node still exist at mirror completion */ mirror_start(job_id, bs, target, - has_replaces ? replaces : NULL, + has_replaces ? replaces : NULL, job_flags, speed, granularity, buf_size, sync, backing_mode, on_source_error, on_target_error, unmap, filter_node_name, copy_mode, errp); -- 2.17.1 From: John Snow <jsnow@redhat.com> Add support for taking and passing forward job creation flags. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Jeff Cody <jcody@redhat.com> Message-id: 20180906130225.5118-4-jsnow@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block_int.h | 5 ++++- block/stream.c | 5 +++-- blockdev.c | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ int is_windows_drive(const char *filename); * flatten the whole backing file chain onto @bs. * @backing_file_str: The file name that will be written to @bs as the * the new backing file if the job completes. Ignored if @base is %NULL. + * @creation_flags: Flags that control the behavior of the Job lifetime. + * See @BlockJobCreateFlags * @speed: The maximum speed, in bytes per second, or 0 for unlimited. * @on_error: The action to take upon error. * @errp: Error object. @@ -XXX,XX +XXX,XX @@ int is_windows_drive(const char *filename); */ void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, - int64_t speed, BlockdevOnError on_error, Error **errp); + int creation_flags, int64_t speed, + BlockdevOnError on_error, Error **errp); /** * commit_start: diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, - int64_t speed, BlockdevOnError on_error, Error **errp) + int creation_flags, int64_t speed, + BlockdevOnError on_error, Error **errp) { StreamBlockJob *s; BlockDriverState *iter; @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, BLK_PERM_GRAPH_MOD, BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE, - speed, JOB_DEFAULT, NULL, NULL, errp); + speed, creation_flags, NULL, NULL, errp); if (!s) { goto fail; } diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, AioContext *aio_context; Error *local_err = NULL; const char *base_name = NULL; + int job_flags = JOB_DEFAULT; if (!has_on_error) { on_error = BLOCKDEV_ON_ERROR_REPORT; @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, base_name = has_backing_file ? backing_file : base_name; stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name, - has_speed ? speed : 0, on_error, &local_err); + job_flags, has_speed ? speed : 0, on_error, &local_err); if (local_err) { error_propagate(errp, local_err); goto out; -- 2.17.1 From: John Snow <jsnow@redhat.com> Use the component callbacks; prepare, abort, and clean. NB: prepare is only called when the job has not yet failed; and abort can be called after prepare. complete -> prepare -> abort -> clean complete -> abort -> clean During refactor, a potential problem with bdrv_drop_intermediate was identified, the patched behavior is no worse than the pre-patch behavior, so leave a FIXME for now to be fixed in a future patch. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-5-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/commit.c | 92 ++++++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/block/commit.c b/block/commit.c index XXXXXXX..XXXXXXX 100644 --- a/block/commit.c +++ b/block/commit.c @@ -XXX,XX +XXX,XX @@ typedef struct CommitBlockJob { BlockDriverState *commit_top_bs; BlockBackend *top; BlockBackend *base; + BlockDriverState *base_bs; BlockdevOnError on_error; int base_flags; char *backing_file_str; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn commit_populate(BlockBackend *bs, BlockBackend *base, return 0; } -static void commit_exit(Job *job) +static int commit_prepare(Job *job) { CommitBlockJob *s = container_of(job, CommitBlockJob, common.job); - BlockJob *bjob = &s->common; - BlockDriverState *top = blk_bs(s->top); - BlockDriverState *base = blk_bs(s->base); - BlockDriverState *commit_top_bs = s->commit_top_bs; - bool remove_commit_top_bs = false; - - /* Make sure commit_top_bs and top stay around until bdrv_replace_node() */ - bdrv_ref(top); - bdrv_ref(commit_top_bs); /* Remove base node parent that still uses BLK_PERM_WRITE/RESIZE before * the normal backing chain can be restored. */ blk_unref(s->base); + s->base = NULL; + + /* FIXME: bdrv_drop_intermediate treats total failures and partial failures + * identically. Further work is needed to disambiguate these cases. */ + return bdrv_drop_intermediate(s->commit_top_bs, s->base_bs, + s->backing_file_str); +} - if (!job_is_cancelled(job) && job->ret == 0) { - /* success */ - job->ret = bdrv_drop_intermediate(s->commit_top_bs, base, - s->backing_file_str); - } else { - /* XXX Can (or should) we somehow keep 'consistent read' blocked even - * after the failed/cancelled commit job is gone? If we already wrote - * something to base, the intermediate images aren't valid any more. */ - remove_commit_top_bs = true; +static void commit_abort(Job *job) +{ + CommitBlockJob *s = container_of(job, CommitBlockJob, common.job); + BlockDriverState *top_bs = blk_bs(s->top); + + /* Make sure commit_top_bs and top stay around until bdrv_replace_node() */ + bdrv_ref(top_bs); + bdrv_ref(s->commit_top_bs); + + if (s->base) { + blk_unref(s->base); } + /* free the blockers on the intermediate nodes so that bdrv_replace_nodes + * can succeed */ + block_job_remove_all_bdrv(&s->common); + + /* If bdrv_drop_intermediate() failed (or was not invoked), remove the + * commit filter driver from the backing chain now. Do this as the final + * step so that the 'consistent read' permission can be granted. + * + * XXX Can (or should) we somehow keep 'consistent read' blocked even + * after the failed/cancelled commit job is gone? If we already wrote + * something to base, the intermediate images aren't valid any more. */ + bdrv_child_try_set_perm(s->commit_top_bs->backing, 0, BLK_PERM_ALL, + &error_abort); + bdrv_replace_node(s->commit_top_bs, backing_bs(s->commit_top_bs), + &error_abort); + + bdrv_unref(s->commit_top_bs); + bdrv_unref(top_bs); +} + +static void commit_clean(Job *job) +{ + CommitBlockJob *s = container_of(job, CommitBlockJob, common.job); + /* restore base open flags here if appropriate (e.g., change the base back * to r/o). These reopens do not need to be atomic, since we won't abort * even on failure here */ - if (s->base_flags != bdrv_get_flags(base)) { - bdrv_reopen(base, s->base_flags, NULL); + if (s->base_flags != bdrv_get_flags(s->base_bs)) { + bdrv_reopen(s->base_bs, s->base_flags, NULL); } + g_free(s->backing_file_str); blk_unref(s->top); - - /* If there is more than one reference to the job (e.g. if called from - * job_finish_sync()), job_completed() won't free it and therefore the - * blockers on the intermediate nodes remain. This would cause - * bdrv_set_backing_hd() to fail. */ - block_job_remove_all_bdrv(bjob); - - /* If bdrv_drop_intermediate() didn't already do that, remove the commit - * filter driver from the backing chain. Do this as the final step so that - * the 'consistent read' permission can be granted. */ - if (remove_commit_top_bs) { - bdrv_child_try_set_perm(commit_top_bs->backing, 0, BLK_PERM_ALL, - &error_abort); - bdrv_replace_node(commit_top_bs, backing_bs(commit_top_bs), - &error_abort); - } - - bdrv_unref(commit_top_bs); - bdrv_unref(top); } static int coroutine_fn commit_run(Job *job, Error **errp) @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver commit_job_driver = { .user_resume = block_job_user_resume, .drain = block_job_drain, .run = commit_run, - .exit = commit_exit, + .prepare = commit_prepare, + .abort = commit_abort, + .clean = commit_clean }, }; @@ -XXX,XX +XXX,XX @@ void commit_start(const char *job_id, BlockDriverState *bs, if (ret < 0) { goto fail; } + s->base_bs = base; /* Required permissions are already taken with block_job_add_bdrv() */ s->top = blk_new(0, BLK_PERM_ALL); -- 2.17.1 From: John Snow <jsnow@redhat.com> In cases where we abort the block/mirror job, there's no point in installing the new backing chain before we finish aborting. Signed-off-by: John Snow <jsnow@redhat.com> Message-id: 20180906130225.5118-6-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/mirror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/mirror.c b/block/mirror.c index XXXXXXX..XXXXXXX 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job) * required before it could become a backing file of target_bs. */ bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL, &error_abort); - if (s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) { + if (ret == 0 && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) { BlockDriverState *backing = s->is_none_mode ? src : s->base; if (backing_bs(target_bs) != backing) { bdrv_set_backing_hd(target_bs, backing, &local_err); -- 2.17.1 From: John Snow <jsnow@redhat.com> For purposes of minimum code movement, refactor the mirror_exit callback to use the post-finalization callbacks in a trivial way. Signed-off-by: John Snow <jsnow@redhat.com> Message-id: 20180906130225.5118-7-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> [mreitz: Added comment for the mirror_exit() function] Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/mirror.c | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index XXXXXXX..XXXXXXX 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -XXX,XX +XXX,XX @@ typedef struct MirrorBlockJob { int max_iov; bool initial_zeroing_ongoing; int in_active_write_counter; + bool prepared; } MirrorBlockJob; typedef struct MirrorBDSOpaque { @@ -XXX,XX +XXX,XX @@ static void mirror_wait_for_all_io(MirrorBlockJob *s) } } -static void mirror_exit(Job *job) +/** + * mirror_exit_common: handle both abort() and prepare() cases. + * for .prepare, returns 0 on success and -errno on failure. + * for .abort cases, denoted by abort = true, MUST return 0. + */ +static int mirror_exit_common(Job *job) { MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job); BlockJob *bjob = &s->common; @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job) BlockDriverState *target_bs = blk_bs(s->target); BlockDriverState *mirror_top_bs = s->mirror_top_bs; Error *local_err = NULL; - int ret = job->ret; + bool abort = job->ret < 0; + int ret = 0; + + if (s->prepared) { + return 0; + } + s->prepared = true; bdrv_release_dirty_bitmap(src, s->dirty_bitmap); @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job) * required before it could become a backing file of target_bs. */ bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL, &error_abort); - if (ret == 0 && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) { + if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) { BlockDriverState *backing = s->is_none_mode ? src : s->base; if (backing_bs(target_bs) != backing) { bdrv_set_backing_hd(target_bs, backing, &local_err); @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job) aio_context_acquire(replace_aio_context); } - if (s->should_complete && ret == 0) { - BlockDriverState *to_replace = src; - if (s->to_replace) { - to_replace = s->to_replace; - } + if (s->should_complete && !abort) { + BlockDriverState *to_replace = s->to_replace ?: src; if (bdrv_get_flags(target_bs) != bdrv_get_flags(to_replace)) { bdrv_reopen(target_bs, bdrv_get_flags(to_replace), NULL); @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job) bdrv_unref(mirror_top_bs); bdrv_unref(src); - job->ret = ret; + return ret; +} + +static int mirror_prepare(Job *job) +{ + return mirror_exit_common(job); +} + +static void mirror_abort(Job *job) +{ + int ret = mirror_exit_common(job); + assert(ret == 0); } static void mirror_throttle(MirrorBlockJob *s) @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver mirror_job_driver = { .user_resume = block_job_user_resume, .drain = block_job_drain, .run = mirror_run, - .exit = mirror_exit, + .prepare = mirror_prepare, + .abort = mirror_abort, .pause = mirror_pause, .complete = mirror_complete, }, @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver commit_active_job_driver = { .user_resume = block_job_user_resume, .drain = block_job_drain, .run = mirror_run, - .exit = mirror_exit, + .prepare = mirror_prepare, + .abort = mirror_abort, .pause = mirror_pause, .complete = mirror_complete, }, -- 2.17.1 From: John Snow <jsnow@redhat.com> Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-8-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/stream.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_populate(BlockBackend *blk, return blk_co_preadv(blk, offset, qiov.size, &qiov, BDRV_REQ_COPY_ON_READ); } -static void stream_exit(Job *job) +static int stream_prepare(Job *job) { StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); BlockJob *bjob = &s->common; BlockDriverState *bs = blk_bs(bjob->blk); BlockDriverState *base = s->base; Error *local_err = NULL; - int ret = job->ret; + int ret = 0; - if (!job_is_cancelled(job) && bs->backing && ret == 0) { + if (bs->backing) { const char *base_id = NULL, *base_fmt = NULL; if (base) { base_id = s->backing_file_str; @@ -XXX,XX +XXX,XX @@ static void stream_exit(Job *job) bdrv_set_backing_hd(bs, base, &local_err); if (local_err) { error_report_err(local_err); - ret = -EPERM; - goto out; + return -EPERM; } } -out: + return ret; +} + +static void stream_clean(Job *job) +{ + StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); + BlockJob *bjob = &s->common; + BlockDriverState *bs = blk_bs(bjob->blk); + /* Reopen the image back in read-only mode if necessary */ if (s->bs_flags != bdrv_get_flags(bs)) { /* Give up write permissions before making it read-only */ @@ -XXX,XX +XXX,XX @@ out: } g_free(s->backing_file_str); - job->ret = ret; } static int coroutine_fn stream_run(Job *job, Error **errp) @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { .job_type = JOB_TYPE_STREAM, .free = block_job_free, .run = stream_run, - .exit = stream_exit, + .prepare = stream_prepare, + .clean = stream_clean, .user_resume = block_job_user_resume, .drain = block_job_drain, }, -- 2.17.1 From: John Snow <jsnow@redhat.com> These tests don't actually test blockjobs anymore, they test generic Job lifetimes. Change the types accordingly. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-9-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/test-blockjob.c | 98 ++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-blockjob.c +++ b/tests/test-blockjob.c @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_cancel_driver = { }, }; -static CancelJob *create_common(BlockJob **pjob) +static CancelJob *create_common(Job **pjob) { BlockBackend *blk; - BlockJob *job; + Job *job; + BlockJob *bjob; CancelJob *s; blk = create_blk(NULL); - job = mk_job(blk, "Steve", &test_cancel_driver, true, - JOB_MANUAL_FINALIZE | JOB_MANUAL_DISMISS); - job_ref(&job->job); - assert(job->job.status == JOB_STATUS_CREATED); - s = container_of(job, CancelJob, common); + bjob = mk_job(blk, "Steve", &test_cancel_driver, true, + JOB_MANUAL_FINALIZE | JOB_MANUAL_DISMISS); + job = &bjob->job; + job_ref(job); + assert(job->status == JOB_STATUS_CREATED); + s = container_of(bjob, CancelJob, common); s->blk = blk; *pjob = job; @@ -XXX,XX +XXX,XX @@ static void cancel_common(CancelJob *s) static void test_cancel_created(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); @@ -XXX,XX +XXX,XX @@ static void test_cancel_created(void) static void test_cancel_running(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); - job_start(&job->job); - assert(job->job.status == JOB_STATUS_RUNNING); + job_start(job); + assert(job->status == JOB_STATUS_RUNNING); cancel_common(s); } static void test_cancel_paused(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); - job_start(&job->job); - assert(job->job.status == JOB_STATUS_RUNNING); + job_start(job); + assert(job->status == JOB_STATUS_RUNNING); - job_user_pause(&job->job, &error_abort); - job_enter(&job->job); - assert(job->job.status == JOB_STATUS_PAUSED); + job_user_pause(job, &error_abort); + job_enter(job); + assert(job->status == JOB_STATUS_PAUSED); cancel_common(s); } static void test_cancel_ready(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); - job_start(&job->job); - assert(job->job.status == JOB_STATUS_RUNNING); + job_start(job); + assert(job->status == JOB_STATUS_RUNNING); s->should_converge = true; - job_enter(&job->job); - assert(job->job.status == JOB_STATUS_READY); + job_enter(job); + assert(job->status == JOB_STATUS_READY); cancel_common(s); } static void test_cancel_standby(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); - job_start(&job->job); - assert(job->job.status == JOB_STATUS_RUNNING); + job_start(job); + assert(job->status == JOB_STATUS_RUNNING); s->should_converge = true; - job_enter(&job->job); - assert(job->job.status == JOB_STATUS_READY); + job_enter(job); + assert(job->status == JOB_STATUS_READY); - job_user_pause(&job->job, &error_abort); - job_enter(&job->job); - assert(job->job.status == JOB_STATUS_STANDBY); + job_user_pause(job, &error_abort); + job_enter(job); + assert(job->status == JOB_STATUS_STANDBY); cancel_common(s); } static void test_cancel_pending(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); - job_start(&job->job); - assert(job->job.status == JOB_STATUS_RUNNING); + job_start(job); + assert(job->status == JOB_STATUS_RUNNING); s->should_converge = true; - job_enter(&job->job); - assert(job->job.status == JOB_STATUS_READY); + job_enter(job); + assert(job->status == JOB_STATUS_READY); - job_complete(&job->job, &error_abort); - job_enter(&job->job); + job_complete(job, &error_abort); + job_enter(job); while (!s->completed) { aio_poll(qemu_get_aio_context(), true); } - assert(job->job.status == JOB_STATUS_PENDING); + assert(job->status == JOB_STATUS_PENDING); cancel_common(s); } static void test_cancel_concluded(void) { - BlockJob *job; + Job *job; CancelJob *s; s = create_common(&job); - job_start(&job->job); - assert(job->job.status == JOB_STATUS_RUNNING); + job_start(job); + assert(job->status == JOB_STATUS_RUNNING); s->should_converge = true; - job_enter(&job->job); - assert(job->job.status == JOB_STATUS_READY); + job_enter(job); + assert(job->status == JOB_STATUS_READY); - job_complete(&job->job, &error_abort); - job_enter(&job->job); + job_complete(job, &error_abort); + job_enter(job); while (!s->completed) { aio_poll(qemu_get_aio_context(), true); } - assert(job->job.status == JOB_STATUS_PENDING); + assert(job->status == JOB_STATUS_PENDING); - job_finalize(&job->job, &error_abort); - assert(job->job.status == JOB_STATUS_CONCLUDED); + job_finalize(job, &error_abort); + assert(job->status == JOB_STATUS_CONCLUDED); cancel_common(s); } -- 2.17.1 From: John Snow <jsnow@redhat.com> We remove the exit callback and the completed boolean along with it. We can simulate it just fine by waiting for the job to defer to the main loop, and then giving it one final kick to get the main loop portion to run. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-10-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/test-blockjob.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-blockjob.c +++ b/tests/test-blockjob.c @@ -XXX,XX +XXX,XX @@ typedef struct CancelJob { BlockBackend *blk; bool should_converge; bool should_complete; - bool completed; } CancelJob; -static void cancel_job_exit(Job *job) -{ - CancelJob *s = container_of(job, CancelJob, common.job); - s->completed = true; -} - static void cancel_job_complete(Job *job, Error **errp) { CancelJob *s = container_of(job, CancelJob, common.job); @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_cancel_driver = { .user_resume = block_job_user_resume, .drain = block_job_drain, .run = cancel_job_run, - .exit = cancel_job_exit, .complete = cancel_job_complete, }, }; @@ -XXX,XX +XXX,XX @@ static void test_cancel_pending(void) job_complete(job, &error_abort); job_enter(job); - while (!s->completed) { + while (!job->deferred_to_main_loop) { aio_poll(qemu_get_aio_context(), true); } + assert(job->status == JOB_STATUS_READY); + aio_poll(qemu_get_aio_context(), true); assert(job->status == JOB_STATUS_PENDING); cancel_common(s); @@ -XXX,XX +XXX,XX @@ static void test_cancel_concluded(void) job_complete(job, &error_abort); job_enter(job); - while (!s->completed) { + while (!job->deferred_to_main_loop) { aio_poll(qemu_get_aio_context(), true); } + assert(job->status == JOB_STATUS_READY); + aio_poll(qemu_get_aio_context(), true); assert(job->status == JOB_STATUS_PENDING); job_finalize(job, &error_abort); -- 2.17.1 From: John Snow <jsnow@redhat.com> The exit callback in this test actually only performs cleanup. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-11-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/test-blockjob-txn.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-blockjob-txn.c b/tests/test-blockjob-txn.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-blockjob-txn.c +++ b/tests/test-blockjob-txn.c @@ -XXX,XX +XXX,XX @@ typedef struct { int *result; } TestBlockJob; -static void test_block_job_exit(Job *job) +static void test_block_job_clean(Job *job) { BlockJob *bjob = container_of(job, BlockJob, job); BlockDriverState *bs = blk_bs(bjob->blk); @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_block_job_driver = { .user_resume = block_job_user_resume, .drain = block_job_drain, .run = test_block_job_run, - .exit = test_block_job_exit, + .clean = test_block_job_clean, }, }; -- 2.17.1 From: John Snow <jsnow@redhat.com> Now that all of the jobs use the component finalization callbacks, there's no use for the heavy-hammer .exit callback anymore. job_exit becomes a glorified type shim so that we can call job_completed from aio_bh_schedule_oneshot. Move these three functions down into job.c to eliminate a forward reference. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-12-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/qemu/job.h | 11 ------- job.c | 77 ++++++++++++++++++++-------------------------- 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/include/qemu/job.h b/include/qemu/job.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/job.h +++ b/include/qemu/job.h @@ -XXX,XX +XXX,XX @@ struct JobDriver { */ void (*drain)(Job *job); - /** - * If the callback is not NULL, exit will be invoked from the main thread - * when the job's coroutine has finished, but before transactional - * convergence; before @prepare or @abort. - * - * FIXME TODO: This callback is only temporary to transition remaining jobs - * to prepare/commit/abort/clean callbacks and will be removed before 3.1. - * is released. - */ - void (*exit)(Job *job); - /** * If the callback is not NULL, prepare will be invoked when all the jobs * belonging to the same transaction complete; or upon this job's completion diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ void job_drain(Job *job) } } -static void job_completed(Job *job); - -static void job_exit(void *opaque) -{ - Job *job = (Job *)opaque; - AioContext *aio_context = job->aio_context; - - if (job->driver->exit) { - aio_context_acquire(aio_context); - job->driver->exit(job); - aio_context_release(aio_context); - } - job_completed(job); -} - -/** - * All jobs must allow a pause point before entering their job proper. This - * ensures that jobs can be paused prior to being started, then resumed later. - */ -static void coroutine_fn job_co_entry(void *opaque) -{ - Job *job = opaque; - - assert(job && job->driver && job->driver->run); - job_pause_point(job); - job->ret = job->driver->run(job, &job->err); - job->deferred_to_main_loop = true; - aio_bh_schedule_oneshot(qemu_get_aio_context(), job_exit, job); -} - - -void job_start(Job *job) -{ - assert(job && !job_started(job) && job->paused && - job->driver && job->driver->run); - job->co = qemu_coroutine_create(job_co_entry, job); - job->pause_count--; - job->busy = true; - job->paused = false; - job_state_transition(job, JOB_STATUS_RUNNING); - aio_co_enter(job->aio_context, job->co); -} - /* Assumes the block_job_mutex is held */ static bool job_timer_not_pending(Job *job) { @@ -XXX,XX +XXX,XX @@ static void job_completed(Job *job) } } +/** Useful only as a type shim for aio_bh_schedule_oneshot. */ +static void job_exit(void *opaque) +{ + Job *job = (Job *)opaque; + job_completed(job); +} + +/** + * All jobs must allow a pause point before entering their job proper. This + * ensures that jobs can be paused prior to being started, then resumed later. + */ +static void coroutine_fn job_co_entry(void *opaque) +{ + Job *job = opaque; + + assert(job && job->driver && job->driver->run); + job_pause_point(job); + job->ret = job->driver->run(job, &job->err); + job->deferred_to_main_loop = true; + aio_bh_schedule_oneshot(qemu_get_aio_context(), job_exit, job); +} + +void job_start(Job *job) +{ + assert(job && !job_started(job) && job->paused && + job->driver && job->driver->run); + job->co = qemu_coroutine_create(job_co_entry, job); + job->pause_count--; + job->busy = true; + job->paused = false; + job_state_transition(job, JOB_STATUS_RUNNING); + aio_co_enter(job->aio_context, job->co); +} + void job_cancel(Job *job, bool force) { if (job->status == JOB_STATUS_CONCLUDED) { -- 2.17.1 From: John Snow <jsnow@redhat.com> Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-13-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 16 +++++++++++++++- blockdev.c | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # above @top. If this option is not given, a node name is # autogenerated. (Since: 2.9) # +# @auto-finalize: When false, this job will wait in a PENDING state after it has +# finished its work, waiting for @block-job-finalize before +# making any block graph changes. +# When true, this job will automatically +# perform its abort or commit actions. +# Defaults to true. (Since 3.1) +# +# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it +# has completely ceased all work, and awaits @block-job-dismiss. +# When true, this job will automatically disappear from the query +# list without user intervention. +# Defaults to true. (Since 3.1) +# # Returns: Nothing on success # If @device does not exist, DeviceNotFound # Any other error returns a GenericError. @@ -XXX,XX +XXX,XX @@ { 'command': 'block-commit', 'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', '*top': 'str', '*backing-file': 'str', '*speed': 'int', - '*filter-node-name': 'str' } } + '*filter-node-name': 'str', + '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } ## # @drive-backup: diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, bool has_backing_file, const char *backing_file, bool has_speed, int64_t speed, bool has_filter_node_name, const char *filter_node_name, + bool has_auto_finalize, bool auto_finalize, + bool has_auto_dismiss, bool auto_dismiss, Error **errp) { BlockDriverState *bs; @@ -XXX,XX +XXX,XX @@ void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, if (!has_filter_node_name) { filter_node_name = NULL; } + if (has_auto_finalize && !auto_finalize) { + job_flags |= JOB_MANUAL_FINALIZE; + } + if (has_auto_dismiss && !auto_dismiss) { + job_flags |= JOB_MANUAL_DISMISS; + } /* Important Note: * libvirt relies on the DeviceNotFound error class in order to probe for -- 2.17.1 From: John Snow <jsnow@redhat.com> Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-14-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 30 ++++++++++++++++++++++++++++-- blockdev.c | 14 ++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # @copy-mode: when to copy data to the destination; defaults to 'background' # (Since: 3.0) # +# @auto-finalize: When false, this job will wait in a PENDING state after it has +# finished its work, waiting for @block-job-finalize before +# making any block graph changes. +# When true, this job will automatically +# perform its abort or commit actions. +# Defaults to true. (Since 3.1) +# +# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it +# has completely ceased all work, and awaits @block-job-dismiss. +# When true, this job will automatically disappear from the query +# list without user intervention. +# Defaults to true. (Since 3.1) # Since: 1.3 ## { 'struct': 'DriveMirror', @@ -XXX,XX +XXX,XX @@ '*speed': 'int', '*granularity': 'uint32', '*buf-size': 'int', '*on-source-error': 'BlockdevOnError', '*on-target-error': 'BlockdevOnError', - '*unmap': 'bool', '*copy-mode': 'MirrorCopyMode' } } + '*unmap': 'bool', '*copy-mode': 'MirrorCopyMode', + '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } ## # @BlockDirtyBitmap: @@ -XXX,XX +XXX,XX @@ # @copy-mode: when to copy data to the destination; defaults to 'background' # (Since: 3.0) # +# @auto-finalize: When false, this job will wait in a PENDING state after it has +# finished its work, waiting for @block-job-finalize before +# making any block graph changes. +# When true, this job will automatically +# perform its abort or commit actions. +# Defaults to true. (Since 3.1) +# +# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it +# has completely ceased all work, and awaits @block-job-dismiss. +# When true, this job will automatically disappear from the query +# list without user intervention. +# Defaults to true. (Since 3.1) # Returns: nothing on success. # # Since: 2.6 @@ -XXX,XX +XXX,XX @@ '*buf-size': 'int', '*on-source-error': 'BlockdevOnError', '*on-target-error': 'BlockdevOnError', '*filter-node-name': 'str', - '*copy-mode': 'MirrorCopyMode' } } + '*copy-mode': 'MirrorCopyMode', + '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } ## # @block_set_io_throttle: diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs, bool has_filter_node_name, const char *filter_node_name, bool has_copy_mode, MirrorCopyMode copy_mode, + bool has_auto_finalize, bool auto_finalize, + bool has_auto_dismiss, bool auto_dismiss, Error **errp) { int job_flags = JOB_DEFAULT; @@ -XXX,XX +XXX,XX @@ static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs, if (!has_copy_mode) { copy_mode = MIRROR_COPY_MODE_BACKGROUND; } + if (has_auto_finalize && !auto_finalize) { + job_flags |= JOB_MANUAL_FINALIZE; + } + if (has_auto_dismiss && !auto_dismiss) { + job_flags |= JOB_MANUAL_DISMISS; + } if (granularity != 0 && (granularity < 512 || granularity > 1048576 * 64)) { error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "granularity", @@ -XXX,XX +XXX,XX @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp) arg->has_unmap, arg->unmap, false, NULL, arg->has_copy_mode, arg->copy_mode, + arg->has_auto_finalize, arg->auto_finalize, + arg->has_auto_dismiss, arg->auto_dismiss, &local_err); bdrv_unref(target_bs); error_propagate(errp, local_err); @@ -XXX,XX +XXX,XX @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id, bool has_filter_node_name, const char *filter_node_name, bool has_copy_mode, MirrorCopyMode copy_mode, + bool has_auto_finalize, bool auto_finalize, + bool has_auto_dismiss, bool auto_dismiss, Error **errp) { BlockDriverState *bs; @@ -XXX,XX +XXX,XX @@ void qmp_blockdev_mirror(bool has_job_id, const char *job_id, true, true, has_filter_node_name, filter_node_name, has_copy_mode, copy_mode, + has_auto_finalize, auto_finalize, + has_auto_dismiss, auto_dismiss, &local_err); error_propagate(errp, local_err); -- 2.17.1 From: John Snow <jsnow@redhat.com> Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-15-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 16 +++++++++++++++- blockdev.c | 9 +++++++++ hmp.c | 5 +++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # 'stop' and 'enospc' can only be used if the block device # supports io-status (see BlockInfo). Since 1.3. # +# @auto-finalize: When false, this job will wait in a PENDING state after it has +# finished its work, waiting for @block-job-finalize before +# making any block graph changes. +# When true, this job will automatically +# perform its abort or commit actions. +# Defaults to true. (Since 3.1) +# +# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it +# has completely ceased all work, and awaits @block-job-dismiss. +# When true, this job will automatically disappear from the query +# list without user intervention. +# Defaults to true. (Since 3.1) +# # Returns: Nothing on success. If @device does not exist, DeviceNotFound. # # Since: 1.1 @@ -XXX,XX +XXX,XX @@ { 'command': 'block-stream', 'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', '*base-node': 'str', '*backing-file': 'str', '*speed': 'int', - '*on-error': 'BlockdevOnError' } } + '*on-error': 'BlockdevOnError', + '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } ## # @block-job-set-speed: diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, bool has_backing_file, const char *backing_file, bool has_speed, int64_t speed, bool has_on_error, BlockdevOnError on_error, + bool has_auto_finalize, bool auto_finalize, + bool has_auto_dismiss, bool auto_dismiss, Error **errp) { BlockDriverState *bs, *iter; @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, /* backing_file string overrides base bs filename */ base_name = has_backing_file ? backing_file : base_name; + if (has_auto_finalize && !auto_finalize) { + job_flags |= JOB_MANUAL_FINALIZE; + } + if (has_auto_dismiss && !auto_dismiss) { + job_flags |= JOB_MANUAL_DISMISS; + } + stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name, job_flags, has_speed ? speed : 0, on_error, &local_err); if (local_err) { diff --git a/hmp.c b/hmp.c index XXXXXXX..XXXXXXX 100644 --- a/hmp.c +++ b/hmp.c @@ -XXX,XX +XXX,XX @@ void hmp_block_stream(Monitor *mon, const QDict *qdict) int64_t speed = qdict_get_try_int(qdict, "speed", 0); qmp_block_stream(true, device, device, base != NULL, base, false, NULL, - false, NULL, qdict_haskey(qdict, "speed"), speed, - true, BLOCKDEV_ON_ERROR_REPORT, &error); + false, NULL, qdict_haskey(qdict, "speed"), speed, true, + BLOCKDEV_ON_ERROR_REPORT, false, false, false, false, + &error); hmp_handle_error(mon, &error); } -- 2.17.1 From: John Snow <jsnow@redhat.com> Fix documentation to match the other jobs amended for 3.1. Signed-off-by: John Snow <jsnow@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20180906130225.5118-16-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # a different block device than @device). # # @auto-finalize: When false, this job will wait in a PENDING state after it has -# finished its work, waiting for @block-job-finalize. -# When true, this job will automatically perform its abort or -# commit actions. +# finished its work, waiting for @block-job-finalize before +# making any block graph changes. +# When true, this job will automatically +# perform its abort or commit actions. # Defaults to true. (Since 2.12) # # @auto-dismiss: When false, this job will wait in a CONCLUDED state after it -# has completed ceased all work, and wait for @block-job-dismiss. +# has completely ceased all work, and awaits @block-job-dismiss. # When true, this job will automatically disappear from the query # list without user intervention. # Defaults to true. (Since 2.12) @@ -XXX,XX +XXX,XX @@ # a different block device than @device). # # @auto-finalize: When false, this job will wait in a PENDING state after it has -# finished its work, waiting for @block-job-finalize. -# When true, this job will automatically perform its abort or -# commit actions. +# finished its work, waiting for @block-job-finalize before +# making any block graph changes. +# When true, this job will automatically +# perform its abort or commit actions. # Defaults to true. (Since 2.12) # # @auto-dismiss: When false, this job will wait in a CONCLUDED state after it -# has completed ceased all work, and wait for @block-job-dismiss. +# has completely ceased all work, and awaits @block-job-dismiss. # When true, this job will automatically disappear from the query # list without user intervention. # Defaults to true. (Since 2.12) -- 2.17.1 From: John Snow <jsnow@redhat.com> Presently only the backup job really guarantees what one would consider transactional semantics. To guard against someone helpfully adding them in the future, document that there are shortcomings in the model that would need to be audited at that time. Signed-off-by: John Snow <jsnow@redhat.com> Message-id: 20180906130225.5118-17-jsnow@redhat.com Reviewed-by: Jeff Cody <jcody@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- blockdev.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static const BlkActionOps actions[] = { .instance_size = sizeof(BlockDirtyBitmapState), .prepare = block_dirty_bitmap_disable_prepare, .abort = block_dirty_bitmap_disable_abort, - } + }, + /* Where are transactions for MIRROR, COMMIT and STREAM? + * Although these blockjobs use transaction callbacks like the backup job, + * these jobs do not necessarily adhere to transaction semantics. + * These jobs may not fully undo all of their actions on abort, nor do they + * necessarily work in transactions with more than one job in them. + */ }; /** -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> The block-commit QMP command required specifying the top and base nodes of the commit jobs using the file name of that node. While this works in simple cases (local files with absolute paths), the file names generated for more complicated setups can be hard to predict. The block-commit command has more problems than just this, so we want to replace it altogether in the long run, but libvirt needs a reliable way to address nodes now. So we don't want to wait for a new, cleaner command, but just add the minimal thing needed right now. This adds two new options top-node and base-node to the command, which allow specifying node names instead. They are mutually exclusive with the old options. Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- qapi/block-core.json | 24 ++++++++++++++++++------ blockdev.c | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # # @device: the device name or node-name of a root node # -# @base: The file name of the backing image to write data into. -# If not specified, this is the deepest backing image. +# @base-node: The node name of the backing image to write data into. +# If not specified, this is the deepest backing image. +# (since: 3.1) # -# @top: The file name of the backing image within the image chain, -# which contains the topmost data to be committed down. If -# not specified, this is the active layer. +# @base: Same as @base-node, except that it is a file name rather than a node +# name. This must be the exact filename string that was used to open the +# node; other strings, even if addressing the same file, are not +# accepted (deprecated, use @base-node instead) +# +# @top-node: The node name of the backing image within the image chain +# which contains the topmost data to be committed down. If +# not specified, this is the active layer. (since: 3.1) +# +# @top: Same as @top-node, except that it is a file name rather than a node +# name. This must be the exact filename string that was used to open the +# node; other strings, even if addressing the same file, are not +# accepted (deprecated, use @base-node instead) # # @backing-file: The backing file string to write into the overlay # image of 'top'. If 'top' is the active layer, @@ -XXX,XX +XXX,XX @@ # ## { 'command': 'block-commit', - 'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', '*top': 'str', + 'data': { '*job-id': 'str', 'device': 'str', '*base-node': 'str', + '*base': 'str', '*top-node': 'str', '*top': 'str', '*backing-file': 'str', '*speed': 'int', '*filter-node-name': 'str', '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ out: } void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, + bool has_base_node, const char *base_node, bool has_base, const char *base, + bool has_top_node, const char *top_node, bool has_top, const char *top, bool has_backing_file, const char *backing_file, bool has_speed, int64_t speed, @@ -XXX,XX +XXX,XX @@ void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, /* default top_bs is the active layer */ top_bs = bs; - if (has_top && top) { + if (has_top_node && has_top) { + error_setg(errp, "'top-node' and 'top' are mutually exclusive"); + goto out; + } else if (has_top_node) { + top_bs = bdrv_lookup_bs(NULL, top_node, errp); + if (top_bs == NULL) { + goto out; + } + if (!bdrv_chain_contains(bs, top_bs)) { + error_setg(errp, "'%s' is not in this backing file chain", + top_node); + goto out; + } + } else if (has_top && top) { if (strcmp(bs->filename, top) != 0) { top_bs = bdrv_find_backing_image(bs, top); } @@ -XXX,XX +XXX,XX @@ void qmp_block_commit(bool has_job_id, const char *job_id, const char *device, assert(bdrv_get_aio_context(top_bs) == aio_context); - if (has_base && base) { + if (has_base_node && has_base) { + error_setg(errp, "'base-node' and 'base' are mutually exclusive"); + goto out; + } else if (has_base_node) { + base_bs = bdrv_lookup_bs(NULL, base_node, errp); + if (base_bs == NULL) { + goto out; + } + if (!bdrv_chain_contains(top_bs, base_bs)) { + error_setg(errp, "'%s' is not in this backing file chain", + base_node); + goto out; + } + } else if (has_base && base) { base_bs = bdrv_find_backing_image(top_bs, base); } else { base_bs = bdrv_find_base(top_bs); -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> This adds some tests for block-commit with the new options top-node and base-node (taking node names) instead of top and base (taking file names). Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- tests/qemu-iotests/040 | 52 ++++++++++++++++++++++++++++++++++++-- tests/qemu-iotests/040.out | 4 +-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/040 +++ b/tests/qemu-iotests/040 @@ -XXX,XX +XXX,XX @@ class ImageCommitTestCase(iotests.QMPTestCase): self.assert_no_active_block_jobs() self.vm.shutdown() - def run_commit_test(self, top, base, need_ready=False): + def run_commit_test(self, top, base, need_ready=False, node_names=False): self.assert_no_active_block_jobs() - result = self.vm.qmp('block-commit', device='drive0', top=top, base=base) + if node_names: + result = self.vm.qmp('block-commit', device='drive0', top_node=top, base_node=base) + else: + result = self.vm.qmp('block-commit', device='drive0', top=top, base=base) self.assert_qmp(result, 'return', {}) self.wait_for_complete(need_ready) @@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase): self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed")) self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed")) + def test_commit_node(self): + self.run_commit_test("mid", "base", node_names=True) + self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed")) + self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed")) + def test_device_not_found(self): result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % mid_img) self.assert_qmp(result, 'error/class', 'DeviceNotFound') @@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase): self.assert_qmp(result, 'error/class', 'GenericError') self.assert_qmp(result, 'error/desc', 'Base \'badfile\' not found') + def test_top_node_invalid(self): + self.assert_no_active_block_jobs() + result = self.vm.qmp('block-commit', device='drive0', top_node='badfile', base_node='base') + self.assert_qmp(result, 'error/class', 'GenericError') + self.assert_qmp(result, 'error/desc', "Cannot find device= nor node_name=badfile") + + def test_base_node_invalid(self): + self.assert_no_active_block_jobs() + result = self.vm.qmp('block-commit', device='drive0', top_node='mid', base_node='badfile') + self.assert_qmp(result, 'error/class', 'GenericError') + self.assert_qmp(result, 'error/desc', "Cannot find device= nor node_name=badfile") + + def test_top_path_and_node(self): + self.assert_no_active_block_jobs() + result = self.vm.qmp('block-commit', device='drive0', top_node='mid', base_node='base', top='%s' % mid_img) + self.assert_qmp(result, 'error/class', 'GenericError') + self.assert_qmp(result, 'error/desc', "'top-node' and 'top' are mutually exclusive") + + def test_base_path_and_node(self): + self.assert_no_active_block_jobs() + result = self.vm.qmp('block-commit', device='drive0', top_node='mid', base_node='base', base='%s' % backing_img) + self.assert_qmp(result, 'error/class', 'GenericError') + self.assert_qmp(result, 'error/desc', "'base-node' and 'base' are mutually exclusive") + def test_top_is_active(self): self.run_commit_test(test_img, backing_img, need_ready=True) self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed")) @@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase): self.assert_qmp(result, 'error/class', 'GenericError') self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % mid_img) + def test_top_and_base_node_reversed(self): + self.assert_no_active_block_jobs() + result = self.vm.qmp('block-commit', device='drive0', top_node='base', base_node='top') + self.assert_qmp(result, 'error/class', 'GenericError') + self.assert_qmp(result, 'error/desc', "'top' is not in this backing file chain") + + def test_top_node_in_wrong_chain(self): + self.assert_no_active_block_jobs() + + result = self.vm.qmp('blockdev-add', driver='null-co', node_name='null') + self.assert_qmp(result, 'return', {}) + + result = self.vm.qmp('block-commit', device='drive0', top_node='null', base_node='base') + self.assert_qmp(result, 'error/class', 'GenericError') + self.assert_qmp(result, 'error/desc', "'null' is not in this backing file chain") + # When the job is running on a BB that is automatically deleted on hot # unplug, the job is cancelled when the device disappears def test_hot_unplug(self): diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/040.out +++ b/tests/qemu-iotests/040.out @@ -XXX,XX +XXX,XX @@ -............................. +........................................... ---------------------------------------------------------------------- -Ran 29 tests +Ran 43 tests OK -- 2.17.1 From: Sergio Lopez <slp@redhat.com> In qemu_laio_process_completions_and_submit, the AioContext is acquired before the ioq_submit iteration and after qemu_laio_process_completions, but the latter is not thread safe either. This change avoids a number of random crashes when the Main Thread and an IO Thread collide processing completions for the same AioContext. This is an example of such crash: - The IO Thread is trying to acquire the AioContext at aio_co_enter, which evidences that it didn't lock it before: Thread 3 (Thread 0x7fdfd8bd8700 (LWP 36743)): #0 0x00007fdfe0dd542d in __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007fdfe0dd0de6 in _L_lock_870 () at /lib64/libpthread.so.0 #2 0x00007fdfe0dd0cdf in __GI___pthread_mutex_lock (mutex=mutex@entry=0x5631fde0e6c0) at ../nptl/pthread_mutex_lock.c:114 #3 0x00005631fc0603a7 in qemu_mutex_lock_impl (mutex=0x5631fde0e6c0, file=0x5631fc23520f "util/async.c", line=511) at util/qemu-thread-posix.c:66 #4 0x00005631fc05b558 in aio_co_enter (ctx=0x5631fde0e660, co=0x7fdfcc0c2b40) at util/async.c:493 #5 0x00005631fc05b5ac in aio_co_wake (co=<optimized out>) at util/async.c:478 #6 0x00005631fbfc51ad in qemu_laio_process_completion (laiocb=<optimized out>) at block/linux-aio.c:104 #7 0x00005631fbfc523c in qemu_laio_process_completions (s=s@entry=0x7fdfc0297670) at block/linux-aio.c:222 #8 0x00005631fbfc5499 in qemu_laio_process_completions_and_submit (s=0x7fdfc0297670) at block/linux-aio.c:237 #9 0x00005631fc05d978 in aio_dispatch_handlers (ctx=ctx@entry=0x5631fde0e660) at util/aio-posix.c:406 #10 0x00005631fc05e3ea in aio_poll (ctx=0x5631fde0e660, blocking=blocking@entry=true) at util/aio-posix.c:693 #11 0x00005631fbd7ad96 in iothread_run (opaque=0x5631fde0e1c0) at iothread.c:64 #12 0x00007fdfe0dcee25 in start_thread (arg=0x7fdfd8bd8700) at pthread_create.c:308 #13 0x00007fdfe0afc34d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113 - The Main Thread is also processing completions from the same AioContext, and crashes due to failed assertion at util/iov.c:78: Thread 1 (Thread 0x7fdfeb5eac80 (LWP 36740)): #0 0x00007fdfe0a391f7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56 #1 0x00007fdfe0a3a8e8 in __GI_abort () at abort.c:90 #2 0x00007fdfe0a32266 in __assert_fail_base (fmt=0x7fdfe0b84e68 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x5631fc238ccb "offset == 0", file=file@entry=0x5631fc23698e "util/iov.c", line=line@entry=78, function=function@entry=0x5631fc236adc <__PRETTY_FUNCTION__.15220> "iov_memset") at assert.c:92 #3 0x00007fdfe0a32312 in __GI___assert_fail (assertion=assertion@entry=0x5631fc238ccb "offset == 0", file=file@entry=0x5631fc23698e "util/iov.c", line=line@entry=78, function=function@entry=0x5631fc236adc <__PRETTY_FUNCTION__.15220> "iov_memset") at assert.c:101 #4 0x00005631fc065287 in iov_memset (iov=<optimized out>, iov_cnt=<optimized out>, offset=<optimized out>, offset@entry=65536, fillc=fillc@entry=0, bytes=15515191315812405248) at util/iov.c:78 #5 0x00005631fc065a63 in qemu_iovec_memset (qiov=<optimized out>, offset=offset@entry=65536, fillc=fillc@entry=0, bytes=<optimized out>) at util/iov.c:410 #6 0x00005631fbfc5178 in qemu_laio_process_completion (laiocb=0x7fdd920df630) at block/linux-aio.c:88 #7 0x00005631fbfc523c in qemu_laio_process_completions (s=s@entry=0x7fdfc0297670) at block/linux-aio.c:222 #8 0x00005631fbfc5499 in qemu_laio_process_completions_and_submit (s=0x7fdfc0297670) at block/linux-aio.c:237 #9 0x00005631fbfc54ed in qemu_laio_poll_cb (opaque=<optimized out>) at block/linux-aio.c:272 #10 0x00005631fc05d85e in run_poll_handlers_once (ctx=ctx@entry=0x5631fde0e660) at util/aio-posix.c:497 #11 0x00005631fc05e2ca in aio_poll (blocking=false, ctx=0x5631fde0e660) at util/aio-posix.c:574 #12 0x00005631fc05e2ca in aio_poll (ctx=0x5631fde0e660, blocking=blocking@entry=false) at util/aio-posix.c:604 #13 0x00005631fbfcb8a3 in bdrv_do_drained_begin (ignore_parent=<optimized out>, recursive=<optimized out>, bs=<optimized out>) at block/io.c:273 #14 0x00005631fbfcb8a3 in bdrv_do_drained_begin (bs=0x5631fe8b6200, recursive=<optimized out>, parent=0x0, ignore_bds_parents=<optimized out>, poll=<optimized out>) at block/io.c:390 #15 0x00005631fbfbcd2e in blk_drain (blk=0x5631fe83ac80) at block/block-backend.c:1590 #16 0x00005631fbfbe138 in blk_remove_bs (blk=blk@entry=0x5631fe83ac80) at block/block-backend.c:774 #17 0x00005631fbfbe3d6 in blk_unref (blk=0x5631fe83ac80) at block/block-backend.c:401 #18 0x00005631fbfbe3d6 in blk_unref (blk=0x5631fe83ac80) at block/block-backend.c:449 #19 0x00005631fbfc9a69 in commit_complete (job=0x5631fe8b94b0, opaque=0x7fdfcc1bb080) at block/commit.c:92 #20 0x00005631fbf7d662 in job_defer_to_main_loop_bh (opaque=0x7fdfcc1b4560) at job.c:973 #21 0x00005631fc05ad41 in aio_bh_poll (bh=0x7fdfcc01ad90) at util/async.c:90 #22 0x00005631fc05ad41 in aio_bh_poll (ctx=ctx@entry=0x5631fddffdb0) at util/async.c:118 #23 0x00005631fc05e210 in aio_dispatch (ctx=0x5631fddffdb0) at util/aio-posix.c:436 #24 0x00005631fc05ac1e in aio_ctx_dispatch (source=<optimized out>, callback=<optimized out>, user_data=<optimized out>) at util/async.c:261 #25 0x00007fdfeaae44c9 in g_main_context_dispatch (context=0x5631fde00140) at gmain.c:3201 #26 0x00007fdfeaae44c9 in g_main_context_dispatch (context=context@entry=0x5631fde00140) at gmain.c:3854 #27 0x00005631fc05d503 in main_loop_wait () at util/main-loop.c:215 #28 0x00005631fc05d503 in main_loop_wait (timeout=<optimized out>) at util/main-loop.c:238 #29 0x00005631fc05d503 in main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:497 #30 0x00005631fbd81412 in main_loop () at vl.c:1866 #31 0x00005631fbc18ff3 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4647 - A closer examination shows that s->io_q.in_flight appears to have gone backwards: (gdb) frame 7 #7 0x00005631fbfc523c in qemu_laio_process_completions (s=s@entry=0x7fdfc0297670) at block/linux-aio.c:222 222 qemu_laio_process_completion(laiocb); (gdb) p s $2 = (LinuxAioState *) 0x7fdfc0297670 (gdb) p *s $3 = {aio_context = 0x5631fde0e660, ctx = 0x7fdfeb43b000, e = {rfd = 33, wfd = 33}, io_q = {plugged = 0, in_queue = 0, in_flight = 4294967280, blocked = false, pending = {sqh_first = 0x0, sqh_last = 0x7fdfc0297698}}, completion_bh = 0x7fdfc0280ef0, event_idx = 21, event_max = 241} (gdb) p/x s->io_q.in_flight $4 = 0xfffffff0 Signed-off-by: Sergio Lopez <slp@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- block/linux-aio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/linux-aio.c b/block/linux-aio.c index XXXXXXX..XXXXXXX 100644 --- a/block/linux-aio.c +++ b/block/linux-aio.c @@ -XXX,XX +XXX,XX @@ static void qemu_laio_process_completions(LinuxAioState *s) static void qemu_laio_process_completions_and_submit(LinuxAioState *s) { + aio_context_acquire(s->aio_context); qemu_laio_process_completions(s); - aio_context_acquire(s->aio_context); if (!s->io_q.plugged && !QSIMPLEQ_EMPTY(&s->io_q.pending)) { ioq_submit(s); } -- 2.17.1 From: Alberto Garcia <berto@igalia.com> When a block device is opened with BDRV_O_SNAPSHOT and the bdrv_append_temp_snapshot() call fails then the error code path tries to unref the already destroyed 'options' QDict. This can be reproduced easily by setting TMPDIR to a location where the QEMU process can't write: $ TMPDIR=/nonexistent $QEMU -drive driver=null-co,snapshot=on Signed-off-by: Alberto Garcia <berto@igalia.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- block.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block.c b/block.c index XXXXXXX..XXXXXXX 100644 --- a/block.c +++ b/block.c @@ -XXX,XX +XXX,XX @@ static BlockDriverState *bdrv_open_inherit(const char *filename, bdrv_parent_cb_change_media(bs, true); qobject_unref(options); + options = NULL; /* For snapshot=on, create a temporary qcow2 overlay. bs points to the * temporary snapshot afterwards. */ -- 2.17.1 From: Alberto Garcia <berto@igalia.com> We just fixed a bug that was causing a use-after-free when QEMU was unable to create a temporary snapshot. This is a test case for this scenario. Signed-off-by: Alberto Garcia <berto@igalia.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- tests/qemu-iotests/051 | 3 +++ tests/qemu-iotests/051.out | 3 +++ tests/qemu-iotests/051.pc.out | 3 +++ 3 files changed, 9 insertions(+) diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/051 +++ b/tests/qemu-iotests/051 @@ -XXX,XX +XXX,XX @@ printf %b "qemu-io $device_id \"write -P 0x33 0 4k\"\ncommit $device_id\n" | $QEMU_IO -c "read -P 0x33 0 4k" "$TEST_IMG" | _filter_qemu_io +# Using snapshot=on with a non-existent TMPDIR +TMPDIR=/nonexistent run_qemu -drive driver=null-co,snapshot=on + # success, all done echo "*** done" rm -f $seq.full diff --git a/tests/qemu-iotests/051.out b/tests/qemu-iotests/051.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/051.out +++ b/tests/qemu-iotests/051.out @@ -XXX,XX +XXX,XX @@ wrote 4096/4096 bytes at offset 0 read 4096/4096 bytes at offset 0 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Testing: -drive driver=null-co,snapshot=on +QEMU_PROG: -drive driver=null-co,snapshot=on: Could not get temporary filename: No such file or directory + *** done diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/051.pc.out +++ b/tests/qemu-iotests/051.pc.out @@ -XXX,XX +XXX,XX @@ wrote 4096/4096 bytes at offset 0 read 4096/4096 bytes at offset 0 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +Testing: -drive driver=null-co,snapshot=on +QEMU_PROG: -drive driver=null-co,snapshot=on: Could not get temporary filename: No such file or directory + *** done -- 2.17.1 From: Sergio Lopez <slp@redhat.com> AIO Coroutines shouldn't by managed by an AioContext different than the one assigned when they are created. aio_co_enter avoids entering a coroutine from a different AioContext, calling aio_co_schedule instead. Scheduled coroutines are then entered by co_schedule_bh_cb using qemu_coroutine_enter, which just calls qemu_aio_coroutine_enter with the current AioContext obtained with qemu_get_current_aio_context. Eventually, co->ctx will be set to the AioContext passed as an argument to qemu_aio_coroutine_enter. This means that, if an IO Thread's AioConext is being processed by the Main Thread (due to aio_poll being called with a BDS AioContext, as it happens in AIO_WAIT_WHILE among other places), the AioContext from some coroutines may be wrongly replaced with the one from the Main Thread. This is the root cause behind some crashes, mainly triggered by the drain code at block/io.c. The most common are these abort and failed assertion: util/async.c:aio_co_schedule 456 if (scheduled) { 457 fprintf(stderr, 458 "%s: Co-routine was already scheduled in '%s'\n", 459 __func__, scheduled); 460 abort(); 461 } util/qemu-coroutine-lock.c: 286 assert(mutex->holder == self); But it's also known to cause random errors at different locations, and even SIGSEGV with broken coroutine backtraces. By using qemu_aio_coroutine_enter directly in co_schedule_bh_cb, we can pass the correct AioContext as an argument, making sure co->ctx is not wrongly altered. Signed-off-by: Sergio Lopez <slp@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- util/async.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/async.c b/util/async.c index XXXXXXX..XXXXXXX 100644 --- a/util/async.c +++ b/util/async.c @@ -XXX,XX +XXX,XX @@ static void co_schedule_bh_cb(void *opaque) /* Protected by write barrier in qemu_aio_coroutine_enter */ atomic_set(&co->scheduled, NULL); - qemu_coroutine_enter(co); + qemu_aio_coroutine_enter(ctx, co); aio_context_release(ctx); } } -- 2.17.1 From: Fam Zheng <famz@redhat.com> All callers have acquired ctx already. Doing that again results in aio_poll() hang. This fixes the problem that a BDRV_POLL_WHILE() in the callback cannot make progress because ctx is recursively locked, for example, when drive-backup finishes. There are two callers of job_finalize(): fam@lemon:~/work/qemu [master]$ git grep -w -A1 '^\s*job_finalize' blockdev.c: job_finalize(&job->job, errp); blockdev.c- aio_context_release(aio_context); -- job-qmp.c: job_finalize(job, errp); job-qmp.c- aio_context_release(aio_context); -- tests/test-blockjob.c: job_finalize(&job->job, &error_abort); tests/test-blockjob.c- assert(job->job.status == JOB_STATUS_CONCLUDED); Ignoring the test, it's easy to see both callers to job_finalize (and job_do_finalize) have acquired the context. Cc: qemu-stable@nongnu.org Reported-by: Gu Nini <ngu@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Signed-off-by: Fam Zheng <famz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com> --- job.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ static void job_txn_del_job(Job *job) } } -static int job_txn_apply(JobTxn *txn, int fn(Job *), bool lock) +static int job_txn_apply(JobTxn *txn, int fn(Job *)) { - AioContext *ctx; Job *job, *next; int rc = 0; QLIST_FOREACH_SAFE(job, &txn->jobs, txn_list, next) { - if (lock) { - ctx = job->aio_context; - aio_context_acquire(ctx); - } rc = fn(job); - if (lock) { - aio_context_release(ctx); - } if (rc) { break; } @@ -XXX,XX +XXX,XX @@ static void job_do_finalize(Job *job) assert(job && job->txn); /* prepare the transaction to complete */ - rc = job_txn_apply(job->txn, job_prepare, true); + rc = job_txn_apply(job->txn, job_prepare); if (rc) { job_completed_txn_abort(job); } else { - job_txn_apply(job->txn, job_finalize_single, true); + job_txn_apply(job->txn, job_finalize_single); } } @@ -XXX,XX +XXX,XX @@ static void job_completed_txn_success(Job *job) assert(other_job->ret == 0); } - job_txn_apply(txn, job_transition_to_pending, false); + job_txn_apply(txn, job_transition_to_pending); /* If no jobs need manual finalization, automatically do so */ - if (job_txn_apply(txn, job_needs_finalize, false) == 0) { + if (job_txn_apply(txn, job_needs_finalize) == 0) { job_do_finalize(job); } } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> job_completed() had a problem with double locking that was recently fixed independently by two different commits: "job: Fix nested aio_poll() hanging in job_txn_apply" "jobs: add exit shim" One fix removed the first aio_context_acquire(), the other fix removed the other one. Now we have a bug again and the code is run without any locking. Add it back in one of the places. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Reviewed-by: John Snow <jsnow@redhat.com> --- job.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ static void job_completed(Job *job) static void job_exit(void *opaque) { Job *job = (Job *)opaque; + AioContext *ctx = job->aio_context; + + aio_context_acquire(ctx); job_completed(job); + aio_context_release(ctx); } /** -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> In the context of draining a BDS, the .drained_poll callback of block jobs is called. If this returns true (i.e. there is still some activity pending), the drain operation may call aio_poll() with blocking=true to wait for completion. As soon as the pending activity is completed and the job finally arrives in a quiescent state (i.e. its coroutine either yields with busy=false or terminates), the block job must notify the aio_poll() loop to wake up, otherwise we get a deadlock if both are running in different threads. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- include/block/blockjob.h | 13 +++++++++++++ include/qemu/job.h | 3 +++ blockjob.c | 18 ++++++++++++++++++ job.c | 7 +++++++ 4 files changed, 41 insertions(+) diff --git a/include/block/blockjob.h b/include/block/blockjob.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/blockjob.h +++ b/include/block/blockjob.h @@ -XXX,XX +XXX,XX @@ typedef struct BlockJob { /** Called when the job transitions to READY */ Notifier ready_notifier; + /** Called when the job coroutine yields or terminates */ + Notifier idle_notifier; + /** BlockDriverStates that are involved in this block job */ GSList *nodes; } BlockJob; @@ -XXX,XX +XXX,XX @@ int block_job_add_bdrv(BlockJob *job, const char *name, BlockDriverState *bs, */ void block_job_remove_all_bdrv(BlockJob *job); +/** + * block_job_wakeup_all_bdrv: + * @job: The block job + * + * Calls bdrv_wakeup() for all BlockDriverStates that have been added to the + * job. This function is to be called whenever child_job_drained_poll() would + * go from true to false to notify waiting drain requests. + */ +void block_job_wakeup_all_bdrv(BlockJob *job); + /** * block_job_set_speed: * @job: The job to set the speed for. diff --git a/include/qemu/job.h b/include/qemu/job.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/job.h +++ b/include/qemu/job.h @@ -XXX,XX +XXX,XX @@ typedef struct Job { /** Notifiers called when the job transitions to READY */ NotifierList on_ready; + /** Notifiers called when the job coroutine yields or terminates */ + NotifierList on_idle; + /** Element of the list of jobs */ QLIST_ENTRY(Job) job_list; diff --git a/blockjob.c b/blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/blockjob.c +++ b/blockjob.c @@ -XXX,XX +XXX,XX @@ int block_job_add_bdrv(BlockJob *job, const char *name, BlockDriverState *bs, return 0; } +void block_job_wakeup_all_bdrv(BlockJob *job) +{ + GSList *l; + + for (l = job->nodes; l; l = l->next) { + BdrvChild *c = l->data; + bdrv_wakeup(c->bs); + } +} + +static void block_job_on_idle(Notifier *n, void *opaque) +{ + BlockJob *job = opaque; + block_job_wakeup_all_bdrv(job); +} + bool block_job_is_internal(BlockJob *job) { return (job->job.id == NULL); @@ -XXX,XX +XXX,XX @@ void *block_job_create(const char *job_id, const BlockJobDriver *driver, job->finalize_completed_notifier.notify = block_job_event_completed; job->pending_notifier.notify = block_job_event_pending; job->ready_notifier.notify = block_job_event_ready; + job->idle_notifier.notify = block_job_on_idle; notifier_list_add(&job->job.on_finalize_cancelled, &job->finalize_cancelled_notifier); @@ -XXX,XX +XXX,XX @@ void *block_job_create(const char *job_id, const BlockJobDriver *driver, &job->finalize_completed_notifier); notifier_list_add(&job->job.on_pending, &job->pending_notifier); notifier_list_add(&job->job.on_ready, &job->ready_notifier); + notifier_list_add(&job->job.on_idle, &job->idle_notifier); error_setg(&job->blocker, "block device is in use by block job: %s", job_type_str(&job->job)); diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ static void job_event_ready(Job *job) notifier_list_notify(&job->on_ready, job); } +static void job_event_idle(Job *job) +{ + notifier_list_notify(&job->on_idle, job); +} + void job_enter_cond(Job *job, bool(*fn)(Job *job)) { if (!job_started(job)) { @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_do_yield(Job *job, uint64_t ns) timer_mod(&job->sleep_timer, ns); } job->busy = false; + job_event_idle(job); job_unlock(); qemu_coroutine_yield(); @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_co_entry(void *opaque) assert(job && job->driver && job->driver->run); job_pause_point(job); job->ret = job->driver->run(job, &job->err); + job_event_idle(job); job->deferred_to_main_loop = true; aio_bh_schedule_oneshot(qemu_get_aio_context(), job_exit, job); } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> Even if AIO_WAIT_WHILE() is called in the home context of the AioContext, we still want to allow the condition to change depending on other threads as long as they kick the AioWait. Specfically block jobs can be running in an I/O thread and should then be able to kick a drain in the main loop context. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> --- include/block/aio-wait.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/block/aio-wait.h b/include/block/aio-wait.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/aio-wait.h +++ b/include/block/aio-wait.h @@ -XXX,XX +XXX,XX @@ typedef struct { bool waited_ = false; \ AioWait *wait_ = (wait); \ AioContext *ctx_ = (ctx); \ + /* Increment wait_->num_waiters before evaluating cond. */ \ + atomic_inc(&wait_->num_waiters); \ if (ctx_ && in_aio_context_home_thread(ctx_)) { \ while ((cond)) { \ aio_poll(ctx_, true); \ @@ -XXX,XX +XXX,XX @@ typedef struct { } else { \ assert(qemu_get_current_aio_context() == \ qemu_get_aio_context()); \ - /* Increment wait_->num_waiters before evaluating cond. */ \ - atomic_inc(&wait_->num_waiters); \ while ((cond)) { \ if (ctx_) { \ aio_context_release(ctx_); \ @@ -XXX,XX +XXX,XX @@ typedef struct { } \ waited_ = true; \ } \ - atomic_dec(&wait_->num_waiters); \ } \ + atomic_dec(&wait_->num_waiters); \ waited_; }) /** -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> This extends the existing drain test with a block job to include variants where the block job runs in a different AioContext. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> --- tests/test-bdrv-drain.c | 92 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-bdrv-drain.c +++ b/tests/test-bdrv-drain.c @@ -XXX,XX +XXX,XX @@ static void do_drain_end(enum drain_type drain_type, BlockDriverState *bs) } } +static void do_drain_begin_unlocked(enum drain_type drain_type, BlockDriverState *bs) +{ + if (drain_type != BDRV_DRAIN_ALL) { + aio_context_acquire(bdrv_get_aio_context(bs)); + } + do_drain_begin(drain_type, bs); + if (drain_type != BDRV_DRAIN_ALL) { + aio_context_release(bdrv_get_aio_context(bs)); + } +} + +static void do_drain_end_unlocked(enum drain_type drain_type, BlockDriverState *bs) +{ + if (drain_type != BDRV_DRAIN_ALL) { + aio_context_acquire(bdrv_get_aio_context(bs)); + } + do_drain_end(drain_type, bs); + if (drain_type != BDRV_DRAIN_ALL) { + aio_context_release(bdrv_get_aio_context(bs)); + } +} + static void test_drv_cb_common(enum drain_type drain_type, bool recursive) { BlockBackend *blk; @@ -XXX,XX +XXX,XX @@ BlockJobDriver test_job_driver = { }, }; -static void test_blockjob_common(enum drain_type drain_type) +static void test_blockjob_common(enum drain_type drain_type, bool use_iothread) { BlockBackend *blk_src, *blk_target; BlockDriverState *src, *target; BlockJob *job; + IOThread *iothread = NULL; + AioContext *ctx; int ret; src = bdrv_new_open_driver(&bdrv_test, "source", BDRV_O_RDWR, @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type) blk_src = blk_new(BLK_PERM_ALL, BLK_PERM_ALL); blk_insert_bs(blk_src, src, &error_abort); + if (use_iothread) { + iothread = iothread_new(); + ctx = iothread_get_aio_context(iothread); + blk_set_aio_context(blk_src, ctx); + } else { + ctx = qemu_get_aio_context(); + } + target = bdrv_new_open_driver(&bdrv_test, "target", BDRV_O_RDWR, &error_abort); blk_target = blk_new(BLK_PERM_ALL, BLK_PERM_ALL); blk_insert_bs(blk_target, target, &error_abort); + aio_context_acquire(ctx); job = block_job_create("job0", &test_job_driver, NULL, src, 0, BLK_PERM_ALL, 0, 0, NULL, NULL, &error_abort); block_job_add_bdrv(job, "target", target, 0, BLK_PERM_ALL, &error_abort); job_start(&job->job); + aio_context_release(ctx); g_assert_cmpint(job->job.pause_count, ==, 0); g_assert_false(job->job.paused); g_assert_true(job->job.busy); /* We're in job_sleep_ns() */ - do_drain_begin(drain_type, src); + do_drain_begin_unlocked(drain_type, src); if (drain_type == BDRV_DRAIN_ALL) { /* bdrv_drain_all() drains both src and target */ @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type) g_assert_true(job->job.paused); g_assert_false(job->job.busy); /* The job is paused */ - do_drain_end(drain_type, src); + do_drain_end_unlocked(drain_type, src); + + if (use_iothread) { + /* paused is reset in the I/O thread, wait for it */ + while (job->job.paused) { + aio_poll(qemu_get_aio_context(), false); + } + } g_assert_cmpint(job->job.pause_count, ==, 0); g_assert_false(job->job.paused); @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type) do_drain_end(drain_type, target); + if (use_iothread) { + /* paused is reset in the I/O thread, wait for it */ + while (job->job.paused) { + aio_poll(qemu_get_aio_context(), false); + } + } + g_assert_cmpint(job->job.pause_count, ==, 0); g_assert_false(job->job.paused); g_assert_true(job->job.busy); /* We're in job_sleep_ns() */ + aio_context_acquire(ctx); ret = job_complete_sync(&job->job, &error_abort); g_assert_cmpint(ret, ==, 0); + if (use_iothread) { + blk_set_aio_context(blk_src, qemu_get_aio_context()); + } + aio_context_release(ctx); + blk_unref(blk_src); blk_unref(blk_target); bdrv_unref(src); bdrv_unref(target); + + if (iothread) { + iothread_join(iothread); + } } static void test_blockjob_drain_all(void) { - test_blockjob_common(BDRV_DRAIN_ALL); + test_blockjob_common(BDRV_DRAIN_ALL, false); } static void test_blockjob_drain(void) { - test_blockjob_common(BDRV_DRAIN); + test_blockjob_common(BDRV_DRAIN, false); } static void test_blockjob_drain_subtree(void) { - test_blockjob_common(BDRV_SUBTREE_DRAIN); + test_blockjob_common(BDRV_SUBTREE_DRAIN, false); +} + +static void test_blockjob_iothread_drain_all(void) +{ + test_blockjob_common(BDRV_DRAIN_ALL, true); +} + +static void test_blockjob_iothread_drain(void) +{ + test_blockjob_common(BDRV_DRAIN, true); +} + +static void test_blockjob_iothread_drain_subtree(void) +{ + test_blockjob_common(BDRV_SUBTREE_DRAIN, true); } @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv) g_test_add_func("/bdrv-drain/blockjob/drain_subtree", test_blockjob_drain_subtree); + g_test_add_func("/bdrv-drain/blockjob/iothread/drain_all", + test_blockjob_iothread_drain_all); + g_test_add_func("/bdrv-drain/blockjob/iothread/drain", + test_blockjob_iothread_drain); + g_test_add_func("/bdrv-drain/blockjob/iothread/drain_subtree", + test_blockjob_iothread_drain_subtree); + g_test_add_func("/bdrv-drain/deletion/drain", test_delete_by_drain); g_test_add_func("/bdrv-drain/detach/drain_all", test_detach_by_drain_all); g_test_add_func("/bdrv-drain/detach/drain", test_detach_by_drain); -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> All callers in QEMU proper hold the AioContext lock when calling job_finish_sync(). test-blockjob should do the same when it calls the function indirectly through job_cancel_sync(). Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> --- include/qemu/job.h | 6 ++++++ tests/test-blockjob.c | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/include/qemu/job.h b/include/qemu/job.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/job.h +++ b/include/qemu/job.h @@ -XXX,XX +XXX,XX @@ void job_user_cancel(Job *job, bool force, Error **errp); * * Returns the return value from the job if the job actually completed * during the call, or -ECANCELED if it was canceled. + * + * Callers must hold the AioContext lock of job->aio_context. */ int job_cancel_sync(Job *job); @@ -XXX,XX +XXX,XX @@ void job_cancel_sync_all(void); * function). * * Returns the return value from the job. + * + * Callers must hold the AioContext lock of job->aio_context. */ int job_complete_sync(Job *job, Error **errp); @@ -XXX,XX +XXX,XX @@ void job_dismiss(Job **job, Error **errp); * * Returns 0 if the job is successfully completed, -ECANCELED if the job was * cancelled before completing, and -errno in other error cases. + * + * Callers must hold the AioContext lock of job->aio_context. */ int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp); diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-blockjob.c +++ b/tests/test-blockjob.c @@ -XXX,XX +XXX,XX @@ static void cancel_common(CancelJob *s) BlockJob *job = &s->common; BlockBackend *blk = s->blk; JobStatus sts = job->job.status; + AioContext *ctx; + + ctx = job->job.aio_context; + aio_context_acquire(ctx); job_cancel_sync(&job->job); if (sts != JOB_STATUS_CREATED && sts != JOB_STATUS_CONCLUDED) { @@ -XXX,XX +XXX,XX @@ static void cancel_common(CancelJob *s) assert(job->job.status == JOB_STATUS_NULL); job_unref(&job->job); destroy_blk(blk); + + aio_context_release(ctx); } static void test_cancel_created(void) -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> job_finish_sync() needs to release the AioContext lock of the job before calling aio_poll(). Otherwise, callbacks called by aio_poll() would possibly take the lock a second time and run into a deadlock with a nested AIO_WAIT_WHILE() call. Also, job_drain() without aio_poll() isn't necessarily enough to make progress on a job, it could depend on bottom halves to be executed. Combine both open-coded while loops into a single AIO_WAIT_WHILE() call that solves both of these problems. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- job.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ #include "qemu/job.h" #include "qemu/id.h" #include "qemu/main-loop.h" +#include "block/aio-wait.h" #include "trace-root.h" #include "qapi/qapi-events-job.h" @@ -XXX,XX +XXX,XX @@ void job_complete(Job *job, Error **errp) int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp) { Error *local_err = NULL; + AioWait dummy_wait = {}; int ret; job_ref(job); @@ -XXX,XX +XXX,XX @@ int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp) job_unref(job); return -EBUSY; } - /* job_drain calls job_enter, and it should be enough to induce progress - * until the job completes or moves to the main thread. */ - while (!job->deferred_to_main_loop && !job_is_completed(job)) { - job_drain(job); - } - while (!job_is_completed(job)) { - aio_poll(qemu_get_aio_context(), true); - } + + AIO_WAIT_WHILE(&dummy_wait, job->aio_context, + (job_drain(job), !job_is_completed(job))); + ret = (job_is_cancelled(job) && job->ret == 0) ? -ECANCELED : job->ret; job_unref(job); return ret; -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> This is a regression test for a deadlock that occurred in block job completion callbacks (via job_defer_to_main_loop) because the AioContext lock was taken twice: once in job_finish_sync() and then again in job_defer_to_main_loop_bh(). This would cause AIO_WAIT_WHILE() to hang. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> --- tests/test-bdrv-drain.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-bdrv-drain.c +++ b/tests/test-bdrv-drain.c @@ -XXX,XX +XXX,XX @@ typedef struct TestBlockJob { bool should_complete; } TestBlockJob; +static int test_job_prepare(Job *job) +{ + TestBlockJob *s = container_of(job, TestBlockJob, common.job); + + /* Provoke an AIO_WAIT_WHILE() call to verify there is no deadlock */ + blk_flush(s->common.blk); + return 0; +} + static int coroutine_fn test_job_run(Job *job, Error **errp) { TestBlockJob *s = container_of(job, TestBlockJob, common.job); @@ -XXX,XX +XXX,XX @@ BlockJobDriver test_job_driver = { .drain = block_job_drain, .run = test_job_run, .complete = test_job_complete, + .prepare = test_job_prepare, }, }; -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> bdrv_do_drained_begin/end() assume that they are called with the AioContext lock of bs held. If we call drain functions from a coroutine with the AioContext lock held, we yield and schedule a BH to move out of coroutine context. This means that the lock for the home context of the coroutine is released and must be re-acquired in the bottom half. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- include/qemu/coroutine.h | 5 +++++ block/io.c | 15 +++++++++++++++ util/qemu-coroutine.c | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/coroutine.h +++ b/include/qemu/coroutine.h @@ -XXX,XX +XXX,XX @@ void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co); */ void coroutine_fn qemu_coroutine_yield(void); +/** + * Get the AioContext of the given coroutine + */ +AioContext *coroutine_fn qemu_coroutine_get_aio_context(Coroutine *co); + /** * Get the currently executing coroutine */ diff --git a/block/io.c b/block/io.c index XXXXXXX..XXXXXXX 100644 --- a/block/io.c +++ b/block/io.c @@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque) BlockDriverState *bs = data->bs; if (bs) { + AioContext *ctx = bdrv_get_aio_context(bs); + AioContext *co_ctx = qemu_coroutine_get_aio_context(co); + + /* + * When the coroutine yielded, the lock for its home context was + * released, so we need to re-acquire it here. If it explicitly + * acquired a different context, the lock is still held and we don't + * want to lock it a second time (or AIO_WAIT_WHILE() would hang). + */ + if (ctx == co_ctx) { + aio_context_acquire(ctx); + } bdrv_dec_in_flight(bs); if (data->begin) { bdrv_do_drained_begin(bs, data->recursive, data->parent, @@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque) bdrv_do_drained_end(bs, data->recursive, data->parent, data->ignore_bds_parents); } + if (ctx == co_ctx) { + aio_context_release(ctx); + } } else { assert(data->begin); bdrv_drain_all_begin(); diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.c index XXXXXXX..XXXXXXX 100644 --- a/util/qemu-coroutine.c +++ b/util/qemu-coroutine.c @@ -XXX,XX +XXX,XX @@ bool qemu_coroutine_entered(Coroutine *co) { return co->caller; } + +AioContext *coroutine_fn qemu_coroutine_get_aio_context(Coroutine *co) +{ + return co->ctx; +} -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> A bdrv_drain operation must ensure that all parents are quiesced, this includes BlockBackends. Otherwise, callbacks called by requests that are completed on the BDS layer, but not quite yet on the BlockBackend layer could still create new requests. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- block/block-backend.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/block/block-backend.c b/block/block-backend.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -XXX,XX +XXX,XX @@ static void blk_root_inherit_options(int *child_flags, QDict *child_options, abort(); } static void blk_root_drained_begin(BdrvChild *child); +static bool blk_root_drained_poll(BdrvChild *child); static void blk_root_drained_end(BdrvChild *child); static void blk_root_change_media(BdrvChild *child, bool load); @@ -XXX,XX +XXX,XX @@ static const BdrvChildRole child_root = { .get_parent_desc = blk_root_get_parent_desc, .drained_begin = blk_root_drained_begin, + .drained_poll = blk_root_drained_poll, .drained_end = blk_root_drained_end, .activate = blk_root_activate, @@ -XXX,XX +XXX,XX @@ static void blk_root_drained_begin(BdrvChild *child) } } +static bool blk_root_drained_poll(BdrvChild *child) +{ + BlockBackend *blk = child->opaque; + assert(blk->quiesce_counter); + return !!blk->in_flight; +} + static void blk_root_drained_end(BdrvChild *child) { BlockBackend *blk = child->opaque; -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> blk_unref() first decreases the refcount of the BlockBackend and calls blk_delete() if the refcount reaches zero. Requests can still be in flight at this point, they are only drained during blk_delete(): At this point, arbitrary callbacks can run. If any callback takes a temporary BlockBackend reference, it will first increase the refcount to 1 and then decrease it to 0 again, triggering another blk_delete(). This will cause a use-after-free crash in the outer blk_delete(). Fix it by draining the BlockBackend before decreasing to refcount to 0. Assert in blk_ref() that it never takes the first refcount (which would mean that the BlockBackend is already being deleted). Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- block/block-backend.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/block/block-backend.c b/block/block-backend.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -XXX,XX +XXX,XX @@ int blk_get_refcnt(BlockBackend *blk) */ void blk_ref(BlockBackend *blk) { + assert(blk->refcnt > 0); blk->refcnt++; } @@ -XXX,XX +XXX,XX @@ void blk_unref(BlockBackend *blk) { if (blk) { assert(blk->refcnt > 0); - if (!--blk->refcnt) { + if (blk->refcnt > 1) { + blk->refcnt--; + } else { + blk_drain(blk); + /* blk_drain() cannot resurrect blk, nobody held a reference */ + assert(blk->refcnt == 1); + blk->refcnt = 0; blk_delete(blk); } } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> Request callbacks can do pretty much anything, including operations that will yield from the coroutine (such as draining the backend). In that case, a decreased in_flight would be visible to other code and could lead to a drain completing while the callback hasn't actually completed yet. Note that reordering these operations forbids calling drain directly inside an AIO callback. As Paolo explains, indirectly calling it is okay: - Calling it through a coroutine is okay, because then bdrv_drained_begin() goes through bdrv_co_yield_to_drain() and you have in_flight=2 when bdrv_co_yield_to_drain() yields, then soon in_flight=1 when the aio_co_wake() in the AIO callback completes, then in_flight=0 after the bottom half starts. - Calling it through a bottom half would be okay too, as long as the AIO callback remembers to do inc_in_flight/dec_in_flight just like bdrv_co_yield_to_drain() and bdrv_co_drain_bh_cb() do A few more important cases that come to mind: - A coroutine that yields because of I/O is okay, with a sequence similar to bdrv_co_yield_to_drain(). - A coroutine that yields with no I/O pending will correctly decrease in_flight to zero before yielding. - Calling more AIO from the callback won't overflow the counter just because of mutual recursion, because AIO functions always yield at least once before invoking the callback. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com> --- block/block-backend.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/block-backend.c b/block/block-backend.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -XXX,XX +XXX,XX @@ static const AIOCBInfo blk_aio_em_aiocb_info = { static void blk_aio_complete(BlkAioEmAIOCB *acb) { if (acb->has_returned) { - blk_dec_in_flight(acb->rwco.blk); acb->common.cb(acb->common.opaque, acb->rwco.ret); + blk_dec_in_flight(acb->rwco.blk); qemu_aio_unref(acb); } } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> Block jobs claim in .drained_poll() that they are in a quiescent state as soon as job->deferred_to_main_loop is true. This is obviously wrong, they still have a completion BH to run. We only get away with this because commit 91af091f923 added an unconditional aio_poll(false) to the drain functions, but this is bypassing the regular drain mechanisms. However, just removing this and telling that the job is still active doesn't work either: The completion callbacks themselves call drain functions (directly, or indirectly with bdrv_reopen), so they would deadlock then. As a better lie, tell that the job is active as long as the BH is pending, but falsely call it quiescent from the point in the BH when the completion callback is called. At this point, nested drain calls won't deadlock because they ignore the job, and outer drains will wait for the job to really reach a quiescent state because the callback is already running. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- include/qemu/job.h | 3 +++ blockjob.c | 2 +- job.c | 11 ++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/include/qemu/job.h b/include/qemu/job.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/job.h +++ b/include/qemu/job.h @@ -XXX,XX +XXX,XX @@ typedef struct Job { * Set to false by the job while the coroutine has yielded and may be * re-entered by job_enter(). There may still be I/O or event loop activity * pending. Accessed under block_job_mutex (in blockjob.c). + * + * When the job is deferred to the main loop, busy is true as long as the + * bottom half is still pending. */ bool busy; diff --git a/blockjob.c b/blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/blockjob.c +++ b/blockjob.c @@ -XXX,XX +XXX,XX @@ static bool child_job_drained_poll(BdrvChild *c) /* An inactive or completed job doesn't have any pending requests. Jobs * with !job->busy are either already paused or have a pause point after * being reentered, so no job driver code will run before they pause. */ - if (!job->busy || job_is_completed(job) || job->deferred_to_main_loop) { + if (!job->busy || job_is_completed(job)) { return false; } diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ static void job_exit(void *opaque) AioContext *ctx = job->aio_context; aio_context_acquire(ctx); + + /* This is a lie, we're not quiescent, but still doing the completion + * callbacks. However, completion callbacks tend to involve operations that + * drain block nodes, and if .drained_poll still returned true, we would + * deadlock. */ + job->busy = false; + job_event_idle(job); + job_completed(job); + aio_context_release(ctx); } @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_co_entry(void *opaque) assert(job && job->driver && job->driver->run); job_pause_point(job); job->ret = job->driver->run(job, &job->err); - job_event_idle(job); job->deferred_to_main_loop = true; + job->busy = true; aio_bh_schedule_oneshot(qemu_get_aio_context(), job_exit, job); } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> bdrv_drain_poll_top_level() was buggy because it didn't release the AioContext lock of the node to be drained before calling aio_poll(). This way, callbacks called by aio_poll() would possibly take the lock a second time and run into a deadlock with a nested AIO_WAIT_WHILE() call. However, it turns out that the aio_poll() call isn't actually needed any more. It was introduced in commit 91af091f923, which is effectively reverted by this patch. The cases it was supposed to fix are now covered by bdrv_drain_poll(), which waits for block jobs to reach a quiescent state. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- block/io.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/block/io.c b/block/io.c index XXXXXXX..XXXXXXX 100644 --- a/block/io.c +++ b/block/io.c @@ -XXX,XX +XXX,XX @@ bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, BdrvChild *ignore_parent) { - /* Execute pending BHs first and check everything else only after the BHs - * have executed. */ - while (aio_poll(bs->aio_context, false)); - return bdrv_drain_poll(bs, recursive, ignore_parent, false); } @@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_all_poll(void) BlockDriverState *bs = NULL; bool result = false; - /* Execute pending BHs first (may modify the graph) and check everything - * else only after the BHs have executed. */ - while (aio_poll(qemu_get_aio_context(), false)); - /* bdrv_drain_poll() can't make changes to the graph and we are holding the * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ while ((bs = bdrv_next_all_states(bs))) { -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> This is a regression test for a deadlock that could occur in callbacks called from the aio_poll() in bdrv_drain_poll_top_level(). The AioContext lock wasn't released and therefore would be taken a second time in the callback. This would cause a possible AIO_WAIT_WHILE() in the callback to hang. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Fam Zheng <famz@redhat.com> --- tests/test-bdrv-drain.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-bdrv-drain.c +++ b/tests/test-bdrv-drain.c @@ -XXX,XX +XXX,XX @@ static void test_iothread_aio_cb(void *opaque, int ret) qemu_event_set(&done_event); } +static void test_iothread_main_thread_bh(void *opaque) +{ + struct test_iothread_data *data = opaque; + + /* Test that the AioContext is not yet locked in a random BH that is + * executed during drain, otherwise this would deadlock. */ + aio_context_acquire(bdrv_get_aio_context(data->bs)); + bdrv_flush(data->bs); + aio_context_release(bdrv_get_aio_context(data->bs)); +} + /* * Starts an AIO request on a BDS that runs in the AioContext of iothread 1. * The request involves a BH on iothread 2 before it can complete. @@ -XXX,XX +XXX,XX @@ static void test_iothread_common(enum drain_type drain_type, int drain_thread) aio_context_acquire(ctx_a); } + aio_bh_schedule_oneshot(ctx_a, test_iothread_main_thread_bh, &data); + /* The request is running on the IOThread a. Draining its block device * will make sure that it has completed as far as the BDS is concerned, * but the drain in this thread can continue immediately after -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> Amongst others, job_finalize_single() calls the .prepare/.commit/.abort callbacks of the individual job driver. Recently, their use was adapted for all block jobs so that they involve code calling AIO_WAIT_WHILE() now. Such code must be called under the AioContext lock for the respective job, but without holding any other AioContext lock. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- job.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ static void job_cancel_async(Job *job, bool force) static void job_completed_txn_abort(Job *job) { + AioContext *outer_ctx = job->aio_context; AioContext *ctx; JobTxn *txn = job->txn; Job *other_job; @@ -XXX,XX +XXX,XX @@ static void job_completed_txn_abort(Job *job) txn->aborting = true; job_txn_ref(txn); - /* We are the first failed job. Cancel other jobs. */ - QLIST_FOREACH(other_job, &txn->jobs, txn_list) { - ctx = other_job->aio_context; - aio_context_acquire(ctx); - } + /* We can only hold the single job's AioContext lock while calling + * job_finalize_single() because the finalization callbacks can involve + * calls of AIO_WAIT_WHILE(), which could deadlock otherwise. */ + aio_context_release(outer_ctx); /* Other jobs are effectively cancelled by us, set the status for * them; this job, however, may or may not be cancelled, depending * on the caller, so leave it. */ QLIST_FOREACH(other_job, &txn->jobs, txn_list) { if (other_job != job) { + ctx = other_job->aio_context; + aio_context_acquire(ctx); job_cancel_async(other_job, false); + aio_context_release(ctx); } } while (!QLIST_EMPTY(&txn->jobs)) { other_job = QLIST_FIRST(&txn->jobs); ctx = other_job->aio_context; + aio_context_acquire(ctx); if (!job_is_completed(other_job)) { assert(job_is_cancelled(other_job)); job_finish_sync(other_job, NULL, NULL); @@ -XXX,XX +XXX,XX @@ static void job_completed_txn_abort(Job *job) aio_context_release(ctx); } + aio_context_acquire(outer_ctx); + job_txn_unref(txn); } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> This adds tests for calling AIO_WAIT_WHILE() in the .commit and .abort callbacks. Both reasons why .abort could be called for a single job are tested: Either .run or .prepare could return an error. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- tests/test-bdrv-drain.c | 116 +++++++++++++++++++++++++++++++++++----- 1 file changed, 104 insertions(+), 12 deletions(-) diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-bdrv-drain.c +++ b/tests/test-bdrv-drain.c @@ -XXX,XX +XXX,XX @@ static void test_iothread_drain_subtree(void) typedef struct TestBlockJob { BlockJob common; + int run_ret; + int prepare_ret; bool should_complete; } TestBlockJob; @@ -XXX,XX +XXX,XX @@ static int test_job_prepare(Job *job) /* Provoke an AIO_WAIT_WHILE() call to verify there is no deadlock */ blk_flush(s->common.blk); - return 0; + return s->prepare_ret; +} + +static void test_job_commit(Job *job) +{ + TestBlockJob *s = container_of(job, TestBlockJob, common.job); + + /* Provoke an AIO_WAIT_WHILE() call to verify there is no deadlock */ + blk_flush(s->common.blk); +} + +static void test_job_abort(Job *job) +{ + TestBlockJob *s = container_of(job, TestBlockJob, common.job); + + /* Provoke an AIO_WAIT_WHILE() call to verify there is no deadlock */ + blk_flush(s->common.blk); } static int coroutine_fn test_job_run(Job *job, Error **errp) @@ -XXX,XX +XXX,XX @@ static int coroutine_fn test_job_run(Job *job, Error **errp) job_pause_point(&s->common.job); } - return 0; + return s->run_ret; } static void test_job_complete(Job *job, Error **errp) @@ -XXX,XX +XXX,XX @@ BlockJobDriver test_job_driver = { .run = test_job_run, .complete = test_job_complete, .prepare = test_job_prepare, + .commit = test_job_commit, + .abort = test_job_abort, }, }; -static void test_blockjob_common(enum drain_type drain_type, bool use_iothread) +enum test_job_result { + TEST_JOB_SUCCESS, + TEST_JOB_FAIL_RUN, + TEST_JOB_FAIL_PREPARE, +}; + +static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, + enum test_job_result result) { BlockBackend *blk_src, *blk_target; BlockDriverState *src, *target; BlockJob *job; + TestBlockJob *tjob; IOThread *iothread = NULL; AioContext *ctx; int ret; @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread) blk_insert_bs(blk_target, target, &error_abort); aio_context_acquire(ctx); - job = block_job_create("job0", &test_job_driver, NULL, src, 0, BLK_PERM_ALL, - 0, 0, NULL, NULL, &error_abort); + tjob = block_job_create("job0", &test_job_driver, NULL, src, + 0, BLK_PERM_ALL, + 0, 0, NULL, NULL, &error_abort); + job = &tjob->common; block_job_add_bdrv(job, "target", target, 0, BLK_PERM_ALL, &error_abort); + + switch (result) { + case TEST_JOB_SUCCESS: + break; + case TEST_JOB_FAIL_RUN: + tjob->run_ret = -EIO; + break; + case TEST_JOB_FAIL_PREPARE: + tjob->prepare_ret = -EIO; + break; + } + job_start(&job->job); aio_context_release(ctx); @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread) aio_context_acquire(ctx); ret = job_complete_sync(&job->job, &error_abort); - g_assert_cmpint(ret, ==, 0); + g_assert_cmpint(ret, ==, (result == TEST_JOB_SUCCESS ? 0 : -EIO)); if (use_iothread) { blk_set_aio_context(blk_src, qemu_get_aio_context()); @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread) static void test_blockjob_drain_all(void) { - test_blockjob_common(BDRV_DRAIN_ALL, false); + test_blockjob_common(BDRV_DRAIN_ALL, false, TEST_JOB_SUCCESS); } static void test_blockjob_drain(void) { - test_blockjob_common(BDRV_DRAIN, false); + test_blockjob_common(BDRV_DRAIN, false, TEST_JOB_SUCCESS); } static void test_blockjob_drain_subtree(void) { - test_blockjob_common(BDRV_SUBTREE_DRAIN, false); + test_blockjob_common(BDRV_SUBTREE_DRAIN, false, TEST_JOB_SUCCESS); +} + +static void test_blockjob_error_drain_all(void) +{ + test_blockjob_common(BDRV_DRAIN_ALL, false, TEST_JOB_FAIL_RUN); + test_blockjob_common(BDRV_DRAIN_ALL, false, TEST_JOB_FAIL_PREPARE); +} + +static void test_blockjob_error_drain(void) +{ + test_blockjob_common(BDRV_DRAIN, false, TEST_JOB_FAIL_RUN); + test_blockjob_common(BDRV_DRAIN, false, TEST_JOB_FAIL_PREPARE); +} + +static void test_blockjob_error_drain_subtree(void) +{ + test_blockjob_common(BDRV_SUBTREE_DRAIN, false, TEST_JOB_FAIL_RUN); + test_blockjob_common(BDRV_SUBTREE_DRAIN, false, TEST_JOB_FAIL_PREPARE); } static void test_blockjob_iothread_drain_all(void) { - test_blockjob_common(BDRV_DRAIN_ALL, true); + test_blockjob_common(BDRV_DRAIN_ALL, true, TEST_JOB_SUCCESS); } static void test_blockjob_iothread_drain(void) { - test_blockjob_common(BDRV_DRAIN, true); + test_blockjob_common(BDRV_DRAIN, true, TEST_JOB_SUCCESS); } static void test_blockjob_iothread_drain_subtree(void) { - test_blockjob_common(BDRV_SUBTREE_DRAIN, true); + test_blockjob_common(BDRV_SUBTREE_DRAIN, true, TEST_JOB_SUCCESS); +} + +static void test_blockjob_iothread_error_drain_all(void) +{ + test_blockjob_common(BDRV_DRAIN_ALL, true, TEST_JOB_FAIL_RUN); + test_blockjob_common(BDRV_DRAIN_ALL, true, TEST_JOB_FAIL_PREPARE); +} + +static void test_blockjob_iothread_error_drain(void) +{ + test_blockjob_common(BDRV_DRAIN, true, TEST_JOB_FAIL_RUN); + test_blockjob_common(BDRV_DRAIN, true, TEST_JOB_FAIL_PREPARE); +} + +static void test_blockjob_iothread_error_drain_subtree(void) +{ + test_blockjob_common(BDRV_SUBTREE_DRAIN, true, TEST_JOB_FAIL_RUN); + test_blockjob_common(BDRV_SUBTREE_DRAIN, true, TEST_JOB_FAIL_PREPARE); } @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv) g_test_add_func("/bdrv-drain/blockjob/drain_subtree", test_blockjob_drain_subtree); + g_test_add_func("/bdrv-drain/blockjob/error/drain_all", + test_blockjob_error_drain_all); + g_test_add_func("/bdrv-drain/blockjob/error/drain", + test_blockjob_error_drain); + g_test_add_func("/bdrv-drain/blockjob/error/drain_subtree", + test_blockjob_error_drain_subtree); + g_test_add_func("/bdrv-drain/blockjob/iothread/drain_all", test_blockjob_iothread_drain_all); g_test_add_func("/bdrv-drain/blockjob/iothread/drain", @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv) g_test_add_func("/bdrv-drain/blockjob/iothread/drain_subtree", test_blockjob_iothread_drain_subtree); + g_test_add_func("/bdrv-drain/blockjob/iothread/error/drain_all", + test_blockjob_iothread_error_drain_all); + g_test_add_func("/bdrv-drain/blockjob/iothread/error/drain", + test_blockjob_iothread_error_drain); + g_test_add_func("/bdrv-drain/blockjob/iothread/error/drain_subtree", + test_blockjob_iothread_error_drain_subtree); + g_test_add_func("/bdrv-drain/deletion/drain", test_delete_by_drain); g_test_add_func("/bdrv-drain/detach/drain_all", test_detach_by_drain_all); g_test_add_func("/bdrv-drain/detach/drain", test_detach_by_drain); -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> Commit 89bd030533e changed the test case from using job_sleep_ns() to using qemu_co_sleep_ns() instead. Also, block_job_sleep_ns() became job_sleep_ns() in commit 5d43e86e11f. In both cases, some comments in the test case were not updated. Do that now. Reported-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> --- tests/test-bdrv-drain.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-bdrv-drain.c +++ b/tests/test-bdrv-drain.c @@ -XXX,XX +XXX,XX @@ static int coroutine_fn test_job_run(Job *job, Error **errp) job_transition_to_ready(&s->common.job); while (!s->should_complete) { - /* Avoid block_job_sleep_ns() because it marks the job as !busy. We - * want to emulate some actual activity (probably some I/O) here so - * that drain has to wait for this acitivity to stop. */ + /* Avoid job_sleep_ns() because it marks the job as !busy. We want to + * emulate some actual activity (probably some I/O) here so that drain + * has to wait for this activity to stop. */ qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 100000); job_pause_point(&s->common.job); } @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, g_assert_cmpint(job->job.pause_count, ==, 0); g_assert_false(job->job.paused); - g_assert_true(job->job.busy); /* We're in job_sleep_ns() */ + g_assert_true(job->job.busy); /* We're in qemu_co_sleep_ns() */ do_drain_begin_unlocked(drain_type, src); @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, g_assert_cmpint(job->job.pause_count, ==, 0); g_assert_false(job->job.paused); - g_assert_true(job->job.busy); /* We're in job_sleep_ns() */ + g_assert_true(job->job.busy); /* We're in qemu_co_sleep_ns() */ aio_context_acquire(ctx); ret = job_complete_sync(&job->job, &error_abort); -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> When draining a block node, we recurse to its parent and for subtree drains also to its children. A single AIO_WAIT_WHILE() is then used to wait for bdrv_drain_poll() to become true, which depends on all of the nodes we recursed to. However, if the respective child or parent becomes quiescent and calls bdrv_wakeup(), only the AioWait of the child/parent is checked, while AIO_WAIT_WHILE() depends on the AioWait of the original node. Fix this by using a single AioWait for all callers of AIO_WAIT_WHILE(). This may mean that the draining thread gets a few more unnecessary wakeups because an unrelated operation got completed, but we already wake it up when something _could_ have changed rather than only if it has certainly changed. Apart from that, drain is a slow path anyway. In theory it would be possible to use wakeups more selectively and still correctly, but the gains are likely not worth the additional complexity. In fact, this patch is a nice simplification for some places in the code. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- include/block/aio-wait.h | 22 +++++++++++----------- include/block/block.h | 6 +----- include/block/block_int.h | 3 --- include/block/blockjob.h | 10 ---------- block.c | 5 ----- block/block-backend.c | 11 ++++------- block/io.c | 7 ++----- blockjob.c | 13 +------------ job.c | 3 +-- util/aio-wait.c | 11 ++++++----- 10 files changed, 26 insertions(+), 65 deletions(-) diff --git a/include/block/aio-wait.h b/include/block/aio-wait.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/aio-wait.h +++ b/include/block/aio-wait.h @@ -XXX,XX +XXX,XX @@ /** * AioWait: * - * An object that facilitates synchronous waiting on a condition. The main - * loop can wait on an operation running in an IOThread as follows: + * An object that facilitates synchronous waiting on a condition. A single + * global AioWait object (global_aio_wait) is used internally. + * + * The main loop can wait on an operation running in an IOThread as follows: * - * AioWait *wait = ...; * AioContext *ctx = ...; * MyWork work = { .done = false }; * schedule_my_work_in_iothread(ctx, &work); - * AIO_WAIT_WHILE(wait, ctx, !work.done); + * AIO_WAIT_WHILE(ctx, !work.done); * * The IOThread must call aio_wait_kick() to notify the main loop when * work.done changes: @@ -XXX,XX +XXX,XX @@ * { * ... * work.done = true; - * aio_wait_kick(wait); + * aio_wait_kick(); * } */ typedef struct { @@ -XXX,XX +XXX,XX @@ typedef struct { unsigned num_waiters; } AioWait; +extern AioWait global_aio_wait; + /** * AIO_WAIT_WHILE: - * @wait: the aio wait object * @ctx: the aio context, or NULL if multiple aio contexts (for which the * caller does not hold a lock) are involved in the polling condition. * @cond: wait while this conditional expression is true @@ -XXX,XX +XXX,XX @@ typedef struct { * wait on conditions between two IOThreads since that could lead to deadlock, * go via the main loop instead. */ -#define AIO_WAIT_WHILE(wait, ctx, cond) ({ \ +#define AIO_WAIT_WHILE(ctx, cond) ({ \ bool waited_ = false; \ - AioWait *wait_ = (wait); \ + AioWait *wait_ = &global_aio_wait; \ AioContext *ctx_ = (ctx); \ /* Increment wait_->num_waiters before evaluating cond. */ \ atomic_inc(&wait_->num_waiters); \ @@ -XXX,XX +XXX,XX @@ typedef struct { /** * aio_wait_kick: - * @wait: the aio wait object that should re-evaluate its condition - * * Wake up the main thread if it is waiting on AIO_WAIT_WHILE(). During * synchronous operations performed in an IOThread, the main thread lets the * IOThread's event loop run, waiting for the operation to complete. A * aio_wait_kick() call will wake up the main thread. */ -void aio_wait_kick(AioWait *wait); +void aio_wait_kick(void); /** * aio_wait_bh_oneshot: diff --git a/include/block/block.h b/include/block/block.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void); void bdrv_drain_all_end(void); void bdrv_drain_all(void); -/* Returns NULL when bs == NULL */ -AioWait *bdrv_get_aio_wait(BlockDriverState *bs); - #define BDRV_POLL_WHILE(bs, cond) ({ \ BlockDriverState *bs_ = (bs); \ - AIO_WAIT_WHILE(bdrv_get_aio_wait(bs_), \ - bdrv_get_aio_context(bs_), \ + AIO_WAIT_WHILE(bdrv_get_aio_context(bs_), \ cond); }) int bdrv_pdiscard(BdrvChild *child, int64_t offset, int bytes); diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ struct BlockDriverState { unsigned int in_flight; unsigned int serialising_in_flight; - /* Kicked to signal main loop when a request completes. */ - AioWait wait; - /* counter for nested bdrv_io_plug. * Accessed with atomic ops. */ diff --git a/include/block/blockjob.h b/include/block/blockjob.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/blockjob.h +++ b/include/block/blockjob.h @@ -XXX,XX +XXX,XX @@ int block_job_add_bdrv(BlockJob *job, const char *name, BlockDriverState *bs, */ void block_job_remove_all_bdrv(BlockJob *job); -/** - * block_job_wakeup_all_bdrv: - * @job: The block job - * - * Calls bdrv_wakeup() for all BlockDriverStates that have been added to the - * job. This function is to be called whenever child_job_drained_poll() would - * go from true to false to notify waiting drain requests. - */ -void block_job_wakeup_all_bdrv(BlockJob *job); - /** * block_job_set_speed: * @job: The job to set the speed for. diff --git a/block.c b/block.c index XXXXXXX..XXXXXXX 100644 --- a/block.c +++ b/block.c @@ -XXX,XX +XXX,XX @@ AioContext *bdrv_get_aio_context(BlockDriverState *bs) return bs ? bs->aio_context : qemu_get_aio_context(); } -AioWait *bdrv_get_aio_wait(BlockDriverState *bs) -{ - return bs ? &bs->wait : NULL; -} - void bdrv_coroutine_enter(BlockDriverState *bs, Coroutine *co) { aio_co_enter(bdrv_get_aio_context(bs), co); diff --git a/block/block-backend.c b/block/block-backend.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -XXX,XX +XXX,XX @@ struct BlockBackend { * Accessed with atomic ops. */ unsigned int in_flight; - AioWait wait; }; typedef struct BlockBackendAIOCB { @@ -XXX,XX +XXX,XX @@ static void blk_inc_in_flight(BlockBackend *blk) static void blk_dec_in_flight(BlockBackend *blk) { atomic_dec(&blk->in_flight); - aio_wait_kick(&blk->wait); + aio_wait_kick(); } static void error_callback_bh(void *opaque) @@ -XXX,XX +XXX,XX @@ void blk_drain(BlockBackend *blk) } /* We may have -ENOMEDIUM completions in flight */ - AIO_WAIT_WHILE(&blk->wait, - blk_get_aio_context(blk), - atomic_mb_read(&blk->in_flight) > 0); + AIO_WAIT_WHILE(blk_get_aio_context(blk), + atomic_mb_read(&blk->in_flight) > 0); if (bs) { bdrv_drained_end(bs); @@ -XXX,XX +XXX,XX @@ void blk_drain_all(void) aio_context_acquire(ctx); /* We may have -ENOMEDIUM completions in flight */ - AIO_WAIT_WHILE(&blk->wait, ctx, - atomic_mb_read(&blk->in_flight) > 0); + AIO_WAIT_WHILE(ctx, atomic_mb_read(&blk->in_flight) > 0); aio_context_release(ctx); } diff --git a/block/io.c b/block/io.c index XXXXXXX..XXXXXXX 100644 --- a/block/io.c +++ b/block/io.c @@ -XXX,XX +XXX,XX @@ /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) -static AioWait drain_all_aio_wait; - static void bdrv_parent_cb_resize(BlockDriverState *bs); static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes, BdrvRequestFlags flags); @@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void) } /* Now poll the in-flight requests */ - AIO_WAIT_WHILE(&drain_all_aio_wait, NULL, bdrv_drain_all_poll()); + AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); while ((bs = bdrv_next_all_states(bs))) { bdrv_drain_assert_idle(bs); @@ -XXX,XX +XXX,XX @@ void bdrv_inc_in_flight(BlockDriverState *bs) void bdrv_wakeup(BlockDriverState *bs) { - aio_wait_kick(bdrv_get_aio_wait(bs)); - aio_wait_kick(&drain_all_aio_wait); + aio_wait_kick(); } void bdrv_dec_in_flight(BlockDriverState *bs) diff --git a/blockjob.c b/blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/blockjob.c +++ b/blockjob.c @@ -XXX,XX +XXX,XX @@ int block_job_add_bdrv(BlockJob *job, const char *name, BlockDriverState *bs, return 0; } -void block_job_wakeup_all_bdrv(BlockJob *job) -{ - GSList *l; - - for (l = job->nodes; l; l = l->next) { - BdrvChild *c = l->data; - bdrv_wakeup(c->bs); - } -} - static void block_job_on_idle(Notifier *n, void *opaque) { - BlockJob *job = opaque; - block_job_wakeup_all_bdrv(job); + aio_wait_kick(); } bool block_job_is_internal(BlockJob *job) diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ void job_complete(Job *job, Error **errp) int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp) { Error *local_err = NULL; - AioWait dummy_wait = {}; int ret; job_ref(job); @@ -XXX,XX +XXX,XX @@ int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp) return -EBUSY; } - AIO_WAIT_WHILE(&dummy_wait, job->aio_context, + AIO_WAIT_WHILE(job->aio_context, (job_drain(job), !job_is_completed(job))); ret = (job_is_cancelled(job) && job->ret == 0) ? -ECANCELED : job->ret; diff --git a/util/aio-wait.c b/util/aio-wait.c index XXXXXXX..XXXXXXX 100644 --- a/util/aio-wait.c +++ b/util/aio-wait.c @@ -XXX,XX +XXX,XX @@ #include "qemu/main-loop.h" #include "block/aio-wait.h" +AioWait global_aio_wait; + static void dummy_bh_cb(void *opaque) { /* The point is to make AIO_WAIT_WHILE()'s aio_poll() return */ } -void aio_wait_kick(AioWait *wait) +void aio_wait_kick(void) { /* The barrier (or an atomic op) is in the caller. */ - if (atomic_read(&wait->num_waiters)) { + if (atomic_read(&global_aio_wait.num_waiters)) { aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL); } } typedef struct { - AioWait wait; bool done; QEMUBHFunc *cb; void *opaque; @@ -XXX,XX +XXX,XX @@ static void aio_wait_bh(void *opaque) data->cb(data->opaque); data->done = true; - aio_wait_kick(&data->wait); + aio_wait_kick(); } void aio_wait_bh_oneshot(AioContext *ctx, QEMUBHFunc *cb, void *opaque) @@ -XXX,XX +XXX,XX @@ void aio_wait_bh_oneshot(AioContext *ctx, QEMUBHFunc *cb, void *opaque) assert(qemu_get_current_aio_context() == qemu_get_aio_context()); aio_bh_schedule_oneshot(ctx, aio_wait_bh, &data); - AIO_WAIT_WHILE(&data.wait, ctx, !data.done); + AIO_WAIT_WHILE(ctx, !data.done); } -- 2.17.1 From: Kevin Wolf <kwolf@redhat.com> For the block job drain test, don't only test draining the source and the target node, but create a backing chain for the source (source_backing <- source <- source_overlay) and test draining each of the nodes in it. When using iothreads, the source node (and therefore the job) is in a different AioContext than the drain, which happens from the main thread. This way, the main thread waits in AIO_WAIT_WHILE() for the iothread to make process and aio_wait_kick() is required to notify it. The test validates that calling bdrv_wakeup() for a child or a parent node will actually notify AIO_WAIT_WHILE() instead of letting it hang. Increase the sleep time a bit (to 1 ms) because the test case is racy and with the shorter sleep, it didn't reproduce the bug it is supposed to test for me under 'rr record -n'. This was because bdrv_drain_invoke_entry() (in the main thread) was only called after the job had already reached the pause point, so we got a bdrv_dec_in_flight() from the main thread and the additional aio_wait_kick() when the job becomes idle (that we really wanted to test here) wasn't even necessary any more to make progress. Signed-off-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> --- tests/test-bdrv-drain.c | 77 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c index XXXXXXX..XXXXXXX 100644 --- a/tests/test-bdrv-drain.c +++ b/tests/test-bdrv-drain.c @@ -XXX,XX +XXX,XX @@ typedef struct TestBlockJob { BlockJob common; int run_ret; int prepare_ret; + bool running; bool should_complete; } TestBlockJob; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn test_job_run(Job *job, Error **errp) { TestBlockJob *s = container_of(job, TestBlockJob, common.job); + /* We are running the actual job code past the pause point in + * job_co_entry(). */ + s->running = true; + job_transition_to_ready(&s->common.job); while (!s->should_complete) { /* Avoid job_sleep_ns() because it marks the job as !busy. We want to * emulate some actual activity (probably some I/O) here so that drain * has to wait for this activity to stop. */ - qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 100000); + qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 1000000); + job_pause_point(&s->common.job); } @@ -XXX,XX +XXX,XX @@ enum test_job_result { TEST_JOB_FAIL_PREPARE, }; -static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, - enum test_job_result result) +enum test_job_drain_node { + TEST_JOB_DRAIN_SRC, + TEST_JOB_DRAIN_SRC_CHILD, + TEST_JOB_DRAIN_SRC_PARENT, +}; + +static void test_blockjob_common_drain_node(enum drain_type drain_type, + bool use_iothread, + enum test_job_result result, + enum test_job_drain_node drain_node) { BlockBackend *blk_src, *blk_target; - BlockDriverState *src, *target; + BlockDriverState *src, *src_backing, *src_overlay, *target, *drain_bs; BlockJob *job; TestBlockJob *tjob; IOThread *iothread = NULL; @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, src = bdrv_new_open_driver(&bdrv_test, "source", BDRV_O_RDWR, &error_abort); + src_backing = bdrv_new_open_driver(&bdrv_test, "source-backing", + BDRV_O_RDWR, &error_abort); + src_overlay = bdrv_new_open_driver(&bdrv_test, "source-overlay", + BDRV_O_RDWR, &error_abort); + + bdrv_set_backing_hd(src_overlay, src, &error_abort); + bdrv_unref(src); + bdrv_set_backing_hd(src, src_backing, &error_abort); + bdrv_unref(src_backing); + blk_src = blk_new(BLK_PERM_ALL, BLK_PERM_ALL); - blk_insert_bs(blk_src, src, &error_abort); + blk_insert_bs(blk_src, src_overlay, &error_abort); + + switch (drain_node) { + case TEST_JOB_DRAIN_SRC: + drain_bs = src; + break; + case TEST_JOB_DRAIN_SRC_CHILD: + drain_bs = src_backing; + break; + case TEST_JOB_DRAIN_SRC_PARENT: + drain_bs = src_overlay; + break; + default: + g_assert_not_reached(); + } if (use_iothread) { iothread = iothread_new(); @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, job_start(&job->job); aio_context_release(ctx); + if (use_iothread) { + /* job_co_entry() is run in the I/O thread, wait for the actual job + * code to start (we don't want to catch the job in the pause point in + * job_co_entry(). */ + while (!tjob->running) { + aio_poll(qemu_get_aio_context(), false); + } + } + g_assert_cmpint(job->job.pause_count, ==, 0); g_assert_false(job->job.paused); + g_assert_true(tjob->running); g_assert_true(job->job.busy); /* We're in qemu_co_sleep_ns() */ - do_drain_begin_unlocked(drain_type, src); + do_drain_begin_unlocked(drain_type, drain_bs); if (drain_type == BDRV_DRAIN_ALL) { /* bdrv_drain_all() drains both src and target */ @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, g_assert_true(job->job.paused); g_assert_false(job->job.busy); /* The job is paused */ - do_drain_end_unlocked(drain_type, src); + do_drain_end_unlocked(drain_type, drain_bs); if (use_iothread) { /* paused is reset in the I/O thread, wait for it */ @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, blk_unref(blk_src); blk_unref(blk_target); - bdrv_unref(src); + bdrv_unref(src_overlay); bdrv_unref(target); if (iothread) { @@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, } } +static void test_blockjob_common(enum drain_type drain_type, bool use_iothread, + enum test_job_result result) +{ + test_blockjob_common_drain_node(drain_type, use_iothread, result, + TEST_JOB_DRAIN_SRC); + test_blockjob_common_drain_node(drain_type, use_iothread, result, + TEST_JOB_DRAIN_SRC_CHILD); + if (drain_type == BDRV_SUBTREE_DRAIN) { + test_blockjob_common_drain_node(drain_type, use_iothread, result, + TEST_JOB_DRAIN_SRC_PARENT); + } +} + static void test_blockjob_drain_all(void) { test_blockjob_common(BDRV_DRAIN_ALL, false, TEST_JOB_SUCCESS); -- 2.17.1 The following changes since commit eaefea537b476cb853e2edbdc68e969ec777e4bb: Merge remote-tracking branch 'remotes/mjt/tags/trivial-patches-fetch' into staging (2017-12-18 14:17:42 +0000) are available in the Git repository at: git://github.com/stefanha/qemu.git tags/block-pull-request for you to fetch changes up to 7a9dda0d7f9831c2432620dcfefdadbb7ae888dc: qemu-iotests: add 203 savevm with IOThreads test (2017-12-19 10:25:09 +0000) ---------------------------------------------------------------- Pull request v2: * Fixed incorrect virtio_blk_data_plane_create() local_err refactoring in "hw/block: Use errp directly rather than local_err" that broke virtio-blk over virtio-mmio [Peter] ---------------------------------------------------------------- Mao Zhongyi (4): hw/block/nvme: Convert to realize hw/block: Fix the return type hw/block: Use errp directly rather than local_err dev-storage: Fix the unusual function name Mark Kanda (2): virtio-blk: make queue size configurable virtio-blk: reject configs with logical block size > physical block size Paolo Bonzini (1): block: avoid recursive AioContext acquire in bdrv_inactivate_all() Stefan Hajnoczi (16): coroutine: simplify co_aio_sleep_ns() prototype qdev: drop unused #include "sysemu/iothread.h" blockdev: hold AioContext for bdrv_unref() in external_snapshot_clean() block: don't keep AioContext acquired after external_snapshot_prepare() block: don't keep AioContext acquired after drive_backup_prepare() block: don't keep AioContext acquired after blockdev_backup_prepare() block: don't keep AioContext acquired after internal_snapshot_prepare() block: drop unused BlockDirtyBitmapState->aio_context field iothread: add iothread_by_id() API blockdev: add x-blockdev-set-iothread testing command qemu-iotests: add 202 external snapshots IOThread test docs: mark nested AioContext locking as a legacy API blockdev: add x-blockdev-set-iothread force boolean iotests: add VM.add_object() iothread: fix iothread_stop() race condition qemu-iotests: add 203 savevm with IOThreads test docs/devel/multiple-iothreads.txt | 7 +- qapi/block-core.json | 40 ++++++ hw/block/dataplane/virtio-blk.h | 2 +- include/hw/block/block.h | 4 +- include/hw/virtio/virtio-blk.h | 1 + include/qemu/coroutine.h | 6 +- include/sysemu/iothread.h | 4 +- block.c | 14 ++- block/null.c | 3 +- block/sheepdog.c | 3 +- blockdev.c | 259 +++++++++++++++++++++++++++----------- hw/block/block.c | 15 ++- hw/block/dataplane/virtio-blk.c | 12 +- hw/block/fdc.c | 17 +-- hw/block/nvme.c | 23 ++-- hw/block/virtio-blk.c | 30 +++-- hw/core/qdev-properties-system.c | 1 - hw/ide/qdev.c | 12 +- hw/scsi/scsi-disk.c | 13 +- hw/usb/dev-storage.c | 29 ++--- iothread.c | 27 +++- util/qemu-coroutine-sleep.c | 4 +- tests/qemu-iotests/202 | 95 ++++++++++++++ tests/qemu-iotests/202.out | 11 ++ tests/qemu-iotests/203 | 59 +++++++++ tests/qemu-iotests/203.out | 6 + tests/qemu-iotests/group | 2 + tests/qemu-iotests/iotests.py | 5 + 28 files changed, 531 insertions(+), 173 deletions(-) create mode 100755 tests/qemu-iotests/202 create mode 100644 tests/qemu-iotests/202.out create mode 100755 tests/qemu-iotests/203 create mode 100644 tests/qemu-iotests/203.out -- 2.14.3 The AioContext pointer argument to co_aio_sleep_ns() is only used for the sleep timer. It does not affect where the caller coroutine is resumed. Due to changes to coroutine and AIO APIs it is now possible to drop the AioContext pointer argument. This is safe to do since no caller has specific requirements for which AioContext the timer must run in. This patch drops the AioContext pointer argument and renames the function to simplify the API. Reported-by: Paolo Bonzini <pbonzini@redhat.com> Reported-by: Eric Blake <eblake@redhat.com> Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171109102652.6360-1-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- include/qemu/coroutine.h | 6 +----- block/null.c | 3 +-- block/sheepdog.c | 3 +-- util/qemu-coroutine-sleep.c | 4 ++-- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/coroutine.h +++ b/include/qemu/coroutine.h @@ -XXX,XX +XXX,XX @@ void qemu_co_rwlock_unlock(CoRwlock *lock); /** * Yield the coroutine for a given duration - * - * Behaves similarly to co_sleep_ns(), but the sleeping coroutine will be - * resumed when using aio_poll(). */ -void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type, - int64_t ns); +void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns); /** * Yield until a file descriptor becomes readable diff --git a/block/null.c b/block/null.c index XXXXXXX..XXXXXXX 100644 --- a/block/null.c +++ b/block/null.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn int null_co_common(BlockDriverState *bs) BDRVNullState *s = bs->opaque; if (s->latency_ns) { - co_aio_sleep_ns(bdrv_get_aio_context(bs), QEMU_CLOCK_REALTIME, - s->latency_ns); + qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, s->latency_ns); } return 0; } diff --git a/block/sheepdog.c b/block/sheepdog.c index XXXXXXX..XXXXXXX 100644 --- a/block/sheepdog.c +++ b/block/sheepdog.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn void reconnect_to_sdog(void *opaque) if (s->fd < 0) { DPRINTF("Wait for connection to be established\n"); error_report_err(local_err); - co_aio_sleep_ns(bdrv_get_aio_context(s->bs), QEMU_CLOCK_REALTIME, - 1000000000ULL); + qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 1000000000ULL); } }; diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c index XXXXXXX..XXXXXXX 100644 --- a/util/qemu-coroutine-sleep.c +++ b/util/qemu-coroutine-sleep.c @@ -XXX,XX +XXX,XX @@ static void co_sleep_cb(void *opaque) aio_co_wake(sleep_cb->co); } -void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type, - int64_t ns) +void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns) { + AioContext *ctx = qemu_get_current_aio_context(); CoSleepCB sleep_cb = { .co = qemu_coroutine_self(), }; -- 2.14.3 From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> Convert nvme_init() to realize and rename it to nvme_realize(). Cc: John Snow <jsnow@redhat.com> Cc: Keith Busch <keith.busch@intel.com> Cc: Kevin Wolf <kwolf@redhat.com> Cc: Max Reitz <mreitz@redhat.com> Cc: Markus Armbruster <armbru@redhat.com> Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> Message-id: 2882e72d795e04cbe2120f569d551aef2467ac60.1511317952.git.maozy.fnst@cn.fujitsu.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/block/nvme.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/hw/block/nvme.c b/hw/block/nvme.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/nvme.c +++ b/hw/block/nvme.c @@ -XXX,XX +XXX,XX @@ static const MemoryRegionOps nvme_cmb_ops = { }, }; -static int nvme_init(PCIDevice *pci_dev) +static void nvme_realize(PCIDevice *pci_dev, Error **errp) { NvmeCtrl *n = NVME(pci_dev); NvmeIdCtrl *id = &n->id_ctrl; @@ -XXX,XX +XXX,XX @@ static int nvme_init(PCIDevice *pci_dev) Error *local_err = NULL; if (!n->conf.blk) { - return -1; + error_setg(errp, "drive property not set"); + return; } bs_size = blk_getlength(n->conf.blk); if (bs_size < 0) { - return -1; + error_setg(errp, "could not get backing file size"); + return; } blkconf_serial(&n->conf, &n->serial); if (!n->serial) { - return -1; + error_setg(errp, "serial property not set"); + return; } blkconf_blocksizes(&n->conf); blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk), false, &local_err); if (local_err) { - error_report_err(local_err); - return -1; + error_propagate(errp, local_err); + return; } pci_conf = pci_dev->config; @@ -XXX,XX +XXX,XX @@ static int nvme_init(PCIDevice *pci_dev) cpu_to_le64(n->ns_size >> id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds); } - return 0; } static void nvme_exit(PCIDevice *pci_dev) @@ -XXX,XX +XXX,XX @@ static void nvme_class_init(ObjectClass *oc, void *data) DeviceClass *dc = DEVICE_CLASS(oc); PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc); - pc->init = nvme_init; + pc->realize = nvme_realize; pc->exit = nvme_exit; pc->class_id = PCI_CLASS_STORAGE_EXPRESS; pc->vendor_id = PCI_VENDOR_ID_INTEL; -- 2.14.3 From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> When the function no success value to transmit, it usually make the function return void. It has turned out not to be a success, because it means that the extra local_err variable and error_propagate() will be needed. It leads to cumbersome code, therefore, transmit success/ failure in the return value is worth. So fix the return type of blkconf_apply_backend_options(), blkconf_geometry() and virtio_blk_data_plane_create() to avoid it. Cc: John Snow <jsnow@redhat.com> Cc: Kevin Wolf <kwolf@redhat.com> Cc: Max Reitz <mreitz@redhat.com> Cc: Stefan Hajnoczi <stefanha@redhat.com> Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: ac0edc1fc70c4457e5cec94405eb7d1f89f9c2c1.1511317952.git.maozy.fnst@cn.fujitsu.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/block/dataplane/virtio-blk.h | 2 +- include/hw/block/block.h | 4 ++-- hw/block/block.c | 15 +++++++++------ hw/block/dataplane/virtio-blk.c | 12 +++++++----- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h index XXXXXXX..XXXXXXX 100644 --- a/hw/block/dataplane/virtio-blk.h +++ b/hw/block/dataplane/virtio-blk.h @@ -XXX,XX +XXX,XX @@ typedef struct VirtIOBlockDataPlane VirtIOBlockDataPlane; -void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, +bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, VirtIOBlockDataPlane **dataplane, Error **errp); void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s); diff --git a/include/hw/block/block.h b/include/hw/block/block.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/block/block.h +++ b/include/hw/block/block.h @@ -XXX,XX +XXX,XX @@ static inline unsigned int get_physical_block_exp(BlockConf *conf) /* Configuration helpers */ void blkconf_serial(BlockConf *conf, char **serial); -void blkconf_geometry(BlockConf *conf, int *trans, +bool blkconf_geometry(BlockConf *conf, int *trans, unsigned cyls_max, unsigned heads_max, unsigned secs_max, Error **errp); void blkconf_blocksizes(BlockConf *conf); -void blkconf_apply_backend_options(BlockConf *conf, bool readonly, +bool blkconf_apply_backend_options(BlockConf *conf, bool readonly, bool resizable, Error **errp); /* Hard disk geometry */ diff --git a/hw/block/block.c b/hw/block/block.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/block.c +++ b/hw/block/block.c @@ -XXX,XX +XXX,XX @@ void blkconf_blocksizes(BlockConf *conf) } } -void blkconf_apply_backend_options(BlockConf *conf, bool readonly, +bool blkconf_apply_backend_options(BlockConf *conf, bool readonly, bool resizable, Error **errp) { BlockBackend *blk = conf->blk; @@ -XXX,XX +XXX,XX @@ void blkconf_apply_backend_options(BlockConf *conf, bool readonly, ret = blk_set_perm(blk, perm, shared_perm, errp); if (ret < 0) { - return; + return false; } switch (conf->wce) { @@ -XXX,XX +XXX,XX @@ void blkconf_apply_backend_options(BlockConf *conf, bool readonly, blk_set_enable_write_cache(blk, wce); blk_set_on_error(blk, rerror, werror); + + return true; } -void blkconf_geometry(BlockConf *conf, int *ptrans, +bool blkconf_geometry(BlockConf *conf, int *ptrans, unsigned cyls_max, unsigned heads_max, unsigned secs_max, Error **errp) { @@ -XXX,XX +XXX,XX @@ void blkconf_geometry(BlockConf *conf, int *ptrans, if (conf->cyls || conf->heads || conf->secs) { if (conf->cyls < 1 || conf->cyls > cyls_max) { error_setg(errp, "cyls must be between 1 and %u", cyls_max); - return; + return false; } if (conf->heads < 1 || conf->heads > heads_max) { error_setg(errp, "heads must be between 1 and %u", heads_max); - return; + return false; } if (conf->secs < 1 || conf->secs > secs_max) { error_setg(errp, "secs must be between 1 and %u", secs_max); - return; + return false; } } + return true; } diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/dataplane/virtio-blk.c +++ b/hw/block/dataplane/virtio-blk.c @@ -XXX,XX +XXX,XX @@ static void notify_guest_bh(void *opaque) } /* Context: QEMU global mutex held */ -void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, +bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, VirtIOBlockDataPlane **dataplane, Error **errp) { @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, error_setg(errp, "device is incompatible with iothread " "(transport does not support notifiers)"); - return; + return false; } if (!virtio_device_ioeventfd_enabled(vdev)) { error_setg(errp, "ioeventfd is required for iothread"); - return; + return false; } /* If dataplane is (re-)enabled while the guest is running there could @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, */ if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { error_prepend(errp, "cannot start virtio-blk dataplane: "); - return; + return false; } } /* Don't try if transport does not support notifiers. */ if (!virtio_device_ioeventfd_enabled(vdev)) { - return; + return false; } s = g_new0(VirtIOBlockDataPlane, 1); @@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, s->batch_notify_vqs = bitmap_new(conf->num_queues); *dataplane = s; + + return true; } /* Context: QEMU global mutex held */ -- 2.14.3 From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> [Drop virtio_blk_data_plane_create() change that misinterprets return value when the virtio transport does not support dataplane. --Stefan] Cc: John Snow <jsnow@redhat.com> Cc: Kevin Wolf <kwolf@redhat.com> Cc: Max Reitz <mreitz@redhat.com> Cc: Keith Busch <keith.busch@intel.com> Cc: Stefan Hajnoczi <stefanha@redhat.com> Cc: "Michael S. Tsirkin" <mst@redhat.com> Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Gerd Hoffmann <kraxel@redhat.com> Cc: Markus Armbruster <armbru@redhat.com> Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: e77848d3735ba590f23ffbf8094379c646c33d79.1511317952.git.maozy.fnst@cn.fujitsu.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/block/fdc.c | 17 ++++++----------- hw/block/nvme.c | 7 ++----- hw/block/virtio-blk.c | 13 +++++-------- hw/ide/qdev.c | 12 ++++-------- hw/scsi/scsi-disk.c | 13 ++++--------- hw/usb/dev-storage.c | 9 +++------ 6 files changed, 24 insertions(+), 47 deletions(-) diff --git a/hw/block/fdc.c b/hw/block/fdc.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/fdc.c +++ b/hw/block/fdc.c @@ -XXX,XX +XXX,XX @@ static void fd_revalidate(FDrive *drv) static void fd_change_cb(void *opaque, bool load, Error **errp) { FDrive *drive = opaque; - Error *local_err = NULL; if (!load) { blk_set_perm(drive->blk, 0, BLK_PERM_ALL, &error_abort); } else { - blkconf_apply_backend_options(drive->conf, - blk_is_read_only(drive->blk), false, - &local_err); - if (local_err) { - error_propagate(errp, local_err); + if (!blkconf_apply_backend_options(drive->conf, + blk_is_read_only(drive->blk), false, + errp)) { return; } } @@ -XXX,XX +XXX,XX @@ static void floppy_drive_realize(DeviceState *qdev, Error **errp) FloppyDrive *dev = FLOPPY_DRIVE(qdev); FloppyBus *bus = FLOPPY_BUS(qdev->parent_bus); FDrive *drive; - Error *local_err = NULL; int ret; if (dev->unit == -1) { @@ -XXX,XX +XXX,XX @@ static void floppy_drive_realize(DeviceState *qdev, Error **errp) dev->conf.rerror = BLOCKDEV_ON_ERROR_AUTO; dev->conf.werror = BLOCKDEV_ON_ERROR_AUTO; - blkconf_apply_backend_options(&dev->conf, blk_is_read_only(dev->conf.blk), - false, &local_err); - if (local_err) { - error_propagate(errp, local_err); + if (!blkconf_apply_backend_options(&dev->conf, + blk_is_read_only(dev->conf.blk), + false, errp)) { return; } diff --git a/hw/block/nvme.c b/hw/block/nvme.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/nvme.c +++ b/hw/block/nvme.c @@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) int i; int64_t bs_size; uint8_t *pci_conf; - Error *local_err = NULL; if (!n->conf.blk) { error_setg(errp, "drive property not set"); @@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) return; } blkconf_blocksizes(&n->conf); - blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk), - false, &local_err); - if (local_err) { - error_propagate(errp, local_err); + if (!blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk), + false, errp)) { return; } diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) } blkconf_serial(&conf->conf, &conf->serial); - blkconf_apply_backend_options(&conf->conf, - blk_is_read_only(conf->conf.blk), true, - &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_apply_backend_options(&conf->conf, + blk_is_read_only(conf->conf.blk), true, + errp)) { return; } s->original_wce = blk_enable_write_cache(conf->conf.blk); - blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, errp)) { return; } + blkconf_blocksizes(&conf->conf); virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK, diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c index XXXXXXX..XXXXXXX 100644 --- a/hw/ide/qdev.c +++ b/hw/ide/qdev.c @@ -XXX,XX +XXX,XX @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind, Error **errp) { IDEBus *bus = DO_UPCAST(IDEBus, qbus, dev->qdev.parent_bus); IDEState *s = bus->ifs + dev->unit; - Error *err = NULL; int ret; if (!dev->conf.blk) { @@ -XXX,XX +XXX,XX @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind, Error **errp) blkconf_serial(&dev->conf, &dev->serial); if (kind != IDE_CD) { - blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255, &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255, + errp)) { return; } } - blkconf_apply_backend_options(&dev->conf, kind == IDE_CD, kind != IDE_CD, - &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_apply_backend_options(&dev->conf, kind == IDE_CD, + kind != IDE_CD, errp)) { return; } diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/scsi/scsi-disk.c +++ b/hw/scsi/scsi-disk.c @@ -XXX,XX +XXX,XX @@ static void scsi_disk_unit_attention_reported(SCSIDevice *dev) static void scsi_realize(SCSIDevice *dev, Error **errp) { SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev); - Error *err = NULL; if (!s->qdev.conf.blk) { error_setg(errp, "drive property not set"); @@ -XXX,XX +XXX,XX @@ static void scsi_realize(SCSIDevice *dev, Error **errp) } if (dev->type == TYPE_DISK) { - blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, errp)) { return; } } - blkconf_apply_backend_options(&dev->conf, - blk_is_read_only(s->qdev.conf.blk), - dev->type == TYPE_DISK, &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_apply_backend_options(&dev->conf, + blk_is_read_only(s->qdev.conf.blk), + dev->type == TYPE_DISK, errp)) { return; } diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c index XXXXXXX..XXXXXXX 100644 --- a/hw/usb/dev-storage.c +++ b/hw/usb/dev-storage.c @@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp) MSDState *s = USB_STORAGE_DEV(dev); BlockBackend *blk = s->conf.blk; SCSIDevice *scsi_dev; - Error *err = NULL; if (!blk) { error_setg(errp, "drive property not set"); @@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp) blkconf_serial(&s->conf, &dev->serial); blkconf_blocksizes(&s->conf); - blkconf_apply_backend_options(&s->conf, blk_is_read_only(blk), true, &err); - if (err) { - error_propagate(errp, err); + if (!blkconf_apply_backend_options(&s->conf, blk_is_read_only(blk), true, + errp)) { return; } @@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp) &usb_msd_scsi_info_storage, NULL); scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable, s->conf.bootindex, dev->serial, - &err); + errp); blk_unref(blk); if (!scsi_dev) { - error_propagate(errp, err); return; } usb_msd_handle_reset(dev); -- 2.14.3 From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> The function name of usb_msd_{realize,unrealize}_*, usb_msd_class_initfn_* are unusual. Rename it to usb_msd_*_{realize,unrealize}, usb_msd_class_*_initfn. Cc: Gerd Hoffmann <kraxel@redhat.com> Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 11e6003433abce35f3f4970e1acc71ee92dbcf51.1511317952.git.maozy.fnst@cn.fujitsu.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/usb/dev-storage.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c index XXXXXXX..XXXXXXX 100644 --- a/hw/usb/dev-storage.c +++ b/hw/usb/dev-storage.c @@ -XXX,XX +XXX,XX @@ static void usb_msd_unrealize_storage(USBDevice *dev, Error **errp) object_unref(OBJECT(&s->bus)); } -static void usb_msd_realize_storage(USBDevice *dev, Error **errp) +static void usb_msd_storage_realize(USBDevice *dev, Error **errp) { MSDState *s = USB_STORAGE_DEV(dev); BlockBackend *blk = s->conf.blk; @@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp) s->scsi_dev = scsi_dev; } -static void usb_msd_unrealize_bot(USBDevice *dev, Error **errp) +static void usb_msd_bot_unrealize(USBDevice *dev, Error **errp) { MSDState *s = USB_STORAGE_DEV(dev); object_unref(OBJECT(&s->bus)); } -static void usb_msd_realize_bot(USBDevice *dev, Error **errp) +static void usb_msd_bot_realize(USBDevice *dev, Error **errp) { MSDState *s = USB_STORAGE_DEV(dev); DeviceState *d = DEVICE(dev); @@ -XXX,XX +XXX,XX @@ static void usb_msd_class_initfn_common(ObjectClass *klass, void *data) dc->vmsd = &vmstate_usb_msd; } -static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data) +static void usb_msd_class_storage_initfn(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); USBDeviceClass *uc = USB_DEVICE_CLASS(klass); - uc->realize = usb_msd_realize_storage; + uc->realize = usb_msd_storage_realize; uc->unrealize = usb_msd_unrealize_storage; dc->props = msd_properties; } @@ -XXX,XX +XXX,XX @@ static void usb_msd_instance_init(Object *obj) object_property_set_int(obj, -1, "bootindex", NULL); } -static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data) +static void usb_msd_class_bot_initfn(ObjectClass *klass, void *data) { USBDeviceClass *uc = USB_DEVICE_CLASS(klass); - uc->realize = usb_msd_realize_bot; - uc->unrealize = usb_msd_unrealize_bot; + uc->realize = usb_msd_bot_realize; + uc->unrealize = usb_msd_bot_unrealize; uc->attached_settable = true; } static const TypeInfo msd_info = { .name = "usb-storage", .parent = TYPE_USB_STORAGE, - .class_init = usb_msd_class_initfn_storage, + .class_init = usb_msd_class_storage_initfn, .instance_init = usb_msd_instance_init, }; static const TypeInfo bot_info = { .name = "usb-bot", .parent = TYPE_USB_STORAGE, - .class_init = usb_msd_class_initfn_bot, + .class_init = usb_msd_class_bot_initfn, }; static void usb_msd_register_types(void) -- 2.14.3 Commit 1351d1ec89eabebc9fdff20451a62c413d7accc1 ("qdev: drop iothread property type") forgot to remove this include. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: 20171205133954.31006-1-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/core/qdev-properties-system.c | 1 - 1 file changed, 1 deletion(-) diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/qdev-properties-system.c +++ b/hw/core/qdev-properties-system.c @@ -XXX,XX +XXX,XX @@ #include "qapi/visitor.h" #include "chardev/char-fe.h" #include "sysemu/tpm_backend.h" -#include "sysemu/iothread.h" static void get_pointer(Object *obj, Visitor *v, Property *prop, char *(*print)(void *ptr), -- 2.14.3 bdrv_unref() requires the AioContext lock because bdrv_flush() uses BDRV_POLL_WHILE(), which assumes the AioContext is currently held. If BDRV_POLL_WHILE() runs without AioContext held the pthread_mutex_unlock() call in aio_context_release() fails. This patch moves bdrv_unref() into the AioContext locked region to solve the following pthread_mutex_unlock() failure: #0 0x00007f566181969b in raise () at /lib64/libc.so.6 #1 0x00007f566181b3b1 in abort () at /lib64/libc.so.6 #2 0x00005592cd590458 in error_exit (err=<optimized out>, msg=msg@entry=0x5592cdaf6d60 <__func__.23977> "qemu_mutex_unlock") at util/qemu-thread-posix.c:36 #3 0x00005592cd96e738 in qemu_mutex_unlock (mutex=mutex@entry=0x5592ce9505e0) at util/qemu-thread-posix.c:96 #4 0x00005592cd969b69 in aio_context_release (ctx=ctx@entry=0x5592ce950580) at util/async.c:507 #5 0x00005592cd8ead78 in bdrv_flush (bs=bs@entry=0x5592cfa87210) at block/io.c:2478 #6 0x00005592cd89df30 in bdrv_close (bs=0x5592cfa87210) at block.c:3207 #7 0x00005592cd89df30 in bdrv_delete (bs=0x5592cfa87210) at block.c:3395 #8 0x00005592cd89df30 in bdrv_unref (bs=0x5592cfa87210) at block.c:4418 #9 0x00005592cd6b7f86 in qmp_transaction (dev_list=<optimized out>, has_props=<optimized out>, props=<optimized out>, errp=errp@entry=0x7ffe4a1fc9d8) at blockdev.c:2308 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-2-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- blockdev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common) DO_UPCAST(ExternalSnapshotState, common, common); if (state->aio_context) { bdrv_drained_end(state->old_bs); - aio_context_release(state->aio_context); bdrv_unref(state->new_bs); + aio_context_release(state->aio_context); } } -- 2.14.3 It is not necessary to hold AioContext across transactions anymore since bdrv_drained_begin/end() is used to keep the nodes quiesced. In fact, using the AioContext lock for this purpose was always buggy. This patch reduces the scope of AioContext locked regions. This is not just a cleanup but also fixes hangs that occur in BDRV_POLL_WHILE() because it is unware of recursive locking and does not release the AioContext the necessary number of times to allow progress to be made. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-3-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- blockdev.c | 71 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ typedef struct ExternalSnapshotState { BlkActionState common; BlockDriverState *old_bs; BlockDriverState *new_bs; - AioContext *aio_context; bool overlay_appended; } ExternalSnapshotState; @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, ExternalSnapshotState *state = DO_UPCAST(ExternalSnapshotState, common, common); TransactionAction *action = common->action; + AioContext *aio_context; /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar * purpose but a different set of parameters */ @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, return; } - /* Acquire AioContext now so any threads operating on old_bs stop */ - state->aio_context = bdrv_get_aio_context(state->old_bs); - aio_context_acquire(state->aio_context); + aio_context = bdrv_get_aio_context(state->old_bs); + aio_context_acquire(aio_context); + + /* Paired with .clean() */ bdrv_drained_begin(state->old_bs); if (!bdrv_is_inserted(state->old_bs)) { error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device); - return; + goto out; } if (bdrv_op_is_blocked(state->old_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, errp)) { - return; + goto out; } if (!bdrv_is_read_only(state->old_bs)) { if (bdrv_flush(state->old_bs)) { error_setg(errp, QERR_IO_ERROR); - return; + goto out; } } if (!bdrv_is_first_non_filter(state->old_bs)) { error_setg(errp, QERR_FEATURE_DISABLED, "snapshot"); - return; + goto out; } if (action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC) { @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, if (node_name && !snapshot_node_name) { error_setg(errp, "New snapshot node name missing"); - return; + goto out; } if (snapshot_node_name && bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) { error_setg(errp, "New snapshot node name already in use"); - return; + goto out; } flags = state->old_bs->open_flags; @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, int64_t size = bdrv_getlength(state->old_bs); if (size < 0) { error_setg_errno(errp, -size, "bdrv_getlength failed"); - return; + goto out; } bdrv_img_create(new_image_file, format, state->old_bs->filename, @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, NULL, size, flags, false, &local_err); if (local_err) { error_propagate(errp, local_err); - return; + goto out; } } @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, errp); /* We will manually add the backing_hd field to the bs later */ if (!state->new_bs) { - return; + goto out; } if (bdrv_has_blk(state->new_bs)) { error_setg(errp, "The snapshot is already in use"); - return; + goto out; } if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, errp)) { - return; + goto out; } if (state->new_bs->backing != NULL) { error_setg(errp, "The snapshot already has a backing image"); - return; + goto out; } if (!state->new_bs->drv->supports_backing) { error_setg(errp, "The snapshot does not support backing images"); - return; + goto out; } - bdrv_set_aio_context(state->new_bs, state->aio_context); + bdrv_set_aio_context(state->new_bs, aio_context); /* This removes our old bs and adds the new bs. This is an operation that * can fail, so we need to do it in .prepare; undoing it for abort is @@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common, bdrv_append(state->new_bs, state->old_bs, &local_err); if (local_err) { error_propagate(errp, local_err); - return; + goto out; } state->overlay_appended = true; + +out: + aio_context_release(aio_context); } static void external_snapshot_commit(BlkActionState *common) { ExternalSnapshotState *state = DO_UPCAST(ExternalSnapshotState, common, common); + AioContext *aio_context; + + aio_context = bdrv_get_aio_context(state->old_bs); + aio_context_acquire(aio_context); /* We don't need (or want) to use the transactional * bdrv_reopen_multiple() across all the entries at once, because we @@ -XXX,XX +XXX,XX @@ static void external_snapshot_commit(BlkActionState *common) bdrv_reopen(state->old_bs, state->old_bs->open_flags & ~BDRV_O_RDWR, NULL); } + + aio_context_release(aio_context); } static void external_snapshot_abort(BlkActionState *common) @@ -XXX,XX +XXX,XX @@ static void external_snapshot_abort(BlkActionState *common) DO_UPCAST(ExternalSnapshotState, common, common); if (state->new_bs) { if (state->overlay_appended) { + AioContext *aio_context; + + aio_context = bdrv_get_aio_context(state->old_bs); + aio_context_acquire(aio_context); + bdrv_ref(state->old_bs); /* we can't let bdrv_set_backind_hd() close state->old_bs; we need it */ bdrv_set_backing_hd(state->new_bs, NULL, &error_abort); bdrv_replace_node(state->new_bs, state->old_bs, &error_abort); bdrv_unref(state->old_bs); /* bdrv_replace_node() ref'ed old_bs */ + + aio_context_release(aio_context); } } } @@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common) { ExternalSnapshotState *state = DO_UPCAST(ExternalSnapshotState, common, common); - if (state->aio_context) { - bdrv_drained_end(state->old_bs); - bdrv_unref(state->new_bs); - aio_context_release(state->aio_context); + AioContext *aio_context; + + if (!state->old_bs) { + return; } + + aio_context = bdrv_get_aio_context(state->old_bs); + aio_context_acquire(aio_context); + + bdrv_drained_end(state->old_bs); + bdrv_unref(state->new_bs); + + aio_context_release(aio_context); } typedef struct DriveBackupState { -- 2.14.3 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-4-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- blockdev.c | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common) typedef struct DriveBackupState { BlkActionState common; BlockDriverState *bs; - AioContext *aio_context; BlockJob *job; } DriveBackupState; @@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp) DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common); BlockDriverState *bs; DriveBackup *backup; + AioContext *aio_context; Error *local_err = NULL; assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP); @@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp) return; } - /* AioContext is released in .clean() */ - state->aio_context = bdrv_get_aio_context(bs); - aio_context_acquire(state->aio_context); + aio_context = bdrv_get_aio_context(bs); + aio_context_acquire(aio_context); + + /* Paired with .clean() */ bdrv_drained_begin(bs); + state->bs = bs; state->job = do_drive_backup(backup, common->block_job_txn, &local_err); if (local_err) { error_propagate(errp, local_err); - return; + goto out; } + +out: + aio_context_release(aio_context); } static void drive_backup_commit(BlkActionState *common) { DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common); + AioContext *aio_context; + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + assert(state->job); block_job_start(state->job); + + aio_context_release(aio_context); } static void drive_backup_abort(BlkActionState *common) @@ -XXX,XX +XXX,XX @@ static void drive_backup_abort(BlkActionState *common) DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common); if (state->job) { + AioContext *aio_context; + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + block_job_cancel_sync(state->job); + + aio_context_release(aio_context); } } static void drive_backup_clean(BlkActionState *common) { DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common); + AioContext *aio_context; - if (state->aio_context) { - bdrv_drained_end(state->bs); - aio_context_release(state->aio_context); + if (!state->bs) { + return; } + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + + bdrv_drained_end(state->bs); + + aio_context_release(aio_context); } typedef struct BlockdevBackupState { -- 2.14.3 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-5-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- blockdev.c | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ typedef struct BlockdevBackupState { BlkActionState common; BlockDriverState *bs; BlockJob *job; - AioContext *aio_context; } BlockdevBackupState; static BlockJob *do_blockdev_backup(BlockdevBackup *backup, BlockJobTxn *txn, @@ -XXX,XX +XXX,XX @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp) BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common); BlockdevBackup *backup; BlockDriverState *bs, *target; + AioContext *aio_context; Error *local_err = NULL; assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP); @@ -XXX,XX +XXX,XX @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp) return; } - /* AioContext is released in .clean() */ - state->aio_context = bdrv_get_aio_context(bs); - if (state->aio_context != bdrv_get_aio_context(target)) { - state->aio_context = NULL; + aio_context = bdrv_get_aio_context(bs); + if (aio_context != bdrv_get_aio_context(target)) { error_setg(errp, "Backup between two IO threads is not implemented"); return; } - aio_context_acquire(state->aio_context); + aio_context_acquire(aio_context); state->bs = bs; + + /* Paired with .clean() */ bdrv_drained_begin(state->bs); state->job = do_blockdev_backup(backup, common->block_job_txn, &local_err); if (local_err) { error_propagate(errp, local_err); - return; + goto out; } + +out: + aio_context_release(aio_context); } static void blockdev_backup_commit(BlkActionState *common) { BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common); + AioContext *aio_context; + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + assert(state->job); block_job_start(state->job); + + aio_context_release(aio_context); } static void blockdev_backup_abort(BlkActionState *common) @@ -XXX,XX +XXX,XX @@ static void blockdev_backup_abort(BlkActionState *common) BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common); if (state->job) { + AioContext *aio_context; + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + block_job_cancel_sync(state->job); + + aio_context_release(aio_context); } } static void blockdev_backup_clean(BlkActionState *common) { BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common); + AioContext *aio_context; - if (state->aio_context) { - bdrv_drained_end(state->bs); - aio_context_release(state->aio_context); + if (!state->bs) { + return; } + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + + bdrv_drained_end(state->bs); + + aio_context_release(aio_context); } typedef struct BlockDirtyBitmapState { -- 2.14.3 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-6-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- blockdev.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ struct BlkActionState { typedef struct InternalSnapshotState { BlkActionState common; BlockDriverState *bs; - AioContext *aio_context; QEMUSnapshotInfo sn; bool created; } InternalSnapshotState; @@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common, qemu_timeval tv; BlockdevSnapshotInternal *internal; InternalSnapshotState *state; + AioContext *aio_context; int ret1; g_assert(common->action->type == @@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common, return; } - /* AioContext is released in .clean() */ - state->aio_context = bdrv_get_aio_context(bs); - aio_context_acquire(state->aio_context); + aio_context = bdrv_get_aio_context(bs); + aio_context_acquire(aio_context); state->bs = bs; + + /* Paired with .clean() */ bdrv_drained_begin(bs); if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, errp)) { - return; + goto out; } if (bdrv_is_read_only(bs)) { error_setg(errp, "Device '%s' is read only", device); - return; + goto out; } if (!bdrv_can_snapshot(bs)) { error_setg(errp, "Block format '%s' used by device '%s' " "does not support internal snapshots", bs->drv->format_name, device); - return; + goto out; } if (!strlen(name)) { error_setg(errp, "Name is empty"); - return; + goto out; } /* check whether a snapshot with name exist */ @@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common, &local_err); if (local_err) { error_propagate(errp, local_err); - return; + goto out; } else if (ret) { error_setg(errp, "Snapshot with name '%s' already exists on device '%s'", name, device); - return; + goto out; } /* 3. take the snapshot */ @@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common, error_setg_errno(errp, -ret1, "Failed to create snapshot '%s' on device '%s'", name, device); - return; + goto out; } /* 4. succeed, mark a snapshot is created */ state->created = true; + +out: + aio_context_release(aio_context); } static void internal_snapshot_abort(BlkActionState *common) @@ -XXX,XX +XXX,XX @@ static void internal_snapshot_abort(BlkActionState *common) DO_UPCAST(InternalSnapshotState, common, common); BlockDriverState *bs = state->bs; QEMUSnapshotInfo *sn = &state->sn; + AioContext *aio_context; Error *local_error = NULL; if (!state->created) { return; } + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + if (bdrv_snapshot_delete(bs, sn->id_str, sn->name, &local_error) < 0) { error_reportf_err(local_error, "Failed to delete snapshot with id '%s' and " @@ -XXX,XX +XXX,XX @@ static void internal_snapshot_abort(BlkActionState *common) sn->id_str, sn->name, bdrv_get_device_name(bs)); } + + aio_context_release(aio_context); } static void internal_snapshot_clean(BlkActionState *common) { InternalSnapshotState *state = DO_UPCAST(InternalSnapshotState, common, common); + AioContext *aio_context; - if (state->aio_context) { - if (state->bs) { - bdrv_drained_end(state->bs); - } - aio_context_release(state->aio_context); + if (!state->bs) { + return; } + + aio_context = bdrv_get_aio_context(state->bs); + aio_context_acquire(aio_context); + + bdrv_drained_end(state->bs); + + aio_context_release(aio_context); } /* external snapshot private data */ -- 2.14.3 The dirty bitmap actions in qmp_transaction have not used AioContext since the dirty bitmap locking discipline was introduced in commit 2119882c7eb7e2c612b24fc0c8d86f5887d6f1c3 ("block: introduce dirty_bitmap_mutex"). Remove the unused field. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-7-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- blockdev.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ typedef struct BlockDirtyBitmapState { BlkActionState common; BdrvDirtyBitmap *bitmap; BlockDriverState *bs; - AioContext *aio_context; HBitmap *backup; bool prepared; } BlockDirtyBitmapState; @@ -XXX,XX +XXX,XX @@ static void block_dirty_bitmap_clear_prepare(BlkActionState *common, } bdrv_clear_dirty_bitmap(state->bitmap, &state->backup); - /* AioContext is released in .clean() */ } static void block_dirty_bitmap_clear_abort(BlkActionState *common) @@ -XXX,XX +XXX,XX @@ static void block_dirty_bitmap_clear_commit(BlkActionState *common) hbitmap_free(state->backup); } -static void block_dirty_bitmap_clear_clean(BlkActionState *common) -{ - BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState, - common, common); - - if (state->aio_context) { - aio_context_release(state->aio_context); - } -} - static void abort_prepare(BlkActionState *common, Error **errp) { error_setg(errp, "Transaction aborted using Abort action"); @@ -XXX,XX +XXX,XX @@ static const BlkActionOps actions[] = { .prepare = block_dirty_bitmap_clear_prepare, .commit = block_dirty_bitmap_clear_commit, .abort = block_dirty_bitmap_clear_abort, - .clean = block_dirty_bitmap_clear_clean, } }; -- 2.14.3 Encapsulate IOThread QOM object lookup so that callers don't need to know how and where IOThread objects live. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-8-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- include/sysemu/iothread.h | 1 + iothread.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h index XXXXXXX..XXXXXXX 100644 --- a/include/sysemu/iothread.h +++ b/include/sysemu/iothread.h @@ -XXX,XX +XXX,XX @@ typedef struct { OBJECT_CHECK(IOThread, obj, TYPE_IOTHREAD) char *iothread_get_id(IOThread *iothread); +IOThread *iothread_by_id(const char *id); AioContext *iothread_get_aio_context(IOThread *iothread); void iothread_stop_all(void); GMainContext *iothread_get_g_main_context(IOThread *iothread); diff --git a/iothread.c b/iothread.c index XXXXXXX..XXXXXXX 100644 --- a/iothread.c +++ b/iothread.c @@ -XXX,XX +XXX,XX @@ void iothread_destroy(IOThread *iothread) { object_unparent(OBJECT(iothread)); } + +/* Lookup IOThread by its id. Only finds user-created objects, not internal + * iothread_create() objects. */ +IOThread *iothread_by_id(const char *id) +{ + return IOTHREAD(object_resolve_path_type(id, TYPE_IOTHREAD, NULL)); +} -- 2.14.3 Currently there is no easy way for iotests to ensure that a BDS is bound to a particular IOThread. Normally the virtio-blk device calls blk_set_aio_context() when dataplane is enabled during guest driver initialization. This never happens in iotests since -machine accel=qtest means there is no guest activity (including device driver initialization). This patch adds a QMP command to explicitly assign IOThreads in test cases. See qapi/block-core.json for a description of the command. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-9-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- qapi/block-core.json | 36 ++++++++++++++++++++++++++++++++++++ blockdev.c | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ 'data' : { 'parent': 'str', '*child': 'str', '*node': 'str' } } + +## +# @x-blockdev-set-iothread: +# +# Move @node and its children into the @iothread. If @iothread is null then +# move @node and its children into the main loop. +# +# The node must not be attached to a BlockBackend. +# +# @node-name: the name of the block driver node +# +# @iothread: the name of the IOThread object or null for the main loop +# +# Note: this command is experimental and intended for test cases that need +# control over IOThreads only. +# +# Since: 2.12 +# +# Example: +# +# 1. Move a node into an IOThread +# -> { "execute": "x-blockdev-set-iothread", +# "arguments": { "node-name": "disk1", +# "iothread": "iothread0" } } +# <- { "return": {} } +# +# 2. Move a node into the main loop +# -> { "execute": "x-blockdev-set-iothread", +# "arguments": { "node-name": "disk1", +# "iothread": null } } +# <- { "return": {} } +# +## +{ 'command': 'x-blockdev-set-iothread', + 'data' : { 'node-name': 'str', + 'iothread': 'StrOrNull' } } diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ #include "qapi/qmp/qerror.h" #include "qapi/qobject-output-visitor.h" #include "sysemu/sysemu.h" +#include "sysemu/iothread.h" #include "block/block_int.h" #include "qmp-commands.h" #include "block/trace.h" @@ -XXX,XX +XXX,XX @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp) return head; } +void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread, + Error **errp) +{ + AioContext *old_context; + AioContext *new_context; + BlockDriverState *bs; + + bs = bdrv_find_node(node_name); + if (!bs) { + error_setg(errp, "Cannot find node %s", node_name); + return; + } + + /* If we want to allow more extreme test scenarios this guard could be + * removed. For now it protects against accidents. */ + if (bdrv_has_blk(bs)) { + error_setg(errp, "Node %s is in use", node_name); + return; + } + + if (iothread->type == QTYPE_QSTRING) { + IOThread *obj = iothread_by_id(iothread->u.s); + if (!obj) { + error_setg(errp, "Cannot find iothread %s", iothread->u.s); + return; + } + + new_context = iothread_get_aio_context(obj); + } else { + new_context = qemu_get_aio_context(); + } + + old_context = bdrv_get_aio_context(bs); + aio_context_acquire(old_context); + + bdrv_set_aio_context(bs, new_context); + + aio_context_release(old_context); +} + QemuOptsList qemu_common_drive_opts = { .name = "drive", .head = QTAILQ_HEAD_INITIALIZER(qemu_common_drive_opts.head), -- 2.14.3 QMP 'transaction' blockdev-snapshot-sync with multiple disks in an IOThread is an untested code path. Several bugs have been found in connection with this command. This patch adds a test case to prevent future regressions. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171206144550.22295-10-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- tests/qemu-iotests/202 | 95 ++++++++++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/202.out | 11 ++++++ tests/qemu-iotests/group | 1 + 3 files changed, 107 insertions(+) create mode 100755 tests/qemu-iotests/202 create mode 100644 tests/qemu-iotests/202.out diff --git a/tests/qemu-iotests/202 b/tests/qemu-iotests/202 new file mode 100755 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/202 @@ -XXX,XX +XXX,XX @@ +#!/usr/bin/env python +# +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# Creator/Owner: Stefan Hajnoczi <stefanha@redhat.com> +# +# Check that QMP 'transaction' blockdev-snapshot-sync with multiple drives on a +# single IOThread completes successfully. This particular command triggered a +# hang due to recursive AioContext locking and BDRV_POLL_WHILE(). Protect +# against regressions. + +import iotests + +iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.verify_platform(['linux']) + +with iotests.FilePath('disk0.img') as disk0_img_path, \ + iotests.FilePath('disk1.img') as disk1_img_path, \ + iotests.FilePath('disk0-snap.img') as disk0_snap_img_path, \ + iotests.FilePath('disk1-snap.img') as disk1_snap_img_path, \ + iotests.VM() as vm: + + img_size = '10M' + iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk0_img_path, img_size) + iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk1_img_path, img_size) + + iotests.log('Launching VM...') + vm.launch() + + iotests.log('Adding IOThread...') + iotests.log(vm.qmp('object-add', + qom_type='iothread', + id='iothread0')) + + iotests.log('Adding blockdevs...') + iotests.log(vm.qmp('blockdev-add', + driver=iotests.imgfmt, + node_name='disk0', + file={ + 'driver': 'file', + 'filename': disk0_img_path, + })) + iotests.log(vm.qmp('blockdev-add', + driver=iotests.imgfmt, + node_name='disk1', + file={ + 'driver': 'file', + 'filename': disk1_img_path, + })) + + iotests.log('Setting iothread...') + iotests.log(vm.qmp('x-blockdev-set-iothread', + node_name='disk0', + iothread='iothread0')) + iotests.log(vm.qmp('x-blockdev-set-iothread', + node_name='disk1', + iothread='iothread0')) + + iotests.log('Creating external snapshots...') + iotests.log(vm.qmp( + 'transaction', + actions=[ + { + 'data': { + 'node-name': 'disk0', + 'snapshot-file': disk0_snap_img_path, + 'snapshot-node-name': 'disk0-snap', + 'mode': 'absolute-paths', + 'format': iotests.imgfmt, + }, + 'type': 'blockdev-snapshot-sync' + }, { + 'data': { + 'node-name': 'disk1', + 'snapshot-file': disk1_snap_img_path, + 'snapshot-node-name': 'disk1-snap', + 'mode': 'absolute-paths', + 'format': iotests.imgfmt + }, + 'type': 'blockdev-snapshot-sync' + } + ])) diff --git a/tests/qemu-iotests/202.out b/tests/qemu-iotests/202.out new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/202.out @@ -XXX,XX +XXX,XX @@ +Launching VM... +Adding IOThread... +{u'return': {}} +Adding blockdevs... +{u'return': {}} +{u'return': {}} +Setting iothread... +{u'return': {}} +{u'return': {}} +Creating external snapshots... +{u'return': {}} diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -XXX,XX +XXX,XX @@ 197 rw auto quick 198 rw auto 200 rw auto +202 rw auto quick -- 2.14.3 From: Mark Kanda <mark.kanda@oracle.com> Depending on the configuration, it can be beneficial to adjust the virtio-blk queue size to something other than the current default of 128. Add a new property to make the queue size configurable. Signed-off-by: Mark Kanda <mark.kanda@oracle.com> Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com> Reviewed-by: Ameya More <ameya.more@oracle.com> Message-id: 52e6d742811f10dbd16e996e86cf375b9577c187.1513005190.git.mark.kanda@oracle.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- include/hw/virtio/virtio-blk.h | 1 + hw/block/virtio-blk.c | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/virtio/virtio-blk.h +++ b/include/hw/virtio/virtio-blk.h @@ -XXX,XX +XXX,XX @@ struct VirtIOBlkConf uint32_t config_wce; uint32_t request_merging; uint16_t num_queues; + uint16_t queue_size; }; struct VirtIOBlockDataPlane; diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) error_setg(errp, "num-queues property must be larger than 0"); return; } + if (!is_power_of_2(conf->queue_size) || + conf->queue_size > VIRTQUEUE_MAX_SIZE) { + error_setg(errp, "invalid queue-size property (%" PRIu16 "), " + "must be a power of 2 (max %d)", + conf->queue_size, VIRTQUEUE_MAX_SIZE); + return; + } blkconf_serial(&conf->conf, &conf->serial); if (!blkconf_apply_backend_options(&conf->conf, @@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1; for (i = 0; i < conf->num_queues; i++) { - virtio_add_queue(vdev, 128, virtio_blk_handle_output); + virtio_add_queue(vdev, conf->queue_size, virtio_blk_handle_output); } virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err); if (err != NULL) { @@ -XXX,XX +XXX,XX @@ static Property virtio_blk_properties[] = { DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0, true), DEFINE_PROP_UINT16("num-queues", VirtIOBlock, conf.num_queues, 1), + DEFINE_PROP_UINT16("queue-size", VirtIOBlock, conf.queue_size, 128), DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD, IOThread *), DEFINE_PROP_END_OF_LIST(), -- 2.14.3 From: Mark Kanda <mark.kanda@oracle.com> virtio-blk logical block size should never be larger than physical block size because it doesn't make sense to have such configurations. QEMU doesn't have a way to effectively express this condition; the best it can do is report the physical block exponent as 0 - indicating the logical block size equals the physical block size. This is identical to commit 3da023b5827543ee4c022986ea2ad9d1274410b2 but applied to virtio-blk (instead of virtio-scsi). Signed-off-by: Mark Kanda <mark.kanda@oracle.com> Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Reviewed-by: Ameya More <ameya.more@oracle.com> Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: 773169891f9f2deb4cb7c4ef2655580dbe24c1d1.1513005190.git.mark.kanda@oracle.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- hw/block/virtio-blk.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index XXXXXXX..XXXXXXX 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) blkconf_blocksizes(&conf->conf); + if (conf->conf.logical_block_size > + conf->conf.physical_block_size) { + error_setg(errp, + "logical_block_size > physical_block_size not supported"); + return; + } + virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK, sizeof(struct virtio_blk_config)); -- 2.14.3 From: Paolo Bonzini <pbonzini@redhat.com> BDRV_POLL_WHILE() does not support recursive AioContext locking. It only releases the AioContext lock once regardless of how many times the caller has acquired it. This results in a hang since the IOThread does not make progress while the AioContext is still locked. The following steps trigger the hang: $ qemu-system-x86_64 -M accel=kvm -m 1G -cpu host \ -object iothread,id=iothread0 \ -device virtio-scsi-pci,iothread=iothread0 \ -drive if=none,id=drive0,file=test.img,format=raw \ -device scsi-hd,drive=drive0 \ -drive if=none,id=drive1,file=test.img,format=raw \ -device scsi-hd,drive=drive1 $ qemu-system-x86_64 ...same options... \ -incoming tcp::1234 (qemu) migrate tcp:127.0.0.1:1234 ...hang... Tested-by: Stefan Hajnoczi <stefanha@redhat.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171207201320.19284-2-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- block.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/block.c b/block.c index XXXXXXX..XXXXXXX 100644 --- a/block.c +++ b/block.c @@ -XXX,XX +XXX,XX @@ int bdrv_inactivate_all(void) BdrvNextIterator it; int ret = 0; int pass; + GSList *aio_ctxs = NULL, *ctx; for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { - aio_context_acquire(bdrv_get_aio_context(bs)); + AioContext *aio_context = bdrv_get_aio_context(bs); + + if (!g_slist_find(aio_ctxs, aio_context)) { + aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); + aio_context_acquire(aio_context); + } } /* We do two passes of inactivation. The first pass calls to drivers' @@ -XXX,XX +XXX,XX @@ int bdrv_inactivate_all(void) } out: - for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { - aio_context_release(bdrv_get_aio_context(bs)); + for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { + AioContext *aio_context = ctx->data; + aio_context_release(aio_context); } + g_slist_free(aio_ctxs); return ret; } -- 2.14.3 See the patch for why nested AioContext locking is no longer allowed. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171207201320.19284-3-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- docs/devel/multiple-iothreads.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/devel/multiple-iothreads.txt b/docs/devel/multiple-iothreads.txt index XXXXXXX..XXXXXXX 100644 --- a/docs/devel/multiple-iothreads.txt +++ b/docs/devel/multiple-iothreads.txt @@ -XXX,XX +XXX,XX @@ -Copyright (c) 2014 Red Hat Inc. +Copyright (c) 2014-2017 Red Hat Inc. This work is licensed under the terms of the GNU GPL, version 2 or later. See the COPYING file in the top-level directory. @@ -XXX,XX +XXX,XX @@ aio_context_acquire()/aio_context_release() for mutual exclusion. Once the context is acquired no other thread can access it or run event loop iterations in this AioContext. -aio_context_acquire()/aio_context_release() calls may be nested. This -means you can call them if you're not sure whether #2 applies. +Legacy code sometimes nests aio_context_acquire()/aio_context_release() calls. +Do not use nesting anymore, it is incompatible with the BDRV_POLL_WHILE() macro +used in the block layer and can lead to hangs. There is currently no lock ordering rule if a thread needs to acquire multiple AioContexts simultaneously. Therefore, it is only safe for code holding the -- 2.14.3 When a node is already associated with a BlockBackend the x-blockdev-set-iothread command refuses to set the IOThread. This is to prevent accidentally changing the IOThread when the nodes are in use. When the nodes are created with -drive they automatically get a BlockBackend. In that case we know nothing is using them yet and it's safe to set the IOThread. Add a force boolean to override the check. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171207201320.19284-4-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- qapi/block-core.json | 6 +++++- blockdev.c | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # # @iothread: the name of the IOThread object or null for the main loop # +# @force: true if the node and its children should be moved when a BlockBackend +# is already attached +# # Note: this command is experimental and intended for test cases that need # control over IOThreads only. # @@ -XXX,XX +XXX,XX @@ ## { 'command': 'x-blockdev-set-iothread', 'data' : { 'node-name': 'str', - 'iothread': 'StrOrNull' } } + 'iothread': 'StrOrNull', + '*force': 'bool' } } diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp) } void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread, - Error **errp) + bool has_force, bool force, Error **errp) { AioContext *old_context; AioContext *new_context; @@ -XXX,XX +XXX,XX @@ void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread, return; } - /* If we want to allow more extreme test scenarios this guard could be - * removed. For now it protects against accidents. */ - if (bdrv_has_blk(bs)) { - error_setg(errp, "Node %s is in use", node_name); + /* Protects against accidents. */ + if (!(has_force && force) && bdrv_has_blk(bs)) { + error_setg(errp, "Node %s is associated with a BlockBackend and could " + "be in use (use force=true to override this check)", + node_name); return; } -- 2.14.3 The VM.add_object() method can be used to add IOThreads or memory backend objects. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171207201320.19284-5-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- tests/qemu-iotests/iotests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine): socket_scm_helper=socket_scm_helper) self._num_drives = 0 + def add_object(self, opts): + self._args.append('-object') + self._args.append(opts) + return self + def add_device(self, opts): self._args.append('-device') self._args.append(opts) -- 2.14.3 There is a small chance that iothread_stop() hangs as follows: Thread 3 (Thread 0x7f63eba5f700 (LWP 16105)): #0 0x00007f64012c09b6 in ppoll () at /lib64/libc.so.6 #1 0x000055959992eac9 in ppoll (__ss=0x0, __timeout=0x0, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/bits/poll2.h:77 #2 0x000055959992eac9 in qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=<optimized out>) at util/qemu-timer.c:322 #3 0x0000559599930711 in aio_poll (ctx=0x55959bdb83c0, blocking=blocking@entry=true) at util/aio-posix.c:629 #4 0x00005595996806fe in iothread_run (opaque=0x55959bd78400) at iothread.c:59 #5 0x00007f640159f609 in start_thread () at /lib64/libpthread.so.0 #6 0x00007f64012cce6f in clone () at /lib64/libc.so.6 Thread 1 (Thread 0x7f640b45b280 (LWP 16103)): #0 0x00007f64015a0b6d in pthread_join () at /lib64/libpthread.so.0 #1 0x00005595999332ef in qemu_thread_join (thread=<optimized out>) at util/qemu-thread-posix.c:547 #2 0x00005595996808ae in iothread_stop (iothread=<optimized out>) at iothread.c:91 #3 0x000055959968094d in iothread_stop_iter (object=<optimized out>, opaque=<optimized out>) at iothread.c:102 #4 0x0000559599857d97 in do_object_child_foreach (obj=obj@entry=0x55959bdb8100, fn=fn@entry=0x559599680930 <iothread_stop_iter>, opaque=opaque@entry=0x0, recurse=recurse@entry=false) at qom/object.c:852 #5 0x0000559599859477 in object_child_foreach (obj=obj@entry=0x55959bdb8100, fn=fn@entry=0x559599680930 <iothread_stop_iter>, opaque=opaque@entry=0x0) at qom/object.c:867 #6 0x0000559599680a6e in iothread_stop_all () at iothread.c:341 #7 0x000055959955b1d5 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4913 The relevant code from iothread_run() is: while (!atomic_read(&iothread->stopping)) { aio_poll(iothread->ctx, true); and iothread_stop(): iothread->stopping = true; aio_notify(iothread->ctx); ... qemu_thread_join(&iothread->thread); The following scenario can occur: 1. IOThread: while (!atomic_read(&iothread->stopping)) -> stopping=false 2. Main loop: iothread->stopping = true; aio_notify(iothread->ctx); 3. IOThread: aio_poll(iothread->ctx, true); -> hang The bug is explained by the AioContext->notify_me doc comments: "If this field is 0, everything (file descriptors, bottom halves, timers) will be re-evaluated before the next blocking poll(), thus the event_notifier_set call can be skipped." The problem is that "everything" does not include checking iothread->stopping. This means iothread_run() will block in aio_poll() if aio_notify() was called just before aio_poll(). This patch fixes the hang by replacing aio_notify() with aio_bh_schedule_oneshot(). This makes aio_poll() or g_main_loop_run() to return. Implementing this properly required a new bool running flag. The new flag prevents races that are tricky if we try to use iothread->stopping. Now iothread->stopping is purely for iothread_stop() and iothread->running is purely for the iothread_run() thread. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171207201320.19284-6-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- include/sysemu/iothread.h | 3 ++- iothread.c | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h index XXXXXXX..XXXXXXX 100644 --- a/include/sysemu/iothread.h +++ b/include/sysemu/iothread.h @@ -XXX,XX +XXX,XX @@ typedef struct { GOnce once; QemuMutex init_done_lock; QemuCond init_done_cond; /* is thread initialization done? */ - bool stopping; + bool stopping; /* has iothread_stop() been called? */ + bool running; /* should iothread_run() continue? */ int thread_id; /* AioContext poll parameters */ diff --git a/iothread.c b/iothread.c index XXXXXXX..XXXXXXX 100644 --- a/iothread.c +++ b/iothread.c @@ -XXX,XX +XXX,XX @@ static void *iothread_run(void *opaque) qemu_cond_signal(&iothread->init_done_cond); qemu_mutex_unlock(&iothread->init_done_lock); - while (!atomic_read(&iothread->stopping)) { + while (iothread->running) { aio_poll(iothread->ctx, true); if (atomic_read(&iothread->worker_context)) { @@ -XXX,XX +XXX,XX @@ static void *iothread_run(void *opaque) return NULL; } +/* Runs in iothread_run() thread */ +static void iothread_stop_bh(void *opaque) +{ + IOThread *iothread = opaque; + + iothread->running = false; /* stop iothread_run() */ + + if (iothread->main_loop) { + g_main_loop_quit(iothread->main_loop); + } +} + void iothread_stop(IOThread *iothread) { if (!iothread->ctx || iothread->stopping) { return; } iothread->stopping = true; - aio_notify(iothread->ctx); - if (atomic_read(&iothread->main_loop)) { - g_main_loop_quit(iothread->main_loop); - } + aio_bh_schedule_oneshot(iothread->ctx, iothread_stop_bh, iothread); qemu_thread_join(&iothread->thread); } @@ -XXX,XX +XXX,XX @@ static void iothread_complete(UserCreatable *obj, Error **errp) char *name, *thread_name; iothread->stopping = false; + iothread->running = true; iothread->thread_id = -1; iothread->ctx = aio_context_new(&local_error); if (!iothread->ctx) { -- 2.14.3 This test case will prevent future regressions with savevm and IOThreads. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Message-id: 20171207201320.19284-7-stefanha@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- tests/qemu-iotests/203 | 59 ++++++++++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/203.out | 6 +++++ tests/qemu-iotests/group | 1 + 3 files changed, 66 insertions(+) create mode 100755 tests/qemu-iotests/203 create mode 100644 tests/qemu-iotests/203.out diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203 new file mode 100755 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/203 @@ -XXX,XX +XXX,XX @@ +#!/usr/bin/env python +# +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# Creator/Owner: Stefan Hajnoczi <stefanha@redhat.com> +# +# Check that QMP 'migrate' with multiple drives on a single IOThread completes +# successfully. This particular command triggered a hang in the source QEMU +# process due to recursive AioContext locking in bdrv_invalidate_all() and +# BDRV_POLL_WHILE(). + +import iotests + +iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.verify_platform(['linux']) + +with iotests.FilePath('disk0.img') as disk0_img_path, \ + iotests.FilePath('disk1.img') as disk1_img_path, \ + iotests.VM() as vm: + + img_size = '10M' + iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk0_img_path, img_size) + iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk1_img_path, img_size) + + iotests.log('Launching VM...') + (vm.add_object('iothread,id=iothread0') + .add_drive(disk0_img_path, 'node-name=drive0-node', interface='none') + .add_drive(disk1_img_path, 'node-name=drive1-node', interface='none') + .launch()) + + iotests.log('Setting IOThreads...') + iotests.log(vm.qmp('x-blockdev-set-iothread', + node_name='drive0-node', iothread='iothread0', + force=True)) + iotests.log(vm.qmp('x-blockdev-set-iothread', + node_name='drive1-node', iothread='iothread0', + force=True)) + + iotests.log('Starting migration...') + iotests.log(vm.qmp('migrate', uri='exec:cat >/dev/null')) + while True: + vm.get_qmp_event(wait=60.0) + result = vm.qmp('query-migrate') + status = result.get('return', {}).get('status', None) + if status == 'completed': + break diff --git a/tests/qemu-iotests/203.out b/tests/qemu-iotests/203.out new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/203.out @@ -XXX,XX +XXX,XX @@ +Launching VM... +Setting IOThreads... +{u'return': {}} +{u'return': {}} +Starting migration... +{u'return': {}} diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -XXX,XX +XXX,XX @@ 198 rw auto 200 rw auto 202 rw auto quick +203 rw auto -- 2.14.3