From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914068586659.4220561204771; Wed, 13 Jun 2018 11:21:08 -0700 (PDT) Received: from localhost ([::1]:36170 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAOJ-0004qU-Sf for importer@patchew.org; Wed, 13 Jun 2018 14:21:07 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56588) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAM3-0003Ck-Uz for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:49 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTALz-0004rk-In for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:47 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58820 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTALs-0004Sh-Pd; Wed, 13 Jun 2018 14:18:36 -0400 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 5FE9187A7C; Wed, 13 Jun 2018 18:18:36 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 9BDBB1C663; Wed, 13 Jun 2018 18:18:31 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:10 +0200 Message-Id: <20180613181823.13618-2-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.79 on 10.11.54.5 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:36 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:36 +0000 (UTC) for IP:'10.11.54.5' DOMAIN:'int-mx05.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 01/14] block/mirror: Pull out mirror_perform() X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" When converting mirror's I/O to coroutines, we are going to need a point where these coroutines are created. mirror_perform() is going to be that point. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Jeff Cody Reviewed-by: Alberto Garcia --- block/mirror.c | 51 ++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index 435268bbbf..f5e240611b 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -80,6 +80,12 @@ typedef struct MirrorOp { uint64_t bytes; } MirrorOp; =20 +typedef enum MirrorMethod { + MIRROR_METHOD_COPY, + MIRROR_METHOD_ZERO, + MIRROR_METHOD_DISCARD, +} MirrorMethod; + static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, int error) { @@ -319,6 +325,22 @@ static void mirror_do_zero_or_discard(MirrorBlockJob *= s, } } =20 +static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset, + unsigned bytes, MirrorMethod mirror_method) +{ + switch (mirror_method) { + case MIRROR_METHOD_COPY: + return mirror_do_read(s, offset, bytes); + case MIRROR_METHOD_ZERO: + case MIRROR_METHOD_DISCARD: + mirror_do_zero_or_discard(s, offset, bytes, + mirror_method =3D=3D MIRROR_METHOD_DISCA= RD); + return bytes; + default: + abort(); + } +} + static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) { BlockDriverState *source =3D s->source; @@ -385,11 +407,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBl= ockJob *s) int ret; int64_t io_bytes; int64_t io_bytes_acct; - enum MirrorMethod { - MIRROR_METHOD_COPY, - MIRROR_METHOD_ZERO, - MIRROR_METHOD_DISCARD - } mirror_method =3D MIRROR_METHOD_COPY; + MirrorMethod mirror_method =3D MIRROR_METHOD_COPY; =20 assert(!(offset % s->granularity)); ret =3D bdrv_block_status_above(source, NULL, offset, @@ -427,22 +445,11 @@ static uint64_t coroutine_fn mirror_iteration(MirrorB= lockJob *s) } =20 io_bytes =3D mirror_clip_bytes(s, offset, io_bytes); - switch (mirror_method) { - case MIRROR_METHOD_COPY: - io_bytes =3D io_bytes_acct =3D mirror_do_read(s, offset, io_by= tes); - break; - case MIRROR_METHOD_ZERO: - case MIRROR_METHOD_DISCARD: - mirror_do_zero_or_discard(s, offset, io_bytes, - mirror_method =3D=3D MIRROR_METHOD_D= ISCARD); - if (write_zeroes_ok) { - io_bytes_acct =3D 0; - } else { - io_bytes_acct =3D io_bytes; - } - break; - default: - abort(); + io_bytes =3D mirror_perform(s, offset, io_bytes, mirror_method); + if (mirror_method !=3D MIRROR_METHOD_COPY && write_zeroes_ok) { + io_bytes_acct =3D 0; + } else { + io_bytes_acct =3D io_bytes; } assert(io_bytes); offset +=3D io_bytes; @@ -635,7 +642,7 @@ static int coroutine_fn mirror_dirty_init(MirrorBlockJo= b *s) continue; } =20 - mirror_do_zero_or_discard(s, offset, bytes, false); + mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO); offset +=3D bytes; } =20 --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914267891325.5084512599948; Wed, 13 Jun 2018 11:24:27 -0700 (PDT) Received: from localhost ([::1]:36190 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTART-0007WJ-VJ for importer@patchew.org; Wed, 13 Jun 2018 14:24:24 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56672) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAM7-0003G8-IG for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:53 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAM5-0005Cx-Gq for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:51 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58828 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTALu-0004Z8-Sx; Wed, 13 Jun 2018 14:18:38 -0400 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 866A9F68B6; Wed, 13 Jun 2018 18:18:38 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id F281F1C640; Wed, 13 Jun 2018 18:18:37 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:11 +0200 Message-Id: <20180613181823.13618-3-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.79 on 10.11.54.5 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:38 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:38 +0000 (UTC) for IP:'10.11.54.5' DOMAIN:'int-mx05.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 02/14] block/mirror: Convert to coroutines X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" In order to talk to the source BDS (and maybe in the future to the target BDS as well) directly, we need to convert our existing AIO requests into coroutine I/O requests. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng --- block/mirror.c | 152 +++++++++++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 62 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index f5e240611b..47122482ff 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -78,6 +78,10 @@ typedef struct MirrorOp { QEMUIOVector qiov; int64_t offset; uint64_t bytes; + + /* The pointee is set by mirror_co_read(), mirror_co_zero(), and + * mirror_co_discard() before yielding for the first time */ + int64_t *bytes_handled; } MirrorOp; =20 typedef enum MirrorMethod { @@ -99,7 +103,7 @@ static BlockErrorAction mirror_error_action(MirrorBlockJ= ob *s, bool read, } } =20 -static void mirror_iteration_done(MirrorOp *op, int ret) +static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret) { MirrorBlockJob *s =3D op->s; struct iovec *iov; @@ -136,9 +140,8 @@ static void mirror_iteration_done(MirrorOp *op, int ret) } } =20 -static void mirror_write_complete(void *opaque, int ret) +static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret) { - MirrorOp *op =3D opaque; MirrorBlockJob *s =3D op->s; =20 aio_context_acquire(blk_get_aio_context(s->common.blk)); @@ -155,9 +158,8 @@ static void mirror_write_complete(void *opaque, int ret) aio_context_release(blk_get_aio_context(s->common.blk)); } =20 -static void mirror_read_complete(void *opaque, int ret) +static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret) { - MirrorOp *op =3D opaque; MirrorBlockJob *s =3D op->s; =20 aio_context_acquire(blk_get_aio_context(s->common.blk)); @@ -172,8 +174,9 @@ static void mirror_read_complete(void *opaque, int ret) =20 mirror_iteration_done(op, ret); } else { - blk_aio_pwritev(s->target, op->offset, &op->qiov, - 0, mirror_write_complete, op); + ret =3D blk_co_pwritev(s->target, op->offset, + op->qiov.size, &op->qiov, 0); + mirror_write_complete(op, ret); } aio_context_release(blk_get_aio_context(s->common.blk)); } @@ -230,60 +233,57 @@ static inline void mirror_wait_for_io(MirrorBlockJob = *s) s->waiting_for_io =3D false; } =20 -/* Submit async read while handling COW. - * Returns: The number of bytes copied after and including offset, - * excluding any bytes copied prior to offset due to alignment. - * This will be @bytes if no alignment is necessary, or - * (new_end - offset) if tail is rounded up or down due to - * alignment or buffer limit. +/* Perform a mirror copy operation. + * + * *op->bytes_handled is set to the number of bytes copied after and + * including offset, excluding any bytes copied prior to offset due + * to alignment. This will be op->bytes if no alignment is necessary, + * or (new_end - op->offset) if the tail is rounded up or down due to + * alignment or buffer limit. */ -static uint64_t mirror_do_read(MirrorBlockJob *s, int64_t offset, - uint64_t bytes) +static void coroutine_fn mirror_co_read(void *opaque) { + MirrorOp *op =3D opaque; + MirrorBlockJob *s =3D op->s; BlockBackend *source =3D s->common.blk; int nb_chunks; uint64_t ret; - MirrorOp *op; uint64_t max_bytes; =20 max_bytes =3D s->granularity * s->max_iov; =20 /* We can only handle as much as buf_size at a time. */ - bytes =3D MIN(s->buf_size, MIN(max_bytes, bytes)); - assert(bytes); - assert(bytes < BDRV_REQUEST_MAX_BYTES); - ret =3D bytes; + op->bytes =3D MIN(s->buf_size, MIN(max_bytes, op->bytes)); + assert(op->bytes); + assert(op->bytes < BDRV_REQUEST_MAX_BYTES); + *op->bytes_handled =3D op->bytes; =20 if (s->cow_bitmap) { - ret +=3D mirror_cow_align(s, &offset, &bytes); + *op->bytes_handled +=3D mirror_cow_align(s, &op->offset, &op->byte= s); } - assert(bytes <=3D s->buf_size); + /* Cannot exceed BDRV_REQUEST_MAX_BYTES + INT_MAX */ + assert(*op->bytes_handled <=3D UINT_MAX); + assert(op->bytes <=3D s->buf_size); /* The offset is granularity-aligned because: * 1) Caller passes in aligned values; * 2) mirror_cow_align is used only when target cluster is larger. */ - assert(QEMU_IS_ALIGNED(offset, s->granularity)); + assert(QEMU_IS_ALIGNED(op->offset, s->granularity)); /* The range is sector-aligned, since bdrv_getlength() rounds up. */ - assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); - nb_chunks =3D DIV_ROUND_UP(bytes, s->granularity); + assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE)); + nb_chunks =3D DIV_ROUND_UP(op->bytes, s->granularity); =20 while (s->buf_free_count < nb_chunks) { - trace_mirror_yield_in_flight(s, offset, s->in_flight); + trace_mirror_yield_in_flight(s, op->offset, s->in_flight); mirror_wait_for_io(s); } =20 - /* Allocate a MirrorOp that is used as an AIO callback. */ - op =3D g_new(MirrorOp, 1); - op->s =3D s; - op->offset =3D offset; - op->bytes =3D bytes; - /* Now make a QEMUIOVector taking enough granularity-sized chunks * from s->buf_free. */ qemu_iovec_init(&op->qiov, nb_chunks); while (nb_chunks-- > 0) { MirrorBuffer *buf =3D QSIMPLEQ_FIRST(&s->buf_free); - size_t remaining =3D bytes - op->qiov.size; + size_t remaining =3D op->bytes - op->qiov.size; =20 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); s->buf_free_count--; @@ -292,53 +292,81 @@ static uint64_t mirror_do_read(MirrorBlockJob *s, int= 64_t offset, =20 /* Copy the dirty cluster. */ s->in_flight++; - s->bytes_in_flight +=3D bytes; - trace_mirror_one_iteration(s, offset, bytes); + s->bytes_in_flight +=3D op->bytes; + trace_mirror_one_iteration(s, op->offset, op->bytes); =20 - blk_aio_preadv(source, offset, &op->qiov, 0, mirror_read_complete, op); - return ret; + ret =3D blk_co_preadv(source, op->offset, op->bytes, &op->qiov, 0); + mirror_read_complete(op, ret); } =20 -static void mirror_do_zero_or_discard(MirrorBlockJob *s, - int64_t offset, - uint64_t bytes, - bool is_discard) +static void coroutine_fn mirror_co_zero(void *opaque) { - MirrorOp *op; + MirrorOp *op =3D opaque; + int ret; =20 - /* Allocate a MirrorOp that is used as an AIO callback. The qiov is ze= roed - * so the freeing in mirror_iteration_done is nop. */ - op =3D g_new0(MirrorOp, 1); - op->s =3D s; - op->offset =3D offset; - op->bytes =3D bytes; + op->s->in_flight++; + op->s->bytes_in_flight +=3D op->bytes; + *op->bytes_handled =3D op->bytes; =20 - s->in_flight++; - s->bytes_in_flight +=3D bytes; - if (is_discard) { - blk_aio_pdiscard(s->target, offset, - op->bytes, mirror_write_complete, op); - } else { - blk_aio_pwrite_zeroes(s->target, offset, - op->bytes, s->unmap ? BDRV_REQ_MAY_UNMAP : 0, - mirror_write_complete, op); - } + ret =3D blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes, + op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0); + mirror_write_complete(op, ret); +} + +static void coroutine_fn mirror_co_discard(void *opaque) +{ + MirrorOp *op =3D opaque; + int ret; + + op->s->in_flight++; + op->s->bytes_in_flight +=3D op->bytes; + *op->bytes_handled =3D op->bytes; + + ret =3D blk_co_pdiscard(op->s->target, op->offset, op->bytes); + mirror_write_complete(op, ret); } =20 static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset, unsigned bytes, MirrorMethod mirror_method) { + MirrorOp *op; + Coroutine *co; + int64_t bytes_handled =3D -1; + + op =3D g_new(MirrorOp, 1); + *op =3D (MirrorOp){ + .s =3D s, + .offset =3D offset, + .bytes =3D bytes, + .bytes_handled =3D &bytes_handled, + }; + switch (mirror_method) { case MIRROR_METHOD_COPY: - return mirror_do_read(s, offset, bytes); + co =3D qemu_coroutine_create(mirror_co_read, op); + break; case MIRROR_METHOD_ZERO: + co =3D qemu_coroutine_create(mirror_co_zero, op); + break; case MIRROR_METHOD_DISCARD: - mirror_do_zero_or_discard(s, offset, bytes, - mirror_method =3D=3D MIRROR_METHOD_DISCA= RD); - return bytes; + co =3D qemu_coroutine_create(mirror_co_discard, op); + break; default: abort(); } + + qemu_coroutine_enter(co); + /* At this point, ownership of op has been moved to the coroutine + * and the object may already be freed */ + + /* Assert that this value has been set */ + assert(bytes_handled >=3D 0); + + /* Same assertion as in mirror_co_read() (and for mirror_co_read() + * and mirror_co_discard(), bytes_handled =3D=3D op->bytes, which + * is the @bytes parameter given to this function) */ + assert(bytes_handled <=3D UINT_MAX); + return bytes_handled; } =20 static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914144413729.4959868949425; Wed, 13 Jun 2018 11:22:24 -0700 (PDT) Received: from localhost ([::1]:36173 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAPX-0005r7-Ic for importer@patchew.org; Wed, 13 Jun 2018 14:22:23 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56650) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAM6-0003FB-Ra for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:52 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAM5-0005D7-HC for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:50 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:38570 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTALx-0004gt-A2; Wed, 13 Jun 2018 14:18:41 -0400 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id E69828185331; Wed, 13 Jun 2018 18:18:40 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 79E7A10FFE4D; Wed, 13 Jun 2018 18:18:40 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:12 +0200 Message-Id: <20180613181823.13618-4-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.3 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:18:40 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:18:40 +0000 (UTC) for IP:'10.11.54.3' DOMAIN:'int-mx03.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 03/14] block/mirror: Use CoQueue to wait on in-flight ops X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Attach a CoQueue to each in-flight operation so if we need to wait for any we can use it to wait instead of just blindly yielding and hoping for some operation to wake us. A later patch will use this infrastructure to allow requests accessing the same area of the virtual disk to specifically wait for each other. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng --- block/mirror.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index 47122482ff..7fb1074c72 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -13,6 +13,7 @@ =20 #include "qemu/osdep.h" #include "qemu/cutils.h" +#include "qemu/coroutine.h" #include "trace.h" #include "block/blockjob_int.h" #include "block/block_int.h" @@ -33,6 +34,8 @@ typedef struct MirrorBuffer { QSIMPLEQ_ENTRY(MirrorBuffer) next; } MirrorBuffer; =20 +typedef struct MirrorOp MirrorOp; + typedef struct MirrorBlockJob { BlockJob common; BlockBackend *target; @@ -65,15 +68,15 @@ typedef struct MirrorBlockJob { unsigned long *in_flight_bitmap; int in_flight; int64_t bytes_in_flight; + QTAILQ_HEAD(MirrorOpList, MirrorOp) ops_in_flight; int ret; bool unmap; - bool waiting_for_io; int target_cluster_size; int max_iov; bool initial_zeroing_ongoing; } MirrorBlockJob; =20 -typedef struct MirrorOp { +struct MirrorOp { MirrorBlockJob *s; QEMUIOVector qiov; int64_t offset; @@ -82,7 +85,11 @@ typedef struct MirrorOp { /* The pointee is set by mirror_co_read(), mirror_co_zero(), and * mirror_co_discard() before yielding for the first time */ int64_t *bytes_handled; -} MirrorOp; + + CoQueue waiting_requests; + + QTAILQ_ENTRY(MirrorOp) next; +}; =20 typedef enum MirrorMethod { MIRROR_METHOD_COPY, @@ -123,7 +130,9 @@ static void coroutine_fn mirror_iteration_done(MirrorOp= *op, int ret) =20 chunk_num =3D op->offset / s->granularity; nb_chunks =3D DIV_ROUND_UP(op->bytes, s->granularity); + bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); + QTAILQ_REMOVE(&s->ops_in_flight, op, next); if (ret >=3D 0) { if (s->cow_bitmap) { bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); @@ -133,11 +142,9 @@ static void coroutine_fn mirror_iteration_done(MirrorO= p *op, int ret) } } qemu_iovec_destroy(&op->qiov); - g_free(op); =20 - if (s->waiting_for_io) { - qemu_coroutine_enter(s->common.job.co); - } + qemu_co_queue_restart_all(&op->waiting_requests); + g_free(op); } =20 static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret) @@ -227,10 +234,11 @@ static int mirror_cow_align(MirrorBlockJob *s, int64_= t *offset, =20 static inline void mirror_wait_for_io(MirrorBlockJob *s) { - assert(!s->waiting_for_io); - s->waiting_for_io =3D true; - qemu_coroutine_yield(); - s->waiting_for_io =3D false; + MirrorOp *op; + + op =3D QTAILQ_FIRST(&s->ops_in_flight); + assert(op); + qemu_co_queue_wait(&op->waiting_requests, NULL); } =20 /* Perform a mirror copy operation. @@ -340,6 +348,7 @@ static unsigned mirror_perform(MirrorBlockJob *s, int64= _t offset, .bytes =3D bytes, .bytes_handled =3D &bytes_handled, }; + qemu_co_queue_init(&op->waiting_requests); =20 switch (mirror_method) { case MIRROR_METHOD_COPY: @@ -355,6 +364,7 @@ static unsigned mirror_perform(MirrorBlockJob *s, int64= _t offset, abort(); } =20 + QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next); qemu_coroutine_enter(co); /* At this point, ownership of op has been moved to the coroutine * and the object may already be freed */ @@ -1282,6 +1292,8 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, } } =20 + QTAILQ_INIT(&s->ops_in_flight); + trace_mirror_start(bs, s, opaque); job_start(&s->common.job); return; --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914294314368.2409571596811; Wed, 13 Jun 2018 11:24:54 -0700 (PDT) Received: from localhost ([::1]:36195 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTARu-0007tM-KF for importer@patchew.org; Wed, 13 Jun 2018 14:24:50 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56719) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAM9-0003Ho-9f for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:57 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAM7-0005KP-CV for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:53 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58830 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAM0-0004sr-9G; Wed, 13 Jun 2018 14:18:44 -0400 Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.rdu2.redhat.com [10.11.54.6]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id E1EBA87A80; Wed, 13 Jun 2018 18:18:43 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id EC40A2157F42; Wed, 13 Jun 2018 18:18:42 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:13 +0200 Message-Id: <20180613181823.13618-5-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.6 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:43 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:43 +0000 (UTC) for IP:'10.11.54.6' DOMAIN:'int-mx06.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 04/14] block/mirror: Wait for in-flight op conflicts X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This patch makes the mirror code differentiate between simply waiting for any operation to complete (mirror_wait_for_free_in_flight_slot()) and specifically waiting for all operations touching a certain range of the virtual disk to complete (mirror_wait_on_conflicts()). Signed-off-by: Max Reitz Reviewed-by: Fam Zheng --- block/mirror.c | 102 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 18 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index 7fb1074c72..7c1b8ef5b6 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -14,6 +14,7 @@ #include "qemu/osdep.h" #include "qemu/cutils.h" #include "qemu/coroutine.h" +#include "qemu/range.h" #include "trace.h" #include "block/blockjob_int.h" #include "block/block_int.h" @@ -86,6 +87,7 @@ struct MirrorOp { * mirror_co_discard() before yielding for the first time */ int64_t *bytes_handled; =20 + bool is_pseudo_op; CoQueue waiting_requests; =20 QTAILQ_ENTRY(MirrorOp) next; @@ -110,6 +112,41 @@ static BlockErrorAction mirror_error_action(MirrorBloc= kJob *s, bool read, } } =20 +static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self, + MirrorBlockJob *s, + uint64_t offset, + uint64_t bytes) +{ + uint64_t self_start_chunk =3D offset / s->granularity; + uint64_t self_end_chunk =3D DIV_ROUND_UP(offset + bytes, s->granularit= y); + uint64_t self_nb_chunks =3D self_end_chunk - self_start_chunk; + + while (find_next_bit(s->in_flight_bitmap, self_end_chunk, + self_start_chunk) < self_end_chunk && + s->ret >=3D 0) + { + MirrorOp *op; + + QTAILQ_FOREACH(op, &s->ops_in_flight, next) { + uint64_t op_start_chunk =3D op->offset / s->granularity; + uint64_t op_nb_chunks =3D DIV_ROUND_UP(op->offset + op->bytes, + s->granularity) - + op_start_chunk; + + if (op =3D=3D self) { + continue; + } + + if (ranges_overlap(self_start_chunk, self_nb_chunks, + op_start_chunk, op_nb_chunks)) + { + qemu_co_queue_wait(&op->waiting_requests, NULL); + break; + } + } + } +} + static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret) { MirrorBlockJob *s =3D op->s; @@ -232,13 +269,22 @@ static int mirror_cow_align(MirrorBlockJob *s, int64_= t *offset, return ret; } =20 -static inline void mirror_wait_for_io(MirrorBlockJob *s) +static inline void mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s) { MirrorOp *op; =20 - op =3D QTAILQ_FIRST(&s->ops_in_flight); - assert(op); - qemu_co_queue_wait(&op->waiting_requests, NULL); + QTAILQ_FOREACH(op, &s->ops_in_flight, next) { + /* Do not wait on pseudo ops, because it may in turn wait on + * some other operation to start, which may in fact be the + * caller of this function. Since there is only one pseudo op + * at any given time, we will always find some real operation + * to wait on. */ + if (!op->is_pseudo_op) { + qemu_co_queue_wait(&op->waiting_requests, NULL); + return; + } + } + abort(); } =20 /* Perform a mirror copy operation. @@ -282,7 +328,7 @@ static void coroutine_fn mirror_co_read(void *opaque) =20 while (s->buf_free_count < nb_chunks) { trace_mirror_yield_in_flight(s, op->offset, s->in_flight); - mirror_wait_for_io(s); + mirror_wait_for_free_in_flight_slot(s); } =20 /* Now make a QEMUIOVector taking enough granularity-sized chunks @@ -382,8 +428,9 @@ static unsigned mirror_perform(MirrorBlockJob *s, int64= _t offset, static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) { BlockDriverState *source =3D s->source; - int64_t offset, first_chunk; - uint64_t delay_ns =3D 0; + MirrorOp *pseudo_op; + int64_t offset; + uint64_t delay_ns =3D 0, ret =3D 0; /* At least the first dirty chunk is mirrored in one iteration. */ int nb_chunks =3D 1; bool write_zeroes_ok =3D bdrv_can_write_zeroes_with_unmap(blk_bs(s->ta= rget)); @@ -399,11 +446,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBl= ockJob *s) } bdrv_dirty_bitmap_unlock(s->dirty_bitmap); =20 - first_chunk =3D offset / s->granularity; - while (test_bit(first_chunk, s->in_flight_bitmap)) { - trace_mirror_yield_in_flight(s, offset, s->in_flight); - mirror_wait_for_io(s); - } + mirror_wait_on_conflicts(NULL, s, offset, 1); =20 job_pause_point(&s->common.job); =20 @@ -440,6 +483,21 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBl= ockJob *s) nb_chunks * s->granularity); bdrv_dirty_bitmap_unlock(s->dirty_bitmap); =20 + /* Before claiming an area in the in-flight bitmap, we have to + * create a MirrorOp for it so that conflicting requests can wait + * for it. mirror_perform() will create the real MirrorOps later, + * for now we just create a pseudo operation that will wake up all + * conflicting requests once all real operations have been + * launched. */ + pseudo_op =3D g_new(MirrorOp, 1); + *pseudo_op =3D (MirrorOp){ + .offset =3D offset, + .bytes =3D nb_chunks * s->granularity, + .is_pseudo_op =3D true, + }; + qemu_co_queue_init(&pseudo_op->waiting_requests); + QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next); + bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks); while (nb_chunks > 0 && offset < s->bdev_length) { int ret; @@ -475,11 +533,12 @@ static uint64_t coroutine_fn mirror_iteration(MirrorB= lockJob *s) =20 while (s->in_flight >=3D MAX_IN_FLIGHT) { trace_mirror_yield_in_flight(s, offset, s->in_flight); - mirror_wait_for_io(s); + mirror_wait_for_free_in_flight_slot(s); } =20 if (s->ret < 0) { - return 0; + ret =3D 0; + goto fail; } =20 io_bytes =3D mirror_clip_bytes(s, offset, io_bytes); @@ -494,7 +553,14 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBl= ockJob *s) nb_chunks -=3D DIV_ROUND_UP(io_bytes, s->granularity); delay_ns =3D block_job_ratelimit_get_delay(&s->common, io_bytes_ac= ct); } - return delay_ns; + + ret =3D delay_ns; +fail: + QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next); + qemu_co_queue_restart_all(&pseudo_op->waiting_requests); + g_free(pseudo_op); + + return ret; } =20 static void mirror_free_init(MirrorBlockJob *s) @@ -521,7 +587,7 @@ static void mirror_free_init(MirrorBlockJob *s) static void mirror_wait_for_all_io(MirrorBlockJob *s) { while (s->in_flight > 0) { - mirror_wait_for_io(s); + mirror_wait_for_free_in_flight_slot(s); } } =20 @@ -676,7 +742,7 @@ static int coroutine_fn mirror_dirty_init(MirrorBlockJo= b *s) if (s->in_flight >=3D MAX_IN_FLIGHT) { trace_mirror_yield(s, UINT64_MAX, s->buf_free_count, s->in_flight); - mirror_wait_for_io(s); + mirror_wait_for_free_in_flight_slot(s); continue; } =20 @@ -849,7 +915,7 @@ static void coroutine_fn mirror_run(void *opaque) if (s->in_flight >=3D MAX_IN_FLIGHT || s->buf_free_count =3D= =3D 0 || (cnt =3D=3D 0 && s->in_flight > 0)) { trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight= ); - mirror_wait_for_io(s); + mirror_wait_for_free_in_flight_slot(s); continue; } else if (cnt !=3D 0) { delay_ns =3D mirror_iteration(s); --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 152891411223240.19755341126995; Wed, 13 Jun 2018 11:21:52 -0700 (PDT) Received: from localhost ([::1]:36172 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAP1-0005PC-CK for importer@patchew.org; Wed, 13 Jun 2018 14:21:51 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56711) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAM8-0003HM-VM for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:56 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAM7-0005J6-3p for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:52 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58832 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAM2-00051s-JQ; Wed, 13 Jun 2018 14:18:46 -0400 Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 409AE87A80; Wed, 13 Jun 2018 18:18:46 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id BC5BB2026609; Wed, 13 Jun 2018 18:18:45 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:14 +0200 Message-Id: <20180613181823.13618-6-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:46 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:18:46 +0000 (UTC) for IP:'10.11.54.4' DOMAIN:'int-mx04.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 05/14] block/mirror: Use source as a BdrvChild X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" With this, the mirror_top_bs is no longer just a technically required node in the BDS graph but actually represents the block job operation. Also, drop MirrorBlockJob.source, as we can reach it through mirror_top_bs->backing. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: Alberto Garcia --- block/mirror.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/block/mirror.c b/block/mirror.c index 7c1b8ef5b6..fa94fd9d17 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -41,7 +41,6 @@ typedef struct MirrorBlockJob { BlockJob common; BlockBackend *target; BlockDriverState *mirror_top_bs; - BlockDriverState *source; BlockDriverState *base; =20 /* The name of the graph node to replace */ @@ -299,7 +298,6 @@ static void coroutine_fn mirror_co_read(void *opaque) { MirrorOp *op =3D opaque; MirrorBlockJob *s =3D op->s; - BlockBackend *source =3D s->common.blk; int nb_chunks; uint64_t ret; uint64_t max_bytes; @@ -349,7 +347,8 @@ static void coroutine_fn mirror_co_read(void *opaque) s->bytes_in_flight +=3D op->bytes; trace_mirror_one_iteration(s, op->offset, op->bytes); =20 - ret =3D blk_co_preadv(source, op->offset, op->bytes, &op->qiov, 0); + ret =3D bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->byte= s, + &op->qiov, 0); mirror_read_complete(op, ret); } =20 @@ -427,7 +426,7 @@ static unsigned mirror_perform(MirrorBlockJob *s, int64= _t offset, =20 static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s) { - BlockDriverState *source =3D s->source; + BlockDriverState *source =3D s->mirror_top_bs->backing->bs; MirrorOp *pseudo_op; int64_t offset; uint64_t delay_ns =3D 0, ret =3D 0; @@ -601,7 +600,7 @@ static void mirror_exit(Job *job, void *opaque) BlockJob *bjob =3D &s->common; MirrorExitData *data =3D opaque; AioContext *replace_aio_context =3D NULL; - BlockDriverState *src =3D s->source; + BlockDriverState *src =3D s->mirror_top_bs->backing->bs; BlockDriverState *target_bs =3D blk_bs(s->target); BlockDriverState *mirror_top_bs =3D s->mirror_top_bs; Error *local_err =3D NULL; @@ -716,7 +715,7 @@ static int coroutine_fn mirror_dirty_init(MirrorBlockJo= b *s) { int64_t offset; BlockDriverState *base =3D s->base; - BlockDriverState *bs =3D s->source; + BlockDriverState *bs =3D s->mirror_top_bs->backing->bs; BlockDriverState *target_bs =3D blk_bs(s->target); int ret; int64_t count; @@ -798,7 +797,7 @@ static void coroutine_fn mirror_run(void *opaque) { MirrorBlockJob *s =3D opaque; MirrorExitData *data; - BlockDriverState *bs =3D s->source; + BlockDriverState *bs =3D s->mirror_top_bs->backing->bs; BlockDriverState *target_bs =3D blk_bs(s->target); bool need_drain =3D true; int64_t length; @@ -1285,7 +1284,6 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, /* The block job now has a reference to this node */ bdrv_unref(mirror_top_bs); =20 - s->source =3D bs; s->mirror_top_bs =3D mirror_top_bs; =20 /* No resize for the target either; while the mirror is still running,= a --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914319584820.321591815224; Wed, 13 Jun 2018 11:25:19 -0700 (PDT) Received: from localhost ([::1]:36196 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTASM-0008IP-PW for importer@patchew.org; Wed, 13 Jun 2018 14:25:18 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56751) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMB-0003Ib-Tl for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:00 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAM9-0005Rs-DS for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:18:54 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58890 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAM5-0005A1-1v; Wed, 13 Jun 2018 14:18:49 -0400 Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id A8D75400ADE0; Wed, 13 Jun 2018 18:18:48 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 455C42026609; Wed, 13 Jun 2018 18:18:48 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:15 +0200 Message-Id: <20180613181823.13618-7-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.6]); Wed, 13 Jun 2018 18:18:48 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.6]); Wed, 13 Jun 2018 18:18:48 +0000 (UTC) for IP:'10.11.54.4' DOMAIN:'int-mx04.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 06/14] block: Generalize should_update_child() rule X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Currently, bdrv_replace_node() refuses to create loops from one BDS to itself if the BDS to be replaced is the backing node of the BDS to replace it: Say there is a node A and a node B. Replacing B by A means making all references to B point to A. If B is a child of A (i.e. A has a reference to B), that would mean we would have to make this reference point to A itself -- so we'd create a loop. bdrv_replace_node() (through should_update_child()) refuses to do so if B is the backing node of A. There is no reason why we should create loops if B is not the backing node of A, though. The BDS graph should never contain loops, so we should always refuse to create them. If B is a child of A and B is to be replaced by A, we should simply leave B in place there because it is the most sensible choice. A more specific argument would be: Putting filter drivers into the BDS graph is basically the same as appending an overlay to a backing chain. But the main child BDS of a filter driver is not "backing" but "file", so restricting the no-loop rule to backing nodes would fail here. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: Alberto Garcia --- block.c | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/block.c b/block.c index 50887087f3..df6dce6a73 100644 --- a/block.c +++ b/block.c @@ -3394,16 +3394,39 @@ static bool should_update_child(BdrvChild *c, Block= DriverState *to) return false; } =20 - if (c->role =3D=3D &child_backing) { - /* If @from is a backing file of @to, ignore the child to avoid - * creating a loop. We only want to change the pointer of other - * parents. */ - QLIST_FOREACH(to_c, &to->children, next) { - if (to_c =3D=3D c) { - break; - } - } - if (to_c) { + /* If the child @c belongs to the BDS @to, replacing the current + * c->bs by @to would mean to create a loop. + * + * Such a case occurs when appending a BDS to a backing chain. + * For instance, imagine the following chain: + * + * guest device -> node A -> further backing chain... + * + * Now we create a new BDS B which we want to put on top of this + * chain, so we first attach A as its backing node: + * + * node B + * | + * v + * guest device -> node A -> further backing chain... + * + * Finally we want to replace A by B. When doing that, we want to + * replace all pointers to A by pointers to B -- except for the + * pointer from B because (1) that would create a loop, and (2) + * that pointer should simply stay intact: + * + * guest device -> node B + * | + * v + * node A -> further backing chain... + * + * In general, when replacing a node A (c->bs) by a node B (@to), + * if A is a child of B, that means we cannot replace A by B there + * because that would create a loop. Silently detaching A from B + * is also not really an option. So overall just leaving A in + * place there is the most sensible choice. */ + QLIST_FOREACH(to_c, &to->children, next) { + if (to_c =3D=3D c) { return false; } } @@ -3429,6 +3452,7 @@ void bdrv_replace_node(BlockDriverState *from, BlockD= riverState *to, =20 /* Put all parents into @list and calculate their cumulative permissio= ns */ QLIST_FOREACH_SAFE(c, &from->parents, next_parent, next) { + assert(c->bs =3D=3D from); if (!should_update_child(c, to)) { continue; } --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 152891443724421.598099974810225; Wed, 13 Jun 2018 11:27:17 -0700 (PDT) Received: from localhost ([::1]:36215 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAUD-0001pg-26 for importer@patchew.org; Wed, 13 Jun 2018 14:27:13 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56871) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMJ-0003PY-T4 for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:05 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMI-0005yo-FJ for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:03 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:42594 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMB-0005Zx-Qn; Wed, 13 Jun 2018 14:18:55 -0400 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 6BC84401CE88; Wed, 13 Jun 2018 18:18:55 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 9B4241C640; Wed, 13 Jun 2018 18:18:50 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:16 +0200 Message-Id: <20180613181823.13618-8-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.79 on 10.11.54.5 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.5]); Wed, 13 Jun 2018 18:18:55 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.5]); Wed, 13 Jun 2018 18:18:55 +0000 (UTC) for IP:'10.11.54.5' DOMAIN:'int-mx05.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 07/14] hbitmap: Add @advance param to hbitmap_iter_next() X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This new parameter allows the caller to just query the next dirty position without moving the iterator. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: John Snow --- include/qemu/hbitmap.h | 5 ++++- block/backup.c | 2 +- block/dirty-bitmap.c | 2 +- tests/test-hbitmap.c | 26 +++++++++++++------------- util/hbitmap.c | 10 +++++++--- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h index 6b6490ecad..ddca52c48e 100644 --- a/include/qemu/hbitmap.h +++ b/include/qemu/hbitmap.h @@ -324,11 +324,14 @@ void hbitmap_free_meta(HBitmap *hb); /** * hbitmap_iter_next: * @hbi: HBitmapIter to operate on. + * @advance: If true, advance the iterator. Otherwise, the next call + * of this function will return the same result (if that + * position is still dirty). * * Return the next bit that is set in @hbi's associated HBitmap, * or -1 if all remaining bits are zero. */ -int64_t hbitmap_iter_next(HBitmapIter *hbi); +int64_t hbitmap_iter_next(HBitmapIter *hbi, bool advance); =20 /** * hbitmap_iter_next_word: diff --git a/block/backup.c b/block/backup.c index 5661435675..d18be40caf 100644 --- a/block/backup.c +++ b/block/backup.c @@ -354,7 +354,7 @@ static int coroutine_fn backup_run_incremental(BackupBl= ockJob *job) HBitmapIter hbi; =20 hbitmap_iter_init(&hbi, job->copy_bitmap, 0); - while ((cluster =3D hbitmap_iter_next(&hbi)) !=3D -1) { + while ((cluster =3D hbitmap_iter_next(&hbi, true)) !=3D -1) { do { if (yield_and_check(job)) { return 0; diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c index 383d742cdb..cedb971765 100644 --- a/block/dirty-bitmap.c +++ b/block/dirty-bitmap.c @@ -519,7 +519,7 @@ void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter) =20 int64_t bdrv_dirty_iter_next(BdrvDirtyBitmapIter *iter) { - return hbitmap_iter_next(&iter->hbi); + return hbitmap_iter_next(&iter->hbi, true); } =20 /* Called within bdrv_dirty_bitmap_lock..unlock */ diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c index f29631f939..f2158f767d 100644 --- a/tests/test-hbitmap.c +++ b/tests/test-hbitmap.c @@ -46,7 +46,7 @@ static void hbitmap_test_check(TestHBitmapData *data, =20 i =3D first; for (;;) { - next =3D hbitmap_iter_next(&hbi); + next =3D hbitmap_iter_next(&hbi, true); if (next < 0) { next =3D data->size; } @@ -435,25 +435,25 @@ static void test_hbitmap_iter_granularity(TestHBitmap= Data *data, /* Note that hbitmap_test_check has to be invoked manually in this tes= t. */ hbitmap_test_init(data, 131072 << 7, 7); hbitmap_iter_init(&hbi, data->hb, 0); - g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); =20 hbitmap_test_set(data, ((L2 + L1 + 1) << 7) + 8, 8); hbitmap_iter_init(&hbi, data->hb, 0); - g_assert_cmpint(hbitmap_iter_next(&hbi), =3D=3D, (L2 + L1 + 1) << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, (L2 + L1 + 1) <= < 7); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); =20 hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); =20 hbitmap_test_set(data, (131072 << 7) - 8, 8); hbitmap_iter_init(&hbi, data->hb, 0); - g_assert_cmpint(hbitmap_iter_next(&hbi), =3D=3D, (L2 + L1 + 1) << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi), =3D=3D, 131071 << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, (L2 + L1 + 1) <= < 7); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, 131071 << 7); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); =20 hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi), =3D=3D, 131071 << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, 131071 << 7); + g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); } =20 static void hbitmap_test_set_boundary_bits(TestHBitmapData *data, ssize_t = diff) @@ -893,7 +893,7 @@ static void test_hbitmap_serialize_zeroes(TestHBitmapDa= ta *data, for (i =3D 0; i < num_positions; i++) { hbitmap_deserialize_zeroes(data->hb, positions[i], min_l1, true); hbitmap_iter_init(&iter, data->hb, 0); - next =3D hbitmap_iter_next(&iter); + next =3D hbitmap_iter_next(&iter, true); if (i =3D=3D num_positions - 1) { g_assert_cmpint(next, =3D=3D, -1); } else { @@ -919,10 +919,10 @@ static void test_hbitmap_iter_and_reset(TestHBitmapDa= ta *data, =20 hbitmap_iter_init(&hbi, data->hb, BITS_PER_LONG - 1); =20 - hbitmap_iter_next(&hbi); + hbitmap_iter_next(&hbi, true); =20 hbitmap_reset_all(data->hb); - hbitmap_iter_next(&hbi); + hbitmap_iter_next(&hbi, true); } =20 static void test_hbitmap_next_zero_check(TestHBitmapData *data, int64_t st= art) diff --git a/util/hbitmap.c b/util/hbitmap.c index 58a2c93842..bcd304041a 100644 --- a/util/hbitmap.c +++ b/util/hbitmap.c @@ -141,7 +141,7 @@ unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi) return cur; } =20 -int64_t hbitmap_iter_next(HBitmapIter *hbi) +int64_t hbitmap_iter_next(HBitmapIter *hbi, bool advance) { unsigned long cur =3D hbi->cur[HBITMAP_LEVELS - 1] & hbi->hb->levels[HBITMAP_LEVELS - 1][hbi->pos]; @@ -154,8 +154,12 @@ int64_t hbitmap_iter_next(HBitmapIter *hbi) } } =20 - /* The next call will resume work from the next bit. */ - hbi->cur[HBITMAP_LEVELS - 1] =3D cur & (cur - 1); + if (advance) { + /* The next call will resume work from the next bit. */ + hbi->cur[HBITMAP_LEVELS - 1] =3D cur & (cur - 1); + } else { + hbi->cur[HBITMAP_LEVELS - 1] =3D cur; + } item =3D ((uint64_t)hbi->pos << BITS_PER_LEVEL) + ctzl(cur); =20 return item << hbi->granularity; --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 15289144811261008.1703189599326; Wed, 13 Jun 2018 11:28:01 -0700 (PDT) Received: from localhost ([::1]:36217 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAUy-0002dZ-BO for importer@patchew.org; Wed, 13 Jun 2018 14:28:00 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:56923) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMM-0003Rn-Gj for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:11 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAML-00069l-A0 for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:06 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:38574 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMI-0005yJ-KC; Wed, 13 Jun 2018 14:19:02 -0400 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 3D7008185331; Wed, 13 Jun 2018 18:19:02 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 7B58D10FFE4D; Wed, 13 Jun 2018 18:18:57 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:17 +0200 Message-Id: <20180613181823.13618-9-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.3 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:19:02 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:19:02 +0000 (UTC) for IP:'10.11.54.3' DOMAIN:'int-mx03.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 08/14] test-hbitmap: Add non-advancing iter_next tests X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add a function that wraps hbitmap_iter_next() and always calls it in non-advancing mode first, and in advancing mode next. The result should always be the same. By using this function everywhere we called hbitmap_iter_next() before, we should get good test coverage for non-advancing hbitmap_iter_next(). Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: John Snow --- tests/test-hbitmap.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c index f2158f767d..5e67ac1d3a 100644 --- a/tests/test-hbitmap.c +++ b/tests/test-hbitmap.c @@ -30,6 +30,18 @@ typedef struct TestHBitmapData { } TestHBitmapData; =20 =20 +static int64_t check_hbitmap_iter_next(HBitmapIter *hbi) +{ + int next0, next1; + + next0 =3D hbitmap_iter_next(hbi, false); + next1 =3D hbitmap_iter_next(hbi, true); + + g_assert_cmpint(next0, =3D=3D, next1); + + return next0; +} + /* Check that the HBitmap and the shadow bitmap contain the same data, * ignoring the same "first" bits. */ @@ -46,7 +58,7 @@ static void hbitmap_test_check(TestHBitmapData *data, =20 i =3D first; for (;;) { - next =3D hbitmap_iter_next(&hbi, true); + next =3D check_hbitmap_iter_next(&hbi); if (next < 0) { next =3D data->size; } @@ -435,25 +447,25 @@ static void test_hbitmap_iter_granularity(TestHBitmap= Data *data, /* Note that hbitmap_test_check has to be invoked manually in this tes= t. */ hbitmap_test_init(data, 131072 << 7, 7); hbitmap_iter_init(&hbi, data->hb, 0); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), <, 0); =20 hbitmap_test_set(data, ((L2 + L1 + 1) << 7) + 8, 8); hbitmap_iter_init(&hbi, data->hb, 0); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, (L2 + L1 + 1) <= < 7); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), =3D=3D, (L2 + L1 + 1) <= < 7); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), <, 0); =20 hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7); g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); =20 hbitmap_test_set(data, (131072 << 7) - 8, 8); hbitmap_iter_init(&hbi, data->hb, 0); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, (L2 + L1 + 1) <= < 7); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, 131071 << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), =3D=3D, (L2 + L1 + 1) <= < 7); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), =3D=3D, 131071 << 7); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), <, 0); =20 hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), =3D=3D, 131071 << 7); - g_assert_cmpint(hbitmap_iter_next(&hbi, true), <, 0); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), =3D=3D, 131071 << 7); + g_assert_cmpint(check_hbitmap_iter_next(&hbi), <, 0); } =20 static void hbitmap_test_set_boundary_bits(TestHBitmapData *data, ssize_t = diff) @@ -893,7 +905,7 @@ static void test_hbitmap_serialize_zeroes(TestHBitmapDa= ta *data, for (i =3D 0; i < num_positions; i++) { hbitmap_deserialize_zeroes(data->hb, positions[i], min_l1, true); hbitmap_iter_init(&iter, data->hb, 0); - next =3D hbitmap_iter_next(&iter, true); + next =3D check_hbitmap_iter_next(&iter); if (i =3D=3D num_positions - 1) { g_assert_cmpint(next, =3D=3D, -1); } else { @@ -919,10 +931,10 @@ static void test_hbitmap_iter_and_reset(TestHBitmapDa= ta *data, =20 hbitmap_iter_init(&hbi, data->hb, BITS_PER_LONG - 1); =20 - hbitmap_iter_next(&hbi, true); + check_hbitmap_iter_next(&hbi); =20 hbitmap_reset_all(data->hb); - hbitmap_iter_next(&hbi, true); + check_hbitmap_iter_next(&hbi); } =20 static void test_hbitmap_next_zero_check(TestHBitmapData *data, int64_t st= art) --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914665613215.55134369814414; Wed, 13 Jun 2018 11:31:05 -0700 (PDT) Received: from localhost ([::1]:36243 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAXp-0005UF-L9 for importer@patchew.org; Wed, 13 Jun 2018 14:30:57 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:57054) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMT-0003Ya-0k for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:14 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMR-0006XL-OP for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:13 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58838 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAML-00067U-1X; Wed, 13 Jun 2018 14:19:05 -0400 Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.rdu2.redhat.com [10.11.54.6]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id AAA4387A80; Wed, 13 Jun 2018 18:19:04 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 3EC03215672D; Wed, 13 Jun 2018 18:19:04 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:18 +0200 Message-Id: <20180613181823.13618-10-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.6 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:19:04 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:19:04 +0000 (UTC) for IP:'10.11.54.6' DOMAIN:'int-mx06.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 09/14] block/dirty-bitmap: Add bdrv_dirty_iter_next_area X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This new function allows to look for a consecutively dirty area in a dirty bitmap. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: John Snow --- include/block/dirty-bitmap.h | 2 ++ block/dirty-bitmap.c | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h index 02e0cbabd2..288dc6adb6 100644 --- a/include/block/dirty-bitmap.h +++ b/include/block/dirty-bitmap.h @@ -82,6 +82,8 @@ void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap, void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap, int64_t offset, int64_t bytes); int64_t bdrv_dirty_iter_next(BdrvDirtyBitmapIter *iter); +bool bdrv_dirty_iter_next_area(BdrvDirtyBitmapIter *iter, uint64_t max_off= set, + uint64_t *offset, int *bytes); void bdrv_set_dirty_iter(BdrvDirtyBitmapIter *hbi, int64_t offset); int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap); int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap); diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c index cedb971765..db1782ec1f 100644 --- a/block/dirty-bitmap.c +++ b/block/dirty-bitmap.c @@ -522,6 +522,61 @@ int64_t bdrv_dirty_iter_next(BdrvDirtyBitmapIter *iter) return hbitmap_iter_next(&iter->hbi, true); } =20 +/** + * Return the next consecutively dirty area in the dirty bitmap + * belonging to the given iterator @iter. + * + * @max_offset: Maximum value that may be returned for + * *offset + *bytes + * @offset: Will contain the start offset of the next dirty area + * @bytes: Will contain the length of the next dirty area + * + * Returns: True if a dirty area could be found before max_offset + * (which means that *offset and *bytes then contain valid + * values), false otherwise. + * + * Note that @iter is never advanced if false is returned. If an area + * is found (which means that true is returned), it will be advanced + * past that area. + */ +bool bdrv_dirty_iter_next_area(BdrvDirtyBitmapIter *iter, uint64_t max_off= set, + uint64_t *offset, int *bytes) +{ + uint32_t granularity =3D bdrv_dirty_bitmap_granularity(iter->bitmap); + uint64_t gran_max_offset; + int64_t ret; + int size; + + if (max_offset =3D=3D iter->bitmap->size) { + /* If max_offset points to the image end, round it up by the + * bitmap granularity */ + gran_max_offset =3D ROUND_UP(max_offset, granularity); + } else { + gran_max_offset =3D max_offset; + } + + ret =3D hbitmap_iter_next(&iter->hbi, false); + if (ret < 0 || ret + granularity > gran_max_offset) { + return false; + } + + *offset =3D ret; + size =3D 0; + + assert(granularity <=3D INT_MAX); + + do { + /* Advance iterator */ + ret =3D hbitmap_iter_next(&iter->hbi, true); + size +=3D granularity; + } while (ret + granularity <=3D gran_max_offset && + hbitmap_iter_next(&iter->hbi, false) =3D=3D ret + granularity= && + size <=3D INT_MAX - granularity); + + *bytes =3D MIN(size, max_offset - *offset); + return true; +} + /* Called within bdrv_dirty_bitmap_lock..unlock */ void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap, int64_t offset, int64_t bytes) --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914488497321.9249544778746; Wed, 13 Jun 2018 11:28:08 -0700 (PDT) Received: from localhost ([::1]:36218 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAV5-0002jY-Hl for importer@patchew.org; Wed, 13 Jun 2018 14:28:07 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:57082) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMT-0003ZT-Uf for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:14 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMS-0006cn-SG for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:13 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58840 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMN-0006GD-Fw; Wed, 13 Jun 2018 14:19:07 -0400 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 24A4487A80; Wed, 13 Jun 2018 18:19:07 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id B01A41C664; Wed, 13 Jun 2018 18:19:06 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:19 +0200 Message-Id: <20180613181823.13618-11-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.79 on 10.11.54.5 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:19:07 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:19:07 +0000 (UTC) for IP:'10.11.54.5' DOMAIN:'int-mx05.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 10/14] block/mirror: Add MirrorBDSOpaque X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This will allow us to access the block job data when the mirror block driver becomes more complex. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng --- block/mirror.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/block/mirror.c b/block/mirror.c index fa94fd9d17..f060bd8bb4 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -76,6 +76,10 @@ typedef struct MirrorBlockJob { bool initial_zeroing_ongoing; } MirrorBlockJob; =20 +typedef struct MirrorBDSOpaque { + MirrorBlockJob *job; +} MirrorBDSOpaque; + struct MirrorOp { MirrorBlockJob *s; QEMUIOVector qiov; @@ -599,6 +603,7 @@ static void mirror_exit(Job *job, void *opaque) MirrorBlockJob *s =3D container_of(job, MirrorBlockJob, common.job); BlockJob *bjob =3D &s->common; MirrorExitData *data =3D opaque; + MirrorBDSOpaque *bs_opaque =3D s->mirror_top_bs->opaque; AioContext *replace_aio_context =3D NULL; BlockDriverState *src =3D s->mirror_top_bs->backing->bs; BlockDriverState *target_bs =3D blk_bs(s->target); @@ -691,6 +696,7 @@ static void mirror_exit(Job *job, void *opaque) blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort); blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort); =20 + bs_opaque->job =3D NULL; job_completed(job, data->ret, NULL); =20 g_free(data); @@ -1222,6 +1228,7 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, Error **errp) { MirrorBlockJob *s; + MirrorBDSOpaque *bs_opaque; BlockDriverState *mirror_top_bs; bool target_graph_mod; bool target_is_backing; @@ -1257,6 +1264,8 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, mirror_top_bs->total_sectors =3D bs->total_sectors; mirror_top_bs->supported_write_flags =3D BDRV_REQ_WRITE_UNCHANGED; mirror_top_bs->supported_zero_flags =3D BDRV_REQ_WRITE_UNCHANGED; + bs_opaque =3D g_new0(MirrorBDSOpaque, 1); + mirror_top_bs->opaque =3D bs_opaque; bdrv_set_aio_context(mirror_top_bs, bdrv_get_aio_context(bs)); =20 /* bdrv_append takes ownership of the mirror_top_bs reference, need to= keep @@ -1281,6 +1290,8 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, if (!s) { goto fail; } + bs_opaque->job =3D s; + /* The block job now has a reference to this node */ bdrv_unref(mirror_top_bs); =20 @@ -1370,6 +1381,7 @@ fail: =20 g_free(s->replaces); blk_unref(s->target); + bs_opaque->job =3D NULL; job_early_fail(&s->common.job); } =20 --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914860864855.8328746185555; Wed, 13 Jun 2018 11:34:20 -0700 (PDT) Received: from localhost ([::1]:36257 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAb6-000898-1F for importer@patchew.org; Wed, 13 Jun 2018 14:34:20 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:57085) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMT-0003ZW-W5 for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:15 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMS-0006cQ-Oo for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:13 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58844 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMQ-0006Oj-19; Wed, 13 Jun 2018 14:19:10 -0400 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id AB49487A80; Wed, 13 Jun 2018 18:19:09 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 4307910FFE4D; Wed, 13 Jun 2018 18:19:09 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:20 +0200 Message-Id: <20180613181823.13618-12-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.3 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:19:09 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.1]); Wed, 13 Jun 2018 18:19:09 +0000 (UTC) for IP:'10.11.54.3' DOMAIN:'int-mx03.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 11/14] job: Add job_progress_increase_remaining() X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Signed-off-by: Max Reitz Reviewed-by: Kevin Wolf --- include/qemu/job.h | 15 +++++++++++++++ job.c | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/include/qemu/job.h b/include/qemu/job.h index 1d820530fa..18c9223e31 100644 --- a/include/qemu/job.h +++ b/include/qemu/job.h @@ -335,6 +335,21 @@ void job_progress_update(Job *job, uint64_t done); */ void job_progress_set_remaining(Job *job, uint64_t remaining); =20 +/** + * @job: The job whose expected progress end value is updated + * @delta: Value which is to be added to the current expected end + * value + * + * Increases the expected end value of the progress counter of a job. + * This is useful for parenthesis operations: If a job has to + * conditionally perform a high-priority operation as part of its + * progress, it calls this function with the expected operation's + * length before, and job_progress_update() afterwards. + * (So the operation acts as a parenthesis in regards to the main job + * operation running in background.) + */ +void job_progress_increase_remaining(Job *job, uint64_t delta); + /** To be called when a cancelled job is finalised. */ void job_event_cancelled(Job *job); =20 diff --git a/job.c b/job.c index 84e140238b..fa671b431a 100644 --- a/job.c +++ b/job.c @@ -385,6 +385,11 @@ void job_progress_set_remaining(Job *job, uint64_t rem= aining) job->progress_total =3D job->progress_current + remaining; } =20 +void job_progress_increase_remaining(Job *job, uint64_t delta) +{ + job->progress_total +=3D delta; +} + void job_event_cancelled(Job *job) { notifier_list_notify(&job->on_finalize_cancelled, job); --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914288698890.161537480607; Wed, 13 Jun 2018 11:24:48 -0700 (PDT) Received: from localhost ([::1]:36194 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTARr-0007rF-Ov for importer@patchew.org; Wed, 13 Jun 2018 14:24:47 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:57253) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMc-0003gw-V5 for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:25 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMX-0006uC-So for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:22 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:38582 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMS-0006ZR-G2; Wed, 13 Jun 2018 14:19:12 -0400 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 20D9B8185331; Wed, 13 Jun 2018 18:19:12 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 80A5F10FFE4D; Wed, 13 Jun 2018 18:19:11 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:21 +0200 Message-Id: <20180613181823.13618-13-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.3 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:19:12 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:19:12 +0000 (UTC) for IP:'10.11.54.3' DOMAIN:'int-mx03.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 12/14] block/mirror: Add active mirroring X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This patch implements active synchronous mirroring. In active mode, the passive mechanism will still be in place and is used to copy all initially dirty clusters off the source disk; but every write request will write data both to the source and the target disk, so the source cannot be dirtied faster than data is mirrored to the target. Also, once the block job has converged (BLOCK_JOB_READY sent), source and target are guaranteed to stay in sync (unless an error occurs). Active mode is completely optional and currently disabled at runtime. A later patch will add a way for users to enable it. Signed-off-by: Max Reitz Reviewed-by: Fam Zheng --- qapi/block-core.json | 18 ++++ block/mirror.c | 252 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 265 insertions(+), 5 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index fff23fc82b..11b55530b3 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -1050,6 +1050,24 @@ { 'enum': 'MirrorSyncMode', 'data': ['top', 'full', 'none', 'incremental'] } =20 +## +# @MirrorCopyMode: +# +# An enumeration whose values tell the mirror block job when to +# trigger writes to the target. +# +# @background: copy data in background only. +# +# @write-blocking: when data is written to the source, write it +# (synchronously) to the target as well. In +# addition, data is copied in background just like in +# @background mode. +# +# Since: 3.0 +## +{ 'enum': 'MirrorCopyMode', + 'data': ['background', 'write-blocking'] } + ## # @BlockJobInfo: # diff --git a/block/mirror.c b/block/mirror.c index f060bd8bb4..b58d7bbce1 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -51,8 +51,12 @@ typedef struct MirrorBlockJob { Error *replace_blocker; bool is_none_mode; BlockMirrorBackingMode backing_mode; + MirrorCopyMode copy_mode; BlockdevOnError on_source_error, on_target_error; bool synced; + /* Set when the target is synced (dirty bitmap is clean, nothing + * in flight) and the job is running in active mode */ + bool actively_synced; bool should_complete; int64_t granularity; size_t buf_size; @@ -74,6 +78,7 @@ typedef struct MirrorBlockJob { int target_cluster_size; int max_iov; bool initial_zeroing_ongoing; + int in_active_write_counter; } MirrorBlockJob; =20 typedef struct MirrorBDSOpaque { @@ -91,6 +96,7 @@ struct MirrorOp { int64_t *bytes_handled; =20 bool is_pseudo_op; + bool is_active_write; CoQueue waiting_requests; =20 QTAILQ_ENTRY(MirrorOp) next; @@ -106,6 +112,7 @@ static BlockErrorAction mirror_error_action(MirrorBlock= Job *s, bool read, int error) { s->synced =3D false; + s->actively_synced =3D false; if (read) { return block_job_error_action(&s->common, s->on_source_error, true, error); @@ -272,7 +279,7 @@ static int mirror_cow_align(MirrorBlockJob *s, int64_t = *offset, return ret; } =20 -static inline void mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s) +static inline void mirror_wait_for_any_operation(MirrorBlockJob *s, bool a= ctive) { MirrorOp *op; =20 @@ -282,7 +289,7 @@ static inline void mirror_wait_for_free_in_flight_slot(= MirrorBlockJob *s) * caller of this function. Since there is only one pseudo op * at any given time, we will always find some real operation * to wait on. */ - if (!op->is_pseudo_op) { + if (!op->is_pseudo_op && op->is_active_write =3D=3D active) { qemu_co_queue_wait(&op->waiting_requests, NULL); return; } @@ -290,6 +297,12 @@ static inline void mirror_wait_for_free_in_flight_slot= (MirrorBlockJob *s) abort(); } =20 +static inline void mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s) +{ + /* Only non-active operations use up in-flight slots */ + mirror_wait_for_any_operation(s, false); +} + /* Perform a mirror copy operation. * * *op->bytes_handled is set to the number of bytes copied after and @@ -846,6 +859,7 @@ static void coroutine_fn mirror_run(void *opaque) /* Transition to the READY state and wait for complete. */ job_transition_to_ready(&s->common.job); s->synced =3D true; + s->actively_synced =3D true; while (!job_is_cancelled(&s->common.job) && !s->should_complete) { job_yield(&s->common.job); } @@ -897,6 +911,12 @@ static void coroutine_fn mirror_run(void *opaque) int64_t cnt, delta; bool should_complete; =20 + /* Do not start passive operations while there are active + * writes in progress */ + while (s->in_active_write_counter) { + mirror_wait_for_any_operation(s, true); + } + if (s->ret < 0) { ret =3D s->ret; goto immediate_exit; @@ -942,6 +962,9 @@ static void coroutine_fn mirror_run(void *opaque) */ job_transition_to_ready(&s->common.job); s->synced =3D true; + if (s->copy_mode !=3D MIRROR_COPY_MODE_BACKGROUND) { + s->actively_synced =3D true; + } } =20 should_complete =3D s->should_complete || @@ -1132,16 +1155,232 @@ static const BlockJobDriver commit_active_job_driv= er =3D { .drain =3D mirror_drain, }; =20 +static void do_sync_target_write(MirrorBlockJob *job, MirrorMethod method, + uint64_t offset, uint64_t bytes, + QEMUIOVector *qiov, int flags) +{ + BdrvDirtyBitmapIter *iter; + QEMUIOVector target_qiov; + uint64_t dirty_offset; + int dirty_bytes; + + if (qiov) { + qemu_iovec_init(&target_qiov, qiov->niov); + } + + iter =3D bdrv_dirty_iter_new(job->dirty_bitmap); + bdrv_set_dirty_iter(iter, offset); + + while (true) { + bool valid_area; + int ret; + + bdrv_dirty_bitmap_lock(job->dirty_bitmap); + valid_area =3D bdrv_dirty_iter_next_area(iter, offset + bytes, + &dirty_offset, &dirty_bytes= ); + if (!valid_area) { + bdrv_dirty_bitmap_unlock(job->dirty_bitmap); + break; + } + + bdrv_reset_dirty_bitmap_locked(job->dirty_bitmap, + dirty_offset, dirty_bytes); + bdrv_dirty_bitmap_unlock(job->dirty_bitmap); + + job_progress_increase_remaining(&job->common.job, dirty_bytes); + + assert(dirty_offset - offset <=3D SIZE_MAX); + if (qiov) { + qemu_iovec_reset(&target_qiov); + qemu_iovec_concat(&target_qiov, qiov, + dirty_offset - offset, dirty_bytes); + } + + switch (method) { + case MIRROR_METHOD_COPY: + ret =3D blk_co_pwritev(job->target, dirty_offset, dirty_bytes, + qiov ? &target_qiov : NULL, flags); + break; + + case MIRROR_METHOD_ZERO: + assert(!qiov); + ret =3D blk_co_pwrite_zeroes(job->target, dirty_offset, dirty_= bytes, + flags); + break; + + case MIRROR_METHOD_DISCARD: + assert(!qiov); + ret =3D blk_co_pdiscard(job->target, dirty_offset, dirty_bytes= ); + break; + + default: + abort(); + } + + if (ret >=3D 0) { + job_progress_update(&job->common.job, dirty_bytes); + } else { + BlockErrorAction action; + + bdrv_set_dirty_bitmap(job->dirty_bitmap, dirty_offset, dirty_b= ytes); + job->actively_synced =3D false; + + action =3D mirror_error_action(job, false, -ret); + if (action =3D=3D BLOCK_ERROR_ACTION_REPORT) { + if (!job->ret) { + job->ret =3D ret; + } + break; + } + } + } + + bdrv_dirty_iter_free(iter); + if (qiov) { + qemu_iovec_destroy(&target_qiov); + } +} + +static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s, + uint64_t offset, + uint64_t bytes) +{ + MirrorOp *op; + uint64_t start_chunk =3D offset / s->granularity; + uint64_t end_chunk =3D DIV_ROUND_UP(offset + bytes, s->granularity); + + op =3D g_new(MirrorOp, 1); + *op =3D (MirrorOp){ + .s =3D s, + .offset =3D offset, + .bytes =3D bytes, + .is_active_write =3D true, + }; + qemu_co_queue_init(&op->waiting_requests); + QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next); + + s->in_active_write_counter++; + + mirror_wait_on_conflicts(op, s, offset, bytes); + + bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk); + + return op; +} + +static void coroutine_fn active_write_settle(MirrorOp *op) +{ + uint64_t start_chunk =3D op->offset / op->s->granularity; + uint64_t end_chunk =3D DIV_ROUND_UP(op->offset + op->bytes, + op->s->granularity); + + if (!--op->s->in_active_write_counter && op->s->actively_synced) { + BdrvChild *source =3D op->s->mirror_top_bs->backing; + + if (QLIST_FIRST(&source->bs->parents) =3D=3D source && + QLIST_NEXT(source, next_parent) =3D=3D NULL) + { + /* Assert that we are back in sync once all active write + * operations are settled. + * Note that we can only assert this if the mirror node + * is the source node's only parent. */ + assert(!bdrv_get_dirty_count(op->s->dirty_bitmap)); + } + } + bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_c= hunk); + QTAILQ_REMOVE(&op->s->ops_in_flight, op, next); + qemu_co_queue_restart_all(&op->waiting_requests); + g_free(op); +} + static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags) { return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags); } =20 +static int coroutine_fn bdrv_mirror_top_do_write(BlockDriverState *bs, + MirrorMethod method, uint64_t offset, uint64_t bytes, QEMUIOVector *qi= ov, + int flags) +{ + MirrorOp *op =3D NULL; + MirrorBDSOpaque *s =3D bs->opaque; + int ret =3D 0; + bool copy_to_target; + + copy_to_target =3D s->job->ret >=3D 0 && + s->job->copy_mode =3D=3D MIRROR_COPY_MODE_WRITE_BLOCK= ING; + + if (copy_to_target) { + op =3D active_write_prepare(s->job, offset, bytes); + } + + switch (method) { + case MIRROR_METHOD_COPY: + ret =3D bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags); + break; + + case MIRROR_METHOD_ZERO: + ret =3D bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags); + break; + + case MIRROR_METHOD_DISCARD: + ret =3D bdrv_co_pdiscard(bs->backing->bs, offset, bytes); + break; + + default: + abort(); + } + + if (ret < 0) { + goto out; + } + + if (copy_to_target) { + do_sync_target_write(s->job, method, offset, bytes, qiov, flags); + } + +out: + if (copy_to_target) { + active_write_settle(op); + } + return ret; +} + static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags) { - return bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags); + MirrorBDSOpaque *s =3D bs->opaque; + QEMUIOVector bounce_qiov; + void *bounce_buf; + int ret =3D 0; + bool copy_to_target; + + copy_to_target =3D s->job->ret >=3D 0 && + s->job->copy_mode =3D=3D MIRROR_COPY_MODE_WRITE_BLOCK= ING; + + if (copy_to_target) { + /* The guest might concurrently modify the data to write; but + * the data on source and destination must match, so we have + * to use a bounce buffer if we are going to write to the + * target now. */ + bounce_buf =3D qemu_blockalign(bs, bytes); + iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes); + + qemu_iovec_init(&bounce_qiov, 1); + qemu_iovec_add(&bounce_qiov, bounce_buf, bytes); + qiov =3D &bounce_qiov; + } + + ret =3D bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes= , qiov, + flags); + + if (copy_to_target) { + qemu_iovec_destroy(&bounce_qiov); + qemu_vfree(bounce_buf); + } + + return ret; } =20 static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs) @@ -1156,13 +1395,15 @@ static int coroutine_fn bdrv_mirror_top_flush(Block= DriverState *bs) static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes, BdrvRequestFlags flags) { - return bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags); + return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes,= NULL, + flags); } =20 static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) { - return bdrv_co_pdiscard(bs->backing->bs, offset, bytes); + return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, byt= es, + NULL, 0); } =20 static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs, QDict *= opts) @@ -1332,6 +1573,7 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, s->on_target_error =3D on_target_error; s->is_none_mode =3D is_none_mode; s->backing_mode =3D backing_mode; + s->copy_mode =3D MIRROR_COPY_MODE_BACKGROUND; s->base =3D base; s->granularity =3D granularity; s->buf_size =3D ROUND_UP(buf_size, granularity); --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914521668453.0007553564311; Wed, 13 Jun 2018 11:28:41 -0700 (PDT) Received: from localhost ([::1]:36221 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAVc-0003Fv-Lw for importer@patchew.org; Wed, 13 Jun 2018 14:28:40 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:57345) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMg-0003lP-Jt for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:28 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMf-0007JE-1a for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:26 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:58904 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMZ-0006yQ-7D; Wed, 13 Jun 2018 14:19:19 -0400 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id CA3B04057CC9; Wed, 13 Jun 2018 18:19:18 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id B17591C640; Wed, 13 Jun 2018 18:19:13 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:22 +0200 Message-Id: <20180613181823.13618-14-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.79 on 10.11.54.5 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.6]); Wed, 13 Jun 2018 18:19:18 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.6]); Wed, 13 Jun 2018 18:19:18 +0000 (UTC) for IP:'10.11.54.5' DOMAIN:'int-mx05.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 13/14] block/mirror: Add copy mode QAPI interface X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This patch allows the user to specify whether to use active or only background mode for mirror block jobs. Currently, this setting will remain constant for the duration of the entire block job. Signed-off-by: Max Reitz Reviewed-by: Alberto Garcia --- qapi/block-core.json | 11 +++++++++-- include/block/block_int.h | 4 +++- block/mirror.c | 12 +++++++----- blockdev.c | 9 ++++++++- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index 11b55530b3..8fbee5ee54 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -1710,6 +1710,9 @@ # written. Both will result in identical contents. # Default is true. (Since 2.4) # +# @copy-mode: when to copy data to the destination; defaults to 'backgroun= d' +# (Since: 3.0) +# # Since: 1.3 ## { 'struct': 'DriveMirror', @@ -1719,7 +1722,7 @@ '*speed': 'int', '*granularity': 'uint32', '*buf-size': 'int', '*on-source-error': 'BlockdevOnError', '*on-target-error': 'BlockdevOnError', - '*unmap': 'bool' } } + '*unmap': 'bool', '*copy-mode': 'MirrorCopyMode' } } =20 ## # @BlockDirtyBitmap: @@ -1982,6 +1985,9 @@ # above @device. If this option is not given, a node na= me is # autogenerated. (Since: 2.9) # +# @copy-mode: when to copy data to the destination; defaults to 'backgroun= d' +# (Since: 3.0) +# # Returns: nothing on success. # # Since: 2.6 @@ -2002,7 +2008,8 @@ '*speed': 'int', '*granularity': 'uint32', '*buf-size': 'int', '*on-source-error': 'BlockdevOnError', '*on-target-error': 'BlockdevOnError', - '*filter-node-name': 'str' } } + '*filter-node-name': 'str', + '*copy-mode': 'MirrorCopyMode' } } =20 ## # @block_set_io_throttle: diff --git a/include/block/block_int.h b/include/block/block_int.h index 327e478a73..fe9950f019 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -1017,6 +1017,7 @@ void commit_active_start(const char *job_id, BlockDri= verState *bs, * @filter_node_name: The node name that should be assigned to the filter * driver that the mirror job inserts into the graph above @bs. NULL means= that * a node name should be autogenerated. + * @copy_mode: When to trigger writes to the target. * @errp: Error object. * * Start a mirroring operation on @bs. Clusters that are allocated @@ -1030,7 +1031,8 @@ void mirror_start(const char *job_id, BlockDriverStat= e *bs, MirrorSyncMode mode, BlockMirrorBackingMode backing_mode, BlockdevOnError on_source_error, BlockdevOnError on_target_error, - bool unmap, const char *filter_node_name, Error **errp); + bool unmap, const char *filter_node_name, + MirrorCopyMode copy_mode, Error **errp); =20 /* * backup_job_create: diff --git a/block/mirror.c b/block/mirror.c index b58d7bbce1..f0c0003d47 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -1465,7 +1465,7 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, const BlockJobDriver *driver, bool is_none_mode, BlockDriverState *base, bool auto_complete, const char *filter_node_n= ame, - bool is_mirror, + bool is_mirror, MirrorCopyMode copy_mode, Error **errp) { MirrorBlockJob *s; @@ -1573,7 +1573,7 @@ static void mirror_start_job(const char *job_id, Bloc= kDriverState *bs, s->on_target_error =3D on_target_error; s->is_none_mode =3D is_none_mode; s->backing_mode =3D backing_mode; - s->copy_mode =3D MIRROR_COPY_MODE_BACKGROUND; + s->copy_mode =3D copy_mode; s->base =3D base; s->granularity =3D granularity; s->buf_size =3D ROUND_UP(buf_size, granularity); @@ -1640,7 +1640,8 @@ void mirror_start(const char *job_id, BlockDriverStat= e *bs, MirrorSyncMode mode, BlockMirrorBackingMode backing_mode, BlockdevOnError on_source_error, BlockdevOnError on_target_error, - bool unmap, const char *filter_node_name, Error **errp) + bool unmap, const char *filter_node_name, + MirrorCopyMode copy_mode, Error **errp) { bool is_none_mode; BlockDriverState *base; @@ -1655,7 +1656,7 @@ void mirror_start(const char *job_id, BlockDriverStat= e *bs, speed, granularity, buf_size, backing_mode, on_source_error, on_target_error, unmap, NULL, NULL, &mirror_job_driver, is_none_mode, base, false, - filter_node_name, true, errp); + filter_node_name, true, copy_mode, errp); } =20 void commit_active_start(const char *job_id, BlockDriverState *bs, @@ -1678,7 +1679,8 @@ void commit_active_start(const char *job_id, BlockDri= verState *bs, MIRROR_LEAVE_BACKING_CHAIN, on_error, on_error, true, cb, opaque, &commit_active_job_driver, false, base, auto_complete, - filter_node_name, false, &local_err); + filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND, + &local_err); if (local_err) { error_propagate(errp, local_err); goto error_restore_flags; diff --git a/blockdev.c b/blockdev.c index 4862323012..51536e0d6e 100644 --- a/blockdev.c +++ b/blockdev.c @@ -3695,6 +3695,7 @@ static void blockdev_mirror_common(const char *job_id= , BlockDriverState *bs, bool has_unmap, bool unmap, bool has_filter_node_name, const char *filter_node_name, + bool has_copy_mode, MirrorCopyMode copy= _mode, Error **errp) { =20 @@ -3719,6 +3720,9 @@ static void blockdev_mirror_common(const char *job_id= , BlockDriverState *bs, if (!has_filter_node_name) { filter_node_name =3D NULL; } + if (!has_copy_mode) { + copy_mode =3D MIRROR_COPY_MODE_BACKGROUND; + } =20 if (granularity !=3D 0 && (granularity < 512 || granularity > 1048576 = * 64)) { error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "granularity", @@ -3749,7 +3753,7 @@ static void blockdev_mirror_common(const char *job_id= , BlockDriverState *bs, has_replaces ? replaces : NULL, speed, granularity, buf_size, sync, backing_mode, on_source_error, on_target_error, unmap, filter_node_name, - errp); + copy_mode, errp); } =20 void qmp_drive_mirror(DriveMirror *arg, Error **errp) @@ -3895,6 +3899,7 @@ void qmp_drive_mirror(DriveMirror *arg, Error **errp) arg->has_on_target_error, arg->on_target_error, arg->has_unmap, arg->unmap, false, NULL, + arg->has_copy_mode, arg->copy_mode, &local_err); bdrv_unref(target_bs); error_propagate(errp, local_err); @@ -3915,6 +3920,7 @@ void qmp_blockdev_mirror(bool has_job_id, const char = *job_id, BlockdevOnError on_target_error, bool has_filter_node_name, const char *filter_node_name, + bool has_copy_mode, MirrorCopyMode copy_mode, Error **errp) { BlockDriverState *bs; @@ -3947,6 +3953,7 @@ void qmp_blockdev_mirror(bool has_job_id, const char = *job_id, has_on_target_error, on_target_error, true, true, has_filter_node_name, filter_node_name, + has_copy_mode, copy_mode, &local_err); error_propagate(errp, local_err); =20 --=20 2.17.1 From nobody Fri Apr 19 11:25:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1528914665023221.26865726262247; Wed, 13 Jun 2018 11:31:05 -0700 (PDT) Received: from localhost ([::1]:36242 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAXo-0005Sg-Tx for importer@patchew.org; Wed, 13 Jun 2018 14:30:56 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:57355) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fTAMh-0003m6-0c for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:28 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fTAMf-0007LC-Hl for qemu-devel@nongnu.org; Wed, 13 Jun 2018 14:19:27 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:38584 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fTAMb-00076f-Ed; Wed, 13 Jun 2018 14:19:21 -0400 Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 138878185331; Wed, 13 Jun 2018 18:19:21 +0000 (UTC) Received: from localhost (unknown [10.40.205.1]) by smtp.corp.redhat.com (Postfix) with ESMTPS id A9AB82026609; Wed, 13 Jun 2018 18:19:20 +0000 (UTC) From: Max Reitz To: qemu-block@nongnu.org Date: Wed, 13 Jun 2018 20:18:23 +0200 Message-Id: <20180613181823.13618-15-mreitz@redhat.com> In-Reply-To: <20180613181823.13618-1-mreitz@redhat.com> References: <20180613181823.13618-1-mreitz@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:19:21 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Wed, 13 Jun 2018 18:19:21 +0000 (UTC) for IP:'10.11.54.4' DOMAIN:'int-mx04.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'mreitz@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v5 14/14] iotests: Add test for active mirroring X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Kevin Wolf , Jeff Cody , qemu-devel@nongnu.org, Max Reitz , Stefan Hajnoczi , John Snow Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Signed-off-by: Max Reitz Reviewed-by: Fam Zheng Reviewed-by: Alberto Garcia --- tests/qemu-iotests/151 | 120 +++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/151.out | 5 ++ tests/qemu-iotests/group | 1 + 3 files changed, 126 insertions(+) create mode 100755 tests/qemu-iotests/151 create mode 100644 tests/qemu-iotests/151.out diff --git a/tests/qemu-iotests/151 b/tests/qemu-iotests/151 new file mode 100755 index 0000000000..fe53b9f446 --- /dev/null +++ b/tests/qemu-iotests/151 @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# +# Tests for active mirroring +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +import iotests +from iotests import qemu_img + +source_img =3D os.path.join(iotests.test_dir, 'source.' + iotests.imgfmt) +target_img =3D os.path.join(iotests.test_dir, 'target.' + iotests.imgfmt) + +class TestActiveMirror(iotests.QMPTestCase): + image_len =3D 128 * 1024 * 1024 # MB + potential_writes_in_flight =3D True + + def setUp(self): + qemu_img('create', '-f', iotests.imgfmt, source_img, '128M') + qemu_img('create', '-f', iotests.imgfmt, target_img, '128M') + + blk_source =3D {'id': 'source', + 'if': 'none', + 'node-name': 'source-node', + 'driver': iotests.imgfmt, + 'file': {'driver': 'file', + 'filename': source_img}} + + blk_target =3D {'node-name': 'target-node', + 'driver': iotests.imgfmt, + 'file': {'driver': 'file', + 'filename': target_img}} + + self.vm =3D iotests.VM() + self.vm.add_drive_raw(self.vm.qmp_to_opts(blk_source)) + self.vm.add_blockdev(self.vm.qmp_to_opts(blk_target)) + self.vm.add_device('virtio-blk,drive=3Dsource') + self.vm.launch() + + def tearDown(self): + self.vm.shutdown() + + if not self.potential_writes_in_flight: + self.assertTrue(iotests.compare_images(source_img, target_img), + 'mirror target does not match source') + + os.remove(source_img) + os.remove(target_img) + + def doActiveIO(self, sync_source_and_target): + # Fill the source image + self.vm.hmp_qemu_io('source', + 'write -P 1 0 %i' % self.image_len); + + # Start some background requests + for offset in range(1 * self.image_len / 8, 3 * self.image_len / 8= , 1024 * 1024): + self.vm.hmp_qemu_io('source', 'aio_write -P 2 %i 1M' % offset) + for offset in range(2 * self.image_len / 8, 3 * self.image_len / 8= , 1024 * 1024): + self.vm.hmp_qemu_io('source', 'aio_write -z %i 1M' % offset) + + # Start the block job + result =3D self.vm.qmp('blockdev-mirror', + job_id=3D'mirror', + filter_node_name=3D'mirror-node', + device=3D'source-node', + target=3D'target-node', + sync=3D'full', + copy_mode=3D'write-blocking') + self.assert_qmp(result, 'return', {}) + + # Start some more requests + for offset in range(3 * self.image_len / 8, 5 * self.image_len / 8= , 1024 * 1024): + self.vm.hmp_qemu_io('source', 'aio_write -P 3 %i 1M' % offset) + for offset in range(4 * self.image_len / 8, 5 * self.image_len / 8= , 1024 * 1024): + self.vm.hmp_qemu_io('source', 'aio_write -z %i 1M' % offset) + + # Wait for the READY event + self.wait_ready(drive=3D'mirror') + + # Now start some final requests; all of these (which land on + # the source) should be settled using the active mechanism. + # The mirror code itself asserts that the source BDS's dirty + # bitmap will stay clean between READY and COMPLETED. + for offset in range(5 * self.image_len / 8, 7 * self.image_len / 8= , 1024 * 1024): + self.vm.hmp_qemu_io('source', 'aio_write -P 3 %i 1M' % offset) + for offset in range(6 * self.image_len / 8, 7 * self.image_len / 8= , 1024 * 1024): + self.vm.hmp_qemu_io('source', 'aio_write -z %i 1M' % offset) + + if sync_source_and_target: + # If source and target should be in sync after the mirror, + # we have to flush before completion + self.vm.hmp_qemu_io('source', 'aio_flush') + self.potential_writes_in_flight =3D False + + self.complete_and_wait(drive=3D'mirror', wait_ready=3DFalse) + + def testActiveIO(self): + self.doActiveIO(False) + + def testActiveIOFlushed(self): + self.doActiveIO(True) + + + +if __name__ =3D=3D '__main__': + iotests.main(supported_fmts=3D['qcow2', 'raw']) diff --git a/tests/qemu-iotests/151.out b/tests/qemu-iotests/151.out new file mode 100644 index 0000000000..fbc63e62f8 --- /dev/null +++ b/tests/qemu-iotests/151.out @@ -0,0 +1,5 @@ +.. +---------------------------------------------------------------------- +Ran 2 tests + +OK diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 937a3d0a4d..eea75819d2 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -157,6 +157,7 @@ 148 rw auto quick 149 rw auto sudo 150 rw auto quick +151 rw auto 152 rw auto quick 153 rw auto quick 154 rw auto backing quick --=20 2.17.1