From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087951282471.4411148950368; Sat, 29 Dec 2018 04:52:31 -0800 (PST) Received: from localhost ([127.0.0.1]:36459 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdE6Q-0007JI-0r for importer@patchew.org; Sat, 29 Dec 2018 07:52:30 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43497) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv4-00036F-Vp for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:41:03 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcB-0007tT-Gq for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:21 -0500 Received: from relay.sw.ru ([185.231.240.75]:46412) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006Vy-BM; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbQ-00014c-QZ; Sat, 29 Dec 2018 15:20:29 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:17 +0300 Message-Id: <20181229122027.42245-2-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 01/11] block/backup: simplify backup_incremental_init_copy_bitmap X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Simplify backup_incremental_init_copy_bitmap using the function bdrv_dirty_bitmap_next_dirty_area. Note: move to job->len instead of bitmap size: it should not matter but less code. Signed-off-by: Vladimir Sementsov-Ogievskiy --- block/backup.c | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/block/backup.c b/block/backup.c index 435414e964..fbe7ce19e1 100644 --- a/block/backup.c +++ b/block/backup.c @@ -406,43 +406,27 @@ static int coroutine_fn backup_run_incremental(Backup= BlockJob *job) /* init copy_bitmap from sync_bitmap */ static void backup_incremental_init_copy_bitmap(BackupBlockJob *job) { - BdrvDirtyBitmapIter *dbi; - int64_t offset; - int64_t end =3D DIV_ROUND_UP(bdrv_dirty_bitmap_size(job->sync_bitmap), - job->cluster_size); - - dbi =3D bdrv_dirty_iter_new(job->sync_bitmap); - while ((offset =3D bdrv_dirty_iter_next(dbi)) !=3D -1) { - int64_t cluster =3D offset / job->cluster_size; - int64_t next_cluster; - - offset +=3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); - if (offset >=3D bdrv_dirty_bitmap_size(job->sync_bitmap)) { - hbitmap_set(job->copy_bitmap, cluster, end - cluster); - break; - } + uint64_t offset =3D 0; + uint64_t bytes =3D job->len; =20 - offset =3D bdrv_dirty_bitmap_next_zero(job->sync_bitmap, offset, - UINT64_MAX); - if (offset =3D=3D -1) { - hbitmap_set(job->copy_bitmap, cluster, end - cluster); - break; - } + while (bdrv_dirty_bitmap_next_dirty_area(job->sync_bitmap, + &offset, &bytes)) + { + uint64_t cluster =3D offset / job->cluster_size; + uint64_t last_cluster =3D (offset + bytes) / job->cluster_size; =20 - next_cluster =3D DIV_ROUND_UP(offset, job->cluster_size); - hbitmap_set(job->copy_bitmap, cluster, next_cluster - cluster); - if (next_cluster >=3D end) { + hbitmap_set(job->copy_bitmap, cluster, last_cluster - cluster + 1); + + offset =3D (last_cluster + 1) * job->cluster_size; + if (offset >=3D job->len) { break; } - - bdrv_set_dirty_iter(dbi, next_cluster * job->cluster_size); + bytes =3D job->len - offset; } =20 /* TODO job_progress_set_remaining() would make more sense */ job_progress_update(&job->common.job, job->len - hbitmap_count(job->copy_bitmap) * job->cluster_size); - - bdrv_dirty_iter_free(dbi); } =20 static int coroutine_fn backup_run(Job *job, Error **errp) --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087543942136.13220202289006; Sat, 29 Dec 2018 04:45:43 -0800 (PST) Received: from localhost ([127.0.0.1]:36383 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDzq-0000IQ-JS for importer@patchew.org; Sat, 29 Dec 2018 07:45:42 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43012) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv0-0002uv-Bt for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:53 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcC-0007w9-Tp for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:26 -0500 Received: from relay.sw.ru ([185.231.240.75]:46414) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006W0-BU; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbR-00014c-4S; Sat, 29 Dec 2018 15:20:29 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:18 +0300 Message-Id: <20181229122027.42245-3-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 02/11] block/backup: move to copy_bitmap with granularity X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" We are going to share this bitmap between backup and backup-top filter driver, so let's share something more meaningful. It also simplifies some calculations. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Max Reitz --- block/backup.c | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/block/backup.c b/block/backup.c index fbe7ce19e1..0b3fddeb6c 100644 --- a/block/backup.c +++ b/block/backup.c @@ -114,7 +114,8 @@ static int coroutine_fn backup_cow_with_bounce_buffer(B= ackupBlockJob *job, int read_flags =3D is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0; int write_flags =3D job->serialize_target_writes ? BDRV_REQ_SERIALISIN= G : 0; =20 - hbitmap_reset(job->copy_bitmap, start / job->cluster_size, 1); + assert(QEMU_IS_ALIGNED(start, job->cluster_size)); + hbitmap_reset(job->copy_bitmap, start, job->cluster_size); nbytes =3D MIN(job->cluster_size, job->len - start); if (!*bounce_buffer) { *bounce_buffer =3D blk_blockalign(blk, job->cluster_size); @@ -150,7 +151,7 @@ static int coroutine_fn backup_cow_with_bounce_buffer(B= ackupBlockJob *job, =20 return nbytes; fail: - hbitmap_set(job->copy_bitmap, start / job->cluster_size, 1); + hbitmap_set(job->copy_bitmap, start, job->cluster_size); return ret; =20 } @@ -170,16 +171,15 @@ static int coroutine_fn backup_cow_with_offload(Backu= pBlockJob *job, int write_flags =3D job->serialize_target_writes ? BDRV_REQ_SERIALISIN= G : 0; =20 assert(QEMU_IS_ALIGNED(job->copy_range_size, job->cluster_size)); + assert(QEMU_IS_ALIGNED(start, job->cluster_size)); nbytes =3D MIN(job->copy_range_size, end - start); nr_clusters =3D DIV_ROUND_UP(nbytes, job->cluster_size); - hbitmap_reset(job->copy_bitmap, start / job->cluster_size, - nr_clusters); + hbitmap_reset(job->copy_bitmap, start, job->cluster_size * nr_clusters= ); ret =3D blk_co_copy_range(blk, start, job->target, start, nbytes, read_flags, write_flags); if (ret < 0) { trace_backup_do_cow_copy_range_fail(job, start, ret); - hbitmap_set(job->copy_bitmap, start / job->cluster_size, - nr_clusters); + hbitmap_set(job->copy_bitmap, start, job->cluster_size * nr_cluste= rs); return ret; } =20 @@ -207,7 +207,7 @@ static int coroutine_fn backup_do_cow(BackupBlockJob *j= ob, cow_request_begin(&cow_request, job, start, end); =20 while (start < end) { - if (!hbitmap_get(job->copy_bitmap, start / job->cluster_size)) { + if (!hbitmap_get(job->copy_bitmap, start)) { trace_backup_do_cow_skip(job, start); start +=3D job->cluster_size; continue; /* already copied */ @@ -303,6 +303,11 @@ static void backup_clean(Job *job) assert(s->target); blk_unref(s->target); s->target =3D NULL; + + if (s->copy_bitmap) { + hbitmap_free(s->copy_bitmap); + s->copy_bitmap =3D NULL; + } } =20 static void backup_attached_aio_context(BlockJob *job, AioContext *aio_con= text) @@ -315,7 +320,6 @@ static void backup_attached_aio_context(BlockJob *job, = AioContext *aio_context) void backup_do_checkpoint(BlockJob *job, Error **errp) { BackupBlockJob *backup_job =3D container_of(job, BackupBlockJob, commo= n); - int64_t len; =20 assert(block_job_driver(job) =3D=3D &backup_job_driver); =20 @@ -325,8 +329,7 @@ void backup_do_checkpoint(BlockJob *job, Error **errp) return; } =20 - len =3D DIV_ROUND_UP(backup_job->len, backup_job->cluster_size); - hbitmap_set(backup_job->copy_bitmap, 0, len); + hbitmap_set(backup_job->copy_bitmap, 0, backup_job->len); } =20 static void backup_drain(BlockJob *job) @@ -381,16 +384,16 @@ static int coroutine_fn backup_run_incremental(Backup= BlockJob *job) { int ret; bool error_is_read; - int64_t cluster; + int64_t offset; HBitmapIter hbi; =20 hbitmap_iter_init(&hbi, job->copy_bitmap, 0); - while ((cluster =3D hbitmap_iter_next(&hbi)) !=3D -1) { + while ((offset =3D hbitmap_iter_next(&hbi)) !=3D -1) { do { if (yield_and_check(job)) { return 0; } - ret =3D backup_do_cow(job, cluster * job->cluster_size, + ret =3D backup_do_cow(job, offset, job->cluster_size, &error_is_read, false); if (ret < 0 && backup_error_action(job, error_is_read, -ret) = =3D=3D BLOCK_ERROR_ACTION_REPORT) @@ -412,12 +415,9 @@ static void backup_incremental_init_copy_bitmap(Backup= BlockJob *job) while (bdrv_dirty_bitmap_next_dirty_area(job->sync_bitmap, &offset, &bytes)) { - uint64_t cluster =3D offset / job->cluster_size; - uint64_t last_cluster =3D (offset + bytes) / job->cluster_size; + hbitmap_set(job->copy_bitmap, offset, bytes); =20 - hbitmap_set(job->copy_bitmap, cluster, last_cluster - cluster + 1); - - offset =3D (last_cluster + 1) * job->cluster_size; + offset +=3D bytes; if (offset >=3D job->len) { break; } @@ -426,30 +426,27 @@ static void backup_incremental_init_copy_bitmap(Backu= pBlockJob *job) =20 /* TODO job_progress_set_remaining() would make more sense */ job_progress_update(&job->common.job, - job->len - hbitmap_count(job->copy_bitmap) * job->cluster_size); + job->len - hbitmap_count(job->copy_bitmap)); } =20 static int coroutine_fn backup_run(Job *job, Error **errp) { BackupBlockJob *s =3D container_of(job, BackupBlockJob, common.job); BlockDriverState *bs =3D blk_bs(s->common.blk); - int64_t offset, nb_clusters; + int64_t offset; int ret =3D 0; =20 QLIST_INIT(&s->inflight_reqs); qemu_co_rwlock_init(&s->flush_rwlock); =20 - nb_clusters =3D DIV_ROUND_UP(s->len, s->cluster_size); job_progress_set_remaining(job, s->len); =20 - s->copy_bitmap =3D hbitmap_alloc(nb_clusters, 0); if (s->sync_mode =3D=3D MIRROR_SYNC_MODE_INCREMENTAL) { backup_incremental_init_copy_bitmap(s); } else { - hbitmap_set(s->copy_bitmap, 0, nb_clusters); + hbitmap_set(s->copy_bitmap, 0, s->len); } =20 - s->before_write.notify =3D backup_before_write_notify; bdrv_add_before_write_notifier(bs, &s->before_write); =20 @@ -530,7 +527,6 @@ static int coroutine_fn backup_run(Job *job, Error **er= rp) /* wait until pending backup_do_cow() calls have completed */ qemu_co_rwlock_wrlock(&s->flush_rwlock); qemu_co_rwlock_unlock(&s->flush_rwlock); - hbitmap_free(s->copy_bitmap); =20 return ret; } @@ -681,6 +677,8 @@ BlockJob *backup_job_create(const char *job_id, BlockDr= iverState *bs, } else { job->cluster_size =3D MAX(BACKUP_CLUSTER_SIZE_DEFAULT, bdi.cluster= _size); } + + job->copy_bitmap =3D hbitmap_alloc(len, ctz32(job->cluster_size)); job->use_copy_range =3D true; job->copy_range_size =3D MIN_NON_ZERO(blk_get_max_transfer(job->common= .blk), blk_get_max_transfer(job->target)); --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087849784445.20553885329423; Sat, 29 Dec 2018 04:50:49 -0800 (PST) Received: from localhost ([127.0.0.1]:36439 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdE4h-0005sN-8c for importer@patchew.org; Sat, 29 Dec 2018 07:50:43 -0500 Received: from eggs.gnu.org ([208.118.235.92]:42738) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv3-0002hB-Fg for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:56 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcE-0007ym-8i for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:26 -0500 Received: from relay.sw.ru ([185.231.240.75]:46418) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006W1-BB; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbR-00014c-By; Sat, 29 Dec 2018 15:20:29 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:19 +0300 Message-Id: <20181229122027.42245-4-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 03/11] block: improve should_update_child X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" As it already said in the comment, we don't want to create loops in parent->child relations. So, when we try to append @to to @c, we should check that @c is not in @to children subtree, and we should check it recursively, not only the first level. The patch provides BFS-based search, to check the relations. This is needed for further fleecing-hook filter usage: we need to append it to source, when the hook is already a parent of target, and source may be in a backing chain of target (fleecing-scheme). So, on appending, the hook should not became a child (direct or through children subtree) of the target. Signed-off-by: Vladimir Sementsov-Ogievskiy --- block.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/block.c b/block.c index 4f5ff2cc12..8f04f293da 100644 --- a/block.c +++ b/block.c @@ -3536,7 +3536,7 @@ void bdrv_close_all(void) =20 static bool should_update_child(BdrvChild *c, BlockDriverState *to) { - BdrvChild *to_c; + GList *queue =3D NULL, *pos; =20 if (c->role->stay_at_node) { return false; @@ -3572,13 +3572,36 @@ static bool should_update_child(BdrvChild *c, Block= DriverState *to) * if A is a child of B, that means we cannot replace A by B there * because that would create a loop. Silently detaching A from B * is also not really an option. So overall just leaving A in - * place there is the most sensible choice. */ - QLIST_FOREACH(to_c, &to->children, next) { - if (to_c =3D=3D c) { - return false; + * place there is the most sensible choice. + * + * We would also create a loop in any cases where @c is only + * indirectly referenced by @to. Prevent this by returning false + * if @c is found (by breadth-first search) anywhere in the whole + * subtree of @to. + */ + + pos =3D queue =3D g_list_append(queue, to); + while (pos) { + BlockDriverState *v =3D pos->data; + BdrvChild *c2; + + QLIST_FOREACH(c2, &v->children, next) { + if (c2 =3D=3D c) { + g_list_free(queue); + return false; + } + + if (g_list_find(queue, c2->bs)) { + continue; + } + + queue =3D g_list_append(queue, c2->bs); } + + pos =3D pos->next; } =20 + g_list_free(queue); return true; } =20 --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087840933439.2039732035637; Sat, 29 Dec 2018 04:50:40 -0800 (PST) Received: from localhost ([127.0.0.1]:36441 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdE4d-0005sS-K6 for importer@patchew.org; Sat, 29 Dec 2018 07:50:39 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43552) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv3-00036f-KT for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:56 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcB-0007tp-LK for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:26 -0500 Received: from relay.sw.ru ([185.231.240.75]:46416) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006W3-C0; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbR-00014c-LN; Sat, 29 Dec 2018 15:20:29 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:20 +0300 Message-Id: <20181229122027.42245-5-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 04/11] iotests: handle -f argument correctly for qemu_io_silent X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Correctly rewrite default argument. After the patch, the function can be used for other (not only default test-chosen) image format. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Max Reitz --- tests/qemu-iotests/iotests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 9595429fea..76877ad584 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -139,7 +139,12 @@ def qemu_io(*args): =20 def qemu_io_silent(*args): '''Run qemu-io and return the exit code, suppressing stdout''' - args =3D qemu_io_args + list(args) + if '-f' in qemu_io_args and '-f' in args: + ind =3D qemu_io_args.index('-f') + args =3D qemu_io_args[:ind] + qemu_io_args[ind+2:] + list(args) + else: + args =3D qemu_io_args + list(args) + exitcode =3D subprocess.call(args, stdout=3Dopen('/dev/null', 'w')) if exitcode < 0: sys.stderr.write('qemu-io received signal %i: %s\n' % --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087697042123.60811336822928; Sat, 29 Dec 2018 04:48:17 -0800 (PST) Received: from localhost ([127.0.0.1]:36406 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdE2E-00032U-JD for importer@patchew.org; Sat, 29 Dec 2018 07:48:10 -0500 Received: from eggs.gnu.org ([208.118.235.92]:42716) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv1-0002h7-MV for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:54 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcC-0007vG-Bk for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:26 -0500 Received: from relay.sw.ru ([185.231.240.75]:46426) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006W4-Bk; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbR-00014c-U3; Sat, 29 Dec 2018 15:20:30 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:21 +0300 Message-Id: <20181229122027.42245-6-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 05/11] iotests: allow resume_drive by node name X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" After node graph changes, we may not be able to resume_drive by device name (backing files are not recursively searched). So, lets allow to resume by node-name. Set constant name for breakpoints, to avoid introducing extra parameters. Signed-off-by: Vladimir Sementsov-Ogievskiy --- tests/qemu-iotests/iotests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 76877ad584..c9779f432f 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -415,11 +415,11 @@ class VM(qtest.QEMUQtestMachine): self.pause_drive(drive, "write_aio") return self.qmp('human-monitor-command', - command_line=3D'qemu-io %s "break %s bp_%s"' % (drive,= event, drive)) + command_line=3D'qemu-io %s "break %s bp_0"' % (drive, = event)) =20 def resume_drive(self, drive): self.qmp('human-monitor-command', - command_line=3D'qemu-io %s "remove_break bp_%s"' % (dr= ive, drive)) + command_line=3D'qemu-io %s "remove_break bp_0"' % (dri= ve)) =20 def hmp_qemu_io(self, drive, cmd): '''Write to a given drive using an HMP command''' @@ -543,13 +543,14 @@ class QMPTestCase(unittest.TestCase): self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filena= me[5:])), self.vm.flatten_qmp_object(reference)) =20 - def cancel_and_wait(self, drive=3D'drive0', force=3DFalse, resume=3DFa= lse): + def cancel_and_wait(self, drive=3D'drive0', force=3DFalse, resume=3DFa= lse, + resume_node=3DNone): '''Cancel a block job and wait for it to finish, returning the eve= nt''' result =3D self.vm.qmp('block-job-cancel', device=3Ddrive, force= =3Dforce) self.assert_qmp(result, 'return', {}) =20 if resume: - self.vm.resume_drive(drive) + self.vm.resume_drive(resume_node or drive) =20 cancelled =3D False result =3D None --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087541280400.29173188665163; Sat, 29 Dec 2018 04:45:41 -0800 (PST) Received: from localhost ([127.0.0.1]:36376 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDzn-0000D9-Qe for importer@patchew.org; Sat, 29 Dec 2018 07:45:39 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43183) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv0-0002w7-QX for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:54 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcC-0007v8-9l for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:26 -0500 Received: from relay.sw.ru ([185.231.240.75]:46398) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006YE-BR; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbS-00014c-53; Sat, 29 Dec 2018 15:20:30 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:22 +0300 Message-Id: <20181229122027.42245-7-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 06/11] iotests: prepare 055 to graph changes during backup job X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Backup will append fleecing-hook node above source node, so, we can't resume by device name (because resume don't search recursively through backing chain). Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Max Reitz --- tests/qemu-iotests/055 | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055 index 3437c11507..be5451e1c5 100755 --- a/tests/qemu-iotests/055 +++ b/tests/qemu-iotests/055 @@ -48,7 +48,8 @@ class TestSingleDrive(iotests.QMPTestCase): def setUp(self): qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(= image_len)) =20 - self.vm =3D iotests.VM().add_drive('blkdebug::' + test_img) + self.vm =3D iotests.VM().add_drive('blkdebug::' + test_img, + 'node-name=3Dsource') self.vm.add_drive(blockdev_target_img, interface=3D"none") if iotests.qemu_default_machine =3D=3D 'pc': self.vm.add_drive(None, 'media=3Dcdrom', 'ide') @@ -69,7 +70,7 @@ class TestSingleDrive(iotests.QMPTestCase): result =3D self.vm.qmp(cmd, device=3D'drive0', target=3Dtarget, sy= nc=3D'full') self.assert_qmp(result, 'return', {}) =20 - event =3D self.cancel_and_wait(resume=3DTrue) + event =3D self.cancel_and_wait(resume=3DTrue, resume_node=3D'sourc= e') self.assert_qmp(event, 'data/type', 'backup') =20 def test_cancel_drive_backup(self): @@ -87,7 +88,7 @@ class TestSingleDrive(iotests.QMPTestCase): self.assert_qmp(result, 'return', {}) =20 self.pause_job('drive0', wait=3DFalse) - self.vm.resume_drive('drive0') + self.vm.resume_drive('source') self.pause_wait('drive0') =20 result =3D self.vm.qmp('query-block-jobs') @@ -165,7 +166,8 @@ class TestSetSpeed(iotests.QMPTestCase): def setUp(self): qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(= image_len)) =20 - self.vm =3D iotests.VM().add_drive('blkdebug::' + test_img) + self.vm =3D iotests.VM().add_drive('blkdebug::' + test_img, + 'node-name=3Dsource') self.vm.add_drive(blockdev_target_img, interface=3D"none") self.vm.launch() =20 @@ -197,7 +199,7 @@ class TestSetSpeed(iotests.QMPTestCase): self.assert_qmp(result, 'return[0]/device', 'drive0') self.assert_qmp(result, 'return[0]/speed', 8 * 1024 * 1024) =20 - event =3D self.cancel_and_wait(resume=3DTrue) + event =3D self.cancel_and_wait(resume=3DTrue, resume_node=3D'sourc= e') self.assert_qmp(event, 'data/type', 'backup') =20 # Check setting speed option works @@ -210,7 +212,7 @@ class TestSetSpeed(iotests.QMPTestCase): self.assert_qmp(result, 'return[0]/device', 'drive0') self.assert_qmp(result, 'return[0]/speed', 4 * 1024 * 1024) =20 - event =3D self.cancel_and_wait(resume=3DTrue) + event =3D self.cancel_and_wait(resume=3DTrue, resume_node=3D'sourc= e') self.assert_qmp(event, 'data/type', 'backup') =20 def test_set_speed_drive_backup(self): @@ -236,7 +238,7 @@ class TestSetSpeed(iotests.QMPTestCase): result =3D self.vm.qmp('block-job-set-speed', device=3D'drive0', s= peed=3D-1) self.assert_qmp(result, 'error/class', 'GenericError') =20 - event =3D self.cancel_and_wait(resume=3DTrue) + event =3D self.cancel_and_wait(resume=3DTrue, resume_node=3D'sourc= e') self.assert_qmp(event, 'data/type', 'backup') =20 def test_set_speed_invalid_drive_backup(self): @@ -464,7 +466,8 @@ class TestDriveCompression(iotests.QMPTestCase): pass =20 def do_prepare_drives(self, fmt, args, attach_target): - self.vm =3D iotests.VM().add_drive('blkdebug::' + test_img) + self.vm =3D iotests.VM().add_drive('blkdebug::' + test_img, + 'node-name=3Dsource') =20 qemu_img('create', '-f', fmt, blockdev_target_img, str(TestDriveCompression.image_len), *args) @@ -507,7 +510,7 @@ class TestDriveCompression(iotests.QMPTestCase): result =3D self.vm.qmp(cmd, device=3D'drive0', sync=3D'full', comp= ress=3DTrue, **args) self.assert_qmp(result, 'return', {}) =20 - event =3D self.cancel_and_wait(resume=3DTrue) + event =3D self.cancel_and_wait(resume=3DTrue, resume_node=3D'sourc= e') self.assert_qmp(event, 'data/type', 'backup') =20 self.vm.shutdown() @@ -532,7 +535,7 @@ class TestDriveCompression(iotests.QMPTestCase): self.assert_qmp(result, 'return', {}) =20 self.pause_job('drive0', wait=3DFalse) - self.vm.resume_drive('drive0') + self.vm.resume_drive('source') self.pause_wait('drive0') =20 result =3D self.vm.qmp('query-block-jobs') --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087375816475.6362767292229; Sat, 29 Dec 2018 04:42:55 -0800 (PST) Received: from localhost ([127.0.0.1]:36348 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDx8-00059l-93 for importer@patchew.org; Sat, 29 Dec 2018 07:42:54 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43429) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDuz-00035C-QI for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:52 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcH-00085h-Ay for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:31 -0500 Received: from relay.sw.ru ([185.231.240.75]:46394) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006Vw-BS; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbS-00014c-9c; Sat, 29 Dec 2018 15:20:30 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:23 +0300 Message-Id: <20181229122027.42245-8-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 07/11] block: introduce backup-top filter driver X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Backup-top filter does copy-before-write operation. It should be inserted above active disk and has a target node for CBW, like the following: +-------+ | Guest | +---+---+ |r,w v +---+-----------+ target +---------------+ | backup_top |---------->| target(qcow2) | +---+-----------+ CBW +---+-----------+ | backing |r,w v +---+---------+ | Active disk | +-------------+ The driver will be used in backup instead of write-notifiers. Signed-off-by: Vladimir Sementsov-Ogievskiy --- block/backup-top.h | 43 +++++++ block/backup-top.c | 306 ++++++++++++++++++++++++++++++++++++++++++++ block/Makefile.objs | 2 + 3 files changed, 351 insertions(+) create mode 100644 block/backup-top.h create mode 100644 block/backup-top.c diff --git a/block/backup-top.h b/block/backup-top.h new file mode 100644 index 0000000000..abe4dda574 --- /dev/null +++ b/block/backup-top.h @@ -0,0 +1,43 @@ +/* + * backup-top filter driver + * + * The driver performs Copy-Before-Write (CBW) operation: it is injected a= bove + * some node, and before each write it copies _old_ data to the target nod= e. + * + * Copyright (c) 2018 Virtuozzo International GmbH. All rights reserved. + * + * Author: + * Sementsov-Ogievskiy Vladimir + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "qemu/osdep.h" + +#include "block/block_int.h" + +typedef struct BDRVBackupTopState { + HBitmap *copy_bitmap; /* what should be copied to @target on guest wri= te. */ + BdrvChild *target; + + uint64_t bytes_copied; +} BDRVBackupTopState; + +void bdrv_backup_top_drop(BlockDriverState *bs); +uint64_t bdrv_backup_top_progress(BlockDriverState *bs); + +BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, + BlockDriverState *target, + HBitmap *copy_bitmap, + Error **errp); diff --git a/block/backup-top.c b/block/backup-top.c new file mode 100644 index 0000000000..0e7b3e3142 --- /dev/null +++ b/block/backup-top.c @@ -0,0 +1,306 @@ +/* + * backup-top filter driver + * + * The driver performs Copy-Before-Write (CBW) operation: it is injected a= bove + * some node, and before each write it copies _old_ data to the target nod= e. + * + * Copyright (c) 2018 Virtuozzo International GmbH. All rights reserved. + * + * Author: + * Sementsov-Ogievskiy Vladimir + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "qemu/osdep.h" + +#include "qemu/cutils.h" +#include "qapi/error.h" +#include "block/block_int.h" +#include "block/qdict.h" + +#include "block/backup-top.h" + +static coroutine_fn int backup_top_co_preadv( + BlockDriverState *bs, uint64_t offset, uint64_t bytes, + QEMUIOVector *qiov, int flags) +{ + /* + * Features to be implemented: + * F1. COR. save read data to fleecing target for fast access + * (to reduce reads). This possibly may be done with use of copy-o= n-read + * filter, but we need an ability to make COR requests optional: f= or + * example, if target is a ram-cache, and if it is full now, we sh= ould + * skip doing COR request, as it is actually not necessary. + * + * F2. Feature for guest: read from fleecing target if data is in ram-= cache + * and is unchanged + */ + + return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags); +} + +static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offs= et, + uint64_t bytes) +{ + int ret =3D 0; + BDRVBackupTopState *s =3D bs->opaque; + uint64_t gran =3D 1UL << hbitmap_granularity(s->copy_bitmap); + uint64_t end =3D QEMU_ALIGN_UP(offset + bytes, gran); + uint64_t off =3D QEMU_ALIGN_DOWN(offset, gran), len; + size_t align =3D MAX(bdrv_opt_mem_align(bs->backing->bs), + bdrv_opt_mem_align(s->target->bs)); + struct iovec iov =3D { + .iov_base =3D qemu_memalign(align, end - off), + .iov_len =3D end - off + }; + QEMUIOVector qiov; + + qemu_iovec_init_external(&qiov, &iov, 1); + + /* + * Features to be implemented: + * F3. parallelize copying loop + * F4. detect zeros + * F5. use block_status ? + * F6. don't copy clusters which are already cached by COR [see F1] + * F7. if target is ram-cache and it is full, there should be a possib= ility + * to drop not necessary data (cached by COR [see F1]) to handle C= BW + * fast. + */ + + len =3D end - off; + while (hbitmap_next_dirty_area(s->copy_bitmap, &off, &len)) { + iov.iov_len =3D qiov.size =3D len; + + hbitmap_reset(s->copy_bitmap, off, len); + + ret =3D bdrv_co_preadv(bs->backing, off, len, &qiov, + BDRV_REQ_NO_SERIALISING); + if (ret < 0) { + hbitmap_set(s->copy_bitmap, off, len); + goto finish; + } + + ret =3D bdrv_co_pwritev(s->target, off, len, &qiov, BDRV_REQ_SERIA= LISING); + if (ret < 0) { + hbitmap_set(s->copy_bitmap, off, len); + goto finish; + } + + s->bytes_copied +=3D len; + off +=3D len; + if (off >=3D end) { + break; + } + len =3D end - off; + } + +finish: + qemu_vfree(iov.iov_base); + + /* + * F8. we fail guest request in case of error. We can alter it by + * possibility to fail copying process instead, or retry several times= , or + * may be guest pause, etc. + */ + return ret; +} + +static int coroutine_fn backup_top_co_pdiscard(BlockDriverState *bs, + int64_t offset, int byte= s) +{ + int ret =3D backup_top_cbw(bs, offset, bytes); + if (ret < 0) { + return ret; + } + + /* + * Features to be implemented: + * F9. possibility of lazy discard: just defer the discard after fleec= ing + * completion. If write (or new discard) occurs to the same area, = just + * drop deferred discard. + */ + + return bdrv_co_pdiscard(bs->backing, offset, bytes); +} + +static int coroutine_fn backup_top_co_pwrite_zeroes(BlockDriverState *bs, + int64_t offset, int bytes, BdrvRequestFlags flags) +{ + int ret =3D backup_top_cbw(bs, offset, bytes); + if (ret < 0) { + return ret; + } + + return bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags); +} + +static coroutine_fn int backup_top_co_pwritev(BlockDriverState *bs, + uint64_t offset, + uint64_t bytes, + QEMUIOVector *qiov, int f= lags) +{ + int ret =3D backup_top_cbw(bs, offset, bytes); + if (ret < 0) { + return ret; + } + + return bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags); +} + +static int coroutine_fn backup_top_co_flush(BlockDriverState *bs) +{ + if (!bs->backing) { + return 0; + } + + return bdrv_co_flush(bs->backing->bs); +} + +static void backup_top_refresh_filename(BlockDriverState *bs, QDict *opts) +{ + if (bs->backing =3D=3D NULL) { + /* + * we can be here after failed bdrv_attach_child in + * bdrv_set_backing_hd + */ + return; + } + bdrv_refresh_filename(bs->backing->bs); + pstrcpy(bs->exact_filename, sizeof(bs->exact_filename), + bs->backing->bs->filename); +} + +static void backup_top_child_perm(BlockDriverState *bs, BdrvChild *c, + const BdrvChildRole *role, + BlockReopenQueue *reopen_queue, + uint64_t perm, uint64_t shared, + uint64_t *nperm, uint64_t *nshared) +{ + bdrv_filter_default_perms(bs, c, role, reopen_queue, perm, shared, npe= rm, + nshared); + + if (role =3D=3D &child_file) { + /* + * share write to target, to not interfere guest writes to it's di= sk + * which will be in target backing chain + */ + *nshared =3D *nshared | BLK_PERM_WRITE; + *nperm =3D *nperm | BLK_PERM_WRITE; + } else { + *nperm =3D *nperm | BLK_PERM_CONSISTENT_READ; + } +} + +BlockDriver bdrv_backup_top_filter =3D { + .format_name =3D "backup-top", + .instance_size =3D sizeof(BDRVBackupTopState), + + .bdrv_co_preadv =3D backup_top_co_preadv, + .bdrv_co_pwritev =3D backup_top_co_pwritev, + .bdrv_co_pwrite_zeroes =3D backup_top_co_pwrite_zeroes, + .bdrv_co_pdiscard =3D backup_top_co_pdiscard, + .bdrv_co_flush =3D backup_top_co_flush, + + .bdrv_co_block_status =3D bdrv_co_block_status_from_backing, + + .bdrv_refresh_filename =3D backup_top_refresh_filename, + + .bdrv_child_perm =3D backup_top_child_perm, + + .is_filter =3D true, +}; + +BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, + BlockDriverState *target, + HBitmap *copy_bitmap, + Error **errp) +{ + Error *local_err =3D NULL; + BDRVBackupTopState *state; + BlockDriverState *top =3D bdrv_new_open_driver(&bdrv_backup_top_filter, + NULL, BDRV_O_RDWR, errp); + + if (!top) { + return NULL; + } + + top->implicit =3D true; + top->total_sectors =3D source->total_sectors; + top->opaque =3D state =3D g_new0(BDRVBackupTopState, 1); + state->copy_bitmap =3D copy_bitmap; + + bdrv_ref(target); + state->target =3D bdrv_attach_child(top, target, "target", &child_file= , errp); + if (!state->target) { + bdrv_unref(target); + bdrv_unref(top); + return NULL; + } + + bdrv_set_aio_context(top, bdrv_get_aio_context(source)); + bdrv_set_aio_context(target, bdrv_get_aio_context(source)); + + bdrv_drained_begin(source); + + bdrv_ref(top); + bdrv_append(top, source, &local_err); + + if (local_err) { + bdrv_unref(top); + } + + bdrv_drained_end(source); + + if (local_err) { + bdrv_unref_child(top, state->target); + bdrv_unref(top); + error_propagate(errp, local_err); + return NULL; + } + + return top; +} + +void bdrv_backup_top_drop(BlockDriverState *bs) +{ + BDRVBackupTopState *s =3D bs->opaque; + + AioContext *aio_context =3D bdrv_get_aio_context(bs); + + aio_context_acquire(aio_context); + + bdrv_drained_begin(bs); + + bdrv_child_try_set_perm(bs->backing, 0, BLK_PERM_ALL, &error_abort); + bdrv_replace_node(bs, backing_bs(bs), &error_abort); + bdrv_set_backing_hd(bs, NULL, &error_abort); + + bdrv_drained_end(bs); + + if (s->target) { + bdrv_unref_child(bs, s->target); + } + bdrv_unref(bs); + + aio_context_release(aio_context); +} + +uint64_t bdrv_backup_top_progress(BlockDriverState *bs) +{ + BDRVBackupTopState *s =3D bs->opaque; + + return s->bytes_copied; +} diff --git a/block/Makefile.objs b/block/Makefile.objs index 7a81892a52..9072115c09 100644 --- a/block/Makefile.objs +++ b/block/Makefile.objs @@ -40,6 +40,8 @@ block-obj-y +=3D throttle.o copy-on-read.o =20 block-obj-y +=3D crypto.o =20 +block-obj-y +=3D backup-top.o + common-obj-y +=3D stream.o =20 nfs.o-libs :=3D $(LIBNFS_LIBS) --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087691020961.125529541721; Sat, 29 Dec 2018 04:48:11 -0800 (PST) Received: from localhost ([127.0.0.1]:36402 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdE2D-00030f-9G for importer@patchew.org; Sat, 29 Dec 2018 07:48:09 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43448) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv4-00035F-Dr for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:58 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcB-0007th-JW for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:21 -0500 Received: from relay.sw.ru ([185.231.240.75]:46392) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006Vv-BG; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbS-00014c-GC; Sat, 29 Dec 2018 15:20:30 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:24 +0300 Message-Id: <20181229122027.42245-9-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 08/11] block/io: refactor wait_serialising_requests X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Split out do_wait_serialising_requests with additional possibility to not actually wait but just check, that there is something to wait for. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Max Reitz --- block/io.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/block/io.c b/block/io.c index bd9d688f8b..b87c11a6ec 100644 --- a/block/io.c +++ b/block/io.c @@ -720,12 +720,13 @@ void bdrv_dec_in_flight(BlockDriverState *bs) bdrv_wakeup(bs); } =20 -static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *sel= f) +static bool coroutine_fn do_wait_serialising_requests(BdrvTrackedRequest *= self, + bool wait) { BlockDriverState *bs =3D self->bs; BdrvTrackedRequest *req; bool retry; - bool waited =3D false; + bool found =3D false; =20 if (!atomic_read(&bs->serialising_in_flight)) { return false; @@ -751,11 +752,13 @@ static bool coroutine_fn wait_serialising_requests(Bd= rvTrackedRequest *self) * will wait for us as soon as it wakes up, then just go on * (instead of producing a deadlock in the former case). */ if (!req->waiting_for) { - self->waiting_for =3D req; - qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); - self->waiting_for =3D NULL; - retry =3D true; - waited =3D true; + found =3D true; + if (wait) { + self->waiting_for =3D req; + qemu_co_queue_wait(&req->wait_queue, &bs->reqs_loc= k); + self->waiting_for =3D NULL; + retry =3D true; + } break; } } @@ -763,7 +766,12 @@ static bool coroutine_fn wait_serialising_requests(Bdr= vTrackedRequest *self) qemu_co_mutex_unlock(&bs->reqs_lock); } while (retry); =20 - return waited; + return found; +} + +static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *sel= f) +{ + return do_wait_serialising_requests(self, true); } =20 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087696723639.9198644815283; Sat, 29 Dec 2018 04:48:16 -0800 (PST) Received: from localhost ([127.0.0.1]:36404 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdE2E-00031v-9p for importer@patchew.org; Sat, 29 Dec 2018 07:48:10 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43042) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv0-0002vB-Jl for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:53 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcD-0007x9-BD for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:26 -0500 Received: from relay.sw.ru ([185.231.240.75]:46430) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006W5-BT; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbS-00014c-Lz; Sat, 29 Dec 2018 15:20:30 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:25 +0300 Message-Id: <20181229122027.42245-10-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 09/11] block: add lock/unlock range functions X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: Vladimir Sementsov-Ogievskiy Introduce lock/unlock range functionality, based on serialized requests. This is needed to refactor backup, dropping local tracked-request-like synchronization. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Max Reitz --- include/block/block_int.h | 4 ++++ block/io.c | 45 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/include/block/block_int.h b/include/block/block_int.h index f605622216..3475ee5360 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -64,6 +64,7 @@ enum BdrvTrackedRequestType { BDRV_TRACKED_WRITE, BDRV_TRACKED_DISCARD, BDRV_TRACKED_TRUNCATE, + BDRV_TRACKED_LOCK, }; =20 typedef struct BdrvTrackedRequest { @@ -856,6 +857,9 @@ int coroutine_fn bdrv_co_preadv(BdrvChild *child, int coroutine_fn bdrv_co_pwritev(BdrvChild *child, int64_t offset, unsigned int bytes, QEMUIOVector *qiov, BdrvRequestFlags flags); +void *coroutine_fn bdrv_co_try_lock(BdrvChild *child, + int64_t offset, unsigned int bytes); +void coroutine_fn bdrv_co_unlock(void *opaque); =20 extern unsigned int bdrv_drain_all_count; void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_pare= nt); diff --git a/block/io.c b/block/io.c index b87c11a6ec..38a63d16f1 100644 --- a/block/io.c +++ b/block/io.c @@ -720,6 +720,15 @@ void bdrv_dec_in_flight(BlockDriverState *bs) bdrv_wakeup(bs); } =20 +static bool ignore_intersection(BdrvTrackedRequest *a, BdrvTrackedRequest = *b) +{ + return a =3D=3D b || (!a->serialising && !b->serialising) || + (a->type =3D=3D BDRV_TRACKED_LOCK && b->type =3D=3D BDRV_TRACKED_R= EAD && + !b->serialising) || + (b->type =3D=3D BDRV_TRACKED_LOCK && a->type =3D=3D BDRV_TRACKED_R= EAD && + !a->serialising); +} + static bool coroutine_fn do_wait_serialising_requests(BdrvTrackedRequest *= self, bool wait) { @@ -736,7 +745,7 @@ static bool coroutine_fn do_wait_serialising_requests(B= drvTrackedRequest *self, retry =3D false; qemu_co_mutex_lock(&bs->reqs_lock); QLIST_FOREACH(req, &bs->tracked_requests, list) { - if (req =3D=3D self || (!req->serialising && !self->serialisin= g)) { + if (ignore_intersection(self, req)) { continue; } if (tracked_request_overlaps(req, self->overlap_offset, @@ -774,6 +783,12 @@ static bool coroutine_fn wait_serialising_requests(Bdr= vTrackedRequest *self) return do_wait_serialising_requests(self, true); } =20 +static bool coroutine_fn should_wait_serialising_requests( + BdrvTrackedRequest *self) +{ + return do_wait_serialising_requests(self, false); +} + static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, size_t size) { @@ -3250,3 +3265,31 @@ int bdrv_truncate(BdrvChild *child, int64_t offset, = PreallocMode prealloc, =20 return tco.ret; } + +void *coroutine_fn bdrv_co_try_lock(BdrvChild *child, + int64_t offset, unsigned int bytes) +{ + BlockDriverState *bs =3D child->bs; + BdrvTrackedRequest *req =3D g_new(BdrvTrackedRequest, 1); + + tracked_request_begin(req, bs, offset, bytes, BDRV_TRACKED_LOCK); + mark_request_serialising(req, bdrv_get_cluster_size(bs)); + + if (should_wait_serialising_requests(req)) { + tracked_request_end(req); + g_free(req); + return NULL; + } + + return req; +} + +void coroutine_fn bdrv_co_unlock(void *opaque) +{ + BdrvTrackedRequest *req =3D opaque; + + assert(req->type =3D=3D BDRV_TRACKED_LOCK); + + tracked_request_end(req); + g_free(req); +} --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087377619483.4286690720114; Sat, 29 Dec 2018 04:42:57 -0800 (PST) Received: from localhost ([127.0.0.1]:36350 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDx6-0005AN-KN for importer@patchew.org; Sat, 29 Dec 2018 07:42:52 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43094) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDv0-0002vb-9Z for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:52 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcG-00083h-GF for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:29 -0500 Received: from relay.sw.ru ([185.231.240.75]:46436) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDbu-0006g4-J2; Sat, 29 Dec 2018 07:20:59 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbS-00014c-TI; Sat, 29 Dec 2018 15:20:30 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:26 +0300 Message-Id: <20181229122027.42245-11-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 10/11] block/backup: tiny refactor backup_job_create X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Move copy-bitmap find/create code. It's needed for the following commit, as we'll need copy_bitmap before actual block job creation. Do it in a separate commit to simplify review. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Max Reitz --- block/backup.c | 69 +++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/block/backup.c b/block/backup.c index 0b3fddeb6c..88c0242b4e 100644 --- a/block/backup.c +++ b/block/backup.c @@ -561,6 +561,8 @@ BlockJob *backup_job_create(const char *job_id, BlockDr= iverState *bs, BlockDriverInfo bdi; BackupBlockJob *job =3D NULL; int ret; + int64_t cluster_size; + HBitmap *copy_bitmap =3D NULL; =20 assert(bs); assert(target); @@ -615,6 +617,35 @@ BlockJob *backup_job_create(const char *job_id, BlockD= riverState *bs, return NULL; } =20 + /* + * If there is no backing file on the target, we cannot rely on COW if= our + * backup cluster size is smaller than the target cluster size. Even f= or + * targets with a backing file, try to avoid COW if possible. + */ + ret =3D bdrv_get_info(target, &bdi); + if (ret =3D=3D -ENOTSUP && !target->backing) { + /* Cluster size is not defined */ + warn_report("The target block device doesn't provide " + "information about the block size and it doesn't have = a " + "backing file. The default block size of %u bytes is " + "used. If the actual block size of the target exceeds " + "this default, the backup may be unusable", + BACKUP_CLUSTER_SIZE_DEFAULT); + cluster_size =3D BACKUP_CLUSTER_SIZE_DEFAULT; + } else if (ret < 0 && !target->backing) { + error_setg_errno(errp, -ret, + "Couldn't determine the cluster size of the target image, " + "which has no backing file"); + error_append_hint(errp, + "Aborting, since this may create an unusable destination image= \n"); + return NULL; + } else if (ret < 0 && target->backing) { + /* Not fatal; just trudge on ahead. */ + cluster_size =3D BACKUP_CLUSTER_SIZE_DEFAULT; + } else { + cluster_size =3D MAX(BACKUP_CLUSTER_SIZE_DEFAULT, bdi.cluster_size= ); + } + len =3D bdrv_getlength(bs); if (len < 0) { error_setg_errno(errp, -len, "unable to get length for '%s'", @@ -622,6 +653,8 @@ BlockJob *backup_job_create(const char *job_id, BlockDr= iverState *bs, goto error; } =20 + copy_bitmap =3D hbitmap_alloc(len, ctz32(cluster_size)); + /* job->len is fixed, so we can't allow resize */ job =3D block_job_create(job_id, &backup_job_driver, txn, bs, BLK_PERM_CONSISTENT_READ, @@ -650,35 +683,9 @@ BlockJob *backup_job_create(const char *job_id, BlockD= riverState *bs, =20 /* Detect image-fleecing (and similar) schemes */ job->serialize_target_writes =3D bdrv_chain_contains(target, bs); - - /* If there is no backing file on the target, we cannot rely on COW if= our - * backup cluster size is smaller than the target cluster size. Even f= or - * targets with a backing file, try to avoid COW if possible. */ - ret =3D bdrv_get_info(target, &bdi); - if (ret =3D=3D -ENOTSUP && !target->backing) { - /* Cluster size is not defined */ - warn_report("The target block device doesn't provide " - "information about the block size and it doesn't have = a " - "backing file. The default block size of %u bytes is " - "used. If the actual block size of the target exceeds " - "this default, the backup may be unusable", - BACKUP_CLUSTER_SIZE_DEFAULT); - job->cluster_size =3D BACKUP_CLUSTER_SIZE_DEFAULT; - } else if (ret < 0 && !target->backing) { - error_setg_errno(errp, -ret, - "Couldn't determine the cluster size of the target image, " - "which has no backing file"); - error_append_hint(errp, - "Aborting, since this may create an unusable destination image= \n"); - goto error; - } else if (ret < 0 && target->backing) { - /* Not fatal; just trudge on ahead. */ - job->cluster_size =3D BACKUP_CLUSTER_SIZE_DEFAULT; - } else { - job->cluster_size =3D MAX(BACKUP_CLUSTER_SIZE_DEFAULT, bdi.cluster= _size); - } - - job->copy_bitmap =3D hbitmap_alloc(len, ctz32(job->cluster_size)); + job->cluster_size =3D cluster_size; + job->copy_bitmap =3D copy_bitmap; + copy_bitmap =3D NULL; job->use_copy_range =3D true; job->copy_range_size =3D MIN_NON_ZERO(blk_get_max_transfer(job->common= .blk), blk_get_max_transfer(job->target)); @@ -694,6 +701,10 @@ BlockJob *backup_job_create(const char *job_id, BlockD= riverState *bs, return &job->common; =20 error: + if (copy_bitmap) { + assert(!job || !job->copy_bitmap); + hbitmap_free(copy_bitmap); + } if (sync_bitmap) { bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL); } --=20 2.18.0 From nobody Sat May 4 14:54:08 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1546087542275916.9425937697255; Sat, 29 Dec 2018 04:45:42 -0800 (PST) Received: from localhost ([127.0.0.1]:36378 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDzo-0000FF-PF for importer@patchew.org; Sat, 29 Dec 2018 07:45:40 -0500 Received: from eggs.gnu.org ([208.118.235.92]:43246) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gdDuz-0002x8-A4 for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:40:53 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gdDcK-0008AT-7O for qemu-devel@nongnu.org; Sat, 29 Dec 2018 07:21:32 -0500 Received: from relay.sw.ru ([185.231.240.75]:46438) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gdDba-0006g3-C8; Sat, 29 Dec 2018 07:20:38 -0500 Received: from [10.28.8.145] (helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.91) (envelope-from ) id 1gdDbT-00014c-2e; Sat, 29 Dec 2018 15:20:31 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Sat, 29 Dec 2018 15:20:27 +0300 Message-Id: <20181229122027.42245-12-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20181229122027.42245-1-vsementsov@virtuozzo.com> References: <20181229122027.42245-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v5 11/11] block/backup: use backup-top instead of write notifiers X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: fam@euphon.net, kwolf@redhat.com, vsementsov@virtuozzo.com, jsnow@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Drop write notifiers and use filter node instead. Changes: 1. copy-before-writes now handled by filter node, so, drop all is_write_notifier arguments. 2. we don't have intersecting requests, so their handling is dropped. Instead, synchronization works as follows: when backup or backup-top starts copying of some area it firstly clears copy-bitmap bits, and nobody touches areas, not marked with dirty bits in copy-bitmap, so there is no intersection. Also, backup job copy operations are surrounded by bdrv region lock, which is actually serializing request, to not interfer with guest writes and not read changed data from source (before reading we clear corresponding bit in copy-bitmap, so, this area is not more handled by backup-top). 3. To sync with in-flight requests we now just drain hook node, we don't need rw-lock. 4. After the whole backup loop (top, full, incremental modes), we need to check for not copied clusters, which were under backup-top operation and we skipped them, but backup-top operation failed, error returned to the guest and dirty bits set back. 5. Don't create additional blk, use backup-top children for copy operations. Signed-off-by: Vladimir Sementsov-Ogievskiy --- block/backup.c | 285 ++++++++++++++++++++++++++----------------------- 1 file changed, 149 insertions(+), 136 deletions(-) diff --git a/block/backup.c b/block/backup.c index 88c0242b4e..e332909fb7 100644 --- a/block/backup.c +++ b/block/backup.c @@ -26,105 +26,61 @@ #include "qemu/bitmap.h" #include "qemu/error-report.h" =20 -#define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16) +#include "block/backup-top.h" =20 -typedef struct CowRequest { - int64_t start_byte; - int64_t end_byte; - QLIST_ENTRY(CowRequest) list; - CoQueue wait_queue; /* coroutines blocked on this request */ -} CowRequest; +#define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16) =20 typedef struct BackupBlockJob { BlockJob common; - BlockBackend *target; + BdrvChild *source; + BdrvChild *target; /* bitmap for sync=3Dincremental */ BdrvDirtyBitmap *sync_bitmap; MirrorSyncMode sync_mode; BlockdevOnError on_source_error; BlockdevOnError on_target_error; - CoRwlock flush_rwlock; uint64_t len; uint64_t bytes_read; int64_t cluster_size; bool compress; - NotifierWithReturn before_write; - QLIST_HEAD(, CowRequest) inflight_reqs; =20 HBitmap *copy_bitmap; bool use_copy_range; int64_t copy_range_size; =20 bool serialize_target_writes; + + BlockDriverState *backup_top; + uint64_t backup_top_progress; } BackupBlockJob; =20 static const BlockJobDriver backup_job_driver; =20 -/* See if in-flight requests overlap and wait for them to complete */ -static void coroutine_fn wait_for_overlapping_requests(BackupBlockJob *job, - int64_t start, - int64_t end) -{ - CowRequest *req; - bool retry; - - do { - retry =3D false; - QLIST_FOREACH(req, &job->inflight_reqs, list) { - if (end > req->start_byte && start < req->end_byte) { - qemu_co_queue_wait(&req->wait_queue, NULL); - retry =3D true; - break; - } - } - } while (retry); -} - -/* Keep track of an in-flight request */ -static void cow_request_begin(CowRequest *req, BackupBlockJob *job, - int64_t start, int64_t end) -{ - req->start_byte =3D start; - req->end_byte =3D end; - qemu_co_queue_init(&req->wait_queue); - QLIST_INSERT_HEAD(&job->inflight_reqs, req, list); -} - -/* Forget about a completed request */ -static void cow_request_end(CowRequest *req) -{ - QLIST_REMOVE(req, list); - qemu_co_queue_restart_all(&req->wait_queue); -} - /* Copy range to target with a bounce buffer and return the bytes copied. = If * error occurred, return a negative error number */ static int coroutine_fn backup_cow_with_bounce_buffer(BackupBlockJob *job, int64_t start, int64_t end, - bool is_write_notifi= er, bool *error_is_read, void **bounce_buffer) { int ret; struct iovec iov; QEMUIOVector qiov; - BlockBackend *blk =3D job->common.blk; int nbytes; - int read_flags =3D is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0; int write_flags =3D job->serialize_target_writes ? BDRV_REQ_SERIALISIN= G : 0; =20 assert(QEMU_IS_ALIGNED(start, job->cluster_size)); hbitmap_reset(job->copy_bitmap, start, job->cluster_size); nbytes =3D MIN(job->cluster_size, job->len - start); if (!*bounce_buffer) { - *bounce_buffer =3D blk_blockalign(blk, job->cluster_size); + *bounce_buffer =3D qemu_blockalign(job->source->bs, job->cluster_s= ize); } iov.iov_base =3D *bounce_buffer; iov.iov_len =3D nbytes; qemu_iovec_init_external(&qiov, &iov, 1); =20 - ret =3D blk_co_preadv(blk, start, qiov.size, &qiov, read_flags); + ret =3D bdrv_co_preadv(job->source, start, qiov.size, &qiov, 0); if (ret < 0) { trace_backup_do_cow_read_fail(job, start, ret); if (error_is_read) { @@ -134,12 +90,12 @@ static int coroutine_fn backup_cow_with_bounce_buffer(= BackupBlockJob *job, } =20 if (qemu_iovec_is_zero(&qiov)) { - ret =3D blk_co_pwrite_zeroes(job->target, start, - qiov.size, write_flags | BDRV_REQ_MAY_U= NMAP); + ret =3D bdrv_co_pwrite_zeroes(job->target, start, qiov.size, + write_flags | BDRV_REQ_MAY_UNMAP); } else { - ret =3D blk_co_pwritev(job->target, start, - qiov.size, &qiov, write_flags | - (job->compress ? BDRV_REQ_WRITE_COMPRESSED : = 0)); + ret =3D bdrv_co_pwritev(job->target, start, + qiov.size, &qiov, write_flags | + (job->compress ? BDRV_REQ_WRITE_COMPRESSED := 0)); } if (ret < 0) { trace_backup_do_cow_write_fail(job, start, ret); @@ -159,15 +115,11 @@ fail: /* Copy range to target and return the bytes copied. If error occurred, re= turn a * negative error number. */ static int coroutine_fn backup_cow_with_offload(BackupBlockJob *job, - int64_t start, - int64_t end, - bool is_write_notifier) + int64_t start, int64_t end) { int ret; int nr_clusters; - BlockBackend *blk =3D job->common.blk; int nbytes; - int read_flags =3D is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0; int write_flags =3D job->serialize_target_writes ? BDRV_REQ_SERIALISIN= G : 0; =20 assert(QEMU_IS_ALIGNED(job->copy_range_size, job->cluster_size)); @@ -175,8 +127,8 @@ static int coroutine_fn backup_cow_with_offload(BackupB= lockJob *job, nbytes =3D MIN(job->copy_range_size, end - start); nr_clusters =3D DIV_ROUND_UP(nbytes, job->cluster_size); hbitmap_reset(job->copy_bitmap, start, job->cluster_size * nr_clusters= ); - ret =3D blk_co_copy_range(blk, start, job->target, start, nbytes, - read_flags, write_flags); + ret =3D bdrv_co_copy_range(job->source, start, job->target, start, nby= tes, + 0, write_flags); if (ret < 0) { trace_backup_do_cow_copy_range_fail(job, start, ret); hbitmap_set(job->copy_bitmap, start, job->cluster_size * nr_cluste= rs); @@ -188,24 +140,18 @@ static int coroutine_fn backup_cow_with_offload(Backu= pBlockJob *job, =20 static int coroutine_fn backup_do_cow(BackupBlockJob *job, int64_t offset, uint64_t bytes, - bool *error_is_read, - bool is_write_notifier) + bool *error_is_read) { - CowRequest cow_request; int ret =3D 0; int64_t start, end; /* bytes */ void *bounce_buffer =3D NULL; - - qemu_co_rwlock_rdlock(&job->flush_rwlock); + uint64_t backup_top_progress; =20 start =3D QEMU_ALIGN_DOWN(offset, job->cluster_size); end =3D QEMU_ALIGN_UP(bytes + offset, job->cluster_size); =20 trace_backup_do_cow_enter(job, start, offset, bytes); =20 - wait_for_overlapping_requests(job, start, end); - cow_request_begin(&cow_request, job, start, end); - while (start < end) { if (!hbitmap_get(job->copy_bitmap, start)) { trace_backup_do_cow_skip(job, start); @@ -216,13 +162,13 @@ static int coroutine_fn backup_do_cow(BackupBlockJob = *job, trace_backup_do_cow_process(job, start); =20 if (job->use_copy_range) { - ret =3D backup_cow_with_offload(job, start, end, is_write_noti= fier); + ret =3D backup_cow_with_offload(job, start, end); if (ret < 0) { job->use_copy_range =3D false; } } if (!job->use_copy_range) { - ret =3D backup_cow_with_bounce_buffer(job, start, end, is_writ= e_notifier, + ret =3D backup_cow_with_bounce_buffer(job, start, end, error_is_read, &bounce_buf= fer); } if (ret < 0) { @@ -234,7 +180,10 @@ static int coroutine_fn backup_do_cow(BackupBlockJob *= job, */ start +=3D ret; job->bytes_read +=3D ret; - job_progress_update(&job->common.job, ret); + backup_top_progress =3D bdrv_backup_top_progress(job->backup_top); + job_progress_update(&job->common.job, ret + backup_top_progress - + job->backup_top_progress); + job->backup_top_progress =3D backup_top_progress; ret =3D 0; } =20 @@ -242,33 +191,15 @@ static int coroutine_fn backup_do_cow(BackupBlockJob = *job, qemu_vfree(bounce_buffer); } =20 - cow_request_end(&cow_request); - trace_backup_do_cow_return(job, offset, bytes, ret); =20 - qemu_co_rwlock_unlock(&job->flush_rwlock); - return ret; } =20 -static int coroutine_fn backup_before_write_notify( - NotifierWithReturn *notifier, - void *opaque) -{ - BackupBlockJob *job =3D container_of(notifier, BackupBlockJob, before_= write); - BdrvTrackedRequest *req =3D opaque; - - assert(req->bs =3D=3D blk_bs(job->common.blk)); - assert(QEMU_IS_ALIGNED(req->offset, BDRV_SECTOR_SIZE)); - assert(QEMU_IS_ALIGNED(req->bytes, BDRV_SECTOR_SIZE)); - - return backup_do_cow(job, req->offset, req->bytes, NULL, true); -} - static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret) { BdrvDirtyBitmap *bm; - BlockDriverState *bs =3D blk_bs(job->common.blk); + BlockDriverState *bs =3D job->source->bs; =20 if (ret < 0) { /* Merge the successor back into the parent, delete nothing. */ @@ -300,21 +231,23 @@ static void backup_abort(Job *job) static void backup_clean(Job *job) { BackupBlockJob *s =3D container_of(job, BackupBlockJob, common.job); - assert(s->target); - blk_unref(s->target); + + /* We must clean it to not crash in backup_drain. */ s->target =3D NULL; =20 if (s->copy_bitmap) { hbitmap_free(s->copy_bitmap); s->copy_bitmap =3D NULL; } + + bdrv_backup_top_drop(s->backup_top); } =20 static void backup_attached_aio_context(BlockJob *job, AioContext *aio_con= text) { BackupBlockJob *s =3D container_of(job, BackupBlockJob, common); =20 - blk_set_aio_context(s->target, aio_context); + bdrv_set_aio_context(s->target->bs, aio_context); } =20 void backup_do_checkpoint(BlockJob *job, Error **errp) @@ -340,10 +273,10 @@ static void backup_drain(BlockJob *job) * of backup_complete... */ if (s->target) { - BlockBackend *target =3D s->target; - blk_ref(target); - blk_drain(target); - blk_unref(target); + BlockDriverState *target =3D s->target->bs; + bdrv_ref(target); + bdrv_drain(target); + bdrv_unref(target); } } =20 @@ -386,21 +319,45 @@ static int coroutine_fn backup_run_incremental(Backup= BlockJob *job) bool error_is_read; int64_t offset; HBitmapIter hbi; + void *lock =3D NULL; =20 hbitmap_iter_init(&hbi, job->copy_bitmap, 0); - while ((offset =3D hbitmap_iter_next(&hbi)) !=3D -1) { + while (hbitmap_count(job->copy_bitmap)) { + offset =3D hbitmap_iter_next(&hbi); + if (offset =3D=3D -1) { + /* + * we may have skipped some clusters, which were handled by + * backup-top, but failed and finished by returning error to + * the guest and set dirty bit back. + */ + hbitmap_iter_init(&hbi, job->copy_bitmap, 0); + offset =3D hbitmap_iter_next(&hbi); + assert(offset); + } + + lock =3D bdrv_co_try_lock(job->source, offset, job->cluster_size); + /* + * Dirty bit is set, which means that there are no in-flight + * write requests on this area. We must succeed. + */ + assert(lock); + do { if (yield_and_check(job)) { + bdrv_co_unlock(lock); return 0; } - ret =3D backup_do_cow(job, offset, - job->cluster_size, &error_is_read, false); + ret =3D backup_do_cow(job, offset, job->cluster_size, &error_i= s_read); if (ret < 0 && backup_error_action(job, error_is_read, -ret) = =3D=3D BLOCK_ERROR_ACTION_REPORT) { + bdrv_co_unlock(lock); return ret; } } while (ret < 0); + + bdrv_co_unlock(lock); + lock =3D NULL; } =20 return 0; @@ -432,12 +389,10 @@ static void backup_incremental_init_copy_bitmap(Backu= pBlockJob *job) static int coroutine_fn backup_run(Job *job, Error **errp) { BackupBlockJob *s =3D container_of(job, BackupBlockJob, common.job); - BlockDriverState *bs =3D blk_bs(s->common.blk); + BlockDriverState *bs =3D s->source->bs; int64_t offset; int ret =3D 0; - - QLIST_INIT(&s->inflight_reqs); - qemu_co_rwlock_init(&s->flush_rwlock); + uint64_t backup_top_progress; =20 job_progress_set_remaining(job, s->len); =20 @@ -447,26 +402,39 @@ static int coroutine_fn backup_run(Job *job, Error **= errp) hbitmap_set(s->copy_bitmap, 0, s->len); } =20 - s->before_write.notify =3D backup_before_write_notify; - bdrv_add_before_write_notifier(bs, &s->before_write); - if (s->sync_mode =3D=3D MIRROR_SYNC_MODE_NONE) { /* All bits are set in copy_bitmap to allow any cluster to be copi= ed. * This does not actually require them to be copied. */ while (!job_is_cancelled(job)) { - /* Yield until the job is cancelled. We just let our before_w= rite - * notify callback service CoW requests. */ + /* + * Yield until the job is cancelled. We just let our backup-t= op + * fileter driver service CbW requests. + */ job_yield(job); } } else if (s->sync_mode =3D=3D MIRROR_SYNC_MODE_INCREMENTAL) { ret =3D backup_run_incremental(s); } else { + bool retry; + void *lock; + +iteration: + retry =3D false; + lock =3D NULL; + /* Both FULL and TOP SYNC_MODE's require copying.. */ for (offset =3D 0; offset < s->len; offset +=3D s->cluster_size) { bool error_is_read; int alloced =3D 0; =20 + if (retry) { + retry =3D false; + } else if (lock) { + bdrv_co_unlock(lock); + lock =3D NULL; + } + if (yield_and_check(s)) { break; } @@ -498,6 +466,7 @@ static int coroutine_fn backup_run(Job *job, Error **er= rp) /* If the above loop never found any sectors that are in * the topmost image, skip this backup. */ if (alloced =3D=3D 0) { + hbitmap_reset(s->copy_bitmap, offset, s->cluster_size); continue; } } @@ -505,8 +474,20 @@ static int coroutine_fn backup_run(Job *job, Error **e= rrp) if (alloced < 0) { ret =3D alloced; } else { + if (!hbitmap_get(s->copy_bitmap, offset)) { + trace_backup_do_cow_skip(job, offset); + continue; /* already copied */ + } + if (!lock) { + lock =3D bdrv_co_try_lock(s->source, offset, s->cluste= r_size); + /* + * Dirty bit is set, which means that there are no in-= flight + * write requests on this area. We must succeed. + */ + assert(lock); + } ret =3D backup_do_cow(s, offset, s->cluster_size, - &error_is_read, false); + &error_is_read); } if (ret < 0) { /* Depending on error action, fail now or retry cluster */ @@ -516,17 +497,34 @@ static int coroutine_fn backup_run(Job *job, Error **= errp) break; } else { offset -=3D s->cluster_size; + retry =3D true; continue; } } } + if (lock) { + bdrv_co_unlock(lock); + lock =3D NULL; + } + if (ret =3D=3D 0 && !job_is_cancelled(job) && + hbitmap_count(s->copy_bitmap)) + { + /* + * we may have skipped some clusters, which were handled by + * backup-top, but failed and finished by returning error to + * the guest and set dirty bit back. + */ + goto iteration; + } } =20 - notifier_with_return_remove(&s->before_write); + /* wait pending CBW operations in backup-top */ + bdrv_drain(s->backup_top); =20 - /* wait until pending backup_do_cow() calls have completed */ - qemu_co_rwlock_wrlock(&s->flush_rwlock); - qemu_co_rwlock_unlock(&s->flush_rwlock); + backup_top_progress =3D bdrv_backup_top_progress(s->backup_top); + job_progress_update(job, ret + backup_top_progress - + s->backup_top_progress); + s->backup_top_progress =3D backup_top_progress; =20 return ret; } @@ -563,6 +561,9 @@ BlockJob *backup_job_create(const char *job_id, BlockDr= iverState *bs, int ret; int64_t cluster_size; HBitmap *copy_bitmap =3D NULL; + BlockDriverState *backup_top =3D NULL; + uint64_t all_except_resize =3D BLK_PERM_CONSISTENT_READ | BLK_PERM_WRI= TE | + BLK_PERM_WRITE_UNCHANGED | BLK_PERM_GRAPH= _MOD; =20 assert(bs); assert(target); @@ -655,25 +656,31 @@ BlockJob *backup_job_create(const char *job_id, Block= DriverState *bs, =20 copy_bitmap =3D hbitmap_alloc(len, ctz32(cluster_size)); =20 - /* job->len is fixed, so we can't allow resize */ - job =3D block_job_create(job_id, &backup_job_driver, txn, bs, - BLK_PERM_CONSISTENT_READ, - BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | - BLK_PERM_WRITE_UNCHANGED | BLK_PERM_GRAPH_MOD, - speed, creation_flags, cb, opaque, errp); - if (!job) { + /* + * bdrv_get_device_name will not help to find device name starting from + * @bs after backup-top append, so let's calculate job_id before. Do + * it in the same way like block_job_create + */ + if (job_id =3D=3D NULL && !(creation_flags & JOB_INTERNAL)) { + job_id =3D bdrv_get_device_name(bs); + } + + backup_top =3D bdrv_backup_top_append(bs, target, copy_bitmap, errp); + if (!backup_top) { goto error; } =20 - /* The target must match the source in size, so no resize here either = */ - job->target =3D blk_new(BLK_PERM_WRITE, - BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | - BLK_PERM_WRITE_UNCHANGED | BLK_PERM_GRAPH_MOD); - ret =3D blk_insert_bs(job->target, target, errp); - if (ret < 0) { + /* job->len is fixed, so we can't allow resize */ + job =3D block_job_create(job_id, &backup_job_driver, txn, bs, 0, + all_except_resize, speed, creation_flags, + cb, opaque, errp); + if (!job) { goto error; } =20 + job->source =3D backup_top->backing; + job->target =3D ((BDRVBackupTopState *)backup_top->opaque)->target; + job->on_source_error =3D on_source_error; job->on_target_error =3D on_target_error; job->sync_mode =3D sync_mode; @@ -687,16 +694,19 @@ BlockJob *backup_job_create(const char *job_id, Block= DriverState *bs, job->copy_bitmap =3D copy_bitmap; copy_bitmap =3D NULL; job->use_copy_range =3D true; - job->copy_range_size =3D MIN_NON_ZERO(blk_get_max_transfer(job->common= .blk), - blk_get_max_transfer(job->target)); + job->copy_range_size =3D + MIN_NON_ZERO(MIN_NON_ZERO(INT_MAX, + job->source->bs->bl.max_transfer), + job->target->bs->bl.max_transfer); job->copy_range_size =3D MAX(job->cluster_size, QEMU_ALIGN_UP(job->copy_range_size, job->cluster_size)); =20 - /* Required permissions are already taken with target's blk_new() */ - block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL, + /* The target must match the source in size, so no resize here either = */ + block_job_add_bdrv(&job->common, "target", target, 0, all_except_resiz= e, &error_abort); job->len =3D len; + job->backup_top =3D backup_top; =20 return &job->common; =20 @@ -712,6 +722,9 @@ BlockJob *backup_job_create(const char *job_id, BlockDr= iverState *bs, backup_clean(&job->common.job); job_early_fail(&job->common.job); } + if (backup_top) { + bdrv_backup_top_drop(backup_top); + } =20 return NULL; } --=20 2.18.0