From nobody Mon May 6 07:18:54 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@gnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@gnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1506955506420173.36079051817615; Mon, 2 Oct 2017 07:45:06 -0700 (PDT) Received: from localhost ([::1]:52665 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1xq-0006BZ-PG for importer@patchew.org; Mon, 02 Oct 2017 10:44:58 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:52143) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1sX-0001YG-Hw for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:33 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1dz1sT-00030e-JN for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:29 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:5764 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1dz1sT-0002ys-7C for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:25 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v92EdJQ5007920; Mon, 2 Oct 2017 17:39:20 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Mon, 2 Oct 2017 17:39:15 +0300 Message-Id: <20171002143919.207741-2-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171002143919.207741-1-vsementsov@virtuozzo.com> References: <20171002143919.207741-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 1/5] hbitmap: add next_zero function X-BeenThere: qemu-devel@gnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@gnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" The function searches for next zero bit. Also add interface for BdrvDirtyBitmap. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow --- include/block/dirty-bitmap.h | 1 + include/qemu/hbitmap.h | 8 ++++++++ block/dirty-bitmap.c | 5 +++++ util/hbitmap.c | 29 +++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h index 3579a7597c..a591c27213 100644 --- a/include/block/dirty-bitmap.h +++ b/include/block/dirty-bitmap.h @@ -91,5 +91,6 @@ bool bdrv_has_changed_persistent_bitmaps(BlockDriverState= *bs); BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs, BdrvDirtyBitmap *bitmap); char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp= ); +int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t star= t); =20 #endif diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h index 81e78043d1..6b6490ecad 100644 --- a/include/qemu/hbitmap.h +++ b/include/qemu/hbitmap.h @@ -292,6 +292,14 @@ void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap= *hb, uint64_t first); */ unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi); =20 +/* hbitmap_next_zero: + * @hb: The HBitmap to operate on + * @start: The bit to start from. + * + * Find next not dirty bit. + */ +int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start); + /* hbitmap_create_meta: * Create a "meta" hbitmap to track dirtiness of the bits in this HBitmap. * The caller owns the created bitmap and must call hbitmap_free_meta(hb) = to diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c index bd04e991b1..7879d13ddb 100644 --- a/block/dirty-bitmap.c +++ b/block/dirty-bitmap.c @@ -715,3 +715,8 @@ char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *b= itmap, Error **errp) { return hbitmap_sha256(bitmap->bitmap, errp); } + +int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offs= et) +{ + return hbitmap_next_zero(bitmap->bitmap, offset); +} diff --git a/util/hbitmap.c b/util/hbitmap.c index 2f9d0fdbd0..ffcdbc5587 100644 --- a/util/hbitmap.c +++ b/util/hbitmap.c @@ -188,6 +188,35 @@ void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap= *hb, uint64_t first) } } =20 +int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start) +{ + size_t pos =3D (start >> hb->granularity) >> BITS_PER_LEVEL; + unsigned long *last_lev =3D hb->levels[HBITMAP_LEVELS - 1]; + uint64_t sz =3D hb->sizes[HBITMAP_LEVELS - 1]; + unsigned long cur =3D last_lev[pos]; + unsigned start_bit_offset =3D + (start >> hb->granularity) & (BITS_PER_LONG - 1); + int64_t res; + cur |=3D (1UL << start_bit_offset) - 1; + assert((start >> hb->granularity) < hb->size); + + if (cur =3D=3D (unsigned long)-1) { + do { + pos++; + } while (pos < sz && last_lev[pos] =3D=3D (unsigned long)-1); + + if (pos >=3D sz) { + return -1; + } + + cur =3D last_lev[pos]; + } + + res =3D (pos << BITS_PER_LEVEL) + ctol(cur); + + return res < hb->size ? (res << hb->granularity) : -1; +} + bool hbitmap_empty(const HBitmap *hb) { return hb->count =3D=3D 0; --=20 2.11.1 From nobody Mon May 6 07:18:54 2024 Delivered-To: importer@patchew.org Received-SPF: temperror (zoho.com: Error in retrieving data from DNS) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@gnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=temperror (zoho.com: Error in retrieving data from DNS) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@gnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1506955413879818.4709235570999; Mon, 2 Oct 2017 07:43:33 -0700 (PDT) Received: from localhost ([::1]:52652 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1wL-0004my-R8 for importer@patchew.org; Mon, 02 Oct 2017 10:43:25 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:52150) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1sX-0001Yd-V9 for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:35 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1dz1sW-00032K-L1 for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:29 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:15308 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1dz1sW-00031Z-8l for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:28 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v92EdJQ6007920; Mon, 2 Oct 2017 17:39:20 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Mon, 2 Oct 2017 17:39:16 +0300 Message-Id: <20171002143919.207741-3-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171002143919.207741-1-vsementsov@virtuozzo.com> References: <20171002143919.207741-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 2/5] backup: move from done_bitmap to copy_bitmap X-BeenThere: qemu-devel@gnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@gnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_6 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Use HBitmap copy_bitmap instead of done_bitmap. This is needed to improve incremental backup in following patches and to unify backup loop for full/incremental modes in future patches. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Jeff Cody Reviewed-by: Stefan Hajnoczi Reviewed-by: John Snow --- block/backup.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/block/backup.c b/block/backup.c index 06ddbfd03d..08efec639f 100644 --- a/block/backup.c +++ b/block/backup.c @@ -40,11 +40,12 @@ typedef struct BackupBlockJob { BlockdevOnError on_target_error; CoRwlock flush_rwlock; uint64_t bytes_read; - unsigned long *done_bitmap; int64_t cluster_size; bool compress; NotifierWithReturn before_write; QLIST_HEAD(, CowRequest) inflight_reqs; + + HBitmap *copy_bitmap; } BackupBlockJob; =20 /* See if in-flight requests overlap and wait for them to complete */ @@ -109,10 +110,11 @@ static int coroutine_fn backup_do_cow(BackupBlockJob = *job, cow_request_begin(&cow_request, job, start, end); =20 for (; start < end; start +=3D job->cluster_size) { - if (test_bit(start / job->cluster_size, job->done_bitmap)) { + if (!hbitmap_get(job->copy_bitmap, start / job->cluster_size)) { trace_backup_do_cow_skip(job, start); continue; /* already copied */ } + hbitmap_reset(job->copy_bitmap, start / job->cluster_size, 1); =20 trace_backup_do_cow_process(job, start); =20 @@ -132,6 +134,7 @@ static int coroutine_fn backup_do_cow(BackupBlockJob *j= ob, if (error_is_read) { *error_is_read =3D true; } + hbitmap_set(job->copy_bitmap, start / job->cluster_size, 1); goto out; } =20 @@ -148,11 +151,10 @@ static int coroutine_fn backup_do_cow(BackupBlockJob = *job, if (error_is_read) { *error_is_read =3D false; } + hbitmap_set(job->copy_bitmap, start / job->cluster_size, 1); goto out; } =20 - set_bit(start / job->cluster_size, job->done_bitmap); - /* Publish progress, guest I/O counts as progress too. Note that = the * offset field is an opaque progress value, it is not a disk offs= et. */ @@ -260,7 +262,7 @@ void backup_do_checkpoint(BlockJob *job, Error **errp) } =20 len =3D DIV_ROUND_UP(backup_job->common.len, backup_job->cluster_size); - bitmap_zero(backup_job->done_bitmap, len); + hbitmap_set(backup_job->copy_bitmap, 0, len); } =20 void backup_wait_for_overlapping_requests(BlockJob *job, int64_t offset, @@ -425,14 +427,15 @@ static void coroutine_fn backup_run(void *opaque) BackupBlockJob *job =3D opaque; BackupCompleteData *data; BlockDriverState *bs =3D blk_bs(job->common.blk); - int64_t offset; + int64_t offset, nb_clusters; int ret =3D 0; =20 QLIST_INIT(&job->inflight_reqs); qemu_co_rwlock_init(&job->flush_rwlock); =20 - job->done_bitmap =3D bitmap_new(DIV_ROUND_UP(job->common.len, - job->cluster_size)); + nb_clusters =3D DIV_ROUND_UP(job->common.len, job->cluster_size); + job->copy_bitmap =3D hbitmap_alloc(nb_clusters, 0); + hbitmap_set(job->copy_bitmap, 0, nb_clusters); =20 job->before_write.notify =3D backup_before_write_notify; bdrv_add_before_write_notifier(bs, &job->before_write); @@ -512,7 +515,7 @@ static void coroutine_fn backup_run(void *opaque) /* wait until pending backup_do_cow() calls have completed */ qemu_co_rwlock_wrlock(&job->flush_rwlock); qemu_co_rwlock_unlock(&job->flush_rwlock); - g_free(job->done_bitmap); + hbitmap_free(job->copy_bitmap); =20 data =3D g_malloc(sizeof(*data)); data->ret =3D ret; --=20 2.11.1 From nobody Mon May 6 07:18:54 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@gnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@gnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 150695528061823.877697283141742; Mon, 2 Oct 2017 07:41:20 -0700 (PDT) Received: from localhost ([::1]:52639 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1uG-0002rQ-LR for importer@patchew.org; Mon, 02 Oct 2017 10:41:16 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:52141) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1sX-0001YF-HL for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:30 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1dz1sW-000326-Ei for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:29 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:24581 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1dz1sW-00031R-21 for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:28 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v92EdJQ7007920; Mon, 2 Oct 2017 17:39:20 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Mon, 2 Oct 2017 17:39:17 +0300 Message-Id: <20171002143919.207741-4-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171002143919.207741-1-vsementsov@virtuozzo.com> References: <20171002143919.207741-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 3/5] backup: init copy_bitmap from sync_bitmap for incremental X-BeenThere: qemu-devel@gnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@gnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" We should not copy non-dirty clusters in write notifiers. So, initialize copy_bitmap from sync_bitmap. Signed-off-by: Vladimir Sementsov-Ogievskiy --- block/backup.c | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/block/backup.c b/block/backup.c index 08efec639f..07f4ae846b 100644 --- a/block/backup.c +++ b/block/backup.c @@ -422,6 +422,43 @@ out: return ret; } =20 +/* init copy_bitmap from sync_bitmap */ +static void backup_incremental_init_copy_bitmap(BackupBlockJob *job) +{ + BdrvDirtyBitmapIter *dbi; + int64_t offset; + int64_t end =3D DIV_ROUND_UP(bdrv_dirty_bitmap_size(job->sync_bitmap), + job->cluster_size); + + dbi =3D bdrv_dirty_iter_new(job->sync_bitmap); + while ((offset =3D bdrv_dirty_iter_next(dbi)) !=3D -1) { + int64_t cluster =3D offset / job->cluster_size; + int64_t next_cluster; + + offset +=3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); + if (offset >=3D bdrv_dirty_bitmap_size(job->sync_bitmap)) { + hbitmap_set(job->copy_bitmap, cluster, end - cluster); + break; + } + + offset =3D bdrv_dirty_bitmap_next_zero(job->sync_bitmap, offset); + if (offset =3D=3D -1) { + hbitmap_set(job->copy_bitmap, cluster, end - cluster); + break; + } + + next_cluster =3D DIV_ROUND_UP(offset, job->cluster_size); + hbitmap_set(job->copy_bitmap, cluster, next_cluster - cluster); + if (next_cluster >=3D end) { + break; + } + + bdrv_set_dirty_iter(dbi, next_cluster * job->cluster_size); + } + + bdrv_dirty_iter_free(dbi); +} + static void coroutine_fn backup_run(void *opaque) { BackupBlockJob *job =3D opaque; @@ -435,20 +472,26 @@ static void coroutine_fn backup_run(void *opaque) =20 nb_clusters =3D DIV_ROUND_UP(job->common.len, job->cluster_size); job->copy_bitmap =3D hbitmap_alloc(nb_clusters, 0); - hbitmap_set(job->copy_bitmap, 0, nb_clusters); =20 job->before_write.notify =3D backup_before_write_notify; bdrv_add_before_write_notifier(bs, &job->before_write); =20 if (job->sync_mode =3D=3D MIRROR_SYNC_MODE_NONE) { + /* Here we set all bits in job->copy_bitmap to allow copying clust= ers + * which are going to be changed (none-mode semantics). It doesn't= mean + * that we want to copy _all_ clusters. + */ + hbitmap_set(job->copy_bitmap, 0, nb_clusters); while (!block_job_is_cancelled(&job->common)) { /* Yield until the job is cancelled. We just let our before_w= rite * notify callback service CoW requests. */ block_job_yield(&job->common); } } else if (job->sync_mode =3D=3D MIRROR_SYNC_MODE_INCREMENTAL) { + backup_incremental_init_copy_bitmap(job); ret =3D backup_run_incremental(job); } else { + hbitmap_set(job->copy_bitmap, 0, nb_clusters); /* Both FULL and TOP SYNC_MODE's require copying.. */ for (offset =3D 0; offset < job->common.len; offset +=3D job->cluster_size) { --=20 2.11.1 From nobody Mon May 6 07:18:54 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@gnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@gnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1506955570046290.3288181569791; Mon, 2 Oct 2017 07:46:10 -0700 (PDT) Received: from localhost ([::1]:52672 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1yr-0006pe-B5 for importer@patchew.org; Mon, 02 Oct 2017 10:46:01 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:52128) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1sW-0001Xk-Uq for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:34 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1dz1sU-000310-0q for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:28 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:34903 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1dz1sT-0002yz-JB for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:25 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v92EdJQ8007920; Mon, 2 Oct 2017 17:39:20 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Mon, 2 Oct 2017 17:39:18 +0300 Message-Id: <20171002143919.207741-5-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171002143919.207741-1-vsementsov@virtuozzo.com> References: <20171002143919.207741-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 4/5] backup: simplify non-dirty bits progress processing X-BeenThere: qemu-devel@gnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@gnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Set fake progress for non-dirty clusters in copy_bitmap initialization, to. It simplifies code and allows further refactoring. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow --- Motivation (some of these paragraphs may be needed in commit message...) 1. Current behavior is not good: setting fake progress leads to progress hopping, so actually for sparse disk progress say nothing. We just move all these hops to the beginning. 2. Big hop in the beginning is possible now too: if there is a hole at the beginning of virtual disk. 3. Now, there are some excuses for old behavior in comparison to new one: backup progress is linear, cluster by cluster. But in future cluster copying will be done by several coroutine-workers, several requests in parallel, so it will make no sense to publish fake progress by parts in parallel with non-sequential requests. 4. Actually it's just a point of view: when we are actually skip clusters? If go through disk sequentially, it's logical to say, that we skip clusters between copied portions to the left and to the right of them. But even know copying progress is not sequential because of write notifiers. If copying is async, non-sequential, we can say in the very beginning, that we skip these clusters and do not think about them later. So, I don't want to maintain portions of fake progress in the new backup architecture. I understand, that this patch will change user's view of backup progress, but formally it doesn't change: progress hops are just moved to the beginning. Progress was meaningless for incremental backup and it remains meaningless. But what should we do with progress, really? It's obvious, that we need at least one more field for block job status, for example, "skipped", which would be amount of bytes which are not really processed but skipped by the block job. So, more real progress may be calculated (by management tool) li= ke (job.offset - job.skipped) / (job.len - job.skipped) And this progress will be more precise if information about skipped bytes is provided earlier, and current patch is close to this idea. So, if you agree, I can make a patch for 'skipped' too, but it actually not related to the series, so I hope that current patch will be merged anyway. (I need it for backup refactoring, not for accurate progress) block/backup.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/block/backup.c b/block/backup.c index 07f4ae846b..52df7bb19e 100644 --- a/block/backup.c +++ b/block/backup.c @@ -369,7 +369,6 @@ static int coroutine_fn backup_run_incremental(BackupBl= ockJob *job) int64_t offset; int64_t cluster; int64_t end; - int64_t last_cluster =3D -1; BdrvDirtyBitmapIter *dbi; =20 granularity =3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); @@ -380,12 +379,6 @@ static int coroutine_fn backup_run_incremental(BackupB= lockJob *job) while ((offset =3D bdrv_dirty_iter_next(dbi)) >=3D 0) { cluster =3D offset / job->cluster_size; =20 - /* Fake progress updates for any clusters we skipped */ - if (cluster !=3D last_cluster + 1) { - job->common.offset +=3D ((cluster - last_cluster - 1) * - job->cluster_size); - } - for (end =3D cluster + clusters_per_iter; cluster < end; cluster++= ) { do { if (yield_and_check(job)) { @@ -407,14 +400,6 @@ static int coroutine_fn backup_run_incremental(BackupB= lockJob *job) if (granularity < job->cluster_size) { bdrv_set_dirty_iter(dbi, cluster * job->cluster_size); } - - last_cluster =3D cluster - 1; - } - - /* Play some final catchup with the progress meter */ - end =3D DIV_ROUND_UP(job->common.len, job->cluster_size); - if (last_cluster + 1 < end) { - job->common.offset +=3D ((end - last_cluster - 1) * job->cluster_s= ize); } =20 out: @@ -456,6 +441,9 @@ static void backup_incremental_init_copy_bitmap(BackupB= lockJob *job) bdrv_set_dirty_iter(dbi, next_cluster * job->cluster_size); } =20 + job->common.offset =3D job->common.len - + hbitmap_count(job->copy_bitmap) * job->cluster_si= ze; + bdrv_dirty_iter_free(dbi); } =20 --=20 2.11.1 From nobody Mon May 6 07:18:54 2024 Delivered-To: importer@patchew.org Received-SPF: temperror (zoho.com: Error in retrieving data from DNS) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@gnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=temperror (zoho.com: Error in retrieving data from DNS) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@gnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1506955413889583.3930052957114; Mon, 2 Oct 2017 07:43:33 -0700 (PDT) Received: from localhost ([::1]:52651 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1wE-0004gm-2i for importer@patchew.org; Mon, 02 Oct 2017 10:43:18 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:52126) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1dz1sW-0001Xj-UP for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:29 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1dz1sU-00031D-BP for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:28 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:3879 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1dz1sT-0002yr-7H for qemu-devel@nongnu.org; Mon, 02 Oct 2017 10:39:26 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v92EdJQ9007920; Mon, 2 Oct 2017 17:39:20 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-block@nongnu.org, qemu-devel@nongnu.org Date: Mon, 2 Oct 2017 17:39:19 +0300 Message-Id: <20171002143919.207741-6-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171002143919.207741-1-vsementsov@virtuozzo.com> References: <20171002143919.207741-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 5/5] backup: use copy_bitmap in incremental backup X-BeenThere: qemu-devel@gnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@gnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_6 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" We can use copy_bitmap instead of sync_bitmap. copy_bitmap is initialized from sync_bitmap and it is more informative: we will not try to process data, that is already in progress (by write notifier). Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Stefan Hajnoczi Reviewed-by: John Snow --- block/backup.c | 55 +++++++++++++++++-------------------------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/block/backup.c b/block/backup.c index 52df7bb19e..c933cfafc3 100644 --- a/block/backup.c +++ b/block/backup.c @@ -362,49 +362,28 @@ static bool coroutine_fn yield_and_check(BackupBlockJ= ob *job) =20 static int coroutine_fn backup_run_incremental(BackupBlockJob *job) { + int ret; bool error_is_read; - int ret =3D 0; - int clusters_per_iter; - uint32_t granularity; - int64_t offset; int64_t cluster; - int64_t end; - BdrvDirtyBitmapIter *dbi; - - granularity =3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); - clusters_per_iter =3D MAX((granularity / job->cluster_size), 1); - dbi =3D bdrv_dirty_iter_new(job->sync_bitmap); - - /* Find the next dirty sector(s) */ - while ((offset =3D bdrv_dirty_iter_next(dbi)) >=3D 0) { - cluster =3D offset / job->cluster_size; - - for (end =3D cluster + clusters_per_iter; cluster < end; cluster++= ) { - do { - if (yield_and_check(job)) { - goto out; - } - ret =3D backup_do_cow(job, cluster * job->cluster_size, - job->cluster_size, &error_is_read, - false); - if ((ret < 0) && - backup_error_action(job, error_is_read, -ret) =3D=3D - BLOCK_ERROR_ACTION_REPORT) { - goto out; - } - } while (ret < 0); - } + HBitmapIter hbi; =20 - /* If the bitmap granularity is smaller than the backup granularit= y, - * we need to advance the iterator pointer to the next cluster. */ - if (granularity < job->cluster_size) { - bdrv_set_dirty_iter(dbi, cluster * job->cluster_size); - } + hbitmap_iter_init(&hbi, job->copy_bitmap, 0); + while ((cluster =3D hbitmap_iter_next(&hbi)) !=3D -1) { + do { + if (yield_and_check(job)) { + return 0; + } + ret =3D backup_do_cow(job, cluster * job->cluster_size, + job->cluster_size, &error_is_read, false); + if (ret < 0 && backup_error_action(job, error_is_read, -ret) = =3D=3D + BLOCK_ERROR_ACTION_REPORT) + { + return ret; + } + } while (ret < 0); } =20 -out: - bdrv_dirty_iter_free(dbi); - return ret; + return 0; } =20 /* init copy_bitmap from sync_bitmap */ --=20 2.11.1