From nobody Thu Apr 25 07:05:25 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1507816703705662.582733799801; Thu, 12 Oct 2017 06:58:23 -0700 (PDT) Received: from localhost ([::1]:45704 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2e09-0002rM-SQ for importer@patchew.org; Thu, 12 Oct 2017 09:58:17 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59907) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dvb-00080V-0A for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1e2dvW-0004w8-Sq for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:35 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:43302 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1e2dvW-0004vG-DV for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:30 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v9CDrD2c019629; Thu, 12 Oct 2017 16:53:16 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Thu, 12 Oct 2017 16:53:09 +0300 Message-Id: <20171012135313.227864-2-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171012135313.227864-1-vsementsov@virtuozzo.com> References: <20171012135313.227864-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 1/5] hbitmap: add next_zero function X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" The function searches for next zero bit. Also add interface for BdrvDirtyBitmap and unit test. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow --- include/block/dirty-bitmap.h | 1 + include/qemu/hbitmap.h | 8 ++++++ block/dirty-bitmap.c | 5 ++++ tests/test-hbitmap.c | 61 ++++++++++++++++++++++++++++++++++++++++= ++++ util/hbitmap.c | 39 ++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+) diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h index 3579a7597c..a591c27213 100644 --- a/include/block/dirty-bitmap.h +++ b/include/block/dirty-bitmap.h @@ -91,5 +91,6 @@ bool bdrv_has_changed_persistent_bitmaps(BlockDriverState= *bs); BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs, BdrvDirtyBitmap *bitmap); char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp= ); +int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t star= t); =20 #endif diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h index 81e78043d1..6b6490ecad 100644 --- a/include/qemu/hbitmap.h +++ b/include/qemu/hbitmap.h @@ -292,6 +292,14 @@ void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap= *hb, uint64_t first); */ unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi); =20 +/* hbitmap_next_zero: + * @hb: The HBitmap to operate on + * @start: The bit to start from. + * + * Find next not dirty bit. + */ +int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start); + /* hbitmap_create_meta: * Create a "meta" hbitmap to track dirtiness of the bits in this HBitmap. * The caller owns the created bitmap and must call hbitmap_free_meta(hb) = to diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c index bd04e991b1..7879d13ddb 100644 --- a/block/dirty-bitmap.c +++ b/block/dirty-bitmap.c @@ -715,3 +715,8 @@ char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *b= itmap, Error **errp) { return hbitmap_sha256(bitmap->bitmap, errp); } + +int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offs= et) +{ + return hbitmap_next_zero(bitmap->bitmap, offset); +} diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c index af41642346..9091c639b3 100644 --- a/tests/test-hbitmap.c +++ b/tests/test-hbitmap.c @@ -925,6 +925,61 @@ static void test_hbitmap_iter_and_reset(TestHBitmapDat= a *data, hbitmap_iter_next(&hbi); } =20 +static void test_hbitmap_next_zero_check(TestHBitmapData *data, int64_t st= art) +{ + int64_t ret1 =3D hbitmap_next_zero(data->hb, start); + int64_t ret2 =3D start; + for ( ; ret2 < data->size && hbitmap_get(data->hb, ret2); ret2++) { + ; + } + if (ret2 =3D=3D data->size) { + ret2 =3D -1; + } + + g_assert_cmpint(ret1, =3D=3D, ret2); +} + +static void test_hbitmap_next_zero_do(TestHBitmapData *data, int granulari= ty) +{ + hbitmap_test_init(data, L3, granularity); + test_hbitmap_next_zero_check(data, 0); + test_hbitmap_next_zero_check(data, L3 - 1); + + hbitmap_set(data->hb, L2, 1); + test_hbitmap_next_zero_check(data, 0); + test_hbitmap_next_zero_check(data, L2 - 1); + test_hbitmap_next_zero_check(data, L2); + test_hbitmap_next_zero_check(data, L2 + 1); + + hbitmap_set(data->hb, L2 + 5, L1); + test_hbitmap_next_zero_check(data, 0); + test_hbitmap_next_zero_check(data, L2 + 1); + test_hbitmap_next_zero_check(data, L2 + 2); + test_hbitmap_next_zero_check(data, L2 + 5); + test_hbitmap_next_zero_check(data, L2 + L1 - 1); + test_hbitmap_next_zero_check(data, L2 + L1); + + hbitmap_set(data->hb, L2 * 2, L3 - L2 * 2); + test_hbitmap_next_zero_check(data, L2 * 2 - L1); + test_hbitmap_next_zero_check(data, L2 * 2 - 2); + test_hbitmap_next_zero_check(data, L2 * 2 - 1); + test_hbitmap_next_zero_check(data, L2 * 2); + test_hbitmap_next_zero_check(data, L3 - 1); + + hbitmap_set(data->hb, 0, L3); + test_hbitmap_next_zero_check(data, 0); +} + +static void test_hbitmap_next_zero_0(TestHBitmapData *data, const void *un= used) +{ + test_hbitmap_next_zero_do(data, 0); +} + +static void test_hbitmap_next_zero_4(TestHBitmapData *data, const void *un= used) +{ + test_hbitmap_next_zero_do(data, 4); +} + int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); @@ -985,6 +1040,12 @@ int main(int argc, char **argv) =20 hbitmap_test_add("/hbitmap/iter/iter_and_reset", test_hbitmap_iter_and_reset); + + hbitmap_test_add("/hbitmap/next_zero/next_zero_0", + test_hbitmap_next_zero_0); + hbitmap_test_add("/hbitmap/next_zero/next_zero_4", + test_hbitmap_next_zero_4); + g_test_run(); =20 return 0; diff --git a/util/hbitmap.c b/util/hbitmap.c index 2f9d0fdbd0..289778a55c 100644 --- a/util/hbitmap.c +++ b/util/hbitmap.c @@ -188,6 +188,45 @@ void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap= *hb, uint64_t first) } } =20 +int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start) +{ + size_t pos =3D (start >> hb->granularity) >> BITS_PER_LEVEL; + unsigned long *last_lev =3D hb->levels[HBITMAP_LEVELS - 1]; + uint64_t sz =3D hb->sizes[HBITMAP_LEVELS - 1]; + unsigned long cur =3D last_lev[pos]; + unsigned start_bit_offset =3D + (start >> hb->granularity) & (BITS_PER_LONG - 1); + int64_t res; + + cur |=3D (1UL << start_bit_offset) - 1; + assert((start >> hb->granularity) < hb->size); + + if (cur =3D=3D (unsigned long)-1) { + do { + pos++; + } while (pos < sz && last_lev[pos] =3D=3D (unsigned long)-1); + + if (pos >=3D sz) { + return -1; + } + + cur =3D last_lev[pos]; + } + + res =3D (pos << BITS_PER_LEVEL) + ctol(cur); + if (res >=3D hb->size) { + return -1; + } + + res =3D res << hb->granularity; + if (res < start) { + assert(((start - res) >> hb->granularity) =3D=3D 0); + return start; + } + + return res; +} + bool hbitmap_empty(const HBitmap *hb) { return hb->count =3D=3D 0; --=20 2.11.1 From nobody Thu Apr 25 07:05:25 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1507816559676536.7487521338011; Thu, 12 Oct 2017 06:55:59 -0700 (PDT) Received: from localhost ([::1]:45699 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dxm-0000xG-Tr for importer@patchew.org; Thu, 12 Oct 2017 09:55:50 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59908) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dvb-00080W-0J for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1e2dvX-0004wu-QW for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:35 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:33119 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1e2dvX-0004vb-EQ for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:31 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v9CDrD2d019629; Thu, 12 Oct 2017 16:53:16 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Thu, 12 Oct 2017 16:53:10 +0300 Message-Id: <20171012135313.227864-3-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171012135313.227864-1-vsementsov@virtuozzo.com> References: <20171012135313.227864-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 2/5] backup: move from done_bitmap to copy_bitmap X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Use HBitmap copy_bitmap instead of done_bitmap. This is needed to improve incremental backup in following patches and to unify backup loop for full/incremental modes in future patches. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Jeff Cody Reviewed-by: Stefan Hajnoczi Reviewed-by: John Snow --- block/backup.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/block/backup.c b/block/backup.c index 06ddbfd03d..6fdb4c33f7 100644 --- a/block/backup.c +++ b/block/backup.c @@ -40,11 +40,12 @@ typedef struct BackupBlockJob { BlockdevOnError on_target_error; CoRwlock flush_rwlock; uint64_t bytes_read; - unsigned long *done_bitmap; int64_t cluster_size; bool compress; NotifierWithReturn before_write; QLIST_HEAD(, CowRequest) inflight_reqs; + + HBitmap *copy_bitmap; } BackupBlockJob; =20 /* See if in-flight requests overlap and wait for them to complete */ @@ -109,10 +110,11 @@ static int coroutine_fn backup_do_cow(BackupBlockJob = *job, cow_request_begin(&cow_request, job, start, end); =20 for (; start < end; start +=3D job->cluster_size) { - if (test_bit(start / job->cluster_size, job->done_bitmap)) { + if (!hbitmap_get(job->copy_bitmap, start / job->cluster_size)) { trace_backup_do_cow_skip(job, start); continue; /* already copied */ } + hbitmap_reset(job->copy_bitmap, start / job->cluster_size, 1); =20 trace_backup_do_cow_process(job, start); =20 @@ -132,6 +134,7 @@ static int coroutine_fn backup_do_cow(BackupBlockJob *j= ob, if (error_is_read) { *error_is_read =3D true; } + hbitmap_set(job->copy_bitmap, start / job->cluster_size, 1); goto out; } =20 @@ -148,11 +151,10 @@ static int coroutine_fn backup_do_cow(BackupBlockJob = *job, if (error_is_read) { *error_is_read =3D false; } + hbitmap_set(job->copy_bitmap, start / job->cluster_size, 1); goto out; } =20 - set_bit(start / job->cluster_size, job->done_bitmap); - /* Publish progress, guest I/O counts as progress too. Note that = the * offset field is an opaque progress value, it is not a disk offs= et. */ @@ -260,7 +262,7 @@ void backup_do_checkpoint(BlockJob *job, Error **errp) } =20 len =3D DIV_ROUND_UP(backup_job->common.len, backup_job->cluster_size); - bitmap_zero(backup_job->done_bitmap, len); + hbitmap_set(backup_job->copy_bitmap, 0, len); } =20 void backup_wait_for_overlapping_requests(BlockJob *job, int64_t offset, @@ -425,19 +427,22 @@ static void coroutine_fn backup_run(void *opaque) BackupBlockJob *job =3D opaque; BackupCompleteData *data; BlockDriverState *bs =3D blk_bs(job->common.blk); - int64_t offset; + int64_t offset, nb_clusters; int ret =3D 0; =20 QLIST_INIT(&job->inflight_reqs); qemu_co_rwlock_init(&job->flush_rwlock); =20 - job->done_bitmap =3D bitmap_new(DIV_ROUND_UP(job->common.len, - job->cluster_size)); + nb_clusters =3D DIV_ROUND_UP(job->common.len, job->cluster_size); + job->copy_bitmap =3D hbitmap_alloc(nb_clusters, 0); + hbitmap_set(job->copy_bitmap, 0, nb_clusters); =20 job->before_write.notify =3D backup_before_write_notify; bdrv_add_before_write_notifier(bs, &job->before_write); =20 if (job->sync_mode =3D=3D MIRROR_SYNC_MODE_NONE) { + /* All bits are set in copy_bitmap to allow any cluster to be copi= ed. + * This does not actually require them to be copied. */ while (!block_job_is_cancelled(&job->common)) { /* Yield until the job is cancelled. We just let our before_w= rite * notify callback service CoW requests. */ @@ -512,7 +517,7 @@ static void coroutine_fn backup_run(void *opaque) /* wait until pending backup_do_cow() calls have completed */ qemu_co_rwlock_wrlock(&job->flush_rwlock); qemu_co_rwlock_unlock(&job->flush_rwlock); - g_free(job->done_bitmap); + hbitmap_free(job->copy_bitmap); =20 data =3D g_malloc(sizeof(*data)); data->ret =3D ret; --=20 2.11.1 From nobody Thu Apr 25 07:05:25 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 15078165552831004.904081502704; Thu, 12 Oct 2017 06:55:55 -0700 (PDT) Received: from localhost ([::1]:45695 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dxc-0000j9-HK for importer@patchew.org; Thu, 12 Oct 2017 09:55:40 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59858) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dvX-0007xy-U6 for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1e2dvW-0004w1-S2 for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:31 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:46410 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1e2dvW-0004v9-F6 for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:30 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v9CDrD2e019629; Thu, 12 Oct 2017 16:53:16 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Thu, 12 Oct 2017 16:53:11 +0300 Message-Id: <20171012135313.227864-4-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171012135313.227864-1-vsementsov@virtuozzo.com> References: <20171012135313.227864-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 3/5] backup: init copy_bitmap from sync_bitmap for incremental X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" We should not copy non-dirty clusters in write notifiers. So, initialize copy_bitmap from sync_bitmap. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow --- block/backup.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/block/backup.c b/block/backup.c index 6fdb4c33f7..71ad59b417 100644 --- a/block/backup.c +++ b/block/backup.c @@ -422,6 +422,43 @@ out: return ret; } =20 +/* init copy_bitmap from sync_bitmap */ +static void backup_incremental_init_copy_bitmap(BackupBlockJob *job) +{ + BdrvDirtyBitmapIter *dbi; + int64_t offset; + int64_t end =3D DIV_ROUND_UP(bdrv_dirty_bitmap_size(job->sync_bitmap), + job->cluster_size); + + dbi =3D bdrv_dirty_iter_new(job->sync_bitmap); + while ((offset =3D bdrv_dirty_iter_next(dbi)) !=3D -1) { + int64_t cluster =3D offset / job->cluster_size; + int64_t next_cluster; + + offset +=3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); + if (offset >=3D bdrv_dirty_bitmap_size(job->sync_bitmap)) { + hbitmap_set(job->copy_bitmap, cluster, end - cluster); + break; + } + + offset =3D bdrv_dirty_bitmap_next_zero(job->sync_bitmap, offset); + if (offset =3D=3D -1) { + hbitmap_set(job->copy_bitmap, cluster, end - cluster); + break; + } + + next_cluster =3D DIV_ROUND_UP(offset, job->cluster_size); + hbitmap_set(job->copy_bitmap, cluster, next_cluster - cluster); + if (next_cluster >=3D end) { + break; + } + + bdrv_set_dirty_iter(dbi, next_cluster * job->cluster_size); + } + + bdrv_dirty_iter_free(dbi); +} + static void coroutine_fn backup_run(void *opaque) { BackupBlockJob *job =3D opaque; @@ -435,7 +472,12 @@ static void coroutine_fn backup_run(void *opaque) =20 nb_clusters =3D DIV_ROUND_UP(job->common.len, job->cluster_size); job->copy_bitmap =3D hbitmap_alloc(nb_clusters, 0); - hbitmap_set(job->copy_bitmap, 0, nb_clusters); + if (job->sync_mode =3D=3D MIRROR_SYNC_MODE_INCREMENTAL) { + backup_incremental_init_copy_bitmap(job); + } else { + hbitmap_set(job->copy_bitmap, 0, nb_clusters); + } + =20 job->before_write.notify =3D backup_before_write_notify; bdrv_add_before_write_notifier(bs, &job->before_write); --=20 2.11.1 From nobody Thu Apr 25 07:05:25 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1507816711203361.2912458290326; Thu, 12 Oct 2017 06:58:31 -0700 (PDT) Received: from localhost ([::1]:45708 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2e0M-00032o-Fa for importer@patchew.org; Thu, 12 Oct 2017 09:58:30 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59862) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dvX-0007y0-Vb for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1e2dvW-0004wA-Sw for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:32 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:39222 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1e2dvW-0004vL-Gx for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:30 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v9CDrD2f019629; Thu, 12 Oct 2017 16:53:16 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Thu, 12 Oct 2017 16:53:12 +0300 Message-Id: <20171012135313.227864-5-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171012135313.227864-1-vsementsov@virtuozzo.com> References: <20171012135313.227864-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 4/5] backup: simplify non-dirty bits progress processing X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Set fake progress for non-dirty clusters in copy_bitmap initialization, to. It simplifies code and allows further refactoring. This patch changes user's view of backup progress, but formally it doesn't changed: progress hops are just moved to the beginning. Actually it's just a point of view: when do we actually skip clusters? We can say in the very beginning, that we skip these clusters and do not think about them later. Of course, if go through disk sequentially, it's logical to say, that we skip clusters between copied portions to the left and to the right of them. But even now copying progress is not sequential because of write notifiers. Future patches will introduce new backup architecture which will do copying in several coroutines in parallel, so it will make no sense to publish fake progress by parts in parallel with other copying requests. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow --- block/backup.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/block/backup.c b/block/backup.c index 71ad59b417..04608836d7 100644 --- a/block/backup.c +++ b/block/backup.c @@ -369,7 +369,6 @@ static int coroutine_fn backup_run_incremental(BackupBl= ockJob *job) int64_t offset; int64_t cluster; int64_t end; - int64_t last_cluster =3D -1; BdrvDirtyBitmapIter *dbi; =20 granularity =3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); @@ -380,12 +379,6 @@ static int coroutine_fn backup_run_incremental(BackupB= lockJob *job) while ((offset =3D bdrv_dirty_iter_next(dbi)) >=3D 0) { cluster =3D offset / job->cluster_size; =20 - /* Fake progress updates for any clusters we skipped */ - if (cluster !=3D last_cluster + 1) { - job->common.offset +=3D ((cluster - last_cluster - 1) * - job->cluster_size); - } - for (end =3D cluster + clusters_per_iter; cluster < end; cluster++= ) { do { if (yield_and_check(job)) { @@ -407,14 +400,6 @@ static int coroutine_fn backup_run_incremental(BackupB= lockJob *job) if (granularity < job->cluster_size) { bdrv_set_dirty_iter(dbi, cluster * job->cluster_size); } - - last_cluster =3D cluster - 1; - } - - /* Play some final catchup with the progress meter */ - end =3D DIV_ROUND_UP(job->common.len, job->cluster_size); - if (last_cluster + 1 < end) { - job->common.offset +=3D ((end - last_cluster - 1) * job->cluster_s= ize); } =20 out: @@ -456,6 +441,9 @@ static void backup_incremental_init_copy_bitmap(BackupB= lockJob *job) bdrv_set_dirty_iter(dbi, next_cluster * job->cluster_size); } =20 + job->common.offset =3D job->common.len - + hbitmap_count(job->copy_bitmap) * job->cluster_si= ze; + bdrv_dirty_iter_free(dbi); } =20 --=20 2.11.1 From nobody Thu Apr 25 07:05:25 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1507816630162470.7489155424157; Thu, 12 Oct 2017 06:57:10 -0700 (PDT) Received: from localhost ([::1]:45702 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dz0-0001rL-En for importer@patchew.org; Thu, 12 Oct 2017 09:57:06 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59906) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dva-00080U-Vj for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1e2dvY-0004xd-Rh for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:35 -0400 Received: from mailhub.sw.ru ([195.214.232.25]:41936 helo=relay.sw.ru) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1e2dvY-0004wY-F7 for qemu-devel@nongnu.org; Thu, 12 Oct 2017 09:53:32 -0400 Received: from kvm.sw.ru (msk-vpn.virtuozzo.com [195.214.232.6]) by relay.sw.ru (8.13.4/8.13.4) with ESMTP id v9CDrD2g019629; Thu, 12 Oct 2017 16:53:16 +0300 (MSK) From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Thu, 12 Oct 2017 16:53:13 +0300 Message-Id: <20171012135313.227864-6-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20171012135313.227864-1-vsementsov@virtuozzo.com> References: <20171012135313.227864-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: OpenBSD 3.x [fuzzy] X-Received-From: 195.214.232.25 Subject: [Qemu-devel] [PATCH 5/5] backup: use copy_bitmap in incremental backup X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, famz@redhat.com, jcody@redhat.com, mreitz@redhat.com, stefanha@redhat.com, den@openvz.org, jsnow@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" We can use copy_bitmap instead of sync_bitmap. copy_bitmap is initialized from sync_bitmap and it is more informative: we will not try to process data, that is already in progress (by write notifier). Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Stefan Hajnoczi Reviewed-by: John Snow --- block/backup.c | 55 +++++++++++++++++-------------------------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/block/backup.c b/block/backup.c index 04608836d7..6bf5fe6b32 100644 --- a/block/backup.c +++ b/block/backup.c @@ -362,49 +362,28 @@ static bool coroutine_fn yield_and_check(BackupBlockJ= ob *job) =20 static int coroutine_fn backup_run_incremental(BackupBlockJob *job) { + int ret; bool error_is_read; - int ret =3D 0; - int clusters_per_iter; - uint32_t granularity; - int64_t offset; int64_t cluster; - int64_t end; - BdrvDirtyBitmapIter *dbi; - - granularity =3D bdrv_dirty_bitmap_granularity(job->sync_bitmap); - clusters_per_iter =3D MAX((granularity / job->cluster_size), 1); - dbi =3D bdrv_dirty_iter_new(job->sync_bitmap); - - /* Find the next dirty sector(s) */ - while ((offset =3D bdrv_dirty_iter_next(dbi)) >=3D 0) { - cluster =3D offset / job->cluster_size; - - for (end =3D cluster + clusters_per_iter; cluster < end; cluster++= ) { - do { - if (yield_and_check(job)) { - goto out; - } - ret =3D backup_do_cow(job, cluster * job->cluster_size, - job->cluster_size, &error_is_read, - false); - if ((ret < 0) && - backup_error_action(job, error_is_read, -ret) =3D=3D - BLOCK_ERROR_ACTION_REPORT) { - goto out; - } - } while (ret < 0); - } + HBitmapIter hbi; =20 - /* If the bitmap granularity is smaller than the backup granularit= y, - * we need to advance the iterator pointer to the next cluster. */ - if (granularity < job->cluster_size) { - bdrv_set_dirty_iter(dbi, cluster * job->cluster_size); - } + hbitmap_iter_init(&hbi, job->copy_bitmap, 0); + while ((cluster =3D hbitmap_iter_next(&hbi)) !=3D -1) { + do { + if (yield_and_check(job)) { + return 0; + } + ret =3D backup_do_cow(job, cluster * job->cluster_size, + job->cluster_size, &error_is_read, false); + if (ret < 0 && backup_error_action(job, error_is_read, -ret) = =3D=3D + BLOCK_ERROR_ACTION_REPORT) + { + return ret; + } + } while (ret < 0); } =20 -out: - bdrv_dirty_iter_free(dbi); - return ret; + return 0; } =20 /* init copy_bitmap from sync_bitmap */ --=20 2.11.1