From nobody Sun May 5 09:57:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com ARC-Seal: i=1; a=rsa-sha256; t=1569958626; cv=none; d=zoho.com; s=zohoarc; b=IHS5bM+tg7+vUQdXmiylcOTRuAP1OdFEfqIaLPDN+K63yPSIZhqLit3ExsWRSrx2WtNzWrnM7drOslUkQsJQv2cCeUDsGm441Ez1gODCbnGeMLZxsOz4PGouA3l8MScEO1br90Msyj08sc60ve4f6hi77q60exOGrNUD2WwrhB8= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1569958626; h=Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:Message-ID:References:Sender:Subject:To:ARC-Authentication-Results; bh=OxJXjYDe1Fyb3dpJsFknZd55h+AgVrYEyaIcUFhWU8w=; b=Qras8fflf6VTlnEWqPmTYstAQcV0WAqskoXHe9gBWymuCr1qsBoOy3/WBQN+G2ic3Bs0/1468KAZElcXOPnEe0H9/rAkb5bhH/j/N9Tkpn/7xDwxACJsvzH8sMSTqG+wQcZFJvf9oO+woI+R2IbJ5eNc8WqGTWqxUtLiz9gjRQY= ARC-Authentication-Results: i=1; mx.zoho.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1569958626441629.4154430216421; Tue, 1 Oct 2019 12:37:06 -0700 (PDT) Received: from localhost ([::1]:47136 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNxG-0002vP-LP for importer@patchew.org; Tue, 01 Oct 2019 15:37:02 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:48313) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNo8-0004iF-Hk for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1iFNo7-0001SC-10 for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:36 -0400 Received: from relay.sw.ru ([185.231.240.75]:45300) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1iFNo6-0001Qi-PF; Tue, 01 Oct 2019 15:27:34 -0400 Received: from [172.16.25.136] (helo=dhcp-172-16-25-136.sw.ru) by relay.sw.ru with esmtp (Exim 4.92.2) (envelope-from ) id 1iFNo2-00060a-Vx; Tue, 01 Oct 2019 22:27:31 +0300 From: Andrey Shinkevich To: qemu-devel@nongnu.org, qemu-block@nongnu.org Subject: [PATCH 1/6] qcow2: multiple clusters write compressed Date: Tue, 1 Oct 2019 22:27:15 +0300 Message-Id: <1569958040-697220-2-git-send-email-andrey.shinkevich@virtuozzo.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> References: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, fam@euphon.net, vsementsov@virtuozzo.com, jsnow@redhat.com, armbru@redhat.com, dgilbert@redhat.com, stefanha@redhat.com, andrey.shinkevich@virtuozzo.com, den@openvz.org, mreitz@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" QEMU currently supports writing compressed data of size less than or equal to one cluster. This patch allows writing QCOW2 compressed data that exceed one cluster. The implementation is simple, we split buffered data into separate clusters and write them using the existing functionality. For unaligned requests, we use a workaround that writes the data without compression. Suggested-by: Pavel Butsykin Suggested-by: Vladimir Sementsov-Ogievskiy Signed-off-by: Andrey Shinkevich --- block/qcow2.c | 113 +++++++++++++++++++++++++++++++++++++++++++-----------= ---- 1 file changed, 85 insertions(+), 28 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index 7961c05..54ccaf6 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -4152,10 +4152,8 @@ fail: return ret; } =20 -/* XXX: put compressed sectors first, then all the cluster aligned - tables to avoid losing bytes in alignment */ static coroutine_fn int -qcow2_co_pwritev_compressed_part(BlockDriverState *bs, +qcow2_co_pwritev_compressed_task(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, size_t qiov_offset) { @@ -4165,36 +4163,14 @@ qcow2_co_pwritev_compressed_part(BlockDriverState *= bs, uint8_t *buf, *out_buf; uint64_t cluster_offset; =20 - if (has_data_file(bs)) { - return -ENOTSUP; - } - - if (bytes =3D=3D 0) { - /* align end of file to a sector boundary to ease reading with - sector based I/Os */ - int64_t len =3D bdrv_getlength(bs->file->bs); - if (len < 0) { - return len; - } - return bdrv_co_truncate(bs->file, len, PREALLOC_MODE_OFF, NULL); - } - - if (offset_into_cluster(s, offset)) { - return -EINVAL; - } + assert(bytes <=3D s->cluster_size); =20 buf =3D qemu_blockalign(bs, s->cluster_size); - if (bytes !=3D s->cluster_size) { - if (bytes > s->cluster_size || - offset + bytes !=3D bs->total_sectors << BDRV_SECTOR_BITS) - { - qemu_vfree(buf); - return -EINVAL; - } + if (bytes < s->cluster_size) { /* Zero-pad last write if image size is not cluster aligned */ memset(buf + bytes, 0, s->cluster_size - bytes); } - qemu_iovec_to_buf(qiov, qiov_offset, buf, bytes); + qemu_iovec_to_buf(qiov, qiov_offset, buf, s->cluster_size); =20 out_buf =3D g_malloc(s->cluster_size); =20 @@ -4228,6 +4204,9 @@ qcow2_co_pwritev_compressed_part(BlockDriverState *bs, =20 BLKDBG_EVENT(s->data_file, BLKDBG_WRITE_COMPRESSED); ret =3D bdrv_co_pwrite(s->data_file, cluster_offset, out_len, out_buf,= 0); + if (ret =3D=3D -ENOTSUP) { + ret =3D qcow2_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset= , 0); + } if (ret < 0) { goto fail; } @@ -4239,6 +4218,84 @@ fail: return ret; } =20 +static coroutine_fn int qcow2_co_pwritev_compressed_task_entry(AioTask *ta= sk) +{ + Qcow2AioTask *t =3D container_of(task, Qcow2AioTask, task); + + assert(!t->cluster_type); + + return qcow2_co_pwritev_compressed_task(t->bs, t->offset, t->bytes, t-= >qiov, + t->qiov_offset); +} + +/* + * XXX: put compressed sectors first, then all the cluster aligned + tables to avoid losing bytes in alignment + */ +static coroutine_fn int +qcow2_co_pwritev_compressed_part(BlockDriverState *bs, + uint64_t offset, uint64_t bytes, + QEMUIOVector *qiov, size_t qiov_offset) +{ + BDRVQcow2State *s =3D bs->opaque; + QCowL2Meta *l2meta =3D NULL; + AioTaskPool *aio =3D NULL; + uint64_t curr_off =3D 0; + int ret; + + if (has_data_file(bs)) { + return -ENOTSUP; + } + + if (bytes =3D=3D 0) { + /* + * align end of file to a sector boundary to ease reading with + * sector based I/Os + */ + int64_t cluster_offset =3D bdrv_getlength(bs->file->bs); + if (cluster_offset < 0) { + return cluster_offset; + } + return bdrv_co_truncate(bs->file, cluster_offset, PREALLOC_MODE_OF= F, + NULL); + } + + if (offset_into_cluster(s, offset)) { + return -EINVAL; + } + + while (bytes && aio_task_pool_status(aio) =3D=3D 0) { + uint32_t chunk_size =3D MIN(bytes, s->cluster_size); + + assert((curr_off & (BDRV_SECTOR_SIZE - 1)) =3D=3D 0); + assert((chunk_size & (BDRV_SECTOR_SIZE - 1)) =3D=3D 0); + + if (!aio && chunk_size !=3D bytes) { + aio =3D aio_task_pool_new(QCOW2_MAX_WORKERS); + } + + ret =3D qcow2_add_task(bs, aio, qcow2_co_pwritev_compressed_task_e= ntry, + 0, 0, offset + curr_off, chunk_size, + qiov, qiov_offset, l2meta); + if (ret < 0) { + break; + } + qiov_offset +=3D chunk_size; + curr_off +=3D chunk_size; + bytes -=3D chunk_size; + } + + if (aio) { + aio_task_pool_wait_all(aio); + if (ret =3D=3D 0) { + ret =3D aio_task_pool_status(aio); + } + g_free(aio); + } + + return ret; +} + static int coroutine_fn qcow2_co_preadv_compressed(BlockDriverState *bs, uint64_t file_cluster_offset, --=20 1.8.3.1 From nobody Sun May 5 09:57:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com ARC-Seal: i=1; a=rsa-sha256; t=1569958784; cv=none; d=zoho.com; s=zohoarc; b=Pw9qDP8qV7ljhPiz0buJE+SIh2kgSNF48Yg79eUrcOWtE1JImaG8Hnsyxnh69ghpMM6hCb5PP/MX9dqh2pg9bkGnxWrkFIBLiUH3DHq8EETf7wbAuqoF6Isu1bF30Cez3zPDKm/2VgNaHZj6Z/aLYWIfvs/8jMCJdaPVUZjfzqo= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1569958784; h=Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:Message-ID:References:Sender:Subject:To:ARC-Authentication-Results; bh=/DuQQzzAKX91CfYvvTBuoVMYv+F+22n6/aM7HBgIPPA=; b=VTXIHXkaZryuo8w+dIoQKRZNY6SkLNgzajx/8C2Sr11ZLFqCprsT63KKS75+Hf4xzQ3bRPaCiCGBW6WTPGQJTlErtgo5yZVDItcZqhHpVraaWtvxwBfRJIVL9wC2lNhREpsoC+8MXwjFDNDg9k946l41m1RqGAFUbpPYl2n9y30= ARC-Authentication-Results: i=1; mx.zoho.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1569958784986190.57364712173091; Tue, 1 Oct 2019 12:39:44 -0700 (PDT) Received: from localhost ([::1]:47160 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNzr-0005Px-GN for importer@patchew.org; Tue, 01 Oct 2019 15:39:43 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:48303) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNo8-0004hT-Hs for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1iFNo6-0001Ru-Px for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:35 -0400 Received: from relay.sw.ru ([185.231.240.75]:45306) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1iFNo6-0001Qk-In; Tue, 01 Oct 2019 15:27:34 -0400 Received: from [172.16.25.136] (helo=dhcp-172-16-25-136.sw.ru) by relay.sw.ru with esmtp (Exim 4.92.2) (envelope-from ) id 1iFNo3-00060a-4F; Tue, 01 Oct 2019 22:27:31 +0300 From: Andrey Shinkevich To: qemu-devel@nongnu.org, qemu-block@nongnu.org Subject: [PATCH 2/6] tests/qemu-iotests: add case of writing compressed data to multiple clusters. Date: Tue, 1 Oct 2019 22:27:16 +0300 Message-Id: <1569958040-697220-3-git-send-email-andrey.shinkevich@virtuozzo.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> References: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, fam@euphon.net, vsementsov@virtuozzo.com, jsnow@redhat.com, armbru@redhat.com, dgilbert@redhat.com, stefanha@redhat.com, andrey.shinkevich@virtuozzo.com, den@openvz.org, mreitz@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add the test case to the iotest #214 that checks possibility of writing compressed data to more than one cluster. Signed-off-by: Andrey Shinkevich --- tests/qemu-iotests/214 | 9 +++++++++ tests/qemu-iotests/214.out | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/tests/qemu-iotests/214 b/tests/qemu-iotests/214 index 21ec8a2..5f437e4 100755 --- a/tests/qemu-iotests/214 +++ b/tests/qemu-iotests/214 @@ -89,6 +89,15 @@ _check_test_img -r all $QEMU_IO -c "read -P 0x11 0 4M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _fi= lter_testdir $QEMU_IO -c "read -P 0x22 4M 4M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _fi= lter_testdir =20 +echo +echo "=3D=3D=3D Write compressed data to multiple clusters =3D=3D=3D" +echo +# Create an empty image and fill three and half clusters with compressed d= ata. +_make_test_img 2M -o cluster_size=3D0x10000 +data_size=3D3*0x10000+0x8000 +$QEMU_IO -c "write -c -P 0x11 0 256k" "$TEST_IMG" \ + 2>&1 | _filter_qemu_io | _filter_testdir + # success, all done echo '*** done' rm -f $seq.full diff --git a/tests/qemu-iotests/214.out b/tests/qemu-iotests/214.out index 0fcd8dc..2b908f6 100644 --- a/tests/qemu-iotests/214.out +++ b/tests/qemu-iotests/214.out @@ -32,4 +32,10 @@ read 4194304/4194304 bytes at offset 0 4 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) read 4194304/4194304 bytes at offset 4194304 4 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +=3D=3D=3D Write compressed data to multiple clusters =3D=3D=3D + +Formatting 'TEST_DIR/t.IMGFMT', fmt=3DIMGFMT size=3D2097152 +wrote 262144/262144 bytes at offset 0 +256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) *** done --=20 1.8.3.1 From nobody Sun May 5 09:57:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com ARC-Seal: i=1; a=rsa-sha256; t=1569959067; cv=none; d=zoho.com; s=zohoarc; b=dQeu09PCuA8FRWeWbIJsPNbxkA9Jip4CJpz23EhaJmPRG2oxRfYNwkWfTJdy22eAAk30XyU1st5wjglDJ32xuCnr8PUd3gc1HHRE2eMB2b/UARCm1KASI3nmWs/D23F5hwxRgPWLJaoTDjmmR7Lmq1cK2z3wxHyHrW8Rp3h95EY= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1569959067; h=Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:Message-ID:References:Sender:Subject:To:ARC-Authentication-Results; bh=2NhRm3y5iaa9Z3EZWE0TOjeuEksJAJXw56leja7uOok=; b=PtH5rWj0sMuVaeogTPAepy8l4yx9maOSI14op5gO/xsH1a5/3WpmJ72qZqr+YpWfqIO9K+ilKyGUeQf5KKoNV/7XnwcjLdPy5rovbOe1EPeGUXtEp15YqImr9KQMw1cr+HrIVm79Jg6GxZubcRZgZOnVKzRjSpl5Ciuj94UPcH8= ARC-Authentication-Results: i=1; mx.zoho.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1569959067612687.1156514455602; Tue, 1 Oct 2019 12:44:27 -0700 (PDT) Received: from localhost ([::1]:47200 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFO4K-0000sW-T5 for importer@patchew.org; Tue, 01 Oct 2019 15:44:20 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:48312) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNo8-0004i9-Gp for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:38 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1iFNo6-0001S6-UT for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:36 -0400 Received: from relay.sw.ru ([185.231.240.75]:45292) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1iFNo6-0001Qj-N8; Tue, 01 Oct 2019 15:27:34 -0400 Received: from [172.16.25.136] (helo=dhcp-172-16-25-136.sw.ru) by relay.sw.ru with esmtp (Exim 4.92.2) (envelope-from ) id 1iFNo3-00060a-Ay; Tue, 01 Oct 2019 22:27:31 +0300 From: Andrey Shinkevich To: qemu-devel@nongnu.org, qemu-block@nongnu.org Subject: [PATCH 3/6] block: support compressed write for copy-on-read Date: Tue, 1 Oct 2019 22:27:17 +0300 Message-Id: <1569958040-697220-4-git-send-email-andrey.shinkevich@virtuozzo.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> References: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, fam@euphon.net, vsementsov@virtuozzo.com, jsnow@redhat.com, armbru@redhat.com, dgilbert@redhat.com, stefanha@redhat.com, andrey.shinkevich@virtuozzo.com, den@openvz.org, mreitz@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Signed-off-by: Anton Nefedov Signed-off-by: Denis V. Lunev Signed-off-by: Andrey Shinkevich --- block/io.c | 21 ++++++++++++++++----- block/trace-events | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/block/io.c b/block/io.c index f8c3596..a7cd24f 100644 --- a/block/io.c +++ b/block/io.c @@ -1264,12 +1264,13 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(Bd= rvChild *child, * allocating cluster in the image file. Note that this value may exc= eed * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which * is one reason we loop rather than doing it all at once. + * Also, this is crucial for compressed copy-on-read. */ bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_by= tes); skip_bytes =3D offset - cluster_offset; =20 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, - cluster_offset, cluster_bytes); + cluster_offset, cluster_bytes, flags); =20 while (cluster_bytes) { int64_t pnum; @@ -1328,9 +1329,15 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(Bdr= vChild *child, /* This does not change the data on the disk, it is not * necessary to flush even in cache=3Dwritethrough mode. */ - ret =3D bdrv_driver_pwritev(bs, cluster_offset, pnum, - &local_qiov, 0, - BDRV_REQ_WRITE_UNCHANGED); + if (flags & BDRV_REQ_WRITE_COMPRESSED) { + ret =3D bdrv_driver_pwritev_compressed(bs, cluster_off= set, + pnum, &local_qiov, + qiov_offset); + } else { + ret =3D bdrv_driver_pwritev(bs, cluster_offset, pnum, + &local_qiov, 0, + BDRV_REQ_WRITE_UNCHANGED); + } } =20 if (ret < 0) { @@ -1396,7 +1403,11 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChil= d *child, * to pass through to drivers. For now, there aren't any * passthrough flags. */ assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ | - BDRV_REQ_PREFETCH))); + BDRV_REQ_PREFETCH | BDRV_REQ_WRITE_COMPRESSED))); + + /* write compressed only makes sense with copy on read */ + assert(!(flags & BDRV_REQ_WRITE_COMPRESSED) || + (flags & BDRV_REQ_COPY_ON_READ)); =20 /* Handle Copy on Read and associated serialisation */ if (flags & BDRV_REQ_COPY_ON_READ) { diff --git a/block/trace-events b/block/trace-events index 3aa27e6..f444548 100644 --- a/block/trace-events +++ b/block/trace-events @@ -14,7 +14,7 @@ blk_root_detach(void *child, void *blk, void *bs) "child = %p blk %p bs %p" bdrv_co_preadv(void *bs, int64_t offset, int64_t nbytes, unsigned int flag= s) "bs %p offset %"PRId64" nbytes %"PRId64" flags 0x%x" bdrv_co_pwritev(void *bs, int64_t offset, int64_t nbytes, unsigned int fla= gs) "bs %p offset %"PRId64" nbytes %"PRId64" flags 0x%x" bdrv_co_pwrite_zeroes(void *bs, int64_t offset, int count, int flags) "bs = %p offset %"PRId64" count %d flags 0x%x" -bdrv_co_do_copy_on_readv(void *bs, int64_t offset, unsigned int bytes, int= 64_t cluster_offset, int64_t cluster_bytes) "bs %p offset %"PRId64" bytes %= u cluster_offset %"PRId64" cluster_bytes %"PRId64 +bdrv_co_do_copy_on_readv(void *bs, int64_t offset, unsigned int bytes, int= 64_t cluster_offset, int64_t cluster_bytes, int flags) "bs %p offset %"PRId= 64" bytes %u cluster_offset %"PRId64" cluster_bytes %"PRId64" flags 0x%x" bdrv_co_copy_range_from(void *src, uint64_t src_offset, void *dst, uint64_= t dst_offset, uint64_t bytes, int read_flags, int write_flags) "src %p offs= et %"PRIu64" dst %p offset %"PRIu64" bytes %"PRIu64" rw flags 0x%x 0x%x" bdrv_co_copy_range_to(void *src, uint64_t src_offset, void *dst, uint64_t = dst_offset, uint64_t bytes, int read_flags, int write_flags) "src %p offset= %"PRIu64" dst %p offset %"PRIu64" bytes %"PRIu64" rw flags 0x%x 0x%x" =20 --=20 1.8.3.1 From nobody Sun May 5 09:57:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com ARC-Seal: i=1; a=rsa-sha256; t=1569958725; cv=none; d=zoho.com; s=zohoarc; b=WyjVUkPPHY5Kfe1+hcNfg7zPaBtQSmechdr90JFLbzjLfVQf6cp/v2Iow2QczDMkFFooWdBtpe1WO1i4Jl5N4ZCeLf9XCEmo5Pa0I13qAhXE5KfPPFpcvfPXNLzJX5OVbnq2IBytFJkYAW5+AmVtwewK0s8oovKXcNJ5YRptKY0= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1569958725; h=Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:Message-ID:References:Sender:Subject:To:ARC-Authentication-Results; bh=QIp3Qc4MPL9tM3iljDr+d2ZYHazheEXwUMwA3JGaIv8=; b=ce8GBHWuX6bIndc05BhXojvMNdoLZsKyamdirI60C1j7kvFs+y5PPisJ3gvzJF9cvbbI2R8W3Lv6gK9jDEqrliIFdJGc9lZRwyNYGl8ePMG5+WtG8MzrVgGOhfLkX1eS+e+B4VepOfRBXSfOV/B6Kf7reuTXqri0+ynbMSIF6zw= ARC-Authentication-Results: i=1; mx.zoho.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1569958725057865.0147832286367; Tue, 1 Oct 2019 12:38:45 -0700 (PDT) Received: from localhost ([::1]:47144 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNys-0004LG-Ve for importer@patchew.org; Tue, 01 Oct 2019 15:38:43 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:48340) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNo9-0004jS-9h for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:38 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1iFNo7-0001Sa-7w for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:37 -0400 Received: from relay.sw.ru ([185.231.240.75]:45302) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1iFNo6-0001Qg-UZ; Tue, 01 Oct 2019 15:27:35 -0400 Received: from [172.16.25.136] (helo=dhcp-172-16-25-136.sw.ru) by relay.sw.ru with esmtp (Exim 4.92.2) (envelope-from ) id 1iFNo3-00060a-In; Tue, 01 Oct 2019 22:27:31 +0300 From: Andrey Shinkevich To: qemu-devel@nongnu.org, qemu-block@nongnu.org Subject: [PATCH 4/6] qemu-nbd: add compression flag support Date: Tue, 1 Oct 2019 22:27:18 +0300 Message-Id: <1569958040-697220-5-git-send-email-andrey.shinkevich@virtuozzo.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> References: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, fam@euphon.net, vsementsov@virtuozzo.com, jsnow@redhat.com, armbru@redhat.com, dgilbert@redhat.com, stefanha@redhat.com, andrey.shinkevich@virtuozzo.com, den@openvz.org, mreitz@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Added possibility to write compressed data by using the blk_write_compressed. This action has the limitations of the format driver. For example we can't write compressed data over other. $ ./qemu-img create -f qcow2 -o size=3D10G ./image.qcow2 $ sudo ./qemu-nbd -c /dev/nbd0 ./image.qcow2 $ sudo dd if=3D/dev/sda1 of=3D/dev/nbd0 bs=3D10M count=3D10 10+0 records in 10+0 records out 104857600 bytes (105 MB) copied, 0,0890581 s, 1,2 GB/s $ sudo ./qemu-nbd -d /dev/nbd0 $ du -sh ./image.qcow2 101M ./image.qcow2 $ ./qemu-img create -f qcow2 -o size=3D10G ./image.qcow2 $ sudo ./qemu-nbd -C -c /dev/nbd0 ./image.qcow2 $ sudo dd if=3D/dev/sda1 of=3D/dev/nbd0 bs=3D10M count=3D10 10+0 records in 10+0 records out 104857600 bytes (105 MB) copied, 0,076046 s, 1,4 GB/s $ sudo ./qemu-nbd -d /dev/nbd0 $ du -sh ./image.qcow2 5,3M ./image.qcow2 Signed-off-by: Pavel Butsykin Signed-off-by: Vladimir Sementsov-Ogievskiy Signed-off-by: Denis V. Lunev Signed-off-by: Andrey Shinkevich --- blockdev-nbd.c | 2 +- include/block/nbd.h | 7 ++++++- nbd/server.c | 8 +++++++- qemu-nbd.c | 18 ++++++++++++++++-- qemu-nbd.texi | 2 ++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/blockdev-nbd.c b/blockdev-nbd.c index 6a8b206..2a01a04 100644 --- a/blockdev-nbd.c +++ b/blockdev-nbd.c @@ -191,7 +191,7 @@ void qmp_nbd_server_add(const char *device, bool has_na= me, const char *name, } =20 exp =3D nbd_export_new(bs, 0, len, name, NULL, bitmap, !writable, !wri= table, - NULL, false, on_eject_blk, errp); + 0, NULL, false, on_eject_blk, errp); if (!exp) { goto out; } diff --git a/include/block/nbd.h b/include/block/nbd.h index 316fd70..0032f73 100644 --- a/include/block/nbd.h +++ b/include/block/nbd.h @@ -25,6 +25,10 @@ #include "crypto/tlscreds.h" #include "qapi/error.h" =20 +enum { + NBD_INTERNAL_FLAG_COMPRESS =3D 1 << 1, /* Use write compressed */ +}; + /* Handshake phase structs - this struct is passed on the wire */ =20 struct NBDOption { @@ -330,7 +334,8 @@ typedef struct NBDClient NBDClient; =20 NBDExport *nbd_export_new(BlockDriverState *bs, uint64_t dev_offset, uint64_t size, const char *name, const char *des= c, - const char *bitmap, bool readonly, bool shared, + const char *bitmap, bool readonly, + bool shared, uint32_t iflags, void (*close)(NBDExport *), bool writethrough, BlockBackend *on_eject_blk, Error **errp); void nbd_export_close(NBDExport *exp); diff --git a/nbd/server.c b/nbd/server.c index d8d1e62..96d581d 100644 --- a/nbd/server.c +++ b/nbd/server.c @@ -91,6 +91,7 @@ struct NBDExport { uint16_t nbdflags; QTAILQ_HEAD(, NBDClient) clients; QTAILQ_ENTRY(NBDExport) next; + uint32_t iflags; =20 AioContext *ctx; =20 @@ -1471,7 +1472,8 @@ static void nbd_eject_notifier(Notifier *n, void *dat= a) =20 NBDExport *nbd_export_new(BlockDriverState *bs, uint64_t dev_offset, uint64_t size, const char *name, const char *des= c, - const char *bitmap, bool readonly, bool shared, + const char *bitmap, bool readonly, + bool shared, uint32_t iflags, void (*close)(NBDExport *), bool writethrough, BlockBackend *on_eject_blk, Error **errp) { @@ -1525,6 +1527,7 @@ NBDExport *nbd_export_new(BlockDriverState *bs, uint6= 4_t dev_offset, exp->nbdflags |=3D (NBD_FLAG_SEND_TRIM | NBD_FLAG_SEND_WRITE_ZEROE= S | NBD_FLAG_SEND_FAST_ZERO); } + exp->iflags =3D iflags; assert(size <=3D INT64_MAX - dev_offset); exp->size =3D QEMU_ALIGN_DOWN(size, BDRV_SECTOR_SIZE); =20 @@ -2312,6 +2315,9 @@ static coroutine_fn int nbd_handle_request(NBDClient = *client, if (request->flags & NBD_CMD_FLAG_FUA) { flags |=3D BDRV_REQ_FUA; } + if (exp->iflags & NBD_INTERNAL_FLAG_COMPRESS) { + flags |=3D BDRV_REQ_WRITE_COMPRESSED; + } ret =3D blk_pwrite(exp->blk, request->from + exp->dev_offset, data, request->len, flags); return nbd_send_generic_reply(client, request->handle, ret, diff --git a/qemu-nbd.c b/qemu-nbd.c index 9032b6d..3765c4b 100644 --- a/qemu-nbd.c +++ b/qemu-nbd.c @@ -139,6 +139,7 @@ static void usage(const char *name) " --discard=3DMODE set discard mode (ignore, unmap)\n" " --detect-zeroes=3DMODE set detect-zeroes mode (off, on, unmap)\n" " --image-opts treat FILE as a full set of image options\n" +" -C, --compress use data compression (if the target format su= pports it)\n" "\n" QEMU_HELP_BOTTOM "\n" , name, name, NBD_DEFAULT_PORT, "DEVICE"); @@ -602,6 +603,7 @@ int main(int argc, char **argv) BlockDriverState *bs; uint64_t dev_offset =3D 0; bool readonly =3D false; + uint32_t iflags =3D 0; bool disconnect =3D false; const char *bindto =3D NULL; const char *port =3D NULL; @@ -610,7 +612,7 @@ int main(int argc, char **argv) int64_t fd_size; QemuOpts *sn_opts =3D NULL; const char *sn_id_or_name =3D NULL; - const char *sopt =3D "hVb:o:p:rsnP:c:dvk:e:f:tl:x:T:D:B:L"; + const char *sopt =3D "hVb:o:p:CrsnP:c:dvk:e:f:tl:x:T:D:B:L"; struct option lopt[] =3D { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, 'V' }, @@ -619,6 +621,7 @@ int main(int argc, char **argv) { "socket", required_argument, NULL, 'k' }, { "offset", required_argument, NULL, 'o' }, { "read-only", no_argument, NULL, 'r' }, + { "compress", no_argument, NULL, 'C'}, { "partition", required_argument, NULL, 'P' }, { "bitmap", required_argument, NULL, 'B' }, { "connect", required_argument, NULL, 'c' }, @@ -786,6 +789,9 @@ int main(int argc, char **argv) readonly =3D true; flags &=3D ~BDRV_O_RDWR; break; + case 'C': + iflags |=3D NBD_INTERNAL_FLAG_COMPRESS; + break; case 'P': warn_report("The '-P' option is deprecated; use --image-opts w= ith " "a raw device wrapper for subset exports instead"); @@ -1117,6 +1123,14 @@ int main(int argc, char **argv) =20 blk_set_enable_write_cache(blk, !writethrough); =20 + if ((iflags & NBD_INTERNAL_FLAG_COMPRESS) && + !(bs->drv && bs->drv->bdrv_co_pwritev_compressed_part)) + { + error_report("Compression not supported for this file format %s", + argv[optind]); + exit(EXIT_FAILURE); + } + if (sn_opts) { ret =3D bdrv_snapshot_load_tmp(bs, qemu_opt_get(sn_opts, SNAPSHOT_OPT_ID= ), @@ -1175,7 +1189,7 @@ int main(int argc, char **argv) =20 export =3D nbd_export_new(bs, dev_offset, fd_size, export_name, export_description, bitmap, readonly, shared >= 1, - nbd_export_closed, writethrough, NULL, + iflags, nbd_export_closed, writethrough, NULL, &error_fatal); =20 if (device) { diff --git a/qemu-nbd.texi b/qemu-nbd.texi index 7f55657..26ea1ec 100644 --- a/qemu-nbd.texi +++ b/qemu-nbd.texi @@ -55,6 +55,8 @@ Force the use of the block driver for format @var{fmt} in= stead of auto-detecting. @item -r, --read-only Export the disk as read-only. +@item -C, --compress +Use data compression (if the target format supports it) @item -P, --partition=3D@var{num} Deprecated: Only expose MBR partition @var{num}. Understands physical partitions 1-4 and logical partition 5. New code should instead use --=20 1.8.3.1 From nobody Sun May 5 09:57:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com ARC-Seal: i=1; a=rsa-sha256; t=1569959596; cv=none; d=zoho.com; s=zohoarc; b=ciVFw+T39I4gVb6oWVAgDDyYifvUXML5rDn0cHZ5Ay2UOfp77dLpdT/qt6LYRAcT7cKhCUFNacTyNivlgNeonzGxlqmtNGLpnK2OOR0S1/pAofNfNFg2I+vrxYVystxvzIJvd0y5NsRPAq5FU4zvAI1ytvG47xKWrgCX0ApGRs4= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1569959596; h=Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:Message-ID:References:Sender:Subject:To:ARC-Authentication-Results; bh=U7aAWE8FO1tLShUyb0IDuvX5l3uIevws4WiPEjnw3vw=; b=S4MIyYSDX7XXrR7QaerLU6+tlRM/9jAU5r0X3tTHgwgB/KgGLTINhnkab5GBbZkkbltrLRjEx/Z2ASYtwj3eFSEoezWI7yq29RyGM5CeGTTZe88kT45oZmblYu3d7UyVgfv5thXIm3JJ2akLFAi0Eah43wWRxS0I+WqxPLyA4B4= ARC-Authentication-Results: i=1; mx.zoho.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1569959596670377.03837414302086; Tue, 1 Oct 2019 12:53:16 -0700 (PDT) Received: from localhost ([::1]:47548 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFOCv-000197-CK for importer@patchew.org; Tue, 01 Oct 2019 15:53:13 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:48335) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNo9-0004j2-5E for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:38 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1iFNo7-0001SS-6S for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:36 -0400 Received: from relay.sw.ru ([185.231.240.75]:45312) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1iFNo6-0001Qp-UX; Tue, 01 Oct 2019 15:27:35 -0400 Received: from [172.16.25.136] (helo=dhcp-172-16-25-136.sw.ru) by relay.sw.ru with esmtp (Exim 4.92.2) (envelope-from ) id 1iFNo3-00060a-UZ; Tue, 01 Oct 2019 22:27:32 +0300 From: Andrey Shinkevich To: qemu-devel@nongnu.org, qemu-block@nongnu.org Subject: [PATCH 5/6] block-stream: add compress option Date: Tue, 1 Oct 2019 22:27:19 +0300 Message-Id: <1569958040-697220-6-git-send-email-andrey.shinkevich@virtuozzo.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> References: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, fam@euphon.net, vsementsov@virtuozzo.com, jsnow@redhat.com, armbru@redhat.com, dgilbert@redhat.com, stefanha@redhat.com, andrey.shinkevich@virtuozzo.com, den@openvz.org, mreitz@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Signed-off-by: Anton Nefedov Signed-off-by: Vladimir Sementsov-Ogievskiy Signed-off-by: Denis V. Lunev Signed-off-by: Andrey Shinkevich --- block/stream.c | 19 +++++++++++++------ blockdev.c | 14 +++++++++++++- hmp-commands.hx | 4 ++-- include/block/block_int.h | 3 ++- monitor/hmp-cmds.c | 5 +++-- qapi/block-core.json | 5 ++++- 6 files changed, 37 insertions(+), 13 deletions(-) diff --git a/block/stream.c b/block/stream.c index 5562ccb..51cc49e 100644 --- a/block/stream.c +++ b/block/stream.c @@ -36,15 +36,21 @@ typedef struct StreamBlockJob { char *backing_file_str; bool bs_read_only; bool chain_frozen; + bool compress; } StreamBlockJob; =20 -static int coroutine_fn stream_populate(BlockBackend *blk, - int64_t offset, uint64_t bytes) +static int coroutine_fn stream_populate(BlockBackend *blk, int64_t offset, + uint64_t bytes, bool compress) { + int flags =3D BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH; + + if (compress) { + flags |=3D BDRV_REQ_WRITE_COMPRESSED; + } + assert(bytes < SIZE_MAX); =20 - return blk_co_preadv(blk, offset, bytes, NULL, - BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH); + return blk_co_preadv(blk, offset, bytes, NULL, flags); } =20 static void stream_abort(Job *job) @@ -167,7 +173,7 @@ static int coroutine_fn stream_run(Job *job, Error **er= rp) } trace_stream_one_iteration(s, offset, n, ret); if (copy) { - ret =3D stream_populate(blk, offset, n); + ret =3D stream_populate(blk, offset, n, s->compress); } if (ret < 0) { BlockErrorAction action =3D @@ -217,7 +223,7 @@ static const BlockJobDriver stream_job_driver =3D { =20 void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, - int creation_flags, int64_t speed, + int creation_flags, int64_t speed, bool compress, BlockdevOnError on_error, Error **errp) { StreamBlockJob *s; @@ -267,6 +273,7 @@ void stream_start(const char *job_id, BlockDriverState = *bs, s->backing_file_str =3D g_strdup(backing_file_str); s->bs_read_only =3D bs_read_only; s->chain_frozen =3D true; + s->compress =3D compress; =20 s->on_error =3D on_error; trace_stream_start(bs, base, s); diff --git a/blockdev.c b/blockdev.c index fbef684..290ee4b 100644 --- a/blockdev.c +++ b/blockdev.c @@ -3238,6 +3238,7 @@ void qmp_block_stream(bool has_job_id, const char *jo= b_id, const char *device, bool has_base_node, const char *base_node, bool has_backing_file, const char *backing_file, bool has_speed, int64_t speed, + bool has_compress, bool compress, bool has_on_error, BlockdevOnError on_error, bool has_auto_finalize, bool auto_finalize, bool has_auto_dismiss, bool auto_dismiss, @@ -3254,6 +3255,10 @@ void qmp_block_stream(bool has_job_id, const char *j= ob_id, const char *device, on_error =3D BLOCKDEV_ON_ERROR_REPORT; } =20 + if (!has_compress) { + compress =3D false; + } + bs =3D bdrv_lookup_bs(device, device, errp); if (!bs) { return; @@ -3308,6 +3313,12 @@ void qmp_block_stream(bool has_job_id, const char *j= ob_id, const char *device, goto out; } =20 + if (compress && bs->drv->bdrv_co_pwritev_compressed_part =3D=3D NULL) { + error_setg(errp, "Compression is not supported for this drive %s", + bdrv_get_device_name(bs)); + goto out; + } + /* backing_file string overrides base bs filename */ base_name =3D has_backing_file ? backing_file : base_name; =20 @@ -3319,7 +3330,8 @@ void qmp_block_stream(bool has_job_id, const char *jo= b_id, const char *device, } =20 stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name, - job_flags, has_speed ? speed : 0, on_error, &local_err); + job_flags, has_speed ? speed : 0, compress, on_error, + &local_err); if (local_err) { error_propagate(errp, local_err); goto out; diff --git a/hmp-commands.hx b/hmp-commands.hx index cfcc044..3a347fd 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -95,8 +95,8 @@ ETEXI =20 { .name =3D "block_stream", - .args_type =3D "device:B,speed:o?,base:s?", - .params =3D "device [speed [base]]", + .args_type =3D "device:B,speed:o?,base:s?,compress:o?", + .params =3D "device [speed [base]] [compress]", .help =3D "copy data from a backing file into a block device= ", .cmd =3D hmp_block_stream, }, diff --git a/include/block/block_int.h b/include/block/block_int.h index 0422acd..5e7fce8 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -1065,6 +1065,7 @@ int is_windows_drive(const char *filename); * @creation_flags: Flags that control the behavior of the Job lifetime. * See @BlockJobCreateFlags * @speed: The maximum speed, in bytes per second, or 0 for unlimited. + * @compress: True to compress data. * @on_error: The action to take upon error. * @errp: Error object. * @@ -1077,7 +1078,7 @@ int is_windows_drive(const char *filename); */ void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, - int creation_flags, int64_t speed, + int creation_flags, int64_t speed, bool compress, BlockdevOnError on_error, Error **errp); =20 /** diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index b2551c1..91201fe 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -2025,11 +2025,12 @@ void hmp_block_stream(Monitor *mon, const QDict *qd= ict) const char *device =3D qdict_get_str(qdict, "device"); const char *base =3D qdict_get_try_str(qdict, "base"); int64_t speed =3D qdict_get_try_int(qdict, "speed", 0); + bool compress =3D qdict_get_try_bool(qdict, "compress", false); =20 qmp_block_stream(true, device, device, base !=3D NULL, base, false, NU= LL, false, NULL, qdict_haskey(qdict, "speed"), speed, tru= e, - BLOCKDEV_ON_ERROR_REPORT, false, false, false, false, - &error); + compress, true, BLOCKDEV_ON_ERROR_REPORT, + false, false, false, false, &error); =20 hmp_handle_error(mon, &error); } diff --git a/qapi/block-core.json b/qapi/block-core.json index e6edd64..f41513f 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -2544,6 +2544,9 @@ # # @speed: the maximum speed, in bytes per second # +# @compress: true to compress data, if the target format supports it. +# (default: false). Since 4.1. +# # @on-error: the action to take on an error (default report). # 'stop' and 'enospc' can only be used if the block device # supports io-status (see BlockInfo). Since 1.3. @@ -2576,7 +2579,7 @@ { 'command': 'block-stream', 'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', '*base-node': 'str', '*backing-file': 'str', '*speed': 'int', - '*on-error': 'BlockdevOnError', + '*compress': 'bool', '*on-error': 'BlockdevOnError', '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } =20 ## --=20 1.8.3.1 From nobody Sun May 5 09:57:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com ARC-Seal: i=1; a=rsa-sha256; t=1569959354; cv=none; d=zoho.com; s=zohoarc; b=izgIovxZfX9AvY0ZGWNnHRkEkFcppkFU533fSitGIgGG2K0Gq5jHrE+PuiF0vww79tM6uCMWpvQMbaA8Snyayvga0Hu8x6DC/VFj9rD61DZ9RphO9J3a1GdaucFjwDUCv4BrH0GLMkQPeqBYnoI+D+49XToY/hRIgTNgEb90chw= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1569959354; h=Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:Message-ID:References:Sender:Subject:To:ARC-Authentication-Results; bh=WuoAkT3biPPM+T/Rekmb2WzoWUhIZRku4smtIhDwUTk=; b=VkMmEwnO4oXQaUSSKhKXzywssAeS4brV0UTi6m43/MM/8R2xD94OBgOPMra22i0pAH6BMtwlls9+JVJnK2Z1/8m+7FO2VZrRDOWhnQigxABLKm0ydKC0gnBtf3uyu++QKxT/zlbRkYBmWWUrCYQor8Mk+zr8ncDmQiS48mVeYOc= ARC-Authentication-Results: i=1; mx.zoho.com; spf=pass (zoho.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1569959354835969.7012024964644; Tue, 1 Oct 2019 12:49:14 -0700 (PDT) Received: from localhost ([::1]:47330 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFO8y-0004nj-Jb for importer@patchew.org; Tue, 01 Oct 2019 15:49:08 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:48327) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1iFNo8-0004il-Rq for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:38 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1iFNo7-0001Sk-CF for qemu-devel@nongnu.org; Tue, 01 Oct 2019 15:27:36 -0400 Received: from relay.sw.ru ([185.231.240.75]:45314) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1iFNo7-0001RR-2P; Tue, 01 Oct 2019 15:27:35 -0400 Received: from [172.16.25.136] (helo=dhcp-172-16-25-136.sw.ru) by relay.sw.ru with esmtp (Exim 4.92.2) (envelope-from ) id 1iFNo4-00060a-7j; Tue, 01 Oct 2019 22:27:32 +0300 From: Andrey Shinkevich To: qemu-devel@nongnu.org, qemu-block@nongnu.org Subject: [PATCH 6/6] tests/qemu-iotests: add case for block-stream compress Date: Tue, 1 Oct 2019 22:27:20 +0300 Message-Id: <1569958040-697220-7-git-send-email-andrey.shinkevich@virtuozzo.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> References: <1569958040-697220-1-git-send-email-andrey.shinkevich@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, fam@euphon.net, vsementsov@virtuozzo.com, jsnow@redhat.com, armbru@redhat.com, dgilbert@redhat.com, stefanha@redhat.com, andrey.shinkevich@virtuozzo.com, den@openvz.org, mreitz@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add a test case to the iotest #030 that checks 'compress' option for a block-stream job. Signed-off-by: Andrey Shinkevich --- tests/qemu-iotests/030 | 49 ++++++++++++++++++++++++++++++++++++++++++= +++- tests/qemu-iotests/030.out | 4 ++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030 index f3766f2..13fe5a2 100755 --- a/tests/qemu-iotests/030 +++ b/tests/qemu-iotests/030 @@ -21,7 +21,8 @@ import time import os import iotests -from iotests import qemu_img, qemu_io +from iotests import qemu_img, qemu_io, qemu_img_pipe +import json =20 backing_img =3D os.path.join(iotests.test_dir, 'backing.img') mid_img =3D os.path.join(iotests.test_dir, 'mid.img') @@ -956,6 +957,52 @@ class TestSetSpeed(iotests.QMPTestCase): =20 self.cancel_and_wait(resume=3DTrue) =20 +class TestCompressed(iotests.QMPTestCase): + + def setUp(self): + qemu_img('create', '-f', iotests.imgfmt, backing_img, '1M') + qemu_img('create', '-f', iotests.imgfmt, '-o', + 'backing_file=3D%s' % backing_img, mid_img) + qemu_img('create', '-f', iotests.imgfmt, '-o', + 'backing_file=3D%s' % mid_img, test_img) + qemu_io('-c', 'write -P 0x1 0 512k', backing_img) + self.vm =3D iotests.VM().add_drive(test_img, "backing.node-name=3D= mid," + + "backing.backing.node-name=3Dbase= ") + self.vm.launch() + + def tearDown(self): + self.vm.shutdown() + os.remove(test_img) + os.remove(mid_img) + os.remove(backing_img) + + def test_stream_compress(self): + self.assert_no_active_block_jobs() + + result =3D self.vm.qmp('block-stream', device=3D'mid', job_id=3D's= tream-mid') + self.assert_qmp(result, 'return', {}) + + self.wait_until_completed(drive=3D'stream-mid') + for event in self.vm.get_qmp_events(wait=3DTrue): + if event['event'] =3D=3D 'BLOCK_JOB_COMPLETED': + self.dictpath(event, 'data/device') + self.assert_qmp_absent(event, 'data/error') + + result =3D self.vm.qmp('block-stream', device=3D'drive0', base=3Dm= id_img, + job_id=3D'stream-top', compress=3DTrue) + self.assert_qmp(result, 'return', {}) + + self.wait_until_completed(drive=3D'stream-top') + self.assert_no_active_block_jobs() + self.vm.shutdown() + + top =3D json.loads(qemu_img_pipe('info', '--output=3Djson', test_i= mg)) + mid =3D json.loads(qemu_img_pipe('info', '--output=3Djson', mid_im= g)) + base =3D json.loads(qemu_img_pipe('info', '--output=3Djson', backi= ng_img)) + + self.assertEqual(mid['actual-size'], base['actual-size']) + self.assertLess(top['actual-size'], mid['actual-size']) + if __name__ =3D=3D '__main__': iotests.main(supported_fmts=3D['qcow2', 'qed'], supported_protocols=3D['file']) diff --git a/tests/qemu-iotests/030.out b/tests/qemu-iotests/030.out index 6d9bee1..af8dac1 100644 --- a/tests/qemu-iotests/030.out +++ b/tests/qemu-iotests/030.out @@ -1,5 +1,5 @@ -........................... +............................ ---------------------------------------------------------------------- -Ran 27 tests +Ran 28 tests =20 OK --=20 1.8.3.1