The following changes since commit 8c1ecb590497b0349c550607db923972b37f6963:

Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-next-280519-2' into staging (2019-05-28 17:38:32 +0100)

are available in the Git repository at:

https://github.com/XanClic/qemu.git tags/pull-block-2019-05-28

for you to fetch changes up to a2d665c1bc3624a8375e2f9a7d569f7565cc1358:

blockdev: loosen restrictions on drive-backup source node (2019-05-28 20:30:55 +0200)

Andrey Shinkevich (1):

qcow2-bitmap: initialize bitmap directory alignment

Anton Nefedov (1):

qcow2: skip writing zero buffers to empty COW areas

John Snow (1):

blockdev: loosen restrictions on drive-backup source node

Sam Eiderman (3):

qemu-img: rebase: Reuse parent BlockDriverState

qemu-img: rebase: Reduce reads on in-chain rebase

qemu-img: rebase: Reuse in-chain BlockDriverState

Vladimir Sementsov-Ogievskiy (13):

qcow2.h: add missing include

qcow2: add separate file for threaded data processing functions

qcow2-threads: use thread_pool_submit_co

qcow2-threads: qcow2_co_do_compress: protect queuing by mutex

qcow2-threads: split out generic path

qcow2: qcow2_co_preadv: improve locking

qcow2: bdrv_co_pwritev: move encryption code out of the lock

qcow2: do encryption in threads

block/backup: simplify backup_incremental_init_copy_bitmap

block/backup: move to copy_bitmap with granularity

block/backup: refactor and tolerate unallocated cluster skipping

block/backup: unify different modes code path

block/backup: refactor: split out backup_calculate_cluster_size

block/Makefile.objs | 2 +-

qapi/block-core.json | 4 +-

block/qcow2.h | 26 ++-

block.c | 46 +++---

block/backup.c | 243 ++++++++++++---------------

block/block-backend.c | 3 +-

block/qcow2-bitmap.c | 3 +-

block/qcow2-cache.c | 1 -

block/qcow2-cluster.c | 10 +-

block/qcow2-refcount.c | 1 -

block/qcow2-snapshot.c | 1 -

block/qcow2-threads.c | 268 ++++++++++++++++++++++++++++++

block/qcow2.c | 320 +++++++++++++-----------------------

block/quorum.c | 1 -

blockdev.c | 7 +-

blockjob.c | 2 +-

qemu-img.c | 85 ++++++----

tests/test-bdrv-drain.c | 6 -

tests/test-bdrv-graph-mod.c | 1 -

block/trace-events | 1 +

tests/qemu-iotests/056 | 2 +-

tests/qemu-iotests/060 | 7 +-

tests/qemu-iotests/060.out | 5 +-

23 files changed, 615 insertions(+), 430 deletions(-)

create mode 100644 block/qcow2-threads.c

2.21.0

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

qcow2.h depends on block_int.h. Compilation isn't broken currently only

due to block_int.h always included before qcow2.h. Though, it seems

better to directly include block_int.h in qcow2.h.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Reviewed-by: Alberto Garcia <berto@igalia.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-id: 20190506142741.41731-2-vsementsov@virtuozzo.com

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

block/qcow2.h | 1 +

block/qcow2-bitmap.c | 1 -

block/qcow2-cache.c | 1 -

block/qcow2-cluster.c | 1 -

block/qcow2-refcount.c | 1 -

block/qcow2-snapshot.c | 1 -

block/qcow2.c | 1 -

7 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.h

+++ b/block/qcow2.h

@@ -XXX,XX +XXX,XX @@

#include "crypto/block.h"

#include "qemu/coroutine.h"

#include "qemu/units.h"

+#include "block/block_int.h"

//#define DEBUG_ALLOC

//#define DEBUG_ALLOC2

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-bitmap.c

+++ b/block/qcow2-bitmap.c

@@ -XXX,XX +XXX,XX @@

#include "qapi/error.h"

#include "qemu/cutils.h"

-#include "block/block_int.h"

#include "qcow2.h"

/* NOTICE: BME here means Bitmaps Extension and used as a namespace for

diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-cache.c

+++ b/block/qcow2-cache.c

@@ -XXX,XX +XXX,XX @@

*/

#include "qemu/osdep.h"

-#include "block/block_int.h"

#include "qemu-common.h"

#include "qcow2.h"

#include "trace.h"

@@ -XXX,XX +XXX,XX @@

#include "qapi/error.h"

#include "qemu-common.h"

-#include "block/block_int.h"

#include "qcow2.h"

#include "qemu/bswap.h"

#include "trace.h"

diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-refcount.c

+++ b/block/qcow2-refcount.c

@@ -XXX,XX +XXX,XX @@

#include "qemu/osdep.h"

#include "qapi/error.h"

#include "qemu-common.h"

-#include "block/block_int.h"

#include "qcow2.h"

#include "qemu/range.h"

#include "qemu/bswap.h"

diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-snapshot.c

+++ b/block/qcow2-snapshot.c

@@ -XXX,XX +XXX,XX @@

#include "qemu/osdep.h"

#include "qapi/error.h"

-#include "block/block_int.h"

#include "qcow2.h"

#include "qemu/bswap.h"

#include "qemu/error-report.h"

@@ -XXX,XX +XXX,XX @@

#define ZLIB_CONST

#include <zlib.h>

-#include "block/block_int.h"

#include "block/qdict.h"

#include "sysemu/block-backend.h"

#include "qemu/module.h"

2.21.0

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Move compression-on-threads to separate file. Encryption will be in it

too.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Reviewed-by: Alberto Garcia <berto@igalia.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-id: 20190506142741.41731-3-vsementsov@virtuozzo.com

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/Makefile.objs | 2 +-

block/qcow2.h | 7 ++

block/qcow2-threads.c | 201 ++++++++++++++++++++++++++++++++++++++++++

block/qcow2.c | 169 -----------------------------------

4 files changed, 209 insertions(+), 170 deletions(-)

create mode 100644 block/qcow2-threads.c

diff --git a/block/Makefile.objs b/block/Makefile.objs

index XXXXXXX..XXXXXXX 100644

--- a/block/Makefile.objs

+++ b/block/Makefile.objs

@@ -XXX,XX +XXX,XX @@ block-obj-$(CONFIG_BOCHS) += bochs.o

block-obj-$(CONFIG_VVFAT) += vvfat.o

block-obj-$(CONFIG_DMG) += dmg.o

-block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o qcow2-bitmap.o

+block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o qcow2-bitmap.o qcow2-threads.o

block-obj-$(CONFIG_QED) += qed.o qed-l2-cache.o qed-table.o qed-cluster.o

block-obj-$(CONFIG_QED) += qed-check.o

block-obj-y += vhdx.o vhdx-endian.o vhdx-log.o

diff --git a/block/qcow2.h b/block/qcow2.h

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.h

+++ b/block/qcow2.h

@@ -XXX,XX +XXX,XX @@ void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,

const char *name,

Error **errp);

+ssize_t coroutine_fn

+qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,

+ const void *src, size_t src_size);

+ssize_t coroutine_fn

+qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,

+ const void *src, size_t src_size);

+

#endif

diff --git a/block/qcow2-threads.c b/block/qcow2-threads.c

+++ b/block/qcow2-threads.c

+/*

+ * Threaded data processing for Qcow2: compression, encryption

+ *

+ * Copyright (c) 2004-2006 Fabrice Bellard

+ * Copyright (c) 2018 Virtuozzo International GmbH. All rights reserved.

+ *

+ * Permission is hereby granted, free of charge, to any person obtaining a copy

+ * of this software and associated documentation files (the "Software"), to deal

+ * in the Software without restriction, including without limitation the rights

+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ * copies of the Software, and to permit persons to whom the Software is

+ * furnished to do so, subject to the following conditions:

+ *

+ * The above copyright notice and this permission notice shall be included in

+ * all copies or substantial portions of the Software.

+ *

+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

+ * THE SOFTWARE.

+ */

+

+#include "qemu/osdep.h"

+

+#define ZLIB_CONST

+#include <zlib.h>

+

+#include "qcow2.h"

+#include "block/thread-pool.h"

+

+#define MAX_COMPRESS_THREADS 4

+

+typedef ssize_t (*Qcow2CompressFunc)(void *dest, size_t dest_size,

+ const void *src, size_t src_size);

+typedef struct Qcow2CompressData {

+ void *dest;

+ size_t dest_size;

+ const void *src;

+ size_t src_size;

+ ssize_t ret;

+

+ Qcow2CompressFunc func;

+} Qcow2CompressData;

+

+/*

+ * qcow2_compress()

+ *

+ * @dest - destination buffer, @dest_size bytes

+ * @src - source buffer, @src_size bytes

+ *

+ * Returns: compressed size on success

+ * -ENOMEM destination buffer is not enough to store compressed data

+ * -EIO on any other error

+ */

+static ssize_t qcow2_compress(void *dest, size_t dest_size,

+ const void *src, size_t src_size)

+{

+ ssize_t ret;

+ z_stream strm;

+

+ /* best compression, small window, no zlib header */

+ memset(&strm, 0, sizeof(strm));

+ ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,

+ -12, 9, Z_DEFAULT_STRATEGY);

+ if (ret != Z_OK) {

+ return -EIO;

+ }

+

+ /*

+ * strm.next_in is not const in old zlib versions, such as those used on

+ * OpenBSD/NetBSD, so cast the const away

+ */

+ strm.avail_in = src_size;

+ strm.next_in = (void *) src;

+ strm.avail_out = dest_size;

+ strm.next_out = dest;

+

+ ret = deflate(&strm, Z_FINISH);

+ if (ret == Z_STREAM_END) {

+ ret = dest_size - strm.avail_out;

+ } else {

+ ret = (ret == Z_OK ? -ENOMEM : -EIO);

+ }

+

+ deflateEnd(&strm);

+

+ return ret;

+}

+

+/*

+ * qcow2_decompress()

+ *

+ * Decompress some data (not more than @src_size bytes) to produce exactly

+ * @dest_size bytes.

+ *

+ * @dest - destination buffer, @dest_size bytes

+ * @src - source buffer, @src_size bytes

+ *

+ * Returns: 0 on success

+ * -1 on fail

+ */

+static ssize_t qcow2_decompress(void *dest, size_t dest_size,

+ const void *src, size_t src_size)

+{

+ int ret = 0;

+ z_stream strm;

+

+ memset(&strm, 0, sizeof(strm));

+ strm.avail_in = src_size;

+ strm.next_in = (void *) src;

+ strm.avail_out = dest_size;

+ strm.next_out = dest;

+

+ ret = inflateInit2(&strm, -12);

+ if (ret != Z_OK) {

+ return -1;

+ }

+

+ ret = inflate(&strm, Z_FINISH);

+ if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || strm.avail_out != 0) {

+ /*

+ * We approve Z_BUF_ERROR because we need @dest buffer to be filled, but

+ * @src buffer may be processed partly (because in qcow2 we know size of

+ * compressed data with precision of one sector)

+ */

+ ret = -1;

+ }

+

+ inflateEnd(&strm);

+

+ return ret;

+}

+

+static int qcow2_compress_pool_func(void *opaque)

+{

+ Qcow2CompressData *data = opaque;

+

+ data->ret = data->func(data->dest, data->dest_size,

+ data->src, data->src_size);

+

+ return 0;

+}

+

+static void qcow2_compress_complete(void *opaque, int ret)

+{

+ qemu_coroutine_enter(opaque);

+}

+

+static ssize_t coroutine_fn

+qcow2_co_do_compress(BlockDriverState *bs, void *dest, size_t dest_size,

+ const void *src, size_t src_size, Qcow2CompressFunc func)

+{

+ BDRVQcow2State *s = bs->opaque;

+ BlockAIOCB *acb;

+ ThreadPool *pool = aio_get_thread_pool(bdrv_get_aio_context(bs));

+ Qcow2CompressData arg = {

+ .dest = dest,

+ .dest_size = dest_size,

+ .src = src,

+ .src_size = src_size,

+ .func = func,

+ };

+

+ while (s->nb_compress_threads >= MAX_COMPRESS_THREADS) {

+ qemu_co_queue_wait(&s->compress_wait_queue, NULL);

+ }

+

+ s->nb_compress_threads++;

+ acb = thread_pool_submit_aio(pool, qcow2_compress_pool_func, &arg,

+ qcow2_compress_complete,

+ qemu_coroutine_self());

+

+ if (!acb) {

+ s->nb_compress_threads--;

+ return -EINVAL;

+ }

+ qemu_coroutine_yield();

+ s->nb_compress_threads--;

+ qemu_co_queue_next(&s->compress_wait_queue);

+

+ return arg.ret;

+}

+

+ssize_t coroutine_fn

+qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,

+ const void *src, size_t src_size)

+{

+ return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,

+ qcow2_compress);

+}

+

+ssize_t coroutine_fn

+qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,

+ const void *src, size_t src_size)

+{

+ return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,

+ qcow2_decompress);

+}

diff --git a/block/qcow2.c b/block/qcow2.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.c

+++ b/block/qcow2.c

#include "qemu/osdep.h"

-#define ZLIB_CONST

-#include <zlib.h>

-

#include "block/qdict.h"

#include "sysemu/block-backend.h"

#include "qemu/module.h"

@@ -XXX,XX +XXX,XX @@

#include "qapi/qobject-input-visitor.h"

#include "qapi/qapi-visit-block-core.h"

#include "crypto.h"

-#include "block/thread-pool.h"

/*

Differences with QCOW:

@@ -XXX,XX +XXX,XX @@ fail:

return ret;

}

-/*

- * qcow2_compress()

- *

- * @dest - destination buffer, @dest_size bytes

- * @src - source buffer, @src_size bytes

- *

- * Returns: compressed size on success

- * -ENOMEM destination buffer is not enough to store compressed data

- * -EIO on any other error

- */

-static ssize_t qcow2_compress(void *dest, size_t dest_size,

- const void *src, size_t src_size)

-{

- ssize_t ret;

- z_stream strm;

-

- /* best compression, small window, no zlib header */

- memset(&strm, 0, sizeof(strm));

- ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,

- -12, 9, Z_DEFAULT_STRATEGY);

- if (ret != Z_OK) {

- return -EIO;

- }

-

- /* strm.next_in is not const in old zlib versions, such as those used on

- * OpenBSD/NetBSD, so cast the const away */

- strm.avail_in = src_size;

- strm.next_in = (void *) src;

- strm.avail_out = dest_size;

- strm.next_out = dest;

-

- ret = deflate(&strm, Z_FINISH);

- if (ret == Z_STREAM_END) {

- ret = dest_size - strm.avail_out;

- } else {

- ret = (ret == Z_OK ? -ENOMEM : -EIO);

- }

-

- deflateEnd(&strm);

-

- return ret;

-}

-

-/*

- * qcow2_decompress()

- *

- * Decompress some data (not more than @src_size bytes) to produce exactly

- * @dest_size bytes.

- *

- * @dest - destination buffer, @dest_size bytes

- * @src - source buffer, @src_size bytes

- *

- * Returns: 0 on success

- * -1 on fail

- */

-static ssize_t qcow2_decompress(void *dest, size_t dest_size,

- const void *src, size_t src_size)

-{

- int ret = 0;

- z_stream strm;

-

- memset(&strm, 0, sizeof(strm));

- strm.avail_in = src_size;

- strm.next_in = (void *) src;

- strm.avail_out = dest_size;

- strm.next_out = dest;

-

- ret = inflateInit2(&strm, -12);

- if (ret != Z_OK) {

- return -1;

- }

-

- ret = inflate(&strm, Z_FINISH);

- if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || strm.avail_out != 0) {

- /* We approve Z_BUF_ERROR because we need @dest buffer to be filled, but

- * @src buffer may be processed partly (because in qcow2 we know size of

- * compressed data with precision of one sector) */

- ret = -1;

- }

-

- inflateEnd(&strm);

-

- return ret;

-}

-

-#define MAX_COMPRESS_THREADS 4

-

-typedef ssize_t (*Qcow2CompressFunc)(void *dest, size_t dest_size,

- const void *src, size_t src_size);

-typedef struct Qcow2CompressData {

- void *dest;

- size_t dest_size;

- const void *src;

- size_t src_size;

- ssize_t ret;

-

- Qcow2CompressFunc func;

-} Qcow2CompressData;

-

-static int qcow2_compress_pool_func(void *opaque)

-{

- Qcow2CompressData *data = opaque;

-

- data->ret = data->func(data->dest, data->dest_size,

- data->src, data->src_size);

-

- return 0;

-}

-

-static void qcow2_compress_complete(void *opaque, int ret)

-{

- qemu_coroutine_enter(opaque);

-}

-

-static ssize_t coroutine_fn

-qcow2_co_do_compress(BlockDriverState *bs, void *dest, size_t dest_size,

- const void *src, size_t src_size, Qcow2CompressFunc func)

-{

- BDRVQcow2State *s = bs->opaque;

- BlockAIOCB *acb;

- ThreadPool *pool = aio_get_thread_pool(bdrv_get_aio_context(bs));

- Qcow2CompressData arg = {

- .dest = dest,

- .dest_size = dest_size,

- .src = src,

- .src_size = src_size,

- .func = func,

- };

-

- while (s->nb_compress_threads >= MAX_COMPRESS_THREADS) {

- qemu_co_queue_wait(&s->compress_wait_queue, NULL);

- }

-

- s->nb_compress_threads++;

- acb = thread_pool_submit_aio(pool, qcow2_compress_pool_func, &arg,

- qcow2_compress_complete,

- qemu_coroutine_self());

-

- if (!acb) {

- s->nb_compress_threads--;

- return -EINVAL;

- }

- qemu_coroutine_yield();

- s->nb_compress_threads--;

- qemu_co_queue_next(&s->compress_wait_queue);

-

- return arg.ret;

-}

-

-static ssize_t coroutine_fn

-qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,

- const void *src, size_t src_size)

-{

- return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,

- qcow2_compress);

-}

-

-static ssize_t coroutine_fn

-qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,

- const void *src, size_t src_size)

-{

- return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,

- qcow2_decompress);

-}

-

/* XXX: put compressed sectors first, then all the cluster aligned

tables to avoid losing bytes in alignment */

static coroutine_fn int