The following changes since commit fe8d2d5737ab20ed0118863f5eb888cae37122ab:

Merge remote-tracking branch 'remotes/vivier2/tags/linux-user-for-3.0-pull-request' into staging (2018-07-04 22:38:10 +0100)

are available in the git repository at:

for you to fetch changes up to 7c20c808a5cbf5d244735bc78fc3138c739c1946:

file-posix: Unlock FD after creation (2018-07-05 11:07:58 +0200)

- qcow2: Use worker threads for compression to improve performance of

'qemu-img convert -W' and compressed backup jobs

- blklogwrites: New filter driver to log write requests to an image in

the dm-log-writes format

- file-posix: Fix image locking during image creation

- crypto: Fix memory leak in error path

- Error out instead of silently truncating node names

Aapo Vienamo (1):

block: Add blklogwrites

Ari Sundholm (4):

block: Move two block permission constants to the relevant enum

block/blklogwrites: Change log_sector_size from int64_t to uint64_t

block/blklogwrites: Add an option for appending to an old log

block/blklogwrites: Add an option for the update interval of the log superblock

Kevin Wolf (2):

block: Don't silently truncate node names

block/crypto: Fix memory leak in create error path

Max Reitz (2):

file-posix: Fix creation locking

file-posix: Unlock FD after creation

Vladimir Sementsov-Ogievskiy (3):

qemu-img: allow compressed not-in-order writes

qcow2: refactor data compression

qcow2: add compress threads

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

No reason to forbid them, and they are needed to improve performance

with compress-threads in further patches.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

qemu-img.c | 5 -----

1 file changed, 5 deletions(-)

@@ -XXX,XX +XXX,XX @@ static int img_convert(int argc, char **argv)

goto fail_getopt;

}

- if (!s.wr_in_order && s.compressed) {

- error_report("Out of order write and compress are mutually exclusive");

- goto fail_getopt;

- }

-

if (tgt_image_opts && !skip_create) {

error_report("--target-image-opts requires use of -n flag");

goto fail_getopt;

2.13.6

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Make a separate function for compression to be parallelized later.

- use .avail_out field instead of .next_out to calculate size of

compressed data. It looks more natural and it allows to keep dest to

be void pointer

- set avail_out to be at least one byte less than input, to be sure

avoid inefficient compression earlier

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

block/qcow2.c | 78 ++++++++++++++++++++++++++++++++++++++---------------------

1 file changed, 51 insertions(+), 27 deletions(-)

@@ -XXX,XX +XXX,XX @@

*/

#include "qemu/osdep.h"

+

+#define ZLIB_CONST

+#include <zlib.h>

+

#include "block/block_int.h"

#include "block/qdict.h"

#include "sysemu/block-backend.h"

#include "qemu/module.h"

-#include <zlib.h>

#include "qcow2.h"

#include "qemu/error-report.h"

#include "qapi/error.h"

@@ -XXX,XX +XXX,XX @@ fail:

return ret;

}

+/*

+ * qcow2_compress()

+ *

+ * @dest - destination buffer, at least of @size-1 bytes

+ * @src - source buffer, @size bytes

+ *

+ * Returns: compressed size on success

+ * -1 if compression is inefficient

+ * -2 on any other error

+ */

+static ssize_t qcow2_compress(void *dest, const void *src, size_t size)

+{

+ ssize_t ret;

+ z_stream strm;

+

+ /* best compression, small window, no zlib header */

+ memset(&strm, 0, sizeof(strm));

+ ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,

+ -12, 9, Z_DEFAULT_STRATEGY);

+ if (ret != 0) {

+ return -2;

+ }

+

+ /* strm.next_in is not const in old zlib versions, such as those used on

+ * OpenBSD/NetBSD, so cast the const away */

+ strm.avail_in = size;

+ strm.next_in = (void *) src;

+ strm.avail_out = size - 1;

+ strm.next_out = dest;

+

+ ret = deflate(&strm, Z_FINISH);

+ if (ret == Z_STREAM_END) {

+ ret = size - 1 - strm.avail_out;

+ } else {

+ ret = (ret == Z_OK ? -1 : -2);

+ }

+

+ deflateEnd(&strm);

+

+ return ret;

+}

+

/* XXX: put compressed sectors first, then all the cluster aligned

tables to avoid losing bytes in alignment */

static coroutine_fn int

@@ -XXX,XX +XXX,XX @@ qcow2_co_pwritev_compressed(BlockDriverState *bs, uint64_t offset,

BDRVQcow2State *s = bs->opaque;

QEMUIOVector hd_qiov;

struct iovec iov;

- z_stream strm;

- int ret, out_len;

+ int ret;

+ size_t out_len;

uint8_t *buf, *out_buf;

int64_t cluster_offset;

@@ -XXX,XX +XXX,XX @@ qcow2_co_pwritev_compressed(BlockDriverState *bs, uint64_t offset,

out_buf = g_malloc(s->cluster_size);

- /* best compression, small window, no zlib header */

- memset(&strm, 0, sizeof(strm));

- ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,

- Z_DEFLATED, -12,

- 9, Z_DEFAULT_STRATEGY);

- if (ret != 0) {

+ out_len = qcow2_compress(out_buf, buf, s->cluster_size);

+ if (out_len == -2) {

ret = -EINVAL;

goto fail;

- }

-

- strm.avail_in = s->cluster_size;

- strm.next_in = (uint8_t *)buf;

- strm.avail_out = s->cluster_size;

- strm.next_out = out_buf;

-

- ret = deflate(&strm, Z_FINISH);

- if (ret != Z_STREAM_END && ret != Z_OK) {

- deflateEnd(&strm);

- ret = -EINVAL;

- goto fail;

- }

- out_len = strm.next_out - out_buf;

-

- deflateEnd(&strm);

-

- if (ret != Z_STREAM_END || out_len >= s->cluster_size) {

+ } else if (out_len == -1) {

/* could not compress: write normal cluster */

ret = qcow2_co_pwritev(bs, offset, bytes, qiov, 0);

2.13.6

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Do data compression in separate threads. This significantly improve

performance for qemu-img convert with -W (allow async writes) and -c

(compressed) options.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

block/qcow2.h | 3 +++

block/qcow2.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-

2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/block/qcow2.h b/block/qcow2.h

--- a/block/qcow2.h

+++ b/block/qcow2.h

@@ -XXX,XX +XXX,XX @@ typedef struct BDRVQcow2State {

* override) */

char *image_backing_file;

char *image_backing_format;

+

+ CoQueue compress_wait_queue;

+ int nb_compress_threads;

} BDRVQcow2State;

typedef struct Qcow2COWRegion {

qcow2_check_refcounts(bs, &result, 0);

#endif

+ qemu_co_queue_init(&s->compress_wait_queue);

return ret;

fail:

@@ -XXX,XX +XXX,XX @@ static ssize_t qcow2_compress(void *dest, const void *src, size_t size)

return ret;

}

+#define MAX_COMPRESS_THREADS 4

+typedef struct Qcow2CompressData {

+ void *dest;

+ const void *src;

+ size_t size;

+ ssize_t ret;

+} Qcow2CompressData;

+static int qcow2_compress_pool_func(void *opaque)

+{

+ Qcow2CompressData *data = opaque;

+

+ data->ret = qcow2_compress(data->dest, data->src, data->size);

+

+ return 0;

+}

+

+static void qcow2_compress_complete(void *opaque, int ret)

+{

+ qemu_coroutine_enter(opaque);

+}

+

+/* See qcow2_compress definition for parameters description */

+static ssize_t qcow2_co_compress(BlockDriverState *bs,

+ void *dest, const void *src, size_t size)

+{

+ BDRVQcow2State *s = bs->opaque;

+ BlockAIOCB *acb;

+ ThreadPool *pool = aio_get_thread_pool(bdrv_get_aio_context(bs));

+ Qcow2CompressData arg = {

+ .dest = dest,

+ .src = src,

+ .size = size,

+ };

+

+ while (s->nb_compress_threads >= MAX_COMPRESS_THREADS) {

+ qemu_co_queue_wait(&s->compress_wait_queue, NULL);

+ s->nb_compress_threads++;

+ acb = thread_pool_submit_aio(pool, qcow2_compress_pool_func, &arg,

+ qcow2_compress_complete,

+ qemu_coroutine_self());

+

+ if (!acb) {

+ s->nb_compress_threads--;

+ return -EINVAL;

+ }

+ qemu_coroutine_yield();

+ s->nb_compress_threads--;

+ qemu_co_queue_next(&s->compress_wait_queue);

+

+ return arg.ret;

+}

+

/* XXX: put compressed sectors first, then all the cluster aligned

tables to avoid losing bytes in alignment */

static coroutine_fn int

@@ -XXX,XX +XXX,XX @@ qcow2_co_pwritev_compressed(BlockDriverState *bs, uint64_t offset,

out_buf = g_malloc(s->cluster_size);

- out_len = qcow2_compress(out_buf, buf, s->cluster_size);

+ out_len = qcow2_co_compress(bs, out_buf, buf, s->cluster_size);

if (out_len == -2) {

ret = -EINVAL;

goto fail;

2.13.6

From: Ari Sundholm <ari@tuxera.com>

This allows using the two constants outside of block.c, which will

happen in a subsequent patch.

Signed-off-by: Ari Sundholm <ari@tuxera.com>

include/block/block.h | 7 +++++++

block.c | 6 ------

2 files changed, 7 insertions(+), 6 deletions(-)

@@ -XXX,XX +XXX,XX @@ enum {

BLK_PERM_GRAPH_MOD = 0x10,

BLK_PERM_ALL = 0x1f,

+

+ DEFAULT_PERM_PASSTHROUGH = BLK_PERM_CONSISTENT_READ

+ | BLK_PERM_WRITE

+ | BLK_PERM_WRITE_UNCHANGED

+ | BLK_PERM_RESIZE,

+

+ DEFAULT_PERM_UNCHANGED = BLK_PERM_ALL & ~DEFAULT_PERM_PASSTHROUGH,

};

char *bdrv_perm_names(uint64_t perm);

diff --git a/block.c b/block.c

--- a/block.c

+++ b/block.c

@@ -XXX,XX +XXX,XX @@ int bdrv_child_try_set_perm(BdrvChild *c, uint64_t perm, uint64_t shared,

return 0;

-#define DEFAULT_PERM_PASSTHROUGH (BLK_PERM_CONSISTENT_READ \

- | BLK_PERM_WRITE \

- | BLK_PERM_WRITE_UNCHANGED \

- | BLK_PERM_RESIZE)

-#define DEFAULT_PERM_UNCHANGED (BLK_PERM_ALL & ~DEFAULT_PERM_PASSTHROUGH)

-

void bdrv_filter_default_perms(BlockDriverState *bs, BdrvChild *c,

const BdrvChildRole *role,

BlockReopenQueue *reopen_queue,

2.13.6

From: Aapo Vienamo <aapo@tuxera.com>

Implements a block device write logging system, similar to Linux kernel

device mapper dm-log-writes. The write operations that are performed

on a block device are logged to a file or another block device. The

write log format is identical to the dm-log-writes format. Currently,

log markers are not supported.

This functionality can be used for crash consistency and fs consistency

testing. By implementing it in qemu, tests utilizing write logs can be

be used to test non-Linux drivers and older kernels.

The driver accepts an optional parameter to set the sector size used

for logging. This makes the driver require all requests to be aligned

to this sector size and also makes offsets and sizes of writes in the

log metadata to be expressed in terms of this value (the log format has

a granularity of one sector for offsets and sizes). This allows

accurate logging of writes to guest block devices that have unusual

sector sizes.

The implementation is based on the blkverify and blkdebug block

drivers.

Signed-off-by: Aapo Vienamo <aapo@tuxera.com>

Signed-off-by: Ari Sundholm <ari@tuxera.com>

qapi/block-core.json | 33 +++-

block/blklogwrites.c | 414 +++++++++++++++++++++++++++++++++++++++++++++++++++

MAINTAINERS | 6 +

block/Makefile.objs | 1 +

4 files changed, 448 insertions(+), 6 deletions(-)

create mode 100644 block/blklogwrites.c

diff --git a/qapi/block-core.json b/qapi/block-core.json

index XXXXXXX..XXXXXXX 100644

--- a/qapi/block-core.json

+++ b/qapi/block-core.json

# @throttle: Since 2.11

# @nvme: Since 2.12

# @copy-on-read: Since 3.0

+# @blklogwrites: Since 3.0

#

# Since: 2.9

##

{ 'enum': 'BlockdevDriver',

- 'data': [ 'blkdebug', 'blkverify', 'bochs', 'cloop', 'copy-on-read',

- 'dmg', 'file', 'ftp', 'ftps', 'gluster', 'host_cdrom',

- 'host_device', 'http', 'https', 'iscsi', 'luks', 'nbd', 'nfs',

- 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow', 'qcow2', 'qed',

- 'quorum', 'raw', 'rbd', 'replication', 'sheepdog', 'ssh',

- 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }

+ 'data': [ 'blkdebug', 'blklogwrites', 'blkverify', 'bochs', 'cloop',

+ 'copy-on-read', 'dmg', 'file', 'ftp', 'ftps', 'gluster',

+ 'host_cdrom', 'host_device', 'http', 'https', 'iscsi', 'luks',

+ 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow',

+ 'qcow2', 'qed', 'quorum', 'raw', 'rbd', 'replication', 'sheepdog',

+ 'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }

##

# @BlockdevOptionsFile:

@@ -XXX,XX +XXX,XX @@

'*set-state': ['BlkdebugSetStateOptions'] } }

##

+# @BlockdevOptionsBlklogwrites:

+# Driver specific block device options for blklogwrites.

+# @file: block device

+# @log: block device used to log writes to @file

+# @log-sector-size: sector size used in logging writes to @file, determines

+# granularity of offsets and sizes of writes (default: 512)

+# Since: 3.0

+##

+{ 'struct': 'BlockdevOptionsBlklogwrites',

+ 'data': { 'file': 'BlockdevRef',

+ 'log': 'BlockdevRef',

+ '*log-sector-size': 'uint32' } }

+

+##

# @BlockdevOptionsBlkverify:

#

# Driver specific block device options for blkverify.