The following changes since commit fe8d2d5737ab20ed0118863f5eb888cae37122ab:

Merge remote-tracking branch 'remotes/vivier2/tags/linux-user-for-3.0-pull-request' into staging (2018-07-04 22:38:10 +0100)

for you to fetch changes up to 7c20c808a5cbf5d244735bc78fc3138c739c1946:

file-posix: Unlock FD after creation (2018-07-05 11:07:58 +0200)

Block layer patches:

- qcow2: Use worker threads for compression to improve performance of

'qemu-img convert -W' and compressed backup jobs

- blklogwrites: New filter driver to log write requests to an image in

the dm-log-writes format

- file-posix: Fix image locking during image creation

- crypto: Fix memory leak in error path

- Error out instead of silently truncating node names

Aapo Vienamo (1):

block: Add blklogwrites

Ari Sundholm (4):

block: Move two block permission constants to the relevant enum

block/blklogwrites: Change log_sector_size from int64_t to uint64_t

block/blklogwrites: Add an option for appending to an old log

block/blklogwrites: Add an option for the update interval of the log superblock

Kevin Wolf (2):

block: Don't silently truncate node names

block/crypto: Fix memory leak in create error path

Max Reitz (2):

file-posix: Fix creation locking

file-posix: Unlock FD after creation

Vladimir Sementsov-Ogievskiy (3):

qemu-img: allow compressed not-in-order writes

qcow2: refactor data compression

qcow2: add compress threads

qapi/block-core.json | 38 ++-

block/qcow2.h | 3 +

include/block/block.h | 7 +

block.c | 12 +-

block/blklogwrites.c | 547 ++++++++++++++++++++++++++++++++++++++++++

block/crypto.c | 2 +-

block/file-posix.c | 21 +-

block/qcow2.c | 138 ++++++++---

qemu-img.c | 5 -

MAINTAINERS | 6 +

block/Makefile.objs | 1 +

tests/qemu-iotests/051 | 15 ++

tests/qemu-iotests/051.out | 23 ++

tests/qemu-iotests/051.pc.out | 23 ++

14 files changed, 791 insertions(+), 50 deletions(-)

create mode 100644 block/blklogwrites.c

Fixes: Coverity CID 1393782

Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>

block/crypto.c | 2 +-

1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/block/crypto.c b/block/crypto.c

--- a/block/crypto.c

+++ b/block/crypto.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename,

/* Create protocol layer */

ret = bdrv_create_file(filename, opts, errp);

if (ret < 0) {

- return ret;

+ goto fail;

bs = bdrv_open(filename, NULL, NULL,

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Do data compression in separate threads. This significantly improve

performance for qemu-img convert with -W (allow async writes) and -c

(compressed) options.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

block/qcow2.h | 3 +++

block/qcow2.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-

2 files changed, 64 insertions(+), 1 deletion(-)

@@ -XXX,XX +XXX,XX @@ typedef struct BDRVQcow2State {

* override) */

char *image_backing_file;

char *image_backing_format;

+

+ CoQueue compress_wait_queue;

+ int nb_compress_threads;

} BDRVQcow2State;

typedef struct Qcow2COWRegion {

@@ -XXX,XX +XXX,XX @@

#include "qapi/qobject-input-visitor.h"

#include "qapi/qapi-visit-block-core.h"

#include "crypto.h"

+#include "block/thread-pool.h"

/*

Differences with QCOW:

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_do_open(BlockDriverState *bs, QDict *options,

qcow2_check_refcounts(bs, &result, 0);

}

#endif

+

+ qemu_co_queue_init(&s->compress_wait_queue);

+

return ret;

fail:

@@ -XXX,XX +XXX,XX @@ static ssize_t qcow2_compress(void *dest, const void *src, size_t size)

return ret;

+#define MAX_COMPRESS_THREADS 4

+

+typedef struct Qcow2CompressData {

+ void *dest;

+ const void *src;

+ size_t size;

+ ssize_t ret;

+} Qcow2CompressData;

+

+static int qcow2_compress_pool_func(void *opaque)

+{

+ Qcow2CompressData *data = opaque;

+

+ data->ret = qcow2_compress(data->dest, data->src, data->size);

+

+ return 0;

+}

+

+static void qcow2_compress_complete(void *opaque, int ret)

+{

+ qemu_coroutine_enter(opaque);

+}

+

+/* See qcow2_compress definition for parameters description */

+static ssize_t qcow2_co_compress(BlockDriverState *bs,

+ void *dest, const void *src, size_t size)

+{

+ BDRVQcow2State *s = bs->opaque;

+ BlockAIOCB *acb;

+ ThreadPool *pool = aio_get_thread_pool(bdrv_get_aio_context(bs));

+ Qcow2CompressData arg = {

+ .dest = dest,

+ .src = src,

+ .size = size,

+ };

+

+ while (s->nb_compress_threads >= MAX_COMPRESS_THREADS) {

+ qemu_co_queue_wait(&s->compress_wait_queue, NULL);

+ }

+

+ s->nb_compress_threads++;

+ acb = thread_pool_submit_aio(pool, qcow2_compress_pool_func, &arg,

+ qcow2_compress_complete,

+ qemu_coroutine_self());

+

+ if (!acb) {

+ s->nb_compress_threads--;

+ return -EINVAL;

+ }

+ qemu_coroutine_yield();

+ s->nb_compress_threads--;

+ qemu_co_queue_next(&s->compress_wait_queue);

+

+ return arg.ret;

+}

+

/* XXX: put compressed sectors first, then all the cluster aligned

tables to avoid losing bytes in alignment */

static coroutine_fn int

@@ -XXX,XX +XXX,XX @@ qcow2_co_pwritev_compressed(BlockDriverState *bs, uint64_t offset,

out_buf = g_malloc(s->cluster_size);

- out_len = qcow2_compress(out_buf, buf, s->cluster_size);

+ out_len = qcow2_co_compress(bs, out_buf, buf, s->cluster_size);

if (out_len == -2) {

ret = -EINVAL;

goto fail;

From: Ari Sundholm <ari@tuxera.com>

This is a way to ensure that the log superblock is periodically

updated. Before, this was only done on flush requests, which may

not be enough if the VM exits abnormally, omitting the final flush.

qapi/block-core.json | 6 +++++-

block/blklogwrites.c | 20 ++++++++++++++++++--

2 files changed, 23 insertions(+), 3 deletions(-)

# @log-sector-size: sector size used in logging writes to @file, determines

# granularity of offsets and sizes of writes (default: 512)

+# @log-super-update-interval: interval of write requests after which the log

+# super block is updated to disk (default: 4096)

# Since: 3.0

##

{ 'struct': 'BlockdevOptionsBlklogwrites',

'data': { 'file': 'BlockdevRef',

'log': 'BlockdevRef',

'*log-sector-size': 'uint32',

- '*log-append': 'bool' } }

+ '*log-append': 'bool',

+ '*log-super-update-interval': 'uint64' } }

##

# @BlockdevOptionsBlkverify:

diff --git a/block/blklogwrites.c b/block/blklogwrites.c

index XXXXXXX..XXXXXXX 100644

--- a/block/blklogwrites.c

+++ b/block/blklogwrites.c

@@ -XXX,XX +XXX,XX @@ typedef struct {

uint32_t sectorbits;

uint64_t cur_log_sector;

uint64_t nr_entries;

+ uint64_t update_interval;

} BDRVBlkLogWritesState;

static QemuOptsList runtime_opts = {

@@ -XXX,XX +XXX,XX @@ static QemuOptsList runtime_opts = {

.type = QEMU_OPT_SIZE,

.help = "Log sector size",

},

+ {

+ .name = "log-super-update-interval",

+ .type = QEMU_OPT_NUMBER,

+ .help = "Log superblock update interval (# of write requests)",

+ },

{ /* end of list */ }

},

};

@@ -XXX,XX +XXX,XX @@ static int blk_log_writes_open(BlockDriverState *bs, QDict *options, int flags,

s->sectorsize = log_sector_size;

s->sectorbits = blk_log_writes_log2(log_sector_size);

+ s->update_interval = qemu_opt_get_number(opts, "log-super-update-interval",

+ 4096);

+ if (!s->update_interval) {

+ ret = -EINVAL;

+ error_setg(errp, "Invalid log superblock update interval %"PRIu64,

+ s->update_interval);

+ goto fail_log;

+ }

ret = 0;

fail_log:

@@ -XXX,XX +XXX,XX @@ static void coroutine_fn blk_log_writes_co_do_log(BlkLogWritesLogReq *lr)

lr->zero_size, 0);

}

- /* Update super block on flush */

- if (lr->log_ret == 0 && lr->entry.flags & LOG_FLUSH_FLAG) {

+ /* Update super block on flush or every update interval */

+ if (lr->log_ret == 0 && ((lr->entry.flags & LOG_FLUSH_FLAG)

+ || (s->nr_entries % s->update_interval == 0)))

+ {

struct log_write_super super = {

.magic = cpu_to_le64(WRITE_LOG_MAGIC),

.version = cpu_to_le64(WRITE_LOG_VERSION),