The following changes since commit 30aa19446d82358a30eac3b556b4d6641e00b7c1:

Merge remote-tracking branch 'remotes/cschoenebeck/tags/pull-9p-20200812' into staging (2020-08-24 16:39:53 +0100)

are available in the Git repository at:

https://github.com/XanClic/qemu.git tags/pull-block-2020-08-25

for you to fetch changes up to c576fd97d4ca77b5a1a27728df11a61083dbfa98:

iotests: Add tests for qcow2 images with extended L2 entries (2020-08-25 10:20:18 +0200)

Block patches:

- qcow2 subclusters (extended L2 entries)

docs/interop/qcow2.txt | 68 ++-

docs/qcow2-cache.txt | 19 +-

qapi/block-core.json | 7 +

block/qcow2.h | 211 ++++++-

include/block/block_int.h | 1 +

block/qcow2-cluster.c | 906 +++++++++++++++++++++----------

block/qcow2-refcount.c | 47 +-

block/qcow2.c | 302 +++++++----

block/trace-events | 2 +-

tests/qemu-iotests/031.out | 8 +-

tests/qemu-iotests/036.out | 4 +-

tests/qemu-iotests/049.out | 102 ++--

tests/qemu-iotests/060.out | 3 +-

tests/qemu-iotests/061 | 6 +

tests/qemu-iotests/061.out | 25 +-

tests/qemu-iotests/065 | 12 +-

tests/qemu-iotests/082.out | 39 +-

tests/qemu-iotests/085.out | 38 +-

tests/qemu-iotests/144.out | 4 +-

tests/qemu-iotests/182.out | 2 +-

tests/qemu-iotests/185.out | 8 +-

tests/qemu-iotests/198 | 2 +

tests/qemu-iotests/206.out | 6 +-

tests/qemu-iotests/242.out | 5 +

tests/qemu-iotests/255.out | 8 +-

tests/qemu-iotests/271 | 901 ++++++++++++++++++++++++++++++

tests/qemu-iotests/271.out | 726 +++++++++++++++++++++++++

tests/qemu-iotests/274.out | 49 +-

tests/qemu-iotests/280.out | 2 +-

tests/qemu-iotests/291.out | 2 +

tests/qemu-iotests/302.out | 1 +

tests/qemu-iotests/303.out | 4 +-

tests/qemu-iotests/common.filter | 1 +

tests/qemu-iotests/group | 1 +

34 files changed, 2952 insertions(+), 570 deletions(-)

create mode 100755 tests/qemu-iotests/271

create mode 100644 tests/qemu-iotests/271.out

2.26.2

From: Alberto Garcia <berto@igalia.com>

The file_cluster_offset field of Qcow2AioTask stores a cluster-aligned

host offset. In practice this is not very useful because all users(*)

of this structure need the final host offset into the cluster, which

they calculate using

host_offset = file_cluster_offset + offset_into_cluster(s, offset)

There is no reason why Qcow2AioTask cannot store host_offset directly

and that is what this patch does.

(*) compressed clusters are the exception: in this case what

file_cluster_offset was storing was the full compressed cluster

descriptor (offset + size). This does not change with this patch

but it is documented now.

Signed-off-by: Alberto Garcia <berto@igalia.com>

Reviewed-by: Eric Blake <eblake@redhat.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Message-Id: <07c4b15c644dcf06c9459f98846ac1c4ea96e26f.1594396418.git.berto@igalia.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

block/qcow2.c | 69 ++++++++++++++++++++++------------------------

block/trace-events | 2 +-

2 files changed, 34 insertions(+), 37 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.c

+++ b/block/qcow2.c

@@ -XXX,XX +XXX,XX @@ typedef struct {

static int coroutine_fn

qcow2_co_preadv_compressed(BlockDriverState *bs,

- uint64_t file_cluster_offset,

+ uint64_t cluster_descriptor,

uint64_t offset,

uint64_t bytes,

QEMUIOVector *qiov,

@@ -XXX,XX +XXX,XX @@ out:

static coroutine_fn int

qcow2_co_preadv_encrypted(BlockDriverState *bs,

- uint64_t file_cluster_offset,

+ uint64_t host_offset,

uint64_t offset,

uint64_t bytes,

QEMUIOVector *qiov,

@@ -XXX,XX +XXX,XX @@ qcow2_co_preadv_encrypted(BlockDriverState *bs,

}

BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);

- ret = bdrv_co_pread(s->data_file,

- file_cluster_offset + offset_into_cluster(s, offset),

- bytes, buf, 0);

+ ret = bdrv_co_pread(s->data_file, host_offset, bytes, buf, 0);

if (ret < 0) {

goto fail;

}

- if (qcow2_co_decrypt(bs,

- file_cluster_offset + offset_into_cluster(s, offset),

- offset, buf, bytes) < 0)

+ if (qcow2_co_decrypt(bs, host_offset, offset, buf, bytes) < 0)

{

ret = -EIO;

goto fail;

@@ -XXX,XX +XXX,XX @@ typedef struct Qcow2AioTask {

BlockDriverState *bs;

QCow2ClusterType cluster_type; /* only for read */

- uint64_t file_cluster_offset;

+ uint64_t host_offset; /* or full descriptor in compressed clusters */

uint64_t offset;

uint64_t bytes;

QEMUIOVector *qiov;

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_add_task(BlockDriverState *bs,

AioTaskPool *pool,

AioTaskFunc func,

QCow2ClusterType cluster_type,

- uint64_t file_cluster_offset,

+ uint64_t host_offset,

uint64_t offset,

uint64_t bytes,

QEMUIOVector *qiov,

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_add_task(BlockDriverState *bs,

.bs = bs,

.cluster_type = cluster_type,

.qiov = qiov,

- .file_cluster_offset = file_cluster_offset,

+ .host_offset = host_offset,

.offset = offset,

.bytes = bytes,

.qiov_offset = qiov_offset,

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_add_task(BlockDriverState *bs,

trace_qcow2_add_task(qemu_coroutine_self(), bs, pool,

func == qcow2_co_preadv_task_entry ? "read" : "write",

- cluster_type, file_cluster_offset, offset, bytes,

+ cluster_type, host_offset, offset, bytes,

qiov, qiov_offset);

if (!pool) {

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_add_task(BlockDriverState *bs,

static coroutine_fn int qcow2_co_preadv_task(BlockDriverState *bs,

QCow2ClusterType cluster_type,

- uint64_t file_cluster_offset,

+ uint64_t host_offset,

uint64_t offset, uint64_t bytes,

QEMUIOVector *qiov,

size_t qiov_offset)

{

BDRVQcow2State *s = bs->opaque;

- int offset_in_cluster = offset_into_cluster(s, offset);

switch (cluster_type) {

case QCOW2_CLUSTER_ZERO_PLAIN:

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_task(BlockDriverState *bs,

qiov, qiov_offset, 0);

case QCOW2_CLUSTER_COMPRESSED:

- return qcow2_co_preadv_compressed(bs, file_cluster_offset,

+ return qcow2_co_preadv_compressed(bs, host_offset,

offset, bytes, qiov, qiov_offset);

case QCOW2_CLUSTER_NORMAL:

- assert(offset_into_cluster(s, file_cluster_offset) == 0);

if (bs->encrypted) {

- return qcow2_co_preadv_encrypted(bs, file_cluster_offset,

+ return qcow2_co_preadv_encrypted(bs, host_offset,

offset, bytes, qiov, qiov_offset);

}

BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);

- return bdrv_co_preadv_part(s->data_file,

- file_cluster_offset + offset_in_cluster,

+ return bdrv_co_preadv_part(s->data_file, host_offset,

bytes, qiov, qiov_offset, 0);

default:

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_task_entry(AioTask *task)

assert(!t->l2meta);

- return qcow2_co_preadv_task(t->bs, t->cluster_type, t->file_cluster_offset,

+ return qcow2_co_preadv_task(t->bs, t->cluster_type, t->host_offset,

t->offset, t->bytes, t->qiov, t->qiov_offset);

}

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_part(BlockDriverState *bs,

{

qemu_iovec_memset(qiov, qiov_offset, 0, cur_bytes);

} else {

+ /*

+ * For compressed clusters the variable cluster_offset

+ * does not actually store the offset but the full

+ * descriptor. We need to leave it unchanged because

+ * that's what qcow2_co_preadv_compressed() expects.

+ */

+ uint64_t host_offset = (ret == QCOW2_CLUSTER_COMPRESSED) ?

+ cluster_offset :

+ cluster_offset + offset_into_cluster(s, offset);

if (!aio && cur_bytes != bytes) {

aio = aio_task_pool_new(QCOW2_MAX_WORKERS);

}

ret = qcow2_add_task(bs, aio, qcow2_co_preadv_task_entry, ret,

- cluster_offset, offset, cur_bytes,

+ host_offset, offset, cur_bytes,

qiov, qiov_offset, NULL);

if (ret < 0) {

goto out;

@@ -XXX,XX +XXX,XX @@ static int handle_alloc_space(BlockDriverState *bs, QCowL2Meta *l2meta)

* not use it somehow after qcow2_co_pwritev_task() call

*/

static coroutine_fn int qcow2_co_pwritev_task(BlockDriverState *bs,

- uint64_t file_cluster_offset,

+ uint64_t host_offset,

uint64_t offset, uint64_t bytes,

QEMUIOVector *qiov,

uint64_t qiov_offset,

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwritev_task(BlockDriverState *bs,

int ret;

BDRVQcow2State *s = bs->opaque;

void *crypt_buf = NULL;

- int offset_in_cluster = offset_into_cluster(s, offset);

QEMUIOVector encrypted_qiov;

if (bs->encrypted) {

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwritev_task(BlockDriverState *bs,

}

qemu_iovec_to_buf(qiov, qiov_offset, crypt_buf, bytes);

- if (qcow2_co_encrypt(bs, file_cluster_offset + offset_in_cluster,

- offset, crypt_buf, bytes) < 0)

- {

+ if (qcow2_co_encrypt(bs, host_offset, offset, crypt_buf, bytes) < 0) {

ret = -EIO;

goto out_unlocked;

}

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwritev_task(BlockDriverState *bs,

*/

if (!merge_cow(offset, bytes, qiov, qiov_offset, l2meta)) {

BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);

- trace_qcow2_writev_data(qemu_coroutine_self(),

- file_cluster_offset + offset_in_cluster);

- ret = bdrv_co_pwritev_part(s->data_file,

- file_cluster_offset + offset_in_cluster,

+ trace_qcow2_writev_data(qemu_coroutine_self(), host_offset);

+ ret = bdrv_co_pwritev_part(s->data_file, host_offset,

bytes, qiov, qiov_offset, 0);

if (ret < 0) {

goto out_unlocked;

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwritev_task_entry(AioTask *task)

assert(!t->cluster_type);

- return qcow2_co_pwritev_task(t->bs, t->file_cluster_offset,

+ return qcow2_co_pwritev_task(t->bs, t->host_offset,

t->offset, t->bytes, t->qiov, t->qiov_offset,

t->l2meta);

}

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwritev_part(

aio = aio_task_pool_new(QCOW2_MAX_WORKERS);

}

ret = qcow2_add_task(bs, aio, qcow2_co_pwritev_task_entry, 0,

- cluster_offset, offset, cur_bytes,

- qiov, qiov_offset, l2meta);

+ cluster_offset + offset_in_cluster, offset,

+ cur_bytes, qiov, qiov_offset, l2meta);

l2meta = NULL; /* l2meta is consumed by qcow2_co_pwritev_task() */

if (ret < 0) {

goto fail_nometa;

@@ -XXX,XX +XXX,XX @@ qcow2_co_pwritev_compressed_part(BlockDriverState *bs,

static int coroutine_fn

qcow2_co_preadv_compressed(BlockDriverState *bs,

- uint64_t file_cluster_offset,

+ uint64_t cluster_descriptor,

uint64_t offset,

uint64_t bytes,

QEMUIOVector *qiov,

@@ -XXX,XX +XXX,XX @@ qcow2_co_preadv_compressed(BlockDriverState *bs,

uint8_t *buf, *out_buf;

int offset_in_cluster = offset_into_cluster(s, offset);

- coffset = file_cluster_offset & s->cluster_offset_mask;

- nb_csectors = ((file_cluster_offset >> s->csize_shift) & s->csize_mask) + 1;

+ coffset = cluster_descriptor & s->cluster_offset_mask;

+ nb_csectors = ((cluster_descriptor >> s->csize_shift) & s->csize_mask) + 1;

csize = nb_csectors * QCOW2_COMPRESSED_SECTOR_SIZE -

(coffset & ~QCOW2_COMPRESSED_SECTOR_MASK);

diff --git a/block/trace-events b/block/trace-events

index XXXXXXX..XXXXXXX 100644

--- a/block/trace-events

+++ b/block/trace-events

@@ -XXX,XX +XXX,XX @@ luring_io_uring_submit(void *s, int ret) "LuringState %p ret %d"

luring_resubmit_short_read(void *s, void *luringcb, int nread) "LuringState %p luringcb %p nread %d"

# qcow2.c

-qcow2_add_task(void *co, void *bs, void *pool, const char *action, int cluster_type, uint64_t file_cluster_offset, uint64_t offset, uint64_t bytes, void *qiov, size_t qiov_offset) "co %p bs %p pool %p: %s: cluster_type %d file_cluster_offset %" PRIu64 " offset %" PRIu64 " bytes %" PRIu64 " qiov %p qiov_offset %zu"

+qcow2_add_task(void *co, void *bs, void *pool, const char *action, int cluster_type, uint64_t host_offset, uint64_t offset, uint64_t bytes, void *qiov, size_t qiov_offset) "co %p bs %p pool %p: %s: cluster_type %d file_cluster_offset %" PRIu64 " offset %" PRIu64 " bytes %" PRIu64 " qiov %p qiov_offset %zu"

qcow2_writev_start_req(void *co, int64_t offset, int bytes) "co %p offset 0x%" PRIx64 " bytes %d"

qcow2_writev_done_req(void *co, int ret) "co %p ret %d"

qcow2_writev_start_part(void *co) "co %p"

--

2.26.2

From: Alberto Garcia <berto@igalia.com>

qcow2_get_cluster_offset() takes an (unaligned) guest offset and

returns the (aligned) offset of the corresponding cluster in the qcow2

image.

In practice none of the callers need to know where the cluster starts

so this patch makes the function calculate and return the final host

offset directly. The function is also renamed accordingly.

There is a pre-existing exception with compressed clusters: in this

case the function returns the complete cluster descriptor (containing

the offset and size of the compressed data). This does not change with

this patch but it is now documented.

Signed-off-by: Alberto Garcia <berto@igalia.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <ffae6cdc5ca8950e8280ac0f696dcc376cb07095.1594396418.git.berto@igalia.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

block/qcow2.h | 4 ++--

block/qcow2-cluster.c | 41 +++++++++++++++++++++++------------------

block/qcow2.c | 24 +++++++-----------------

3 files changed, 32 insertions(+), 37 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.h

+++ b/block/qcow2.h

@@ -XXX,XX +XXX,XX @@ int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index);

int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,

uint8_t *buf, int nb_sectors, bool enc, Error **errp);

-int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,

- unsigned int *bytes, uint64_t *cluster_offset);

+int qcow2_get_host_offset(BlockDriverState *bs, uint64_t offset,

+ unsigned int *bytes, uint64_t *host_offset);

int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,

unsigned int *bytes, uint64_t *host_offset,

QCowL2Meta **m);

diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-cluster.c

+++ b/block/qcow2-cluster.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn do_perform_cow_write(BlockDriverState *bs,

/*

- * get_cluster_offset

+ * get_host_offset

*

- * For a given offset of the virtual disk, find the cluster type and offset in

- * the qcow2 file. The offset is stored in *cluster_offset.

+ * For a given offset of the virtual disk find the equivalent host

+ * offset in the qcow2 file and store it in *host_offset. Neither

+ * offset needs to be aligned to a cluster boundary.

+ *

+ * If the cluster is unallocated then *host_offset will be 0.

+ * If the cluster is compressed then *host_offset will contain the

+ * complete compressed cluster descriptor.

*

* On entry, *bytes is the maximum number of contiguous bytes starting at

* offset that we are interested in.

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn do_perform_cow_write(BlockDriverState *bs,

* Returns the cluster type (QCOW2_CLUSTER_*) on success, -errno in error

* cases.

*/

-int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,

- unsigned int *bytes, uint64_t *cluster_offset)

+int qcow2_get_host_offset(BlockDriverState *bs, uint64_t offset,

+ unsigned int *bytes, uint64_t *host_offset)

{

BDRVQcow2State *s = bs->opaque;

unsigned int l2_index;

- uint64_t l1_index, l2_offset, *l2_slice;

+ uint64_t l1_index, l2_offset, *l2_slice, l2_entry;

int c;

unsigned int offset_in_cluster;

uint64_t bytes_available, bytes_needed, nb_clusters;

@@ -XXX,XX +XXX,XX @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,

bytes_needed = bytes_available;

}

- *cluster_offset = 0;

+ *host_offset = 0;

/* seek to the l2 offset in the l1 table */

@@ -XXX,XX +XXX,XX @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,

/* find the cluster offset for the given disk offset */

l2_index = offset_to_l2_slice_index(s, offset);

- *cluster_offset = be64_to_cpu(l2_slice[l2_index]);

+ l2_entry = be64_to_cpu(l2_slice[l2_index]);

nb_clusters = size_to_clusters(s, bytes_needed);

/* bytes_needed <= *bytes + offset_in_cluster, both of which are unsigned

@@ -XXX,XX +XXX,XX @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,

* true */

assert(nb_clusters <= INT_MAX);

- type = qcow2_get_cluster_type(bs, *cluster_offset);

+ type = qcow2_get_cluster_type(bs, l2_entry);

if (s->qcow_version < 3 && (type == QCOW2_CLUSTER_ZERO_PLAIN ||

type == QCOW2_CLUSTER_ZERO_ALLOC)) {

qcow2_signal_corruption(bs, true, -1, -1, "Zero cluster entry found"

@@ -XXX,XX +XXX,XX @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,

}

/* Compressed clusters can only be processed one by one */

c = 1;

- *cluster_offset &= L2E_COMPRESSED_OFFSET_SIZE_MASK;

+ *host_offset = l2_entry & L2E_COMPRESSED_OFFSET_SIZE_MASK;

break;

case QCOW2_CLUSTER_ZERO_PLAIN:

case QCOW2_CLUSTER_UNALLOCATED:

/* how many empty clusters ? */

c = count_contiguous_clusters_unallocated(bs, nb_clusters,

&l2_slice[l2_index], type);

- *cluster_offset = 0;

break;

case QCOW2_CLUSTER_ZERO_ALLOC:

- case QCOW2_CLUSTER_NORMAL:

+ case QCOW2_CLUSTER_NORMAL: {

+ uint64_t host_cluster_offset = l2_entry & L2E_OFFSET_MASK;

+ *host_offset = host_cluster_offset + offset_in_cluster;

/* how many allocated clusters ? */

c = count_contiguous_clusters(bs, nb_clusters, s->cluster_size,

&l2_slice[l2_index], QCOW_OFLAG_ZERO);

- *cluster_offset &= L2E_OFFSET_MASK;

- if (offset_into_cluster(s, *cluster_offset)) {

+ if (offset_into_cluster(s, host_cluster_offset)) {

qcow2_signal_corruption(bs, true, -1, -1,

"Cluster allocation offset %#"

PRIx64 " unaligned (L2 offset: %#" PRIx64

- ", L2 index: %#x)", *cluster_offset,

+ ", L2 index: %#x)", host_cluster_offset,

l2_offset, l2_index);

ret = -EIO;

goto fail;

}

- if (has_data_file(bs) && *cluster_offset != offset - offset_in_cluster)

- {

+ if (has_data_file(bs) && *host_offset != offset) {

qcow2_signal_corruption(bs, true, -1, -1,

"External data file host cluster offset %#"

PRIx64 " does not match guest cluster "

"offset: %#" PRIx64

- ", L2 index: %#x)", *cluster_offset,

+ ", L2 index: %#x)", host_cluster_offset,

offset - offset_in_cluster, l2_index);

ret = -EIO;

goto fail;

}

break;

+ }

default:

abort();

}

diff --git a/block/qcow2.c b/block/qcow2.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.c

+++ b/block/qcow2.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs,

BlockDriverState **file)

{

BDRVQcow2State *s = bs->opaque;

- uint64_t cluster_offset;

+ uint64_t host_offset;

unsigned int bytes;

int ret, status = 0;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs,

}

bytes = MIN(INT_MAX, count);

- ret = qcow2_get_cluster_offset(bs, offset, &bytes, &cluster_offset);

+ ret = qcow2_get_host_offset(bs, offset, &bytes, &host_offset);

qemu_co_mutex_unlock(&s->lock);

if (ret < 0) {

return ret;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs,

if ((ret == QCOW2_CLUSTER_NORMAL || ret == QCOW2_CLUSTER_ZERO_ALLOC) &&

!s->crypto) {

- *map = cluster_offset | offset_into_cluster(s, offset);

+ *map = host_offset;

*file = s->data_file->bs;

status |= BDRV_BLOCK_OFFSET_VALID;

}

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_part(BlockDriverState *bs,

BDRVQcow2State *s = bs->opaque;

int ret = 0;

unsigned int cur_bytes; /* number of bytes in current iteration */

- uint64_t cluster_offset = 0;

+ uint64_t host_offset = 0;

AioTaskPool *aio = NULL;

while (bytes != 0 && aio_task_pool_status(aio) == 0) {

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_part(BlockDriverState *bs,

}

qemu_co_mutex_lock(&s->lock);

- ret = qcow2_get_cluster_offset(bs, offset, &cur_bytes, &cluster_offset);

+ ret = qcow2_get_host_offset(bs, offset, &cur_bytes, &host_offset);

qemu_co_mutex_unlock(&s->lock);

if (ret < 0) {

goto out;

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_part(BlockDriverState *bs,

{

qemu_iovec_memset(qiov, qiov_offset, 0, cur_bytes);

} else {

- /*

- * For compressed clusters the variable cluster_offset

- * does not actually store the offset but the full

- * descriptor. We need to leave it unchanged because

- * that's what qcow2_co_preadv_compressed() expects.

- */

- uint64_t host_offset = (ret == QCOW2_CLUSTER_COMPRESSED) ?

- cluster_offset :

- cluster_offset + offset_into_cluster(s, offset);

if (!aio && cur_bytes != bytes) {

aio = aio_task_pool_new(QCOW2_MAX_WORKERS);

}

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwrite_zeroes(BlockDriverState *bs,

offset = QEMU_ALIGN_DOWN(offset, s->cluster_size);

bytes = s->cluster_size;

nr = s->cluster_size;

- ret = qcow2_get_cluster_offset(bs, offset, &nr, &off);

+ ret = qcow2_get_host_offset(bs, offset, &nr, &off);

if (ret != QCOW2_CLUSTER_UNALLOCATED &&

ret != QCOW2_CLUSTER_ZERO_PLAIN &&

ret != QCOW2_CLUSTER_ZERO_ALLOC) {

@@ -XXX,XX +XXX,XX @@ qcow2_co_copy_range_from(BlockDriverState *bs,

cur_bytes = MIN(bytes, INT_MAX);

cur_write_flags = write_flags;

- ret = qcow2_get_cluster_offset(bs, src_offset, &cur_bytes, &copy_offset);

+ ret = qcow2_get_host_offset(bs, src_offset, &cur_bytes, &copy_offset);

if (ret < 0) {

goto out;

}

@@ -XXX,XX +XXX,XX @@ qcow2_co_copy_range_from(BlockDriverState *bs,

case QCOW2_CLUSTER_NORMAL:

child = s->data_file;

- copy_offset += offset_into_cluster(s, src_offset);

break;

default:

--

2.26.2

From: Alberto Garcia <berto@igalia.com>

When writing to a qcow2 file there are two functions that take a

virtual offset and return a host offset, possibly allocating new

clusters if necessary:

- handle_copied() looks for normal data clusters that are already

allocated and have a reference count of 1. In those clusters we

can simply write the data and there is no need to perform any

copy-on-write.

- handle_alloc() looks for clusters that do need copy-on-write,

either because they haven't been allocated yet, because their

reference count is != 1 or because they are ZERO_ALLOC clusters.

The ZERO_ALLOC case is a bit special because those are clusters that

are already allocated and they could perfectly be dealt with in

handle_copied() (as long as copy-on-write is performed when required).

In fact, there is extra code specifically for them in handle_alloc()

that tries to reuse the existing allocation if possible and frees them

otherwise.

This patch changes the handling of ZERO_ALLOC clusters so the

semantics of these two functions are now like this:

- handle_copied() looks for clusters that are already allocated and

which we can overwrite (NORMAL and ZERO_ALLOC clusters with a

reference count of 1).

- handle_alloc() looks for clusters for which we need a new

allocation (all other cases).

One important difference after this change is that clusters found

in handle_copied() may now require copy-on-write, but this will be

necessary anyway once we add support for subclusters.

Signed-off-by: Alberto Garcia <berto@igalia.com>

Reviewed-by: Eric Blake <eblake@redhat.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <eb17fc938f6be7be2e8d8ff42763d2c19241f866.1594396418.git.berto@igalia.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

block/qcow2-cluster.c | 252 +++++++++++++++++++++++-------------------

1 file changed, 139 insertions(+), 113 deletions(-)

diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-cluster.c

+++ b/block/qcow2-cluster.c

@@ -XXX,XX +XXX,XX @@ void qcow2_alloc_cluster_abort(BlockDriverState *bs, QCowL2Meta *m)

/*

* For a given write request, create a new QCowL2Meta structure, add

- * it to @m and the BDRVQcow2State.cluster_allocs list.

+ * it to @m and the BDRVQcow2State.cluster_allocs list. If the write

+ * request does not need copy-on-write or changes to the L2 metadata

+ * then this function does nothing.

*

* @host_cluster_offset points to the beginning of the first cluster.

*

* @guest_offset and @bytes indicate the offset and length of the

* request.

*

+ * @l2_slice contains the L2 entries of all clusters involved in this

+ * write request.

+ *

* If @keep_old is true it means that the clusters were already

* allocated and will be overwritten. If false then the clusters are

* new and we have to decrease the reference count of the old ones.

@@ -XXX,XX +XXX,XX @@ void qcow2_alloc_cluster_abort(BlockDriverState *bs, QCowL2Meta *m)

static void calculate_l2_meta(BlockDriverState *bs,

uint64_t host_cluster_offset,

uint64_t guest_offset, unsigned bytes,

- QCowL2Meta **m, bool keep_old)

+ uint64_t *l2_slice, QCowL2Meta **m, bool keep_old)

{

BDRVQcow2State *s = bs->opaque;

- unsigned cow_start_from = 0;

+ int l2_index = offset_to_l2_slice_index(s, guest_offset);

+ uint64_t l2_entry;

+ unsigned cow_start_from, cow_end_to;

unsigned cow_start_to = offset_into_cluster(s, guest_offset);

unsigned cow_end_from = cow_start_to + bytes;

- unsigned cow_end_to = ROUND_UP(cow_end_from, s->cluster_size);

unsigned nb_clusters = size_to_clusters(s, cow_end_from);

QCowL2Meta *old_m = *m;

+ QCow2ClusterType type;

+

+ assert(nb_clusters <= s->l2_slice_size - l2_index);

+

+ /* Return if there's no COW (all clusters are normal and we keep them) */

+ if (keep_old) {

+ int i;

+ for (i = 0; i < nb_clusters; i++) {

+ l2_entry = be64_to_cpu(l2_slice[l2_index + i]);

+ if (qcow2_get_cluster_type(bs, l2_entry) != QCOW2_CLUSTER_NORMAL) {

+ break;

+ }

+ }

+ if (i == nb_clusters) {

+ return;

+ }

+ }

+

+ /* Get the L2 entry of the first cluster */

+ l2_entry = be64_to_cpu(l2_slice[l2_index]);

+ type = qcow2_get_cluster_type(bs, l2_entry);

+

+ if (type == QCOW2_CLUSTER_NORMAL && keep_old) {

+ cow_start_from = cow_start_to;

+ } else {

+ cow_start_from = 0;

+ }

+

+ /* Get the L2 entry of the last cluster */

+ l2_entry = be64_to_cpu(l2_slice[l2_index + nb_clusters - 1]);

+ type = qcow2_get_cluster_type(bs, l2_entry);

+

+ if (type == QCOW2_CLUSTER_NORMAL && keep_old) {

+ cow_end_to = cow_end_from;

+ } else {

+ cow_end_to = ROUND_UP(cow_end_from, s->cluster_size);

+ }

*m = g_malloc0(sizeof(**m));

**m = (QCowL2Meta) {

@@ -XXX,XX +XXX,XX @@ static void calculate_l2_meta(BlockDriverState *bs,

QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);

}

-/* Returns true if writing to a cluster requires COW */

-static bool cluster_needs_cow(BlockDriverState *bs, uint64_t l2_entry)

+/*

+ * Returns true if writing to the cluster pointed to by @l2_entry

+ * requires a new allocation (that is, if the cluster is unallocated

+ * or has refcount > 1 and therefore cannot be written in-place).

+ */

+static bool cluster_needs_new_alloc(BlockDriverState *bs, uint64_t l2_entry)

{

switch (qcow2_get_cluster_type(bs, l2_entry)) {

case QCOW2_CLUSTER_NORMAL:

+ case QCOW2_CLUSTER_ZERO_ALLOC:

if (l2_entry & QCOW_OFLAG_COPIED) {

return false;

}

case QCOW2_CLUSTER_UNALLOCATED:

case QCOW2_CLUSTER_COMPRESSED:

case QCOW2_CLUSTER_ZERO_PLAIN:

- case QCOW2_CLUSTER_ZERO_ALLOC:

return true;

default:

abort();

@@ -XXX,XX +XXX,XX @@ static bool cluster_needs_cow(BlockDriverState *bs, uint64_t l2_entry)

}

/*

- * Returns the number of contiguous clusters that can be used for an allocating

- * write, but require COW to be performed (this includes yet unallocated space,

- * which must copy from the backing file)

+ * Returns the number of contiguous clusters that can be written to

+ * using one single write request, starting from @l2_index.

+ * At most @nb_clusters are checked.

+ *

+ * If @new_alloc is true this counts clusters that are either

+ * unallocated, or allocated but with refcount > 1 (so they need to be

+ * newly allocated and COWed).

+ *

+ * If @new_alloc is false this counts clusters that are already

+ * allocated and can be overwritten in-place (this includes clusters

+ * of type QCOW2_CLUSTER_ZERO_ALLOC).

*/

-static int count_cow_clusters(BlockDriverState *bs, int nb_clusters,

- uint64_t *l2_slice, int l2_index)

+static int count_single_write_clusters(BlockDriverState *bs, int nb_clusters,

+ uint64_t *l2_slice, int l2_index,

+ bool new_alloc)

{

+ BDRVQcow2State *s = bs->opaque;

+ uint64_t l2_entry = be64_to_cpu(l2_slice[l2_index]);

+ uint64_t expected_offset = l2_entry & L2E_OFFSET_MASK;

int i;

for (i = 0; i < nb_clusters; i++) {

- uint64_t l2_entry = be64_to_cpu(l2_slice[l2_index + i]);

- if (!cluster_needs_cow(bs, l2_entry)) {

+ l2_entry = be64_to_cpu(l2_slice[l2_index + i]);

+ if (cluster_needs_new_alloc(bs, l2_entry) != new_alloc) {

break;

}

+ if (!new_alloc) {

+ if (expected_offset != (l2_entry & L2E_OFFSET_MASK)) {

+ break;

+ }

+ expected_offset += s->cluster_size;

+ }

}

assert(i <= nb_clusters);

@@ -XXX,XX +XXX,XX @@ static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,

}

/*

- * Checks how many already allocated clusters that don't require a copy on

- * write there are at the given guest_offset (up to *bytes). If *host_offset is

- * not INV_OFFSET, only physically contiguous clusters beginning at this host

- * offset are counted.

+ * Checks how many already allocated clusters that don't require a new

+ * allocation there are at the given guest_offset (up to *bytes).

+ * If *host_offset is not INV_OFFSET, only physically contiguous clusters

+ * beginning at this host offset are counted.

*

* Note that guest_offset may not be cluster aligned. In this case, the

* returned *host_offset points to exact byte referenced by guest_offset and

@@ -XXX,XX +XXX,XX @@ static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,

* Returns:

* 0: if no allocated clusters are available at the given offset.

* *bytes is normally unchanged. It is set to 0 if the cluster

- * is allocated and doesn't need COW, but doesn't have the right

- * physical offset.

+ * is allocated and can be overwritten in-place but doesn't have

+ * the right physical offset.

*

- * 1: if allocated clusters that don't require a COW are available at

- * the requested offset. *bytes may have decreased and describes

- * the length of the area that can be written to.

+ * 1: if allocated clusters that can be overwritten in place are

+ * available at the requested offset. *bytes may have decreased

+ * and describes the length of the area that can be written to.

*

* -errno: in error cases

*/

@@ -XXX,XX +XXX,XX @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,

{

BDRVQcow2State *s = bs->opaque;

int l2_index;

- uint64_t cluster_offset;

+ uint64_t l2_entry, cluster_offset;

uint64_t *l2_slice;

uint64_t nb_clusters;

unsigned int keep_clusters;

@@ -XXX,XX +XXX,XX @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,

l2_index = offset_to_l2_slice_index(s, guest_offset);

nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);

- assert(nb_clusters <= INT_MAX);

+ /* Limit total byte count to BDRV_REQUEST_MAX_BYTES */

+ nb_clusters = MIN(nb_clusters, BDRV_REQUEST_MAX_BYTES >> s->cluster_bits);

/* Find L2 entry for the first involved cluster */

ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index);

@@ -XXX,XX +XXX,XX @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,

return ret;

}

- cluster_offset = be64_to_cpu(l2_slice[l2_index]);

-

- /* Check how many clusters are already allocated and don't need COW */

- if (qcow2_get_cluster_type(bs, cluster_offset) == QCOW2_CLUSTER_NORMAL

- && (cluster_offset & QCOW_OFLAG_COPIED))

- {

- /* If a specific host_offset is required, check it */

- bool offset_matches =

- (cluster_offset & L2E_OFFSET_MASK) == *host_offset;

-

- if (offset_into_cluster(s, cluster_offset & L2E_OFFSET_MASK)) {

- qcow2_signal_corruption(bs, true, -1, -1, "Data cluster offset "

- "%#llx unaligned (guest offset: %#" PRIx64

- ")", cluster_offset & L2E_OFFSET_MASK,

- guest_offset);

+ l2_entry = be64_to_cpu(l2_slice[l2_index]);

+ cluster_offset = l2_entry & L2E_OFFSET_MASK;

+

+ if (!cluster_needs_new_alloc(bs, l2_entry)) {

+ if (offset_into_cluster(s, cluster_offset)) {

+ qcow2_signal_corruption(bs, true, -1, -1, "%s cluster offset "

+ "%#" PRIx64 " unaligned (guest offset: %#"

+ PRIx64 ")", l2_entry & QCOW_OFLAG_ZERO ?

+ "Preallocated zero" : "Data",

+ cluster_offset, guest_offset);

ret = -EIO;

goto out;

}

- if (*host_offset != INV_OFFSET && !offset_matches) {

+ /* If a specific host_offset is required, check it */

+ if (*host_offset != INV_OFFSET && cluster_offset != *host_offset) {

*bytes = 0;

ret = 0;

goto out;

}

/* We keep all QCOW_OFLAG_COPIED clusters */

- keep_clusters =

- count_contiguous_clusters(bs, nb_clusters, s->cluster_size,

- &l2_slice[l2_index],

- QCOW_OFLAG_COPIED | QCOW_OFLAG_ZERO);

+ keep_clusters = count_single_write_clusters(bs, nb_clusters, l2_slice,

+ l2_index, false);

assert(keep_clusters <= nb_clusters);

*bytes = MIN(*bytes,

keep_clusters * s->cluster_size

- offset_into_cluster(s, guest_offset));

+ assert(*bytes != 0);

+

+ calculate_l2_meta(bs, cluster_offset, guest_offset,

+ *bytes, l2_slice, m, true);

ret = 1;

} else {

@@ -XXX,XX +XXX,XX @@ out:

/* Only return a host offset if we actually made progress. Otherwise we

* would make requirements for handle_alloc() that it can't fulfill */

if (ret > 0) {

- *host_offset = (cluster_offset & L2E_OFFSET_MASK)

- + offset_into_cluster(s, guest_offset);

+ *host_offset = cluster_offset + offset_into_cluster(s, guest_offset);

}

return ret;

@@ -XXX,XX +XXX,XX @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,

}

/*

- * Allocates new clusters for an area that either is yet unallocated or needs a

- * copy on write. If *host_offset is not INV_OFFSET, clusters are only

- * allocated if the new allocation can match the specified host offset.

+ * Allocates new clusters for an area that is either still unallocated or

+ * cannot be overwritten in-place. If *host_offset is not INV_OFFSET,

+ * clusters are only allocated if the new allocation can match the specified

+ * host offset.

*

* Note that guest_offset may not be cluster aligned. In this case, the

* returned *host_offset points to exact byte referenced by guest_offset and

@@ -XXX,XX +XXX,XX @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,

BDRVQcow2State *s = bs->opaque;

int l2_index;

uint64_t *l2_slice;

- uint64_t entry;

uint64_t nb_clusters;

int ret;

- bool keep_old_clusters = false;

- uint64_t alloc_cluster_offset = INV_OFFSET;

+ uint64_t alloc_cluster_offset;

trace_qcow2_handle_alloc(qemu_coroutine_self(), guest_offset, *host_offset,

*bytes);

@@ -XXX,XX +XXX,XX @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,

l2_index = offset_to_l2_slice_index(s, guest_offset);

nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);

- assert(nb_clusters <= INT_MAX);

-

- /* Limit total allocation byte count to INT_MAX */

- nb_clusters = MIN(nb_clusters, INT_MAX >> s->cluster_bits);

+ /* Limit total allocation byte count to BDRV_REQUEST_MAX_BYTES */

+ nb_clusters = MIN(nb_clusters, BDRV_REQUEST_MAX_BYTES >> s->cluster_bits);

/* Find L2 entry for the first involved cluster */

ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index);

@@ -XXX,XX +XXX,XX @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,

return ret;

}

- entry = be64_to_cpu(l2_slice[l2_index]);

- nb_clusters = count_cow_clusters(bs, nb_clusters, l2_slice, l2_index);

+ nb_clusters = count_single_write_clusters(bs, nb_clusters,

+ l2_slice, l2_index, true);

/* This function is only called when there were no non-COW clusters, so if

* we can't find any unallocated or COW clusters either, something is

* wrong with our code. */

assert(nb_clusters > 0);

- if (qcow2_get_cluster_type(bs, entry) == QCOW2_CLUSTER_ZERO_ALLOC &&

- (entry & QCOW_OFLAG_COPIED) &&

- (*host_offset == INV_OFFSET ||

- start_of_cluster(s, *host_offset) == (entry & L2E_OFFSET_MASK)))

- {

- int preallocated_nb_clusters;

-

- if (offset_into_cluster(s, entry & L2E_OFFSET_MASK)) {

- qcow2_signal_corruption(bs, true, -1, -1, "Preallocated zero "

- "cluster offset %#llx unaligned (guest "

- "offset: %#" PRIx64 ")",

- entry & L2E_OFFSET_MASK, guest_offset);

- ret = -EIO;

- goto fail;

- }

-

- /* Try to reuse preallocated zero clusters; contiguous normal clusters

- * would be fine, too, but count_cow_clusters() above has limited

- * nb_clusters already to a range of COW clusters */

- preallocated_nb_clusters =

- count_contiguous_clusters(bs, nb_clusters, s->cluster_size,

- &l2_slice[l2_index], QCOW_OFLAG_COPIED);

- assert(preallocated_nb_clusters > 0);

-

- nb_clusters = preallocated_nb_clusters;

- alloc_cluster_offset = entry & L2E_OFFSET_MASK;

-

- /* We want to reuse these clusters, so qcow2_alloc_cluster_link_l2()

- * should not free them. */

- keep_old_clusters = true;

+ /* Allocate at a given offset in the image file */

+ alloc_cluster_offset = *host_offset == INV_OFFSET ? INV_OFFSET :

+ start_of_cluster(s, *host_offset);

+ ret = do_alloc_cluster_offset(bs, guest_offset, &alloc_cluster_offset,

+ &nb_clusters);

+ if (ret < 0) {

+ goto out;

}

- qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);

-

- if (alloc_cluster_offset == INV_OFFSET) {

- /* Allocate, if necessary at a given offset in the image file */

- alloc_cluster_offset = *host_offset == INV_OFFSET ? INV_OFFSET :

- start_of_cluster(s, *host_offset);

- ret = do_alloc_cluster_offset(bs, guest_offset, &alloc_cluster_offset,

- &nb_clusters);

- if (ret < 0) {

- goto fail;

- }

-

- /* Can't extend contiguous allocation */

- if (nb_clusters == 0) {

- *bytes = 0;

- return 0;

- }

-

- assert(alloc_cluster_offset != INV_OFFSET);

+ /* Can't extend contiguous allocation */

+ if (nb_clusters == 0) {

+ *bytes = 0;

+ ret = 0;

+ goto out;

}

+ assert(alloc_cluster_offset != INV_OFFSET);

+

/*

* Save info needed for meta data update.

*

@@ -XXX,XX +XXX,XX @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,

*bytes = MIN(*bytes, nb_bytes - offset_into_cluster(s, guest_offset));

assert(*bytes != 0);

- calculate_l2_meta(bs, alloc_cluster_offset, guest_offset, *bytes,

- m, keep_old_clusters);

+ calculate_l2_meta(bs, alloc_cluster_offset, guest_offset, *bytes, l2_slice,

+ m, false);

- return 1;

+ ret = 1;

-fail:

- if (*m && (*m)->nb_clusters > 0) {

+out:

+ qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);

+ if (ret < 0 && *m && (*m)->nb_clusters > 0) {

QLIST_REMOVE(*m, next_in_flight);

}

return ret;

--

2.26.2

From: Alberto Garcia <berto@igalia.com>

The size of an L2 entry is 64 bits, but if we want to have subclusters

we need extended L2 entries. This means that we have to access L2

tables and slices differently depending on whether an image has

extended L2 entries or not.

This patch replaces all l2_slice[] accesses with calls to

get_l2_entry() and set_l2_entry().

Signed-off-by: Alberto Garcia <berto@igalia.com>

Reviewed-by: Eric Blake <eblake@redhat.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Message-Id: <9586363531fec125ba1386e561762d3e4224e9fc.1594396418.git.berto@igalia.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

block/qcow2.h | 12 ++++++++

block/qcow2-cluster.c | 63 ++++++++++++++++++++++--------------------

block/qcow2-refcount.c | 17 ++++++------

3 files changed, 54 insertions(+), 38 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.h