The following changes since commit dd25f97c66a75d1508f1d4c6478ed2c95bec428f:

Merge remote-tracking branch 'remotes/pmaydell/tags/pull-target-arm-20190913' into staging (2019-09-16 10:15:15 +0100)

https://github.com/XanClic/qemu.git tags/pull-block-2019-09-16

for you to fetch changes up to 1825cc0783ccf0ec5d9f0b225a99b340bdd4c68f:

qemu-iotests: Add test for bz #1745922 (2019-09-16 15:37:12 +0200)

Block patches:

- Fix for block jobs when used with I/O threads

- Fix for a corruption when using qcow2's LUKS encryption mode

- cURL fix

- check-block.sh cleanups (for make check)

- Refactoring

Maxim Levitsky (3):

block/qcow2: Fix corruption introduced by commit 8ac0f15f335

block/qcow2: refactor encryption code

qemu-iotests: Add test for bz #1745922

Nir Soffer (2):

block: Use QEMU_IS_ALIGNED

block: Remove unused masks

Sergio Lopez (1):

blockjob: update nodes head while removing all bdrv

Thomas Huth (2):

tests/qemu-iotests/check: Replace "tests" with "iotests" in final

status text

tests/Makefile: Do not print the name of the check-block.sh shell

script

Vladimir Sementsov-Ogievskiy (1):

tests/qemu-iotests: Fix qemu-io related output in 026.out.nocache

tests/Makefile.include | 2 +-

block/qcow2.h | 8 +-

include/block/block.h | 2 -

block/bochs.c | 4 +-

block/cloop.c | 4 +-

block/curl.c | 133 ++++++++++-------------

block/dmg.c | 4 +-

block/io.c | 8 +-

block/qcow2-cluster.c | 40 +++----

block/qcow2-threads.c | 63 ++++++++---

block/qcow2.c | 9 +-

block/vvfat.c | 8 +-

blockjob.c | 17 ++-

migration/block.c | 2 +-

qemu-img.c | 2 +-

tests/qemu-iotests/026.out.nocache | 168 ++++++++++++++---------------

tests/qemu-iotests/263 | 91 ++++++++++++++++

tests/qemu-iotests/263.out | 40 +++++++

tests/qemu-iotests/check | 8 +-

tests/qemu-iotests/group | 1 +

20 files changed, 380 insertions(+), 234 deletions(-)

create mode 100755 tests/qemu-iotests/263

create mode 100644 tests/qemu-iotests/263.out

2.21.0

curl_multi_do_locked() currently marks all sockets as ready. That is

not only inefficient, but in fact unsafe (the loop is). A follow-up

patch will change that, but to do so, curl_multi_do_locked() needs to

know exactly which socket is ready; and that is accomplished by this

patch here.

Cc: qemu-stable@nongnu.org

Signed-off-by: Max Reitz <mreitz@redhat.com>

Message-id: 20190910124136.10565-5-mreitz@redhat.com

Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>

Reviewed-by: John Snow <jsnow@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/curl.c | 20 +++++++++++---------

1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/block/curl.c b/block/curl.c

--- a/block/curl.c

+++ b/block/curl.c

@@ -XXX,XX +XXX,XX @@ static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,

switch (action) {

case CURL_POLL_IN:

aio_set_fd_handler(s->aio_context, fd, false,

- curl_multi_do, NULL, NULL, state);

+ curl_multi_do, NULL, NULL, socket);

break;

case CURL_POLL_OUT:

aio_set_fd_handler(s->aio_context, fd, false,

- NULL, curl_multi_do, NULL, state);

+ NULL, curl_multi_do, NULL, socket);

break;

case CURL_POLL_INOUT:

aio_set_fd_handler(s->aio_context, fd, false,

- curl_multi_do, curl_multi_do, NULL, state);

+ curl_multi_do, curl_multi_do, NULL, socket);

break;

case CURL_POLL_REMOVE:

aio_set_fd_handler(s->aio_context, fd, false,

@@ -XXX,XX +XXX,XX @@ static void curl_multi_check_completion(BDRVCURLState *s)

/* Called with s->mutex held. */

-static void curl_multi_do_locked(CURLState *s)

+static void curl_multi_do_locked(CURLSocket *ready_socket)

CURLSocket *socket, *next_socket;

+ CURLState *s = ready_socket->state;

int running;

int r;

@@ -XXX,XX +XXX,XX @@ static void curl_multi_do_locked(CURLState *s)

static void curl_multi_do(void *arg)

- CURLState *s = (CURLState *)arg;

+ CURLSocket *socket = arg;

+ BDRVCURLState *s = socket->state->s;

- qemu_mutex_lock(&s->s->mutex);

- curl_multi_do_locked(s);

- curl_multi_check_completion(s->s);

- qemu_mutex_unlock(&s->s->mutex);

+ qemu_mutex_lock(&s->mutex);

+ curl_multi_do_locked(socket);

+ curl_multi_check_completion(s);

+ qemu_mutex_unlock(&s->mutex);

static void curl_multi_timeout_do(void *arg)

2.21.0

From: Maxim Levitsky <mlevitsk@redhat.com>

This fixes subtle corruption introduced by luks threaded encryption

in commit 8ac0f15f335

Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=1745922

The corruption happens when we do a write that

* writes to two or more unallocated clusters at once

* doesn't fully cover the first sector

* doesn't fully cover the last sector

* uses luks encryption

In this case, when allocating the new clusters we COW both areas

prior to the write and after the write, and we encrypt them.

The above mentioned commit accidentally made it so we encrypt the

second COW area using the physical cluster offset of the first area.

The problem is that offset_in_cluster in do_perform_cow_encrypt

can be larger that the cluster size, thus cluster_offset

will no longer point to the start of the cluster at which encrypted

area starts.

Next patch in this series will refactor the code to avoid all these

assumptions.

In the bugreport that was triggered by rebasing a luks image to new,

zero filled base, which lot of such writes, and causes some files

with zero areas to contain garbage there instead.

But as described above it can happen elsewhere as well

Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Message-id: 20190915203655.21638-2-mlevitsk@redhat.com

Reviewed-by: Max Reitz <mreitz@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

block/qcow2-cluster.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c

--- a/block/qcow2-cluster.c

+++ b/block/qcow2-cluster.c

@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn do_perform_cow_encrypt(BlockDriverState *bs,

assert(QEMU_IS_ALIGNED(offset_in_cluster, BDRV_SECTOR_SIZE));

assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

assert(s->crypto);

- if (qcow2_co_encrypt(bs, cluster_offset,

- src_cluster_offset + offset_in_cluster,

- buffer, bytes) < 0) {

+ if (qcow2_co_encrypt(bs,

+ start_of_cluster(s, cluster_offset + offset_in_cluster),

+ src_cluster_offset + offset_in_cluster,

+ buffer, bytes) < 0) {

return false;

}

2.21.0

Instead of reporting all sockets to cURL, only report the one that has

caused curl_multi_do_locked() to be called. This lets us get rid of the

QLIST_FOREACH_SAFE() list, which was actually wrong: SAFE foreaches are

only safe when the current element is removed in each iteration. If it

possible for the list to be concurrently modified, we cannot guarantee

that only the current element will be removed. Therefore, we must not

use QLIST_FOREACH_SAFE() here.

Fixes: ff5ca1664af85b24a4180d595ea6873fd3deac57

Cc: qemu-stable@nongnu.org

Signed-off-by: Max Reitz <mreitz@redhat.com>

Message-id: 20190910124136.10565-6-mreitz@redhat.com

Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>

Reviewed-by: John Snow <jsnow@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/curl.c | 17 ++++++-----------

1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/block/curl.c b/block/curl.c

--- a/block/curl.c

+++ b/block/curl.c

@@ -XXX,XX +XXX,XX @@ static void curl_multi_check_completion(BDRVCURLState *s)

}

/* Called with s->mutex held. */

-static void curl_multi_do_locked(CURLSocket *ready_socket)

+static void curl_multi_do_locked(CURLSocket *socket)

{

- CURLSocket *socket, *next_socket;

- CURLState *s = ready_socket->state;

+ BDRVCURLState *s = socket->state->s;

int running;

int r;

- if (!s->s->multi) {

+ if (!s->multi) {

- /* Need to use _SAFE because curl_multi_socket_action() may trigger

- * curl_sock_cb() which might modify this list */

- QLIST_FOREACH_SAFE(socket, &s->sockets, next, next_socket) {

- do {

- r = curl_multi_socket_action(s->s->multi, socket->fd, 0, &running);

- } while (r == CURLM_CALL_MULTI_PERFORM);

- }

+ do {

+ r = curl_multi_socket_action(s->multi, socket->fd, 0, &running);

+ } while (r == CURLM_CALL_MULTI_PERFORM);

static void curl_multi_do(void *arg)

2.21.0

From: Maxim Levitsky <mlevitsk@redhat.com>

* Change the qcow2_co_{encrypt|decrypt} to just receive full host and

guest offsets and use this function directly instead of calling

do_perform_cow_encrypt (which is removed by that patch).

* Adjust qcow2_co_encdec to take full host and guest offsets as well.

* Document the qcow2_co_{encrypt|decrypt} arguments

to prevent the bug fixed in former commit from hopefully

happening again.

Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>

Message-id: 20190915203655.21638-3-mlevitsk@redhat.com

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

[mreitz: Let perform_cow() return the error value returned by

qcow2_co_encrypt(), as proposed by Vladimir]

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/qcow2.h | 8 +++---

block/qcow2-cluster.c | 41 +++++++++-------------------

block/qcow2-threads.c | 63 +++++++++++++++++++++++++++++++++----------

block/qcow2.c | 5 ++--

4 files changed, 69 insertions(+), 48 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h

--- a/block/qcow2.h

+++ b/block/qcow2.h

@@ -XXX,XX +XXX,XX @@ ssize_t coroutine_fn

qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,

const void *src, size_t src_size);

int coroutine_fn

-qcow2_co_encrypt(BlockDriverState *bs, uint64_t file_cluster_offset,

- uint64_t offset, void *buf, size_t len);

+qcow2_co_encrypt(BlockDriverState *bs, uint64_t host_offset,

+ uint64_t guest_offset, void *buf, size_t len);

int coroutine_fn

-qcow2_co_decrypt(BlockDriverState *bs, uint64_t file_cluster_offset,

- uint64_t offset, void *buf, size_t len);

+qcow2_co_decrypt(BlockDriverState *bs, uint64_t host_offset,

+ uint64_t guest_offset, void *buf, size_t len);

#endif

diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c

--- a/block/qcow2-cluster.c

+++ b/block/qcow2-cluster.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn do_perform_cow_read(BlockDriverState *bs,

return 0;

}

-static bool coroutine_fn do_perform_cow_encrypt(BlockDriverState *bs,

- uint64_t src_cluster_offset,

- uint64_t cluster_offset,

- unsigned offset_in_cluster,

- uint8_t *buffer,

- unsigned bytes)

-{

- if (bytes && bs->encrypted) {

- BDRVQcow2State *s = bs->opaque;

- assert(QEMU_IS_ALIGNED(offset_in_cluster, BDRV_SECTOR_SIZE));

- assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

- assert(s->crypto);

- if (qcow2_co_encrypt(bs,

- start_of_cluster(s, cluster_offset + offset_in_cluster),

- src_cluster_offset + offset_in_cluster,

- buffer, bytes) < 0) {

- return false;

- }

- }

- return true;

-}

-

static int coroutine_fn do_perform_cow_write(BlockDriverState *bs,

uint64_t cluster_offset,

unsigned offset_in_cluster,

@@ -XXX,XX +XXX,XX @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m)

/* Encrypt the data if necessary before writing it */

if (bs->encrypted) {

- if (!do_perform_cow_encrypt(bs, m->offset, m->alloc_offset,

- start->offset, start_buffer,

- start->nb_bytes) ||

- !do_perform_cow_encrypt(bs, m->offset, m->alloc_offset,

- end->offset, end_buffer, end->nb_bytes)) {

- ret = -EIO;

+ ret = qcow2_co_encrypt(bs,

+ m->alloc_offset + start->offset,

+ m->offset + start->offset,

+ start_buffer, start->nb_bytes);

+ if (ret < 0) {

+ goto fail;

+ }

+ ret = qcow2_co_encrypt(bs,

+ m->alloc_offset + end->offset,

+ m->offset + end->offset,

+ end_buffer, end->nb_bytes);

+ if (ret < 0) {

goto fail;

diff --git a/block/qcow2-threads.c b/block/qcow2-threads.c

--- a/block/qcow2-threads.c

+++ b/block/qcow2-threads.c

@@ -XXX,XX +XXX,XX @@ static int qcow2_encdec_pool_func(void *opaque)

}

static int coroutine_fn

-qcow2_co_encdec(BlockDriverState *bs, uint64_t file_cluster_offset,

- uint64_t offset, void *buf, size_t len, Qcow2EncDecFunc func)

+qcow2_co_encdec(BlockDriverState *bs, uint64_t host_offset,

+ uint64_t guest_offset, void *buf, size_t len,

+ Qcow2EncDecFunc func)

{

BDRVQcow2State *s = bs->opaque;

Qcow2EncDecData arg = {

.block = s->crypto,

- .offset = s->crypt_physical_offset ?

- file_cluster_offset + offset_into_cluster(s, offset) :

- offset,

+ .offset = s->crypt_physical_offset ? host_offset : guest_offset,

.buf = buf,

.len = len,

.func = func,

};

- return qcow2_co_process(bs, qcow2_encdec_pool_func, &arg);

+ assert(QEMU_IS_ALIGNED(guest_offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(host_offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(len, BDRV_SECTOR_SIZE));

+ assert(s->crypto);

+

+ return len == 0 ? 0 : qcow2_co_process(bs, qcow2_encdec_pool_func, &arg);

}

+/*

+ * qcow2_co_encrypt()

+ *

+ * Encrypts one or more contiguous aligned sectors

+ *

+ * @host_offset - underlying storage offset of the first sector of the

+ * data to be encrypted

+ *

+ * @guest_offset - guest (virtual) offset of the first sector of the

+ * data to be encrypted

+ *

+ * @buf - buffer with the data to encrypt, that after encryption

+ * will be written to the underlying storage device at

+ * @host_offset

+ *

+ * @len - length of the buffer (must be a BDRV_SECTOR_SIZE multiple)

+ *

+ * Depending on the encryption method, @host_offset and/or @guest_offset

+ * may be used for generating the initialization vector for

+ * encryption.

+ *

+ * Note that while the whole range must be aligned on sectors, it

+ * does not have to be aligned on clusters and can also cross cluster

+ * boundaries

+ */

int coroutine_fn

-qcow2_co_encrypt(BlockDriverState *bs, uint64_t file_cluster_offset,

- uint64_t offset, void *buf, size_t len)

+qcow2_co_encrypt(BlockDriverState *bs, uint64_t host_offset,

+ uint64_t guest_offset, void *buf, size_t len)

{

- return qcow2_co_encdec(bs, file_cluster_offset, offset, buf, len,

- qcrypto_block_encrypt);

+ return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,

+ qcrypto_block_encrypt);

}

+/*

+ * qcow2_co_decrypt()

+ *

+ * Decrypts one or more contiguous aligned sectors

+ * Similar to qcow2_co_encrypt

+ */

int coroutine_fn

-qcow2_co_decrypt(BlockDriverState *bs, uint64_t file_cluster_offset,

- uint64_t offset, void *buf, size_t len)

+qcow2_co_decrypt(BlockDriverState *bs, uint64_t host_offset,

+ uint64_t guest_offset, void *buf, size_t len)

{

- return qcow2_co_encdec(bs, file_cluster_offset, offset, buf, len,

- qcrypto_block_decrypt);

+ return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,

+ qcrypto_block_decrypt);

}

diff --git a/block/qcow2.c b/block/qcow2.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.c

+++ b/block/qcow2.c

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_part(BlockDriverState *bs,

assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));

assert(QEMU_IS_ALIGNED(cur_bytes, BDRV_SECTOR_SIZE));

- if (qcow2_co_decrypt(bs, cluster_offset, offset,

+ if (qcow2_co_decrypt(bs, cluster_offset + offset_in_cluster,

+ offset,

cluster_data, cur_bytes) < 0) {

ret = -EIO;

goto fail;

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_pwritev_part(

qemu_iovec_to_buf(qiov, qiov_offset + bytes_done,

cluster_data, cur_bytes);

- if (qcow2_co_encrypt(bs, cluster_offset, offset,

+ if (qcow2_co_encrypt(bs, cluster_offset + offset_in_cluster, offset,

cluster_data, cur_bytes) < 0) {

ret = -EIO;

goto out_unlocked;

2.21.0

From: Nir Soffer <nirsof@gmail.com>

Replace instances of:

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,

sector_num = offset >> BDRV_SECTOR_BITS;

nb_sectors = bytes >> BDRV_SECTOR_BITS;

- assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);

- assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

+ assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

assert(bytes <= BDRV_REQUEST_MAX_BYTES);

assert(drv->bdrv_co_readv);

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,

sector_num = offset >> BDRV_SECTOR_BITS;

nb_sectors = bytes >> BDRV_SECTOR_BITS;

- assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);

- assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

+ assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

assert(bytes <= BDRV_REQUEST_MAX_BYTES);

assert(drv->bdrv_co_writev);

diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2-cluster.c

+++ b/block/qcow2-cluster.c

@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn do_perform_cow_encrypt(BlockDriverState *bs,

if (bytes && bs->encrypted) {

BDRVQcow2State *s = bs->opaque;

- assert((offset_in_cluster & ~BDRV_SECTOR_MASK) == 0);

- assert((bytes & ~BDRV_SECTOR_MASK) == 0);

+ assert(QEMU_IS_ALIGNED(offset_in_cluster, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

assert(s->crypto);

if (qcow2_co_encrypt(bs, cluster_offset,

src_cluster_offset + offset_in_cluster,

diff --git a/block/qcow2.c b/block/qcow2.c

index XXXXXXX..XXXXXXX 100644

--- a/block/qcow2.c

+++ b/block/qcow2.c

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv_part(BlockDriverState *bs,

goto fail;

}

- assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);

- assert((cur_bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

+ assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(cur_bytes, BDRV_SECTOR_SIZE));

if (qcow2_co_decrypt(bs, cluster_offset, offset,

cluster_data, cur_bytes) < 0) {

ret = -EIO;

diff --git a/block/vvfat.c b/block/vvfat.c

index XXXXXXX..XXXXXXX 100644

--- a/block/vvfat.c

+++ b/block/vvfat.c

@@ -XXX,XX +XXX,XX @@ vvfat_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,

int nb_sectors = bytes >> BDRV_SECTOR_BITS;

void *buf;

- assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);

- assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

+ assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

buf = g_try_malloc(bytes);

if (bytes && buf == NULL) {

@@ -XXX,XX +XXX,XX @@ vvfat_co_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes,

int nb_sectors = bytes >> BDRV_SECTOR_BITS;

void *buf;

- assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);

- assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

+ assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));

+ assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));

buf = g_try_malloc(bytes);

if (bytes && buf == NULL) {

diff --git a/qemu-img.c b/qemu-img.c

index XXXXXXX..XXXXXXX 100644

--- a/qemu-img.c

+++ b/qemu-img.c

@@ -XXX,XX +XXX,XX @@ static int img_convert(int argc, char **argv)

int64_t sval;

sval = cvtnum(optarg);

- if (sval < 0 || sval & (BDRV_SECTOR_SIZE - 1) ||

+ if (sval < 0 || !QEMU_IS_ALIGNED(sval, BDRV_SECTOR_SIZE) ||

sval / BDRV_SECTOR_SIZE > MAX_BUF_SECTORS) {

error_report("Invalid buffer size for sparse output specified. "

"Valid sizes are multiples of %llu up to %llu. Select "

2.21.0