The following changes since commit e5cd695266c5709308aa95b1baae499e4b5d4544:

Merge remote-tracking branch 'remotes/cody/tags/block-pull-request' into staging (2018-05-08 17:05:58 +0100)

git://github.com/stefanha/qemu.git tags/block-pull-request

for you to fetch changes up to 31be8a2a97ecba7d31a82932286489cac318e9e9:

block/file-posix: add x-check-page-cache=on|off option (2018-05-11 16:43:05 +0100)

Block pull request

* Support -drive cache.direct=off live migration for POSIX files

Joe Perches (4):

checkpatch: add a --strict check for utf-8 in commit logs

checkpatch: ignore email headers better

checkpatch: emit a warning on file add/move/delete

checkpatch: reduce MAINTAINERS update message frequency

Pasi Savanainen (1):

checkpatch: check utf-8 content from a commit log when it's missing

from charset

Stefan Hajnoczi (3):

blockjob: drop block_job_pause/resume_all()

block/file-posix: implement bdrv_co_invalidate_cache() on Linux

block/file-posix: add x-check-page-cache=on|off option

qapi/block-core.json | 7 +-

include/block/blockjob_int.h | 14 ----

block/file-posix.c | 146 ++++++++++++++++++++++++++++++++++-

blockjob.c | 27 -------

scripts/checkpatch.pl | 56 +++++++++++++-

5 files changed, 202 insertions(+), 48 deletions(-)

2.17.0

mincore(2) checks whether pages are resident. Use it to verify that

page cache has been dropped.

You can trigger a verification failure by mmapping the image file from

another process that loads a byte from a page, forcing it to become

resident. bdrv_co_invalidate_cache() will fail while that process is

alive.

qapi/block-core.json | 7 ++-

block/file-posix.c | 100 ++++++++++++++++++++++++++++++++++++++++++-

2 files changed, 104 insertions(+), 3 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json

--- a/qapi/block-core.json

+++ b/qapi/block-core.json

@@ -XXX,XX +XXX,XX @@

# @locking: whether to enable file locking. If set to 'auto', only enable

# when Open File Descriptor (OFD) locking API is available

# (default: auto, since 2.10)

+# @x-check-cache-dropped: whether to check that page cache was dropped on live

+# migration. May cause noticeable delays if the image

+# file is large, do not use in production.

+# (default: off) (since: 2.13)

#

# Since: 2.9

##

@@ -XXX,XX +XXX,XX @@

'data': { 'filename': 'str',

'*pr-manager': 'str',

'*locking': 'OnOffAuto',

- '*aio': 'BlockdevAioOptions' } }

+ '*aio': 'BlockdevAioOptions',

+ '*x-check-cache-dropped': 'bool' } }

##

# @BlockdevOptionsNull:

diff --git a/block/file-posix.c b/block/file-posix.c

index XXXXXXX..XXXXXXX 100644

--- a/block/file-posix.c

+++ b/block/file-posix.c

@@ -XXX,XX +XXX,XX @@ typedef struct BDRVRawState {

bool page_cache_inconsistent:1;

bool has_fallocate;

bool needs_alignment;

+ bool check_cache_dropped;

PRManager *pr_mgr;

} BDRVRawState;

@@ -XXX,XX +XXX,XX @@ typedef struct BDRVRawState {

typedef struct BDRVRawReopenState {

int fd;

int open_flags;

+ bool check_cache_dropped;

} BDRVRawReopenState;

static int fd_open(BlockDriverState *bs);

@@ -XXX,XX +XXX,XX @@ static QemuOptsList raw_runtime_opts = {

.type = QEMU_OPT_STRING,

.help = "id of persistent reservation manager object (default: none)",

},

+ {

+ .name = "x-check-cache-dropped",

+ .type = QEMU_OPT_BOOL,

+ .help = "check that page cache was dropped on live migration (default: off)"

+ },

{ /* end of list */ }

},

};

@@ -XXX,XX +XXX,XX @@ static int raw_open_common(BlockDriverState *bs, QDict *options,

}

}

+ s->check_cache_dropped = qemu_opt_get_bool(opts, "x-check-cache-dropped",

+ false);

s->open_flags = open_flags;

raw_parse_flags(bdrv_flags, &s->open_flags);

@@ -XXX,XX +XXX,XX @@ static int raw_reopen_prepare(BDRVReopenState *state,

BDRVRawState *s;

BDRVRawReopenState *rs;

+ QemuOpts *opts;

int ret = 0;

Error *local_err = NULL;

@@ -XXX,XX +XXX,XX @@ static int raw_reopen_prepare(BDRVReopenState *state,

state->opaque = g_new0(BDRVRawReopenState, 1);

rs = state->opaque;

+ rs->fd = -1;

+ /* Handle options changes */

+ opts = qemu_opts_create(&raw_runtime_opts, NULL, 0, &error_abort);

+ qemu_opts_absorb_qdict(opts, state->options, &local_err);

+ if (local_err) {

+ error_propagate(errp, local_err);

+ ret = -EINVAL;

+ goto out;

+ rs->check_cache_dropped = qemu_opt_get_bool(opts, "x-check-cache-dropped",

+ s->check_cache_dropped);

if (s->type == FTYPE_CD) {

rs->open_flags |= O_NONBLOCK;

@@ -XXX,XX +XXX,XX @@ static int raw_reopen_prepare(BDRVReopenState *state,

raw_parse_flags(state->flags, &rs->open_flags);

- rs->fd = -1;

-

int fcntl_flags = O_APPEND | O_NONBLOCK;

#ifdef O_NOATIME

fcntl_flags |= O_NOATIME;

@@ -XXX,XX +XXX,XX @@ static int raw_reopen_prepare(BDRVReopenState *state,

}

+out:

+ qemu_opts_del(opts);

return ret;

}

@@ -XXX,XX +XXX,XX @@ static void raw_reopen_commit(BDRVReopenState *state)

BDRVRawReopenState *rs = state->opaque;

BDRVRawState *s = state->bs->opaque;

+ s->check_cache_dropped = rs->check_cache_dropped;

s->open_flags = rs->open_flags;

qemu_close(s->fd);

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn raw_co_block_status(BlockDriverState *bs,

return ret | BDRV_BLOCK_OFFSET_VALID;

}

+#if defined(__linux__)

+/* Verify that the file is not in the page cache */

+static void check_cache_dropped(BlockDriverState *bs, Error **errp)

+{

+ const size_t window_size = 128 * 1024 * 1024;

+ BDRVRawState *s = bs->opaque;

+ void *window = NULL;

+ size_t length = 0;

+ unsigned char *vec;

+ size_t page_size;

+ off_t offset;

+ off_t end;

+ /* mincore(2) page status information requires 1 byte per page */

+ page_size = sysconf(_SC_PAGESIZE);

+ vec = g_malloc(DIV_ROUND_UP(window_size, page_size));

+

+ end = raw_getlength(bs);

+

+ for (offset = 0; offset < end; offset += window_size) {

+ void *new_window;

+ size_t new_length;

+ size_t vec_end;

+ size_t i;

+ int ret;

+

+ /* Unmap previous window if size has changed */

+ new_length = MIN(end - offset, window_size);

+ if (new_length != length) {

+ munmap(window, length);

+ window = NULL;

+ length = 0;

+

+ new_window = mmap(window, new_length, PROT_NONE, MAP_PRIVATE,

+ s->fd, offset);

+ if (new_window == MAP_FAILED) {

+ error_setg_errno(errp, errno, "mmap failed");

+ break;

+ }

+

+ window = new_window;

+ length = new_length;

+

+ ret = mincore(window, length, vec);

+ if (ret < 0) {

+ error_setg_errno(errp, errno, "mincore failed");

+ break;

+ }

+

+ vec_end = DIV_ROUND_UP(length, page_size);

+ for (i = 0; i < vec_end; i++) {

+ if (vec[i] & 0x1) {

+ error_setg(errp, "page cache still in use!");

+ break;

+ }

+ }

+ }

+

+ if (window) {

+ munmap(window, length);

+ }

+

+ g_free(vec);

+}

+#endif /* __linux__ */

+

static void coroutine_fn raw_co_invalidate_cache(BlockDriverState *bs,

Error **errp)

{

@@ -XXX,XX +XXX,XX @@ static void coroutine_fn raw_co_invalidate_cache(BlockDriverState *bs,

error_setg_errno(errp, ret, "fadvise failed");

return;

+

+ if (s->check_cache_dropped) {

+ check_cache_dropped(bs, errp);

+ }

#else /* __linux__ */

/* Do nothing. Live migration to a remote host with cache.direct=off is

* unsupported on other host operating systems. Cache consistency issues

2.17.0

On Linux posix_fadvise(POSIX_FADV_DONTNEED) invalidates pages*. Use

this to drop page cache on the destination host during shared storage

migration. This way the destination host will read the latest copy of

the data and will not use stale data from the page cache.

The flow is as follows:

1. Source host writes out all dirty pages and inactivates drives.

2. QEMU_VM_EOF is sent on migration stream.

3. Destination host invalidates caches before accessing drives.

This patch enables live migration even with -drive cache.direct=off.

* Terms and conditions may apply, please see patch for details.

Reviewed-by: Fam Zheng <famz@redhat.com>

Message-id: 20180427162312.18583-2-stefanha@redhat.com

block/file-posix.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++

1 file changed, 46 insertions(+)

diff --git a/block/file-posix.c b/block/file-posix.c

--- a/block/file-posix.c

+++ b/block/file-posix.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn raw_co_block_status(BlockDriverState *bs,

return ret | BDRV_BLOCK_OFFSET_VALID;

}

+static void coroutine_fn raw_co_invalidate_cache(BlockDriverState *bs,

+ Error **errp)

+{

+ BDRVRawState *s = bs->opaque;

+ int ret;

+

+ ret = fd_open(bs);

+ if (ret < 0) {

+ error_setg_errno(errp, -ret, "The file descriptor is not open");

+ return;

+ }

+

+ if (s->open_flags & O_DIRECT) {

+ return; /* No host kernel page cache */

+ }

+

+#if defined(__linux__)

+ /* This sets the scene for the next syscall... */

+ ret = bdrv_co_flush(bs);

+ if (ret < 0) {

+ error_setg_errno(errp, -ret, "flush failed");

+ return;

+ }

+

+ /* Linux does not invalidate pages that are dirty, locked, or mmapped by a

+ * process. These limitations are okay because we just fsynced the file,

+ * we don't use mmap, and the file should not be in use by other processes.

+ */

+ ret = posix_fadvise(s->fd, 0, 0, POSIX_FADV_DONTNEED);

+ if (ret != 0) { /* the return value is a positive errno */

+ error_setg_errno(errp, ret, "fadvise failed");

+ return;

+ }

+#else /* __linux__ */

+ /* Do nothing. Live migration to a remote host with cache.direct=off is

+ * unsupported on other host operating systems. Cache consistency issues

+ * may occur but no error is reported here, partly because that's the

+ * historical behavior and partly because it's hard to differentiate valid

+ * configurations that should not cause errors.

+ */

+#endif /* !__linux__ */

+}

+

static coroutine_fn BlockAIOCB *raw_aio_pdiscard(BlockDriverState *bs,

int64_t offset, int bytes,

BlockCompletionFunc *cb, void *opaque)

@@ -XXX,XX +XXX,XX @@ BlockDriver bdrv_file = {

.bdrv_co_create_opts = raw_co_create_opts,

.bdrv_has_zero_init = bdrv_has_zero_init_1,

.bdrv_co_block_status = raw_co_block_status,

+ .bdrv_co_invalidate_cache = raw_co_invalidate_cache,

.bdrv_co_pwrite_zeroes = raw_co_pwrite_zeroes,

.bdrv_co_preadv = raw_co_preadv,

@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_host_device = {

.bdrv_reopen_abort = raw_reopen_abort,

.bdrv_co_create_opts = hdev_co_create_opts,

.create_opts = &raw_create_opts,

+ .bdrv_co_invalidate_cache = raw_co_invalidate_cache,

.bdrv_co_pwrite_zeroes = hdev_co_pwrite_zeroes,

.bdrv_co_preadv = raw_co_preadv,

@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_host_cdrom = {

.bdrv_reopen_abort = raw_reopen_abort,

.bdrv_co_create_opts = hdev_co_create_opts,

.create_opts = &raw_create_opts,

+ .bdrv_co_invalidate_cache = raw_co_invalidate_cache,

.bdrv_co_preadv = raw_co_preadv,

2.17.0

From: Joe Perches <joe@perches.com>

Some find using utf-8 in commit logs inappropriate.

Some patch commit logs contain unintended utf-8 characters when doing

things like copy/pasting compilation output.

Look for the start of any commit log by skipping initial lines that look

like email headers and "From: " lines.

Stop looking for utf-8 at the first signature line.

Signed-off-by: Joe Perches <joe@perches.com>

Reviewed-by: Markus Armbruster <armbru@redhat.com>

Reviewed-by: Thomas Huth <thuth@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Message-id: 20180430124651.10340-2-stefanha@redhat.com

Suggested-by: Andrew Morton <akpm@linux-foundation.org>

Cc: Andy Whitcroft <apw@shadowen.org>

Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

(cherry picked from commit 15662b3e8644905032c2e26808401a487d4e90c1)

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Conflicts:

QEMU does not have CHK(), use WARN() instead.

QEMU WARN() only takes one argument, drop the 'type' value in the

first argument.

scripts/checkpatch.pl | 30 ++++++++++++++++++++++++++----

1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl

index XXXXXXX..XXXXXXX 100755

--- a/scripts/checkpatch.pl

+++ b/scripts/checkpatch.pl

@@ -XXX,XX +XXX,XX @@ our $NonptrType;

our $Type;

our $Declare;

-our $UTF8    = qr {

-    [\x09\x0A\x0D\x20-\x7E] # ASCII

-    | [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte

+our $NON_ASCII_UTF8    = qr{

+    [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte

    | \xE0[\xA0-\xBF][\x80-\xBF] # excluding overlongs

    | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2} # straight 3-byte

    | \xED[\x80-\x9F][\x80-\xBF] # excluding surrogates

@@ -XXX,XX +XXX,XX @@ our $UTF8    = qr {

    | \xF4[\x80-\x8F][\x80-\xBF]{2} # plane 16

}x;

+our $UTF8    = qr{

+    [\x09\x0A\x0D\x20-\x7E] # ASCII

+    | $NON_ASCII_UTF8

+}x;

# There are still some false positives, but this catches most

# common cases.

our $typeTypedefs = qr{(?x:

@@ -XXX,XX +XXX,XX @@ sub process {

    my $signoff = 0;

    my $is_patch = 0;

+    my $in_header_lines = 1;

+    my $in_commit_log = 0;        #Scanning lines before patch

+

    our @report = ();

    our $cnt_lines = 0;

    our $cnt_error = 0;

@@ -XXX,XX +XXX,XX @@ sub process {

        if ($line =~ /^diff --git.*?(\S+)$/) {

            $realfile = $1;

            $realfile =~ s@^([^/]*)/@@;

-

        } elsif ($line =~ /^\+\+\+\s+(\S+)/) {

            $realfile = $1;

            $realfile =~ s@^([^/]*)/@@;

@@ -XXX,XX +XXX,XX @@ sub process {

        if ($line =~ /^\s*signed-off-by:/i) {

            # This is a signoff, if ugly, so do not double report.

            $signoff++;

+            $in_commit_log = 0;

+

            if (!($line =~ /^\s*Signed-off-by:/)) {

                ERROR("The correct form is \"Signed-off-by\"\n" .

                    $herecurr);

@@ -XXX,XX +XXX,XX @@ sub process {

            ERROR("Invalid UTF-8, patch and commit message should be encoded in UTF-8\n" . $hereptr);

        }

+# Check if it's the start of a commit log

+# (not a header line and we haven't seen the patch filename)

+        if ($in_header_lines && $realfile =~ /^$/ &&

+         $rawline !~ /^(commit\b|from\b|\w+:).+$/i) {

+            $in_header_lines = 0;

+            $in_commit_log = 1;

+        }

+

+# Still not yet in a patch, check for any UTF-8

+        if ($in_commit_log && $realfile =~ /^$/ &&

+         $rawline =~ /$NON_ASCII_UTF8/) {

+            WARN("8-bit UTF-8 used in possible commit log\n" . $herecurr);

+        }

+

# ignore non-hunk lines and lines being removed

        next if (!$hunk_line || $line =~ /^-/);

2.17.0