Series comparison

-[Qemu-devel] [PULL 00/15] Block patches
+[PULL 00/24] Block patches
-The following changes since commit 23919ddfd56135cad3cb468a8f54d5a595f024f4:
+The following changes since commit 5375af3cd7b8adcc10c18d8083b7be63976c9645:
-  Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190827' into staging (2019-08-27 15:52:36 +0100)
+  Merge remote-tracking branch 'remotes/mst/tags/for_upstream' into staging (2020-05-04 15:51:09 +0100)
 are available in the Git repository at:
-  https://github.com/XanClic/qemu.git tags/pull-block-2019-08-27
+  https://github.com/XanClic/qemu.git tags/pull-block-2020-05-05
-for you to fetch changes up to bb043c056cffcc2f3ce88bfdaf2e76e455c09e2c:
+for you to fetch changes up to 4ce5dd3e9b5ee0fac18625860eb3727399ee965e:
-  iotests: Unify cache mode quoting (2019-08-27 19:48:44 +0200)
+  block/block-copy: use aio-task-pool API (2020-05-05 14:03:28 +0200)
 ----------------------------------------------------------------
 Block patches:
-- qemu-io now accepts a file to read a write pattern from
+- Asynchronous copying for block-copy (i.e., the backup job)
-- Ensure that raw files have their first block allocated so we can probe
+- Allow resizing of qcow2 images when they have internal snapshots
-  the O_DIRECT alignment if necessary
+- iotests: Logging improvements for Python tests
-- Various fixes
+- iotest 153 fix, and block comment cleanups
 ----------------------------------------------------------------
-Denis Plotnikov (1):
+Eric Blake (4):
-  qemu-io: add pattern file for write command
+  block: Add blk_new_with_bs() helper
   qcow2: Allow resize of images with internal snapshots
   qcow2: Tweak comment about bitmaps vs. resize
   block: Comment cleanups
-Max Reitz (7):
+John Snow (14):
-  iotests: Fix _filter_img_create()
+  iotests: do a light delinting
-  vmdk: Use bdrv_dirname() for relative extent paths
+  iotests: don't use 'format' for drive_add
-  iotests: Keep testing broken relative extent paths
+  iotests: ignore import warnings from pylint
-  vmdk: Reject invalid compressed writes
+  iotests: replace mutable list default args
-  iotests: Disable broken streamOptimized tests
+  iotests: add pylintrc file
-  iotests: Disable 110 for vmdk.twoGbMaxExtentSparse
+  iotests: alphabetize standard imports
-  iotests: Disable 126 for flat vmdk subformats
+  iotests: drop pre-Python 3.4 compatibility code
   iotests: touch up log function signature
   iotests: limit line length to 79 chars
   iotests: add hmp helper with logging
   iotests: add script_initialize
   iotest 258: use script_main
   iotests: Mark verify functions as private
   iotests: use python logging for iotests.log()
-Nir Soffer (3):
+Maxim Levitsky (1):
-  block: posix: Always allocate the first block
+  Fix iotest 153
   iotests: Test allocate_first_block() with O_DIRECT
   iotests: Unify cache mode quoting
-Stefan Hajnoczi (1):
+Vladimir Sementsov-Ogievskiy (5):
-  file-posix: fix request_alignment typo
+  block/block-copy: rename in-flight requests to tasks
   block/block-copy: alloc task on each iteration
   block/block-copy: add state pointer to BlockCopyTask
   block/block-copy: refactor task creation
   block/block-copy: use aio-task-pool API
-Thomas Huth (2):
+ include/sysemu/block-backend.h |   2 +
-  iotests: Check for enabled drivers before testing them
+ block/block-backend.c          |  23 +++
-  tests/check-block: Skip iotests when sanitizers are enabled
+ block/block-copy.c             | 279 +++++++++++++++++--------
+ block/crypto.c                 |   9 +-
-Vladimir Sementsov-Ogievskiy (1):
+ block/io.c                     |   3 +-
-  block: fix permission update in bdrv_replace_node
+ block/parallels.c              |   8 +-
+ block/qcow.c                   |   8 +-
- block.c                                       |  5 +-
+ block/qcow2-refcount.c         |   2 +-
- block/file-posix.c                            | 53 +++++++++-
+ block/qcow2-snapshot.c         |  20 +-
- block/vmdk.c                                  | 64 ++++++++----
+ block/qcow2.c                  |  45 ++--
- qemu-io-cmds.c                                | 99 +++++++++++++++++--
+ block/qed.c                    |   8 +-
- tests/check-block.sh                          |  5 +
+ block/sheepdog.c               |  10 +-
- tests/qemu-iotests/002                        |  1 +
+ block/vdi.c                    |   8 +-
- tests/qemu-iotests/003                        |  1 +
+ block/vhdx.c                   |   8 +-
- tests/qemu-iotests/005                        |  3 +-
+ block/vmdk.c                   |   9 +-
- tests/qemu-iotests/009                        |  1 +
+ block/vpc.c                    |   8 +-
- tests/qemu-iotests/010                        |  1 +
+ block/vvfat.c                  |  10 +-
- tests/qemu-iotests/011                        |  1 +
+ blockdev.c                     |   8 +-
- tests/qemu-iotests/017                        |  3 +-
+ blockjob.c                     |   7 +-
- tests/qemu-iotests/018                        |  3 +-
+ tests/qemu-iotests/001         |   2 +-
- tests/qemu-iotests/019                        |  3 +-
+ tests/qemu-iotests/030         |   4 +-
- tests/qemu-iotests/020                        |  3 +-
+ tests/qemu-iotests/052         |   2 +-
- tests/qemu-iotests/026                        |  4 +-
+ tests/qemu-iotests/055         |   3 +-
- tests/qemu-iotests/027                        |  1 +
+ tests/qemu-iotests/061         |  35 ++++
- tests/qemu-iotests/032                        |  1 +
+ tests/qemu-iotests/061.out     |  28 +++
- tests/qemu-iotests/033                        |  1 +
+ tests/qemu-iotests/134         |   2 +-
- tests/qemu-iotests/034                        |  3 +-
+ tests/qemu-iotests/149         |   3 +-
- tests/qemu-iotests/037                        |  3 +-
+ tests/qemu-iotests/153         |   2 +-
- tests/qemu-iotests/039                        |  4 +-
+ tests/qemu-iotests/153.out     |  12 +-
- tests/qemu-iotests/052                        |  2 +-
+ tests/qemu-iotests/155         |   2 +-
- tests/qemu-iotests/059                        | 34 ++++++-
+ tests/qemu-iotests/188         |   2 +-
- tests/qemu-iotests/059.out                    | 26 +++--
+ tests/qemu-iotests/194         |   4 +-
- tests/qemu-iotests/063                        |  3 +-
+ tests/qemu-iotests/202         |   4 +-
- tests/qemu-iotests/071                        |  1 +
+ tests/qemu-iotests/203         |   4 +-
- tests/qemu-iotests/072                        |  1 +
+ tests/qemu-iotests/206         |   2 +-
- tests/qemu-iotests/081                        |  4 +-
+ tests/qemu-iotests/207         |   6 +-
- tests/qemu-iotests/091                        |  4 +-
+ tests/qemu-iotests/208         |   2 +-
- tests/qemu-iotests/099                        |  1 +
+ tests/qemu-iotests/209         |   2 +-
- tests/qemu-iotests/105                        |  3 +-
+ tests/qemu-iotests/210         |   6 +-
- tests/qemu-iotests/110                        |  3 +-
+ tests/qemu-iotests/211         |   6 +-
- tests/qemu-iotests/120                        |  1 +
+ tests/qemu-iotests/212         |   6 +-
- tests/qemu-iotests/126                        |  2 +
+ tests/qemu-iotests/213         |   6 +-
- tests/qemu-iotests/{150.out => 150.out.qcow2} |  0
+ tests/qemu-iotests/216         |   4 +-
- tests/qemu-iotests/150.out.raw                | 12 +++
+ tests/qemu-iotests/218         |   2 +-
- tests/qemu-iotests/162                        |  4 +-
+ tests/qemu-iotests/219         |   2 +-
- tests/qemu-iotests/175                        | 47 +++++++--
+ tests/qemu-iotests/222         |   7 +-
- tests/qemu-iotests/175.out                    | 16 ++-
+ tests/qemu-iotests/224         |   4 +-
- tests/qemu-iotests/178.out.qcow2              |  4 +-
+ tests/qemu-iotests/228         |   6 +-
- tests/qemu-iotests/184                        |  1 +
+ tests/qemu-iotests/234         |   4 +-
- tests/qemu-iotests/186                        |  1 +
+ tests/qemu-iotests/235         |   4 +-
- tests/qemu-iotests/197                        |  1 +
+ tests/qemu-iotests/236         |   2 +-
- tests/qemu-iotests/215                        |  1 +
+ tests/qemu-iotests/237         |   2 +-
- tests/qemu-iotests/221.out                    | 12 ++-
+ tests/qemu-iotests/238         |   2 +
- tests/qemu-iotests/251                        |  1 +
+ tests/qemu-iotests/242         |   2 +-
- tests/qemu-iotests/253.out                    | 12 ++-
+ tests/qemu-iotests/245         |   1 +
- tests/qemu-iotests/common.filter              |  4 +-
+ tests/qemu-iotests/245.out     |  10 +-
- tests/qemu-iotests/common.rc                  | 14 +++
+ tests/qemu-iotests/246         |   2 +-
-files changed, 391 insertions(+), 87 deletions(-)
+ tests/qemu-iotests/248         |   2 +-
- rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%)
+ tests/qemu-iotests/254         |   2 +-
- create mode 100644 tests/qemu-iotests/150.out.raw
+ tests/qemu-iotests/255         |   2 +-
  tests/qemu-iotests/256         |   2 +-
  tests/qemu-iotests/258         |  10 +-
  tests/qemu-iotests/260         |   4 +-
  tests/qemu-iotests/262         |   4 +-
  tests/qemu-iotests/264         |   4 +-
  tests/qemu-iotests/274         |   4 +-
  tests/qemu-iotests/277         |   2 +
  tests/qemu-iotests/280         |   8 +-
  tests/qemu-iotests/283         |   4 +-
  tests/qemu-iotests/iotests.py  | 366 ++++++++++++++++++++-------------
  tests/qemu-iotests/pylintrc    |  26 +++
 files changed, 728 insertions(+), 386 deletions(-)
  create mode 100644 tests/qemu-iotests/pylintrc
 --
-.21.0
+.26.2

-New patch
+[PULL 01/24] iotests: do a light delinting
+From: John Snow <jsnow@redhat.com>
 This doesn't fix everything in here, but it does help clean up the
 pylint report considerably.
 This should be 100% style changes only; the intent is to make pylint
 more useful by working on establishing a baseline for iotests that we
 can gate against in the future.
 Signed-off-by: John Snow <jsnow@redhat.com>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200331000014.11581-2-jsnow@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  tests/qemu-iotests/iotests.py | 83 ++++++++++++++++++-----------------
 file changed, 43 insertions(+), 40 deletions(-)
 diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/iotests.py
 +++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@
  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  #
 -import errno
  import os
  import re
  import subprocess
 -import string
  import unittest
  import sys
  import struct
@@ -XXX,XX +XXX,XX @@ import faulthandler
  sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
  from qemu import qtest
 -assert sys.version_info >= (3,6)
 +assert sys.version_info >= (3, 6)
  faulthandler.enable()
@@ -XXX,XX +XXX,XX @@ def qemu_img_log(*args):
      return result
  def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]):
 -    args = [ 'info' ]
 +    args = ['info']
      if imgopts:
          args.append('--image-opts')
      else:
 -        args += [ '-f', imgfmt ]
 +        args += ['-f', imgfmt]
      args += extra_args
      args.append(filename)
@@ -XXX,XX +XXX,XX @@ class QemuIoInteractive:
          # quit command is in close(), '\n' is added automatically
          assert '\n' not in cmd
          cmd = cmd.strip()
 -        assert cmd != 'q' and cmd != 'quit'
 +        assert cmd not in ('q', 'quit')
          self._p.stdin.write(cmd + '\n')
          self._p.stdin.flush()
          return self._read_output()
@@ -XXX,XX +XXX,XX @@ def qemu_nbd_early_pipe(*args):
          sys.stderr.write('qemu-nbd received signal %i: %s\n' %
                           (-exitcode,
                            ' '.join(qemu_nbd_args + ['--fork'] + list(args))))
 -    if exitcode == 0:
 -        return exitcode, ''
 -    else:
 -        return exitcode, subp.communicate()[0]
 +
 +    return exitcode, subp.communicate()[0] if exitcode else ''
  def qemu_nbd_popen(*args):
      '''Run qemu-nbd in daemon mode and return the parent's exit code'''
@@ -XXX,XX +XXX,XX @@ def filter_qmp(qmsg, filter_fn):
          items = qmsg.items()
      for k, v in items:
 -        if isinstance(v, list) or isinstance(v, dict):
 +        if isinstance(v, (dict, list)):
              qmsg[k] = filter_qmp(v, filter_fn)
          else:
              qmsg[k] = filter_fn(k, v)
@@ -XXX,XX +XXX,XX @@ def filter_testfiles(msg):
      return msg.replace(prefix, 'TEST_DIR/PID-')
  def filter_qmp_testfiles(qmsg):
 -    def _filter(key, value):
 +    def _filter(_key, value):
          if is_str(value):
              return filter_testfiles(value)
          return value
@@ -XXX,XX +XXX,XX @@ def filter_imgfmt(msg):
      return msg.replace(imgfmt, 'IMGFMT')
  def filter_qmp_imgfmt(qmsg):
 -    def _filter(key, value):
 +    def _filter(_key, value):
          if is_str(value):
              return filter_imgfmt(value)
          return value
@@ -XXX,XX +XXX,XX @@ def log(msg, filters=[], indent=None):
      If indent is provided, JSON serializable messages are pretty-printed.'''
      for flt in filters:
          msg = flt(msg)
 -    if isinstance(msg, dict) or isinstance(msg, list):
 +    if isinstance(msg, (dict, list)):
          # Python < 3.4 needs to know not to add whitespace when pretty-printing:
          separators = (', ', ': ') if indent is None else (',', ': ')
          # Don't sort if it's already sorted
@@ -XXX,XX +XXX,XX @@ def log(msg, filters=[], indent=None):
          print(msg)
  class Timeout:
 -    def __init__(self, seconds, errmsg = "Timeout"):
 +    def __init__(self, seconds, errmsg="Timeout"):
          self.seconds = seconds
          self.errmsg = errmsg
      def __enter__(self):
          signal.signal(signal.SIGALRM, self.timeout)
          signal.setitimer(signal.ITIMER_REAL, self.seconds)
          return self
 -    def __exit__(self, type, value, traceback):
 +    def __exit__(self, exc_type, value, traceback):
          signal.setitimer(signal.ITIMER_REAL, 0)
          return False
      def timeout(self, signum, frame):
@@ -XXX,XX +XXX,XX @@ class Timeout:
  def file_pattern(name):
      return "{0}-{1}".format(os.getpid(), name)
 -class FilePaths(object):
 +class FilePaths:
      """
      FilePaths is an auto-generated filename that cleans itself up.
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
              self.pause_drive(drive, "write_aio")
              return
          self.qmp('human-monitor-command',
 -                    command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive))
 +                 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive))
      def resume_drive(self, drive):
          self.qmp('human-monitor-command',
 -                    command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive))
 +                 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive))
      def hmp_qemu_io(self, drive, cmd):
          '''Write to a given drive using an HMP command'''
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
          if output is None:
              output = dict()
          if isinstance(obj, list):
 -            for i in range(len(obj)):
 -                self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.')
 +            for i, item in enumerate(obj):
 +                self.flatten_qmp_object(item, output, basestr + str(i) + '.')
          elif isinstance(obj, dict):
              for key in obj:
                  self.flatten_qmp_object(obj[key], output, basestr + key + '.')
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
          for bitmap in bitmaps[node_name]:
              if bitmap.get('name', '') == bitmap_name:
 -                if recording is None:
 -                    return bitmap
 -                elif bitmap.get('recording') == recording:
 +                if recording is None or bitmap.get('recording') == recording:
                      return bitmap
          return None
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
              assert node is not None, 'Cannot follow path %s%s' % (root, path)
              try:
 -                node_id = next(edge['child'] for edge in graph['edges'] \
 -                                             if edge['parent'] == node['id'] and
 -                                                edge['name'] == child_name)
 +                node_id = next(edge['child'] for edge in graph['edges']
 +                               if (edge['parent'] == node['id'] and
 +                                   edge['name'] == child_name))
 +
 +                node = next(node for node in graph['nodes']
 +                            if node['id'] == node_id)
 -                node = next(node for node in graph['nodes'] \
 -                                 if node['id'] == node_id)
              except StopIteration:
                  node = None
@@ -XXX,XX +XXX,XX @@ index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
  class QMPTestCase(unittest.TestCase):
      '''Abstract base class for QMP test cases'''
 +    def __init__(self, *args, **kwargs):
 +        super().__init__(*args, **kwargs)
 +        # Many users of this class set a VM property we rely on heavily
 +        # in the methods below.
 +        self.vm = None
 +
      def dictpath(self, d, path):
          '''Traverse a path in a nested dict'''
          for component in path.split('/'):
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
          else:
              self.assertEqual(result, value,
                               '"%s" is "%s", expected "%s"'
 -                                 % (path, str(result), str(value)))
 +                             % (path, str(result), str(value)))
      def assert_no_active_block_jobs(self):
          result = self.vm.qmp('query-block-jobs')
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
          """Issue a query-named-block-nodes and assert node_name and/or
          file_name is present in the result"""
          def check_equal_or_none(a, b):
 -            return a == None or b == None or a == b
 +            return a is None or b is None or a == b
          assert node_name or file_name
          result = self.vm.qmp('query-named-block-nodes')
          for x in result["return"]:
              if check_equal_or_none(x.get("node-name"), node_name) and \
                      check_equal_or_none(x.get("file"), file_name):
                  return
 -        self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \
 -                (node_name, file_name, result))
 +        self.fail("Cannot find %s %s in result:\n%s" %
 +                  (node_name, file_name, result))
      def assert_json_filename_equal(self, json_filename, reference):
          '''Asserts that the given filename is a json: filename and that its
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
                          self.assert_qmp(event, 'data/error', error)
                      self.assert_no_active_block_jobs()
                      return event
 -                elif event['event'] == 'JOB_STATUS_CHANGE':
 +                if event['event'] == 'JOB_STATUS_CHANGE':
                      self.assert_qmp(event, 'data/id', drive)
      def wait_ready(self, drive='drive0'):
 -        '''Wait until a block job BLOCK_JOB_READY event'''
 -        f = {'data': {'type': 'mirror', 'device': drive } }
 -        event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f)
 +        """Wait until a BLOCK_JOB_READY event, and return the event."""
 +        f = {'data': {'type': 'mirror', 'device': drive}}
 +        return self.vm.event_wait(name='BLOCK_JOB_READY', match=f)
      def wait_ready_and_cancel(self, drive='drive0'):
          self.wait_ready(drive=drive)
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
                  for job in result['return']:
                      if job['device'] == job_id:
                          found = True
 -                        if job['paused'] == True and job['busy'] == False:
 +                        if job['paused'] and not job['busy']:
                              return job
                          break
                  assert found
@@ -XXX,XX +XXX,XX @@ def qemu_pipe(*args):
                              universal_newlines=True)
      exitcode = subp.wait()
      if exitcode < 0:
 -        sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode,
 -                         ' '.join(args)))
 +        sys.stderr.write('qemu received signal %i: %s\n' %
 +                         (-exitcode, ' '.join(args)))
      return subp.communicate()[0]
  def supported_formats(read_only=False):
@@ -XXX,XX +XXX,XX @@ def skip_if_unsupported(required_formats=[], read_only=False):
              if usf_list:
                  test_case.case_skip('{}: formats {} are not whitelisted'.format(
                      test_case, usf_list))
 +                return None
              else:
                  return func(test_case, *args, **kwargs)
          return func_wrapper
@@ -XXX,XX +XXX,XX @@ def skip_if_user_is_root(func):
      def func_wrapper(*args, **kwargs):
          if os.getuid() == 0:
              case_notrun('{}: cannot be run as root'.format(args[0]))
 +            return None
          else:
              return func(*args, **kwargs)
      return func_wrapper
 --
 .26.2

-New patch
+[PULL 02/24] iotests: don't use 'format' for drive_add
+From: John Snow <jsnow@redhat.com>
+It shadows (with a different type) the built-in format.
+Use something else.
+Signed-off-by: John Snow <jsnow@redhat.com>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
+Message-Id: <20200331000014.11581-3-jsnow@redhat.com>
+Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
+---
+ tests/qemu-iotests/055        | 3 ++-
+ tests/qemu-iotests/iotests.py | 6 +++---
+files changed, 5 insertions(+), 4 deletions(-)
+diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055
+index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/055
++++ b/tests/qemu-iotests/055
+@@ -XXX,XX +XXX,XX @@ class TestDriveCompression(iotests.QMPTestCase):
+         qemu_img('create', '-f', fmt, blockdev_target_img,
+                  str(TestDriveCompression.image_len), *args)
+         if attach_target:
+-            self.vm.add_drive(blockdev_target_img, format=fmt, interface="none")
++            self.vm.add_drive(blockdev_target_img,
++                              img_format=fmt, interface="none")
+         self.vm.launch()
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/qemu-iotests/iotests.py
++++ b/tests/qemu-iotests/iotests.py
+@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
+         self._args.append(opts)
+         return self
+-    def add_drive(self, path, opts='', interface='virtio', format=imgfmt):
++    def add_drive(self, path, opts='', interface='virtio', img_format=imgfmt):
+         '''Add a virtio-blk drive to the VM'''
+         options = ['if=%s' % interface,
+                    'id=drive%d' % self._num_drives]
+         if path is not None:
+             options.append('file=%s' % path)
+-            options.append('format=%s' % format)
++            options.append('format=%s' % img_format)
+             options.append('cache=%s' % cachemode)
+             options.append('aio=%s' % aiomode)
+         if opts:
+             options.append(opts)
+-        if format == 'luks' and 'key-secret' not in opts:
++        if img_format == 'luks' and 'key-secret' not in opts:
+             # default luks support
+             if luks_default_secret_object not in self._args:
+                 self.add_object(luks_default_secret_object)
+--
+.26.2

-New patch
+[PULL 03/24] iotests: ignore import warnings from pylint
+From: John Snow <jsnow@redhat.com>
+The right way to solve this is to come up with a virtual environment
+infrastructure that sets all the paths correctly, and/or to create
+installable python modules that can be imported normally.
+That's hard, so just silence this error for now.
+Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
+Signed-off-by: John Snow <jsnow@redhat.com>
+Message-Id: <20200331000014.11581-4-jsnow@redhat.com>
+Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
+---
+ tests/qemu-iotests/iotests.py | 1 +
+file changed, 1 insertion(+)
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/qemu-iotests/iotests.py
++++ b/tests/qemu-iotests/iotests.py
+@@ -XXX,XX +XXX,XX @@ import io
+ from collections import OrderedDict
+ import faulthandler
++# pylint: disable=import-error, wrong-import-position
+ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+ from qemu import qtest
+--
+.26.2

-New patch
+[PULL 04/24] iotests: replace mutable list default args
+From: John Snow <jsnow@redhat.com>
+It's bad hygiene: if we modify this list, it will be modified across all
+invocations.
+(Remaining bad usages are fixed in a subsequent patch which changes the
+function signature anyway.)
+Signed-off-by: John Snow <jsnow@redhat.com>
+Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
+Message-Id: <20200331000014.11581-5-jsnow@redhat.com>
+Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
+---
+ tests/qemu-iotests/iotests.py | 24 ++++++++++++------------
+file changed, 12 insertions(+), 12 deletions(-)
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/qemu-iotests/iotests.py
++++ b/tests/qemu-iotests/iotests.py
+@@ -XXX,XX +XXX,XX @@ def qemu_img_log(*args):
+     log(result, filters=[filter_testfiles])
+     return result
+-def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]):
++def img_info_log(filename, filter_path=None, imgopts=False, extra_args=()):
+     args = ['info']
+     if imgopts:
+         args.append('--image-opts')
+@@ -XXX,XX +XXX,XX @@ def filter_qmp_imgfmt(qmsg):
+         return value
+     return filter_qmp(qmsg, _filter)
+-def log(msg, filters=[], indent=None):
++def log(msg, filters=(), indent=None):
+     '''Logs either a string message or a JSON serializable message (like QMP).
+     If indent is provided, JSON serializable messages are pretty-printed.'''
+     for flt in filters:
+@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
+             result.append(filter_qmp_event(ev))
+         return result
+-    def qmp_log(self, cmd, filters=[], indent=None, **kwargs):
++    def qmp_log(self, cmd, filters=(), indent=None, **kwargs):
+         full_cmd = OrderedDict((
+             ("execute", cmd),
+             ("arguments", ordered_qmp(kwargs))
+@@ -XXX,XX +XXX,XX @@ def case_notrun(reason):
+     open('%s/%s.casenotrun' % (output_dir, seq), 'a').write(
+         '    [case not run] ' + reason + '\n')
+-def verify_image_format(supported_fmts=[], unsupported_fmts=[]):
++def verify_image_format(supported_fmts=(), unsupported_fmts=()):
+     assert not (supported_fmts and unsupported_fmts)
+     if 'generic' in supported_fmts and \
+@@ -XXX,XX +XXX,XX @@ def verify_image_format(supported_fmts=[], unsupported_fmts=[]):
+     if not_sup or (imgfmt in unsupported_fmts):
+         notrun('not suitable for this image format: %s' % imgfmt)
+-def verify_protocol(supported=[], unsupported=[]):
++def verify_protocol(supported=(), unsupported=()):
+     assert not (supported and unsupported)
+     if 'generic' in supported:
+@@ -XXX,XX +XXX,XX @@ def verify_platform(supported=None, unsupported=None):
+         if not any((sys.platform.startswith(x) for x in supported)):
+             notrun('not suitable for this OS: %s' % sys.platform)
+-def verify_cache_mode(supported_cache_modes=[]):
++def verify_cache_mode(supported_cache_modes=()):
+     if supported_cache_modes and (cachemode not in supported_cache_modes):
+         notrun('not suitable for this cache mode: %s' % cachemode)
+-def verify_aio_mode(supported_aio_modes=[]):
++def verify_aio_mode(supported_aio_modes=()):
+     if supported_aio_modes and (aiomode not in supported_aio_modes):
+         notrun('not suitable for this aio mode: %s' % aiomode)
+@@ -XXX,XX +XXX,XX @@ def supported_formats(read_only=False):
+     return supported_formats.formats[read_only]
+-def skip_if_unsupported(required_formats=[], read_only=False):
++def skip_if_unsupported(required_formats=(), read_only=False):
+     '''Skip Test Decorator
+        Runs the test if all the required formats are whitelisted'''
+     def skip_test_decorator(func):
+@@ -XXX,XX +XXX,XX @@ def execute_unittest(output, verbosity, debug):
+             sys.stderr.write(out)
+ def execute_test(test_function=None,
+-                 supported_fmts=[],
++                 supported_fmts=(),
+                  supported_platforms=None,
+-                 supported_cache_modes=[], supported_aio_modes={},
+-                 unsupported_fmts=[], supported_protocols=[],
+-                 unsupported_protocols=[]):
++                 supported_cache_modes=(), supported_aio_modes=(),
++                 unsupported_fmts=(), supported_protocols=(),
++                 unsupported_protocols=()):
+     """Run either unittest or script-style tests."""
+     # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
+--
+.26.2

-New patch
+[PULL 05/24] iotests: add pylintrc file
+From: John Snow <jsnow@redhat.com>
+This allows others to get repeatable results with pylint. If you run
+`pylint iotests.py`, you should see a 100% pass.
+Signed-off-by: John Snow <jsnow@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
+Message-Id: <20200331000014.11581-6-jsnow@redhat.com>
+Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
+---
+ tests/qemu-iotests/pylintrc | 22 ++++++++++++++++++++++
+file changed, 22 insertions(+)
+ create mode 100644 tests/qemu-iotests/pylintrc
+diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc
+new file mode 100644
+index XXXXXXX..XXXXXXX
+--- /dev/null
++++ b/tests/qemu-iotests/pylintrc
+@@ -XXX,XX +XXX,XX @@
++[MESSAGES CONTROL]
++
++# Disable the message, report, category or checker with the given id(s). You
++# can either give multiple identifiers separated by comma (,) or put this
++# option multiple times (only on the command line, not in the configuration
++# file where it should appear only once). You can also use "--disable=all" to
++# disable everything first and then reenable specific checks. For example, if
++# you want to run only the similarities checker, you can use "--disable=all
++# --enable=similarities". If you want to run only the classes checker, but have
++# no Warning level messages displayed, use "--disable=all --enable=classes
++# --disable=W".
++disable=invalid-name,
++        no-else-return,
++        too-few-public-methods,
++        too-many-arguments,
++        too-many-branches,
++        too-many-lines,
++        too-many-locals,
++        too-many-public-methods,
++        # These are temporary, and should be removed:
++        line-too-long,
++        missing-docstring,
+--
+.26.2

-[Qemu-devel] [PULL 14/15] tests/check-block: Skip iotests when sanitizers are enabled
+[PULL 06/24] iotests: alphabetize standard imports
-From: Thomas Huth <thuth@redhat.com>
+From: John Snow <jsnow@redhat.com>
-The sanitizers (especially the address sanitizer from Clang) are
+I had to fix a merge conflict, so do this tiny harmless thing while I'm
-sometimes printing out warnings or false positives - this spoils
+here.
 the output of the iotests, causing some of the tests to fail.
 Thus let's skip the automatic iotests during "make check" when the
 user configured QEMU with --enable-sanitizers.
-Signed-off-by: Thomas Huth <thuth@redhat.com>
+Signed-off-by: John Snow <jsnow@redhat.com>
-Message-id: 20190823084203.29734-1-thuth@redhat.com
+Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200331000014.11581-7-jsnow@redhat.com>
 Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/check-block.sh | 5 +++++
+ tests/qemu-iotests/iotests.py | 18 +++++++++---------
-file changed, 5 insertions(+)
+file changed, 9 insertions(+), 9 deletions(-)
-diff --git a/tests/check-block.sh b/tests/check-block.sh
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
-index XXXXXXX..XXXXXXX 100755
+index XXXXXXX..XXXXXXX 100644
---- a/tests/check-block.sh
+--- a/tests/qemu-iotests/iotests.py
-+++ b/tests/check-block.sh
++++ b/tests/qemu-iotests/iotests.py
-@@ -XXX,XX +XXX,XX @@ if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then
+@@ -XXX,XX +XXX,XX @@
-     exit 0
+ # along with this program.  If not, see <http://www.gnu.org/licenses/>.
- fi
+ #
-+if grep -q "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ; then
++import atexit
-+    echo "Sanitizers are enabled ==> Not running the qemu-iotests."
++from collections import OrderedDict
-+    exit 0
++import faulthandler
-+fi
++import io
-+
++import json
- if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then
++import logging
-     echo "No qemu-system binary available ==> Not running the qemu-iotests."
+ import os
-     exit 0
+ import re
 +import signal
 +import struct
  import subprocess
 -import unittest
  import sys
 -import struct
 -import json
 -import signal
 -import logging
 -import atexit
 -import io
 -from collections import OrderedDict
 -import faulthandler
 +import unittest
  # pylint: disable=import-error, wrong-import-position
  sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 13/15] iotests: Check for enabled drivers before testing them
+[PULL 07/24] iotests: drop pre-Python 3.4 compatibility code
-From: Thomas Huth <thuth@redhat.com>
+From: John Snow <jsnow@redhat.com>
-It is possible to enable only a subset of the block drivers with the
+We no longer need to accommodate <3.4, drop this code.
-"--block-drv-rw-whitelist" option of the "configure" script. All other
+(The lines were > 79 chars and it stood out.)
 drivers are marked as unusable (or only included as read-only with the
 "--block-drv-ro-whitelist" option). If an iotest is now using such a
 disabled block driver, it is failing - which is bad, since at least the
 tests in the "auto" group should be able to deal with this situation.
 Thus let's introduce a "_require_drivers" function that can be used by
 the shell tests to check for the availability of certain drivers first,
 and marks the test as "not run" if one of the drivers is missing.
-This patch mainly targets the test in the "auto" group which should
+Signed-off-by: John Snow <jsnow@redhat.com>
-never fail in such a case, but also improves some of the other tests
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-along the way. Note that we also assume that the "qcow2" and "file"
+Message-Id: <20200331000014.11581-8-jsnow@redhat.com>
-drivers are always available - otherwise it does not make sense to
+Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 run "make check-block" at all (which only tests with qcow2 by default).
 Signed-off-by: Thomas Huth <thuth@redhat.com>
 Message-id: 20190823133552.11680-1-thuth@redhat.com
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/071       |  1 +
+ tests/qemu-iotests/iotests.py | 5 +----
- tests/qemu-iotests/081       |  4 +---
+file changed, 1 insertion(+), 4 deletions(-)
  tests/qemu-iotests/099       |  1 +
  tests/qemu-iotests/120       |  1 +
  tests/qemu-iotests/162       |  4 +---
  tests/qemu-iotests/184       |  1 +
  tests/qemu-iotests/186       |  1 +
  tests/qemu-iotests/common.rc | 14 ++++++++++++++
 files changed, 21 insertions(+), 6 deletions(-)
-diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/071
 +++ b/tests/qemu-iotests/071
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  _supported_fmt qcow2
  _supported_proto file
 +_require_drivers blkdebug blkverify
  do_run_qemu()
  {
 diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/081
 +++ b/tests/qemu-iotests/081
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  _supported_fmt raw
  _supported_proto file
  _supported_os Linux
 +_require_drivers quorum
  do_run_qemu()
  {
@@ -XXX,XX +XXX,XX @@ run_qemu()
                            | _filter_qemu_io | _filter_generated_node_ids
  }
 -test_quorum=$($QEMU_IMG --help|grep quorum)
 -[ "$test_quorum" = "" ] && _supported_fmt quorum
 -
  quorum="driver=raw,file.driver=quorum,file.vote-threshold=2"
  quorum="$quorum,file.children.0.file.filename=$TEST_DIR/1.raw"
  quorum="$quorum,file.children.1.file.filename=$TEST_DIR/2.raw"
 diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/099
 +++ b/tests/qemu-iotests/099
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  _supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc
  _supported_proto file
  _supported_os Linux
 +_require_drivers blkdebug blkverify
  _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
      "subformat=twoGbMaxExtentSparse"
 diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/120
 +++ b/tests/qemu-iotests/120
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  _supported_fmt generic
  _supported_proto file
  _unsupported_fmt luks
 +_require_drivers raw
  _make_test_img 64M
 diff --git a/tests/qemu-iotests/162 b/tests/qemu-iotests/162
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/162
 +++ b/tests/qemu-iotests/162
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  . ./common.filter
  _supported_fmt generic
 -
 -test_ssh=$($QEMU_IMG --help | grep '^Supported formats:.* ssh\( \|$\)')
 -[ "$test_ssh" = "" ] && _notrun "ssh support required"
 +_require_drivers ssh
  echo
  echo '=== NBD ==='
 diff --git a/tests/qemu-iotests/184 b/tests/qemu-iotests/184
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/184
 +++ b/tests/qemu-iotests/184
@@ -XXX,XX +XXX,XX @@ trap "exit \$status" 0 1 2 3 15
  . ./common.filter
  _supported_os Linux
 +_require_drivers throttle
  do_run_qemu()
  {
 diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/186
 +++ b/tests/qemu-iotests/186
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  _supported_fmt qcow2
  _supported_proto file
 +_require_drivers null-co
  if [ "$QEMU_DEFAULT_MACHINE" != "pc" ]; then
      _notrun "Requires a PC machine"
 diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
 index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/common.rc
+--- a/tests/qemu-iotests/iotests.py
-+++ b/tests/qemu-iotests/common.rc
++++ b/tests/qemu-iotests/iotests.py
-@@ -XXX,XX +XXX,XX @@ _require_command()
+@@ -XXX,XX +XXX,XX @@ def log(msg, filters=(), indent=None):
-     [ -x "$c" ] || _notrun "$1 utility required, skipped this test"
+     for flt in filters:
- }
+         msg = flt(msg)
+     if isinstance(msg, (dict, list)):
-+# Check that a set of drivers has been whitelisted in the QEMU binary
+-        # Python < 3.4 needs to know not to add whitespace when pretty-printing:
-+#
+-        separators = (', ', ': ') if indent is None else (',', ': ')
-+_require_drivers()
+         # Don't sort if it's already sorted
-+{
+         do_sort = not isinstance(msg, OrderedDict)
-+    available=$($QEMU -drive format=help | \
+-        print(json.dumps(msg, sort_keys=do_sort,
-+                sed -e '/Supported formats:/!d' -e 's/Supported formats://')
+-                         indent=indent, separators=separators))
-+    for driver
++        print(json.dumps(msg, sort_keys=do_sort, indent=indent))
-+    do
+     else:
-+        if ! echo "$available" | grep -q " $driver\( \|$\)"; then
+         print(msg)
-+            _notrun "$driver not available"
 +        fi
 +    done
 +}
 +
  # make sure this script returns success
  true
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 11/15] iotests: Disable 126 for flat vmdk subformats
+[PULL 08/24] iotests: touch up log function signature
-iotest 126 requires backing file support, which flat vmdks cannot offer.
+From: John Snow <jsnow@redhat.com>
 Skip this test for such subformats.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+Representing nested, recursive data structures in mypy is notoriously
-Message-id: 20190815153638.4600-8-mreitz@redhat.com
+difficult; the best we can reliably do right now is denote the leaf
-Reviewed-by: John Snow <jsnow@redhat.com>
+types as "Any" while describing the general shape of the data.
 Regardless, this fully annotates the log() function.
 Typing notes:
 TypeVar is a Type variable that can optionally be constrained by a
 sequence of possible types. This variable is bound to a specific type
 per-invocation, like a Generic.
 log() behaves as log<Msg>() now, where the incoming type informs the
 signature it expects for any filter arguments passed in. If Msg is a
 str, then filter should take and return a str.
 Signed-off-by: John Snow <jsnow@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200331000014.11581-9-jsnow@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/126 | 2 ++
+ tests/qemu-iotests/iotests.py | 14 +++++++++++---
-file changed, 2 insertions(+)
+file changed, 11 insertions(+), 3 deletions(-)
-diff --git a/tests/qemu-iotests/126 b/tests/qemu-iotests/126
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
-index XXXXXXX..XXXXXXX 100755
+index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/126
+--- a/tests/qemu-iotests/iotests.py
-+++ b/tests/qemu-iotests/126
++++ b/tests/qemu-iotests/iotests.py
-@@ -XXX,XX +XXX,XX @@ status=1    # failure is the default!
+@@ -XXX,XX +XXX,XX @@ import signal
+ import struct
- # Needs backing file support
+ import subprocess
- _supported_fmt qcow qcow2 qed vmdk
+ import sys
-+_unsupported_imgopts "subformat=monolithicFlat" \
++from typing import (Any, Callable, Dict, Iterable, List, Optional, TypeVar)
-+                     "subformat=twoGbMaxExtentFlat"
+ import unittest
- # This is the default protocol (and we want to test the difference between
- # colons which separate a protocol prefix from the rest and colons which are
+ # pylint: disable=import-error, wrong-import-position
- # just part of the filename, so we cannot test protocols which require a prefix)
+@@ -XXX,XX +XXX,XX @@ def filter_qmp_imgfmt(qmsg):
          return value
      return filter_qmp(qmsg, _filter)
 -def log(msg, filters=(), indent=None):
 -    '''Logs either a string message or a JSON serializable message (like QMP).
 -    If indent is provided, JSON serializable messages are pretty-printed.'''
 +
 +Msg = TypeVar('Msg', Dict[str, Any], List[Any], str)
 +
 +def log(msg: Msg,
 +        filters: Iterable[Callable[[Msg], Msg]] = (),
 +        indent: Optional[int] = None) -> None:
 +    """
 +    Logs either a string message or a JSON serializable message (like QMP).
 +    If indent is provided, JSON serializable messages are pretty-printed.
 +    """
      for flt in filters:
          msg = flt(msg)
      if isinstance(msg, (dict, list)):
 --
-.21.0
+.26.2

-New patch
+[PULL 09/24] iotests: limit line length to 79 chars
+From: John Snow <jsnow@redhat.com>
 is the PEP8 recommendation. This recommendation works well for
 reading patch diffs in TUI email clients.
 Signed-off-by: John Snow <jsnow@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200331000014.11581-10-jsnow@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  tests/qemu-iotests/iotests.py | 64 +++++++++++++++++++++++------------
  tests/qemu-iotests/pylintrc   |  6 +++-
 files changed, 47 insertions(+), 23 deletions(-)
 diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/iotests.py
 +++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ luks_default_key_secret_opt = 'key-secret=keysec0'
  def qemu_img(*args):
      '''Run qemu-img and return the exit code'''
      devnull = open('/dev/null', 'r+')
 -    exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
 +    exitcode = subprocess.call(qemu_img_args + list(args),
 +                               stdin=devnull, stdout=devnull)
      if exitcode < 0:
 -        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
 +        sys.stderr.write('qemu-img received signal %i: %s\n'
 +                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
      return exitcode
  def ordered_qmp(qmsg, conv_keys=True):
@@ -XXX,XX +XXX,XX @@ def qemu_img_verbose(*args):
      '''Run qemu-img without suppressing its output and return the exit code'''
      exitcode = subprocess.call(qemu_img_args + list(args))
      if exitcode < 0:
 -        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
 +        sys.stderr.write('qemu-img received signal %i: %s\n'
 +                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
      return exitcode
  def qemu_img_pipe(*args):
@@ -XXX,XX +XXX,XX @@ def qemu_img_pipe(*args):
                              universal_newlines=True)
      exitcode = subp.wait()
      if exitcode < 0:
 -        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
 +        sys.stderr.write('qemu-img received signal %i: %s\n'
 +                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
      return subp.communicate()[0]
  def qemu_img_log(*args):
@@ -XXX,XX +XXX,XX @@ def qemu_io(*args):
                              universal_newlines=True)
      exitcode = subp.wait()
      if exitcode < 0:
 -        sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args)))
 +        sys.stderr.write('qemu-io received signal %i: %s\n'
 +                         % (-exitcode, ' '.join(args)))
      return subp.communicate()[0]
  def qemu_io_log(*args):
@@ -XXX,XX +XXX,XX @@ win32_re = re.compile(r"\r")
  def filter_win32(msg):
      return win32_re.sub("", msg)
 -qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)")
 +qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* "
 +                        r"\([0-9\/.inf]* [EPTGMKiBbytes]*\/sec "
 +                        r"and [0-9\/.inf]* ops\/sec\)")
  def filter_qemu_io(msg):
      msg = filter_win32(msg)
 -    return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg)
 +    return qemu_io_re.sub("X ops; XX:XX:XX.X "
 +                          "(XXX YYY/sec and XXX ops/sec)", msg)
  chown_re = re.compile(r"chown [0-9]+:[0-9]+")
  def filter_chown(msg):
@@ -XXX,XX +XXX,XX @@ def filter_img_info(output, filename):
          line = filter_testfiles(line)
          line = line.replace(imgfmt, 'IMGFMT')
          line = re.sub('iters: [0-9]+', 'iters: XXX', line)
 -        line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line)
 +        line = re.sub('uuid: [-a-f0-9]+',
 +                      'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',
 +                      line)
          line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line)
          lines.append(line)
      return '\n'.join(lines)
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
              self.pause_drive(drive, "write_aio")
              return
          self.qmp('human-monitor-command',
 -                 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive))
 +                 command_line='qemu-io %s "break %s bp_%s"'
 +                 % (drive, event, drive))
      def resume_drive(self, drive):
          self.qmp('human-monitor-command',
 -                 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive))
 +                 command_line='qemu-io %s "remove_break bp_%s"'
 +                 % (drive, drive))
      def hmp_qemu_io(self, drive, cmd):
          '''Write to a given drive using an HMP command'''
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
                  idx = int(idx)
              if not isinstance(d, dict) or component not in d:
 -                self.fail('failed path traversal for "%s" in "%s"' % (path, str(d)))
 +                self.fail(f'failed path traversal for "{path}" in "{d}"')
              d = d[component]
              if m:
                  if not isinstance(d, list):
 -                    self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d)))
 +                    self.fail(f'path component "{component}" in "{path}" '
 +                              f'is not a list in "{d}"')
                  try:
                      d = d[idx]
                  except IndexError:
 -                    self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
 +                    self.fail(f'invalid index "{idx}" in path "{path}" '
 +                              f'in "{d}"')
          return d
      def assert_qmp_absent(self, d, path):
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
          '''Asserts that the given filename is a json: filename and that its
             content is equal to the given reference object'''
          self.assertEqual(json_filename[:5], 'json:')
 -        self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])),
 -                         self.vm.flatten_qmp_object(reference))
 +        self.assertEqual(
 +            self.vm.flatten_qmp_object(json.loads(json_filename[5:])),
 +            self.vm.flatten_qmp_object(reference)
 +        )
 -    def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0):
 +    def cancel_and_wait(self, drive='drive0', force=False,
 +                        resume=False, wait=60.0):
          '''Cancel a block job and wait for it to finish, returning the event'''
          result = self.vm.qmp('block-job-cancel', device=drive, force=force)
          self.assert_qmp(result, 'return', {})
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
          self.assert_no_active_block_jobs()
          return result
 -    def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0,
 -                             error=None):
 +    def wait_until_completed(self, drive='drive0', check_offset=True,
 +                             wait=60.0, error=None):
          '''Wait for a block job to finish, returning the event'''
          while True:
              for event in self.vm.get_qmp_events(wait=wait):
@@ -XXX,XX +XXX,XX @@ def verify_quorum():
          notrun('quorum support missing')
  def qemu_pipe(*args):
 -    '''Run qemu with an option to print something and exit (e.g. a help option),
 -    and return its output'''
 +    """
 +    Run qemu with an option to print something and exit (e.g. a help option).
 +
 +    :return: QEMU's stdout output.
 +    """
      args = [qemu_prog] + qemu_opts + list(args)
      subp = subprocess.Popen(args, stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
@@ -XXX,XX +XXX,XX @@ def skip_if_unsupported(required_formats=(), read_only=False):
              usf_list = list(set(fmts) - set(supported_formats(read_only)))
              if usf_list:
 -                test_case.case_skip('{}: formats {} are not whitelisted'.format(
 -                    test_case, usf_list))
 +                msg = f'{test_case}: formats {usf_list} are not whitelisted'
 +                test_case.case_skip(msg)
                  return None
              else:
                  return func(test_case, *args, **kwargs)
 diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/pylintrc
 +++ b/tests/qemu-iotests/pylintrc
@@ -XXX,XX +XXX,XX @@ disable=invalid-name,
          too-many-locals,
          too-many-public-methods,
          # These are temporary, and should be removed:
 -        line-too-long,
          missing-docstring,
 +
 +[FORMAT]
 +
 +# Maximum number of characters on a single line.
 +max-line-length=79
 --
 .26.2

-[Qemu-devel] [PULL 07/15] iotests: Keep testing broken relative extent paths
+[PULL 10/24] iotests: add hmp helper with logging
-We had a test for a case where relative extent paths did not work, but
+From: John Snow <jsnow@redhat.com>
 unfortunately we just fixed the underlying problem, so it works now.
 This patch adds a new test case that still fails.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+Minor cleanup for HMP functions; helps with line length and consolidates
-Reviewed-by: John Snow <jsnow@redhat.com>
+HMP helpers through one implementation function.
-Message-id: 20190815153638.4600-4-mreitz@redhat.com
-Reviewed-by: John Snow <jsnow@redhat.com>
+Although we are adding a universal toggle to turn QMP logging on or off,
 many existing callers to hmp functions don't expect that output to be
 logged, which causes quite a few changes in the test output.
 For now, offer a use_log parameter.
 Typing notes:
 QMPResponse is just an alias for Dict[str, Any]. It holds no special
 meanings and it is not a formal subtype of Dict[str, Any]. It is best
 thought of as a lexical synonym.
 We may well wish to add stricter subtypes in the future for certain
 shapes of data that are not formalized as Python objects, at which point
 we can simply retire the alias and allow mypy to more strictly check
 usages of the name.
 Signed-off-by: John Snow <jsnow@redhat.com>
 Message-Id: <20200331000014.11581-11-jsnow@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/059     | 27 +++++++++++++++++++++++++++
+ tests/qemu-iotests/iotests.py | 39 +++++++++++++++++++++--------------
- tests/qemu-iotests/059.out |  4 ++++
+file changed, 24 insertions(+), 15 deletions(-)
 files changed, 31 insertions(+)
-diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
-index XXXXXXX..XXXXXXX 100755
+index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/059
+--- a/tests/qemu-iotests/iotests.py
-+++ b/tests/qemu-iotests/059
++++ b/tests/qemu-iotests/iotests.py
-@@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2
+@@ -XXX,XX +XXX,XX @@ from qemu import qtest
- echo
+ assert sys.version_info >= (3, 6)
- echo "=== Testing monolithicFlat with internally generated JSON file name ==="
-+
++# Type Aliases
-+echo '--- blkdebug ---'
++QMPResponse = Dict[str, Any]
  # Should work, because bdrv_dirname() works fine with blkdebug
  IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
  $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
@@ -XXX,XX +XXX,XX @@ $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TE
      | _filter_testdir | _filter_imgfmt | _filter_img_info
  _cleanup_test_img
 +echo '--- quorum ---'
 +# Should not work, because bdrv_dirname() does not work with quorum
 +IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
 +cp "$TEST_IMG" "$TEST_IMG.orig"
 +
 +filename="json:{
 +    \"driver\": \"$IMGFMT\",
 +    \"file\": {
 +        \"driver\": \"quorum\",
 +        \"children\": [ {
 +            \"driver\": \"file\",
 +            \"filename\": \"$TEST_IMG\"
 +        }, {
 +            \"driver\": \"file\",
 +            \"filename\": \"$TEST_IMG.orig\"
 +        } ],
 +        \"vote-threshold\": 1
 +    } }"
 +
 +filename=$(echo "$filename" | tr '\n' ' ' | sed -e 's/\s\+/ /g')
 +$QEMU_IMG info "$filename" 2>&1 \
 +    | sed -e "s/'json:[^']*'/\$QUORUM_FILE/g" \
 +    | _filter_testdir | _filter_imgfmt | _filter_img_info
 +
 +
- echo
+ faulthandler.enable()
- echo "=== Testing version 3 ==="
- _use_sample_img iotest-version3.vmdk.bz2
+ # This will not work if arguments contain spaces but is necessary if we
-diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
+@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
-index XXXXXXX..XXXXXXX 100644
+         self._args.append(addr)
---- a/tests/qemu-iotests/059.out
+         return self
-+++ b/tests/qemu-iotests/059.out
-@@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240
+-    def pause_drive(self, drive, event=None):
-bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+-        '''Pause drive r/w operations'''
++    def hmp(self, command_line: str, use_log: bool = False) -> QMPResponse:
- === Testing monolithicFlat with internally generated JSON file name ===
++        cmd = 'human-monitor-command'
-+--- blkdebug ---
++        kwargs = {'command-line': command_line}
- Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
++        if use_log:
- format name: IMGFMT
++            return self.qmp_log(cmd, **kwargs)
- cluster size: 0 bytes
++        else:
- vm state offset: 0 bytes
++            return self.qmp(cmd, **kwargs)
-+--- quorum ---
++
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
++    def pause_drive(self, drive: str, event: Optional[str] = None) -> None:
-+qemu-img: Could not open $QUORUM_FILE: Cannot use relative paths with VMDK descriptor file $QUORUM_FILE: Cannot generate a base directory for quorum nodes
++        """Pause drive r/w operations"""
+         if not event:
- === Testing version 3 ===
+             self.pause_drive(drive, "read_aio")
- image: TEST_DIR/iotest-version3.IMGFMT
+             self.pause_drive(drive, "write_aio")
              return
 -        self.qmp('human-monitor-command',
 -                 command_line='qemu-io %s "break %s bp_%s"'
 -                 % (drive, event, drive))
 -
 -    def resume_drive(self, drive):
 -        self.qmp('human-monitor-command',
 -                 command_line='qemu-io %s "remove_break bp_%s"'
 -                 % (drive, drive))
 -
 -    def hmp_qemu_io(self, drive, cmd):
 -        '''Write to a given drive using an HMP command'''
 -        return self.qmp('human-monitor-command',
 -                        command_line='qemu-io %s "%s"' % (drive, cmd))
 +        self.hmp(f'qemu-io {drive} "break {event} bp_{drive}"')
 +
 +    def resume_drive(self, drive: str) -> None:
 +        """Resume drive r/w operations"""
 +        self.hmp(f'qemu-io {drive} "remove_break bp_{drive}"')
 +
 +    def hmp_qemu_io(self, drive: str, cmd: str,
 +                    use_log: bool = False) -> QMPResponse:
 +        """Write to a given drive using an HMP command"""
 +        return self.hmp(f'qemu-io {drive} "{cmd}"', use_log=use_log)
      def flatten_qmp_object(self, obj, output=None, basestr=''):
          if output is None:
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 09/15] iotests: Disable broken streamOptimized tests
+[PULL 11/24] iotests: add script_initialize
-streamOptimized does not support writes that do not span exactly one
+From: John Snow <jsnow@redhat.com>
 cluster.  Furthermore, it cannot rewrite already allocated clusters.
 As such, many iotests do not work with it.  Disable them.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+Like script_main, but doesn't require a single point of entry.
-Message-id: 20190815153638.4600-6-mreitz@redhat.com
+Replace all existing initialization sections with this drop-in replacement.
-Reviewed-by: John Snow <jsnow@redhat.com>
 This brings debug support to all existing script-style iotests.
 Signed-off-by: John Snow <jsnow@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200331000014.11581-12-jsnow@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 [mreitz: Give 274 the same treatment]
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/002 | 1 +
+ tests/qemu-iotests/149        |  3 +-
- tests/qemu-iotests/003 | 1 +
+ tests/qemu-iotests/194        |  4 +-
- tests/qemu-iotests/005 | 3 ++-
+ tests/qemu-iotests/202        |  4 +-
- tests/qemu-iotests/009 | 1 +
+ tests/qemu-iotests/203        |  4 +-
- tests/qemu-iotests/010 | 1 +
+ tests/qemu-iotests/206        |  2 +-
- tests/qemu-iotests/011 | 1 +
+ tests/qemu-iotests/207        |  6 ++-
- tests/qemu-iotests/017 | 3 ++-
+ tests/qemu-iotests/208        |  2 +-
- tests/qemu-iotests/018 | 3 ++-
+ tests/qemu-iotests/209        |  2 +-
- tests/qemu-iotests/019 | 3 ++-
+ tests/qemu-iotests/210        |  6 ++-
- tests/qemu-iotests/020 | 3 ++-
+ tests/qemu-iotests/211        |  6 ++-
- tests/qemu-iotests/027 | 1 +
+ tests/qemu-iotests/212        |  6 ++-
- tests/qemu-iotests/032 | 1 +
+ tests/qemu-iotests/213        |  6 ++-
- tests/qemu-iotests/033 | 1 +
+ tests/qemu-iotests/216        |  4 +-
- tests/qemu-iotests/034 | 3 ++-
+ tests/qemu-iotests/218        |  2 +-
- tests/qemu-iotests/037 | 3 ++-
+ tests/qemu-iotests/219        |  2 +-
- tests/qemu-iotests/063 | 3 ++-
+ tests/qemu-iotests/222        |  7 ++--
- tests/qemu-iotests/072 | 1 +
+ tests/qemu-iotests/224        |  4 +-
- tests/qemu-iotests/105 | 3 ++-
+ tests/qemu-iotests/228        |  6 ++-
- tests/qemu-iotests/197 | 1 +
+ tests/qemu-iotests/234        |  4 +-
- tests/qemu-iotests/215 | 1 +
+ tests/qemu-iotests/235        |  4 +-
- tests/qemu-iotests/251 | 1 +
+ tests/qemu-iotests/236        |  2 +-
-files changed, 30 insertions(+), 9 deletions(-)
+ tests/qemu-iotests/237        |  2 +-
  tests/qemu-iotests/238        |  2 +
  tests/qemu-iotests/242        |  2 +-
  tests/qemu-iotests/246        |  2 +-
  tests/qemu-iotests/248        |  2 +-
  tests/qemu-iotests/254        |  2 +-
  tests/qemu-iotests/255        |  2 +-
  tests/qemu-iotests/256        |  2 +-
  tests/qemu-iotests/258        |  7 ++--
  tests/qemu-iotests/260        |  4 +-
  tests/qemu-iotests/262        |  4 +-
  tests/qemu-iotests/264        |  4 +-
  tests/qemu-iotests/274        |  4 +-
  tests/qemu-iotests/277        |  2 +
  tests/qemu-iotests/280        |  8 ++--
  tests/qemu-iotests/283        |  4 +-
  tests/qemu-iotests/iotests.py | 76 +++++++++++++++++++++++------------
 files changed, 132 insertions(+), 83 deletions(-)
-diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002
+diff --git a/tests/qemu-iotests/149 b/tests/qemu-iotests/149
 index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/002
+--- a/tests/qemu-iotests/149
-+++ b/tests/qemu-iotests/002
++++ b/tests/qemu-iotests/149
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+@@ -XXX,XX +XXX,XX @@ def test_once(config, qemu_img=False):
- _supported_fmt generic
- _supported_proto generic
+ # Obviously we only work with the luks image format
-+_unsupported_imgopts "subformat=streamOptimized"
+-iotests.verify_image_format(supported_fmts=['luks'])
+-iotests.verify_platform()
++iotests.script_initialize(supported_fmts=['luks'])
- size=128M
-diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003
+ # We need sudo in order to run cryptsetup to create
-index XXXXXXX..XXXXXXX 100755
+ # dm-crypt devices. This is safe to use on any
---- a/tests/qemu-iotests/003
+diff --git a/tests/qemu-iotests/194 b/tests/qemu-iotests/194
-+++ b/tests/qemu-iotests/003
+index XXXXXXX..XXXXXXX 100755
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+--- a/tests/qemu-iotests/194
++++ b/tests/qemu-iotests/194
- _supported_fmt generic
+@@ -XXX,XX +XXX,XX @@
- _supported_proto generic
-+_unsupported_imgopts "subformat=streamOptimized"
+ import iotests
- size=128M
+-iotests.verify_image_format(supported_fmts=['qcow2', 'qed', 'raw'])
- offset=67M
+-iotests.verify_platform(['linux'])
-diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005
++iotests.script_initialize(supported_fmts=['qcow2', 'qed', 'raw'],
-index XXXXXXX..XXXXXXX 100755
++                          supported_platforms=['linux'])
---- a/tests/qemu-iotests/005
-+++ b/tests/qemu-iotests/005
+ with iotests.FilePath('source.img') as source_img_path, \
-@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
+      iotests.FilePath('dest.img') as dest_img_path, \
- _supported_proto generic
+diff --git a/tests/qemu-iotests/202 b/tests/qemu-iotests/202
- _supported_os Linux
+index XXXXXXX..XXXXXXX 100755
- _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
+--- a/tests/qemu-iotests/202
--                     "subformat=twoGbMaxExtentSparse"
++++ b/tests/qemu-iotests/202
-+                     "subformat=twoGbMaxExtentSparse" \
+@@ -XXX,XX +XXX,XX @@
-+                     "subformat=streamOptimized"
+ import iotests
- # vpc is limited to 127GB, so we can't test it here
- if [ "$IMGFMT" = "vpc" ]; then
+-iotests.verify_image_format(supported_fmts=['qcow2'])
-diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009
+-iotests.verify_platform(['linux'])
-index XXXXXXX..XXXXXXX 100755
++iotests.script_initialize(supported_fmts=['qcow2'],
---- a/tests/qemu-iotests/009
++                          supported_platforms=['linux'])
-+++ b/tests/qemu-iotests/009
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+ with iotests.FilePath('disk0.img') as disk0_img_path, \
+      iotests.FilePath('disk1.img') as disk1_img_path, \
- _supported_fmt generic
+diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203
- _supported_proto generic
+index XXXXXXX..XXXXXXX 100755
-+_unsupported_imgopts "subformat=streamOptimized"
+--- a/tests/qemu-iotests/203
++++ b/tests/qemu-iotests/203
+@@ -XXX,XX +XXX,XX @@
- size=6G
-diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010
+ import iotests
-index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/010
+-iotests.verify_image_format(supported_fmts=['qcow2'])
-+++ b/tests/qemu-iotests/010
+-iotests.verify_platform(['linux'])
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
++iotests.script_initialize(supported_fmts=['qcow2'],
++                          supported_platforms=['linux'])
- _supported_fmt generic
- _supported_proto generic
+ with iotests.FilePath('disk0.img') as disk0_img_path, \
-+_unsupported_imgopts "subformat=streamOptimized"
+      iotests.FilePath('disk1.img') as disk1_img_path, \
+diff --git a/tests/qemu-iotests/206 b/tests/qemu-iotests/206
+index XXXXXXX..XXXXXXX 100755
- size=6G
+--- a/tests/qemu-iotests/206
-diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011
++++ b/tests/qemu-iotests/206
-index XXXXXXX..XXXXXXX 100755
+@@ -XXX,XX +XXX,XX @@
---- a/tests/qemu-iotests/011
+ import iotests
-+++ b/tests/qemu-iotests/011
+ from iotests import imgfmt
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+-iotests.verify_image_format(supported_fmts=['qcow2'])
- _supported_fmt generic
++iotests.script_initialize(supported_fmts=['qcow2'])
- _supported_proto generic
-+_unsupported_imgopts "subformat=streamOptimized"
+ with iotests.FilePath('t.qcow2') as disk_path, \
+      iotests.FilePath('t.qcow2.base') as backing_path, \
+diff --git a/tests/qemu-iotests/207 b/tests/qemu-iotests/207
- size=6G
+index XXXXXXX..XXXXXXX 100755
-diff --git a/tests/qemu-iotests/017 b/tests/qemu-iotests/017
+--- a/tests/qemu-iotests/207
-index XXXXXXX..XXXXXXX 100755
++++ b/tests/qemu-iotests/207
---- a/tests/qemu-iotests/017
+@@ -XXX,XX +XXX,XX @@ import iotests
-+++ b/tests/qemu-iotests/017
+ import subprocess
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+ import re
- _supported_fmt qcow qcow2 vmdk qed
- _supported_proto generic
+-iotests.verify_image_format(supported_fmts=['raw'])
- _unsupported_proto vxhs
+-iotests.verify_protocol(supported=['ssh'])
--_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
++iotests.script_initialize(
-+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
++    supported_fmts=['raw'],
-+                     "subformat=streamOptimized"
++    supported_protocols=['ssh'],
++)
- TEST_OFFSETS="0 4294967296"
+ def filter_hash(qmsg):
-diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018
+     def _filter(key, value):
-index XXXXXXX..XXXXXXX 100755
+diff --git a/tests/qemu-iotests/208 b/tests/qemu-iotests/208
---- a/tests/qemu-iotests/018
+index XXXXXXX..XXXXXXX 100755
-+++ b/tests/qemu-iotests/018
+--- a/tests/qemu-iotests/208
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
++++ b/tests/qemu-iotests/208
- _supported_fmt qcow qcow2 vmdk qed
+@@ -XXX,XX +XXX,XX @@
- _supported_proto file
- _supported_os Linux
+ import iotests
--_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
-+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+-iotests.verify_image_format(supported_fmts=['generic'])
-+                     "streamOptimized"
++iotests.script_initialize(supported_fmts=['generic'])
- TEST_OFFSETS="0 4294967296"
+ with iotests.FilePath('disk.img') as disk_img_path, \
+      iotests.FilePath('disk-snapshot.img') as disk_snapshot_img_path, \
-diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019
+diff --git a/tests/qemu-iotests/209 b/tests/qemu-iotests/209
 index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/019
+--- a/tests/qemu-iotests/209
-+++ b/tests/qemu-iotests/019
++++ b/tests/qemu-iotests/209
-@@ -XXX,XX +XXX,XX @@ _supported_proto file
+@@ -XXX,XX +XXX,XX @@ import iotests
- _supported_os Linux
+ from iotests import qemu_img_create, qemu_io, qemu_img_verbose, qemu_nbd, \
- _unsupported_imgopts "subformat=monolithicFlat" \
+                     file_path
-                      "subformat=twoGbMaxExtentFlat" \
--                     "subformat=twoGbMaxExtentSparse"
+-iotests.verify_image_format(supported_fmts=['qcow2'])
-+                     "subformat=twoGbMaxExtentSparse" \
++iotests.script_initialize(supported_fmts=['qcow2'])
-+                     "subformat=streamOptimized"
+ disk = file_path('disk')
- TEST_OFFSETS="0 4294967296"
+ nbd_sock = file_path('nbd-sock', base_dir=iotests.sock_dir)
- CLUSTER_SIZE=65536
+diff --git a/tests/qemu-iotests/210 b/tests/qemu-iotests/210
-diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020
+index XXXXXXX..XXXXXXX 100755
-index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/210
---- a/tests/qemu-iotests/020
++++ b/tests/qemu-iotests/210
-+++ b/tests/qemu-iotests/020
+@@ -XXX,XX +XXX,XX @@
-@@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed
+ import iotests
- _supported_proto file
+ from iotests import imgfmt
- _unsupported_imgopts "subformat=monolithicFlat" \
-                      "subformat=twoGbMaxExtentFlat" \
+-iotests.verify_image_format(supported_fmts=['luks'])
--                     "subformat=twoGbMaxExtentSparse"
+-iotests.verify_protocol(supported=['file'])
-+                     "subformat=twoGbMaxExtentSparse" \
++iotests.script_initialize(
-+                     "subformat=streamOptimized"
++    supported_fmts=['luks'],
++    supported_protocols=['file'],
- TEST_OFFSETS="0 4294967296"
++)
-diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027
+ with iotests.FilePath('t.luks') as disk_path, \
-index XXXXXXX..XXXXXXX 100755
+      iotests.VM() as vm:
---- a/tests/qemu-iotests/027
+diff --git a/tests/qemu-iotests/211 b/tests/qemu-iotests/211
-+++ b/tests/qemu-iotests/027
+index XXXXXXX..XXXXXXX 100755
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+--- a/tests/qemu-iotests/211
++++ b/tests/qemu-iotests/211
- _supported_fmt vmdk qcow qcow2 qed
+@@ -XXX,XX +XXX,XX @@
- _supported_proto generic
+ import iotests
-+_unsupported_imgopts "subformat=streamOptimized"
+ from iotests import imgfmt
+-iotests.verify_image_format(supported_fmts=['vdi'])
- size=128M
+-iotests.verify_protocol(supported=['file'])
-diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032
++iotests.script_initialize(
-index XXXXXXX..XXXXXXX 100755
++    supported_fmts=['vdi'],
---- a/tests/qemu-iotests/032
++    supported_protocols=['file'],
-+++ b/tests/qemu-iotests/032
++)
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
- # This works for any image format (though unlikely to segfault for raw)
+ def blockdev_create(vm, options):
- _supported_fmt generic
+     error = vm.blockdev_create(options)
- _supported_proto generic
+diff --git a/tests/qemu-iotests/212 b/tests/qemu-iotests/212
-+_unsupported_imgopts "subformat=streamOptimized"
+index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/212
- echo
++++ b/tests/qemu-iotests/212
- echo === Prepare image ===
+@@ -XXX,XX +XXX,XX @@
-diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033
+ import iotests
-index XXXXXXX..XXXXXXX 100755
+ from iotests import imgfmt
---- a/tests/qemu-iotests/033
-+++ b/tests/qemu-iotests/033
+-iotests.verify_image_format(supported_fmts=['parallels'])
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+-iotests.verify_protocol(supported=['file'])
++iotests.script_initialize(
- _supported_fmt generic
++    supported_fmts=['parallels'],
- _supported_proto generic
++    supported_protocols=['file'],
-+_unsupported_imgopts "subformat=streamOptimized"
++)
+ with iotests.FilePath('t.parallels') as disk_path, \
- size=128M
+      iotests.VM() as vm:
-diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034
+diff --git a/tests/qemu-iotests/213 b/tests/qemu-iotests/213
 index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/034
+--- a/tests/qemu-iotests/213
-+++ b/tests/qemu-iotests/034
++++ b/tests/qemu-iotests/213
-@@ -XXX,XX +XXX,XX @@ _supported_proto file
+@@ -XXX,XX +XXX,XX @@
- _supported_os Linux
+ import iotests
- _unsupported_imgopts "subformat=monolithicFlat" \
+ from iotests import imgfmt
-                      "subformat=twoGbMaxExtentFlat" \
--                     "subformat=twoGbMaxExtentSparse"
+-iotests.verify_image_format(supported_fmts=['vhdx'])
-+                     "subformat=twoGbMaxExtentSparse" \
+-iotests.verify_protocol(supported=['file'])
-+                     "subformat=streamOptimized"
++iotests.script_initialize(
++    supported_fmts=['vhdx'],
- CLUSTER_SIZE=4k
++    supported_protocols=['file'],
- size=128M
++)
-diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037
-index XXXXXXX..XXXXXXX 100755
+ with iotests.FilePath('t.vhdx') as disk_path, \
---- a/tests/qemu-iotests/037
+      iotests.VM() as vm:
-+++ b/tests/qemu-iotests/037
+diff --git a/tests/qemu-iotests/216 b/tests/qemu-iotests/216
-@@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed
+index XXXXXXX..XXXXXXX 100755
- _supported_proto file
+--- a/tests/qemu-iotests/216
- _unsupported_imgopts "subformat=monolithicFlat" \
++++ b/tests/qemu-iotests/216
-                      "subformat=twoGbMaxExtentFlat" \
+@@ -XXX,XX +XXX,XX @@ import iotests
--                     "subformat=twoGbMaxExtentSparse"
+ from iotests import log, qemu_img, qemu_io_silent
-+                     "subformat=twoGbMaxExtentSparse" \
-+                     "subformat=streamOptimized"
+ # Need backing file support
+-iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk'])
- CLUSTER_SIZE=4k
+-iotests.verify_platform(['linux'])
- size=128M
++iotests.script_initialize(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk'],
-diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063
++                          supported_platforms=['linux'])
-index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/063
+ log('')
-+++ b/tests/qemu-iotests/063
+ log('=== Copy-on-read across nodes ===')
-@@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed raw
+diff --git a/tests/qemu-iotests/218 b/tests/qemu-iotests/218
- _supported_proto file
+index XXXXXXX..XXXXXXX 100755
- _unsupported_imgopts "subformat=monolithicFlat" \
+--- a/tests/qemu-iotests/218
-                      "subformat=twoGbMaxExtentFlat" \
++++ b/tests/qemu-iotests/218
--                     "subformat=twoGbMaxExtentSparse"
+@@ -XXX,XX +XXX,XX @@
-+                     "subformat=twoGbMaxExtentSparse" \
+ import iotests
-+                     "subformat=streamOptimized"
+ from iotests import log, qemu_img, qemu_io_silent
- _make_test_img 4M
+-iotests.verify_image_format(supported_fmts=['qcow2', 'raw'])
++iotests.script_initialize(supported_fmts=['qcow2', 'raw'])
-diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072
-index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/072
+ # Launches the VM, adds two null-co nodes (source and target), and
-+++ b/tests/qemu-iotests/072
+diff --git a/tests/qemu-iotests/219 b/tests/qemu-iotests/219
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/219
- _supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow
++++ b/tests/qemu-iotests/219
- _supported_proto file
+@@ -XXX,XX +XXX,XX @@
-+_unsupported_imgopts "subformat=streamOptimized"
+ import iotests
- IMG_SIZE=64M
+-iotests.verify_image_format(supported_fmts=['qcow2'])
-diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105
++iotests.script_initialize(supported_fmts=['qcow2'])
-index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/105
+ img_size = 4 * 1024 * 1024
-+++ b/tests/qemu-iotests/105
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+diff --git a/tests/qemu-iotests/222 b/tests/qemu-iotests/222
- _supported_fmt qcow2 vmdk vhdx qed
+index XXXXXXX..XXXXXXX 100755
- _supported_proto generic
+--- a/tests/qemu-iotests/222
- _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
++++ b/tests/qemu-iotests/222
--                     "subformat=twoGbMaxExtentSparse"
+@@ -XXX,XX +XXX,XX @@
-+                     "subformat=twoGbMaxExtentSparse" \
+ import iotests
-+                     "subformat=streamOptimized"
+ from iotests import log, qemu_img, qemu_io, qemu_io_silent
- echo
+-iotests.verify_platform(['linux'])
- echo "creating large image"
+-iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk',
-diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197
+-                                            'vhdx', 'raw'])
-index XXXXXXX..XXXXXXX 100755
++iotests.script_initialize(
---- a/tests/qemu-iotests/197
++    supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'],
-+++ b/tests/qemu-iotests/197
++    supported_platforms=['linux'],
-@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
++)
- _supported_proto generic
- # LUKS support may be possible, but it complicates things.
+ patterns = [("0x5d", "0",         "64k"),
- _unsupported_fmt luks
+             ("0xd5", "1M",        "64k"),
-+_unsupported_imgopts "subformat=streamOptimized"
+diff --git a/tests/qemu-iotests/224 b/tests/qemu-iotests/224
+index XXXXXXX..XXXXXXX 100755
- echo
+--- a/tests/qemu-iotests/224
- echo '=== Copy-on-read ==='
++++ b/tests/qemu-iotests/224
-diff --git a/tests/qemu-iotests/215 b/tests/qemu-iotests/215
+@@ -XXX,XX +XXX,XX @@ from iotests import log, qemu_img, qemu_io_silent, filter_qmp_testfiles, \
-index XXXXXXX..XXXXXXX 100755
+ import json
---- a/tests/qemu-iotests/215
-+++ b/tests/qemu-iotests/215
+ # Need backing file support (for arbitrary backing formats)
-@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
+-iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed'])
- _supported_proto generic
+-iotests.verify_platform(['linux'])
- # LUKS support may be possible, but it complicates things.
++iotests.script_initialize(supported_fmts=['qcow2', 'qcow', 'qed'],
- _unsupported_fmt luks
++                          supported_platforms=['linux'])
-+_unsupported_imgopts "subformat=streamOptimized"
- echo
+ # There are two variations of this test:
- echo '=== Copy-on-read ==='
+diff --git a/tests/qemu-iotests/228 b/tests/qemu-iotests/228
-diff --git a/tests/qemu-iotests/251 b/tests/qemu-iotests/251
+index XXXXXXX..XXXXXXX 100755
-index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/228
---- a/tests/qemu-iotests/251
++++ b/tests/qemu-iotests/228
-+++ b/tests/qemu-iotests/251
+@@ -XXX,XX +XXX,XX @@ from iotests import log, qemu_img, filter_testfiles, filter_imgfmt, \
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+         filter_qmp_testfiles, filter_qmp_imgfmt
- _supported_fmt generic
- _supported_proto file
+ # Need backing file and change-backing-file support
- _supported_os Linux
+-iotests.verify_image_format(supported_fmts=['qcow2', 'qed'])
-+_unsupported_imgopts "subformat=streamOptimized"
+-iotests.verify_platform(['linux'])
++iotests.script_initialize(
- if [ "$IMGOPTSSYNTAX" = "true" ]; then
++    supported_fmts=['qcow2', 'qed'],
-     # We use json:{} filenames here, so we cannot work with additional options.
++    supported_platforms=['linux'],
 +)
  def log_node_info(node):
 diff --git a/tests/qemu-iotests/234 b/tests/qemu-iotests/234
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/234
 +++ b/tests/qemu-iotests/234
@@ -XXX,XX +XXX,XX @@
  import iotests
  import os
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 -iotests.verify_platform(['linux'])
 +iotests.script_initialize(supported_fmts=['qcow2'],
 +                          supported_platforms=['linux'])
  with iotests.FilePath('img') as img_path, \
       iotests.FilePath('backing') as backing_path, \
 diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/235
 +++ b/tests/qemu-iotests/235
@@ -XXX,XX +XXX,XX @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
  from qemu.machine import QEMUMachine
 +iotests.script_initialize(supported_fmts=['qcow2'])
 +
  # Note:
  # This test was added to check that mirror dead-lock was fixed (see previous
  # commit before this test addition).
@@ -XXX,XX +XXX,XX @@ from qemu.machine import QEMUMachine
  size = 1 * 1024 * 1024 * 1024
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 -
  disk = file_path('disk')
  # prepare source image
 diff --git a/tests/qemu-iotests/236 b/tests/qemu-iotests/236
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/236
 +++ b/tests/qemu-iotests/236
@@ -XXX,XX +XXX,XX @@
  import iotests
  from iotests import log
 -iotests.verify_image_format(supported_fmts=['generic'])
 +iotests.script_initialize(supported_fmts=['generic'])
  size = 64 * 1024 * 1024
  granularity = 64 * 1024
 diff --git a/tests/qemu-iotests/237 b/tests/qemu-iotests/237
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/237
 +++ b/tests/qemu-iotests/237
@@ -XXX,XX +XXX,XX @@ import math
  import iotests
  from iotests import imgfmt
 -iotests.verify_image_format(supported_fmts=['vmdk'])
 +iotests.script_initialize(supported_fmts=['vmdk'])
  with iotests.FilePath('t.vmdk') as disk_path, \
       iotests.FilePath('t.vmdk.1') as extent1_path, \
 diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/238
 +++ b/tests/qemu-iotests/238
@@ -XXX,XX +XXX,XX @@ import os
  import iotests
  from iotests import log
 +iotests.script_initialize()
 +
  virtio_scsi_device = iotests.get_virtio_scsi_device()
  vm = iotests.VM()
 diff --git a/tests/qemu-iotests/242 b/tests/qemu-iotests/242
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/242
 +++ b/tests/qemu-iotests/242
@@ -XXX,XX +XXX,XX @@ import struct
  from iotests import qemu_img_create, qemu_io, qemu_img_pipe, \
      file_path, img_info_log, log, filter_qemu_io
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(supported_fmts=['qcow2'])
  disk = file_path('disk')
  chunk = 256 * 1024
 diff --git a/tests/qemu-iotests/246 b/tests/qemu-iotests/246
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/246
 +++ b/tests/qemu-iotests/246
@@ -XXX,XX +XXX,XX @@
  import iotests
  from iotests import log
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(supported_fmts=['qcow2'])
  size = 64 * 1024 * 1024 * 1024
  gran_small = 32 * 1024
  gran_large = 128 * 1024
 diff --git a/tests/qemu-iotests/248 b/tests/qemu-iotests/248
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/248
 +++ b/tests/qemu-iotests/248
@@ -XXX,XX +XXX,XX @@
  import iotests
  from iotests import qemu_img_create, qemu_io, file_path, filter_qmp_testfiles
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(supported_fmts=['qcow2'])
  source, target = file_path('source', 'target')
  size = 5 * 1024 * 1024
 diff --git a/tests/qemu-iotests/254 b/tests/qemu-iotests/254
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/254
 +++ b/tests/qemu-iotests/254
@@ -XXX,XX +XXX,XX @@
  import iotests
  from iotests import qemu_img_create, file_path, log
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(supported_fmts=['qcow2'])
  disk, top = file_path('disk', 'top')
  size = 1024 * 1024
 diff --git a/tests/qemu-iotests/255 b/tests/qemu-iotests/255
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/255
 +++ b/tests/qemu-iotests/255
@@ -XXX,XX +XXX,XX @@
  import iotests
  from iotests import imgfmt
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(supported_fmts=['qcow2'])
  iotests.log('Finishing a commit job with background reads')
  iotests.log('============================================')
 diff --git a/tests/qemu-iotests/256 b/tests/qemu-iotests/256
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/256
 +++ b/tests/qemu-iotests/256
@@ -XXX,XX +XXX,XX @@ import os
  import iotests
  from iotests import log
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(supported_fmts=['qcow2'])
  size = 64 * 1024 * 1024
  with iotests.FilePath('img0') as img0_path, \
 diff --git a/tests/qemu-iotests/258 b/tests/qemu-iotests/258
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/258
 +++ b/tests/qemu-iotests/258
@@ -XXX,XX +XXX,XX @@ from iotests import log, qemu_img, qemu_io_silent, \
          filter_qmp_testfiles, filter_qmp_imgfmt
  # Need backing file and change-backing-file support
 -iotests.verify_image_format(supported_fmts=['qcow2', 'qed'])
 -iotests.verify_platform(['linux'])
 -
 +iotests.script_initialize(
 +    supported_fmts=['qcow2', 'qed'],
 +    supported_platforms=['linux'],
 +)
  # Returns a node for blockdev-add
  def node(node_name, path, backing=None, fmt=None, throttle=None):
 diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/260
 +++ b/tests/qemu-iotests/260
@@ -XXX,XX +XXX,XX @@
  import iotests
  from iotests import qemu_img_create, file_path, log, filter_qmp_event
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(
 +    supported_fmts=['qcow2']
 +)
  base, top = file_path('base', 'top')
  size = 64 * 1024 * 3
 diff --git a/tests/qemu-iotests/262 b/tests/qemu-iotests/262
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/262
 +++ b/tests/qemu-iotests/262
@@ -XXX,XX +XXX,XX @@
  import iotests
  import os
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 -iotests.verify_platform(['linux'])
 +iotests.script_initialize(supported_fmts=['qcow2'],
 +                          supported_platforms=['linux'])
  with iotests.FilePath('img') as img_path, \
       iotests.FilePath('mig_fifo') as fifo, \
 diff --git a/tests/qemu-iotests/264 b/tests/qemu-iotests/264
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/264
 +++ b/tests/qemu-iotests/264
@@ -XXX,XX +XXX,XX @@ import iotests
  from iotests import qemu_img_create, qemu_io_silent_check, file_path, \
          qemu_nbd_popen, log
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(
 +    supported_fmts=['qcow2'],
 +)
  disk_a, disk_b, nbd_sock = file_path('disk_a', 'disk_b', 'nbd-sock')
  nbd_uri = 'nbd+unix:///?socket=' + nbd_sock
 diff --git a/tests/qemu-iotests/274 b/tests/qemu-iotests/274
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/274
 +++ b/tests/qemu-iotests/274
@@ -XXX,XX +XXX,XX @@
  import iotests
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 -iotests.verify_platform(['linux'])
 +iotests.script_initialize(supported_fmts=['qcow2'],
 +                          supported_platforms=['linux'])
  size_short = 1 * 1024 * 1024
  size_long = 2 * 1024 * 1024
 diff --git a/tests/qemu-iotests/277 b/tests/qemu-iotests/277
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/277
 +++ b/tests/qemu-iotests/277
@@ -XXX,XX +XXX,XX @@ import subprocess
  import iotests
  from iotests import file_path, log
 +iotests.script_initialize()
 +
  nbd_sock, conf_file = file_path('nbd-sock', 'nbd-fault-injector.conf')
 diff --git a/tests/qemu-iotests/280 b/tests/qemu-iotests/280
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/280
 +++ b/tests/qemu-iotests/280
@@ -XXX,XX +XXX,XX @@
  import iotests
  import os
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 -iotests.verify_protocol(supported=['file'])
 -iotests.verify_platform(['linux'])
 +iotests.script_initialize(
 +    supported_fmts=['qcow2'],
 +    supported_protocols=['file'],
 +    supported_platforms=['linux'],
 +)
  with iotests.FilePath('base') as base_path , \
       iotests.FilePath('top') as top_path, \
 diff --git a/tests/qemu-iotests/283 b/tests/qemu-iotests/283
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/283
 +++ b/tests/qemu-iotests/283
@@ -XXX,XX +XXX,XX @@
  import iotests
  # The test is unrelated to formats, restrict it to qcow2 to avoid extra runs
 -iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.script_initialize(
 +    supported_fmts=['qcow2'],
 +)
  size = 1024 * 1024
 diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/iotests.py
 +++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ import signal
  import struct
  import subprocess
  import sys
 -from typing import (Any, Callable, Dict, Iterable, List, Optional, TypeVar)
 +from typing import (Any, Callable, Dict, Iterable,
 +                    List, Optional, Sequence, TypeVar)
  import unittest
  # pylint: disable=import-error, wrong-import-position
@@ -XXX,XX +XXX,XX @@ def verify_protocol(supported=(), unsupported=()):
      if not_sup or (imgproto in unsupported):
          notrun('not suitable for this protocol: %s' % imgproto)
 -def verify_platform(supported=None, unsupported=None):
 -    if unsupported is not None:
 -        if any((sys.platform.startswith(x) for x in unsupported)):
 -            notrun('not suitable for this OS: %s' % sys.platform)
 +def verify_platform(supported=(), unsupported=()):
 +    if any((sys.platform.startswith(x) for x in unsupported)):
 +        notrun('not suitable for this OS: %s' % sys.platform)
 -    if supported is not None:
 +    if supported:
          if not any((sys.platform.startswith(x) for x in supported)):
              notrun('not suitable for this OS: %s' % sys.platform)
@@ -XXX,XX +XXX,XX @@ def skip_if_user_is_root(func):
              return func(*args, **kwargs)
      return func_wrapper
 -def execute_unittest(output, verbosity, debug):
 +def execute_unittest(debug=False):
 +    """Executes unittests within the calling module."""
 +
 +    verbosity = 2 if debug else 1
 +
 +    if debug:
 +        output = sys.stdout
 +    else:
 +        # We need to filter out the time taken from the output so that
 +        # qemu-iotest can reliably diff the results against master output.
 +        output = io.StringIO()
 +
      runner = unittest.TextTestRunner(stream=output, descriptions=True,
                                       verbosity=verbosity)
      try:
@@ -XXX,XX +XXX,XX @@ def execute_unittest(output, verbosity, debug):
          # exception
          unittest.main(testRunner=runner)
      finally:
 +        # We need to filter out the time taken from the output so that
 +        # qemu-iotest can reliably diff the results against master output.
          if not debug:
              out = output.getvalue()
              out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out)
@@ -XXX,XX +XXX,XX @@ def execute_unittest(output, verbosity, debug):
              sys.stderr.write(out)
 -def execute_test(test_function=None,
 -                 supported_fmts=(),
 -                 supported_platforms=None,
 -                 supported_cache_modes=(), supported_aio_modes=(),
 -                 unsupported_fmts=(), supported_protocols=(),
 -                 unsupported_protocols=()):
 -    """Run either unittest or script-style tests."""
 +def execute_setup_common(supported_fmts: Sequence[str] = (),
 +                         supported_platforms: Sequence[str] = (),
 +                         supported_cache_modes: Sequence[str] = (),
 +                         supported_aio_modes: Sequence[str] = (),
 +                         unsupported_fmts: Sequence[str] = (),
 +                         supported_protocols: Sequence[str] = (),
 +                         unsupported_protocols: Sequence[str] = ()) -> bool:
 +    """
 +    Perform necessary setup for either script-style or unittest-style tests.
 +
 +    :return: Bool; Whether or not debug mode has been requested via the CLI.
 +    """
 +    # Note: Python 3.6 and pylint do not like 'Collection' so use 'Sequence'.
      # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
      # indicate that we're not being run via "check". There may be
@@ -XXX,XX +XXX,XX @@ def execute_test(test_function=None,
          sys.stderr.write('Please run this test via the "check" script\n')
          sys.exit(os.EX_USAGE)
 -    debug = '-d' in sys.argv
 -    verbosity = 1
      verify_image_format(supported_fmts, unsupported_fmts)
      verify_protocol(supported_protocols, unsupported_protocols)
      verify_platform(supported=supported_platforms)
      verify_cache_mode(supported_cache_modes)
      verify_aio_mode(supported_aio_modes)
 +    debug = '-d' in sys.argv
      if debug:
 -        output = sys.stdout
 -        verbosity = 2
          sys.argv.remove('-d')
 -    else:
 -        # We need to filter out the time taken from the output so that
 -        # qemu-iotest can reliably diff the results against master output.
 -        output = io.StringIO()
 -
      logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN))
 +    return debug
 +
 +def execute_test(*args, test_function=None, **kwargs):
 +    """Run either unittest or script-style tests."""
 +
 +    debug = execute_setup_common(*args, **kwargs)
      if not test_function:
 -        execute_unittest(output, verbosity, debug)
 +        execute_unittest(debug)
      else:
          test_function()
 +# This is called from script-style iotests without a single point of entry
 +def script_initialize(*args, **kwargs):
 +    """Initialize script-style tests without running any tests."""
 +    execute_setup_common(*args, **kwargs)
 +
 +# This is called from script-style iotests with a single point of entry
  def script_main(test_function, *args, **kwargs):
      """Run script-style tests outside of the unittest framework"""
 -    execute_test(test_function, *args, **kwargs)
 +    execute_test(*args, test_function=test_function, **kwargs)
 +# This is called from unittest style iotests
  def main(*args, **kwargs):
      """Run tests using the unittest framework"""
 -    execute_test(None, *args, **kwargs)
 +    execute_test(*args, **kwargs)
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 08/15] vmdk: Reject invalid compressed writes
+[PULL 12/24] iotest 258: use script_main
-Compressed writes generally have to write full clusters, not just in
+From: John Snow <jsnow@redhat.com>
 theory but also in practice when it comes to vmdk's streamOptimized
 subformat.  It currently is just silently broken for writes with
 non-zero in-cluster offsets:
-$ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M
+Since this one is nicely factored to use a single entry point,
-$ qemu-io -c 'write 4k 4k' -c 'read 4k 4k' foo.vmdk
+use script_main to run the tests.
 wrote 4096/4096 bytes at offset 4096
 KiB, 1 ops; 00.01 sec (443.724 KiB/sec and 110.9309 ops/sec)
 read failed: Invalid argument
-(The technical reason is that vmdk_write_extent() just writes the
+Signed-off-by: John Snow <jsnow@redhat.com>
-incomplete compressed data actually to offset 4k.  When reading the
+Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
-data, vmdk_read_extent() looks at offset 0 and finds the compressed data
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-size to be 0, because that is what it reads from there.  This yields an
+Message-Id: <20200331000014.11581-13-jsnow@redhat.com>
-error.)
+Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 For incomplete writes with zero in-cluster offsets, the error path when
 reading the rest of the cluster is a bit different, but the result is
 the same:
 $ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M
 $ qemu-io -c 'write 0k 4k' -c 'read 4k 4k' foo.vmdk
 wrote 4096/4096 bytes at offset 0
 KiB, 1 ops; 00.01 sec (362.641 KiB/sec and 90.6603 ops/sec)
 read failed: Invalid argument
 (Here, vmdk_read_extent() finds the data and then sees that the
 uncompressed data is short.)
 It is better to reject invalid writes than to make the user believe they
 might have succeeded and then fail when trying to read it back.
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 Reviewed-by: John Snow <jsnow@redhat.com>
 Message-id: 20190815153638.4600-5-mreitz@redhat.com
 Reviewed-by: John Snow <jsnow@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- block/vmdk.c | 10 ++++++++++
+ tests/qemu-iotests/258 | 11 ++++-------
-file changed, 10 insertions(+)
+file changed, 4 insertions(+), 7 deletions(-)
-diff --git a/block/vmdk.c b/block/vmdk.c
+diff --git a/tests/qemu-iotests/258 b/tests/qemu-iotests/258
-index XXXXXXX..XXXXXXX 100644
+index XXXXXXX..XXXXXXX 100755
---- a/block/vmdk.c
+--- a/tests/qemu-iotests/258
-+++ b/block/vmdk.c
++++ b/tests/qemu-iotests/258
-@@ -XXX,XX +XXX,XX @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
+@@ -XXX,XX +XXX,XX @@ import iotests
-     if (extent->compressed) {
+ from iotests import log, qemu_img, qemu_io_silent, \
-         void *compressed_data;
+         filter_qmp_testfiles, filter_qmp_imgfmt
-+        /* Only whole clusters */
+-# Need backing file and change-backing-file support
-+        if (offset_in_cluster ||
+-iotests.script_initialize(
-+            n_bytes > (extent->cluster_sectors * SECTOR_SIZE) ||
+-    supported_fmts=['qcow2', 'qed'],
-+            (n_bytes < (extent->cluster_sectors * SECTOR_SIZE) &&
+-    supported_platforms=['linux'],
-+             offset + n_bytes != extent->end_sector * SECTOR_SIZE))
+-)
-+        {
+-
-+            ret = -EINVAL;
+ # Returns a node for blockdev-add
-+            goto out;
+ def node(node_name, path, backing=None, fmt=None, throttle=None):
-+        }
+     if fmt is None:
-+
+@@ -XXX,XX +XXX,XX @@ def main():
-         if (!extent->has_marker) {
+     test_concurrent_finish(False)
-             ret = -EINVAL;
-             goto out;
+ if __name__ == '__main__':
 -    main()
 +    # Need backing file and change-backing-file support
 +    iotests.script_main(main,
 +                        supported_fmts=['qcow2', 'qed'],
 +                        supported_platforms=['linux'])
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 05/15] iotests: Fix _filter_img_create()
+[PULL 13/24] iotests: Mark verify functions as private
-fe646693acc changed qemu-img create's output so that it no longer prints
+From: John Snow <jsnow@redhat.com>
 single quotes around parameter values.  The subformat and adapter_type
 filters in _filter_img_create() have never been adapted to that change.
-Fixes: fe646693acc13ac48b98435d14149ab04dc597bc
+Mark the verify functions as "private" with a leading underscore, to
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+discourage their use. Update type signatures while we're here.
-Reviewed-by: John Snow <jsnow@redhat.com>
-Message-id: 20190815153638.4600-2-mreitz@redhat.com
+(Also, make pending patches not yet using the new entry points fail in a
-Reviewed-by: John Snow <jsnow@redhat.com>
+very obvious way.)
 Signed-off-by: John Snow <jsnow@redhat.com>
 Message-Id: <20200331000014.11581-14-jsnow@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/059.out       | 16 ++++++++--------
+ tests/qemu-iotests/iotests.py | 23 +++++++++++++----------
- tests/qemu-iotests/common.filter |  4 ++--
+file changed, 13 insertions(+), 10 deletions(-)
 files changed, 10 insertions(+), 10 deletions(-)
-diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
+diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
 index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/059.out
+--- a/tests/qemu-iotests/iotests.py
-+++ b/tests/qemu-iotests/059.out
++++ b/tests/qemu-iotests/iotests.py
-@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+@@ -XXX,XX +XXX,XX @@ def case_notrun(reason):
- qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
+     open('%s/%s.casenotrun' % (output_dir, seq), 'a').write(
+         '    [case not run] ' + reason + '\n')
- === Testing monolithicFlat creation and opening ===
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
+-def verify_image_format(supported_fmts=(), unsupported_fmts=()):
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
++def _verify_image_format(supported_fmts: Sequence[str] = (),
- image: TEST_DIR/t.IMGFMT
++                         unsupported_fmts: Sequence[str] = ()) -> None:
- file format: IMGFMT
+     assert not (supported_fmts and unsupported_fmts)
- virtual size: 2 GiB (2147483648 bytes)
+     if 'generic' in supported_fmts and \
- === Testing monolithicFlat with zeroed_grain ===
+@@ -XXX,XX +XXX,XX @@ def verify_image_format(supported_fmts=(), unsupported_fmts=()):
- qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
+     if not_sup or (imgfmt in unsupported_fmts):
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
+         notrun('not suitable for this image format: %s' % imgfmt)
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+-def verify_protocol(supported=(), unsupported=()):
- === Testing big twoGbMaxExtentFlat ===
++def _verify_protocol(supported: Sequence[str] = (),
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat
++                     unsupported: Sequence[str] = ()) -> None:
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000
+     assert not (supported and unsupported)
- image: TEST_DIR/t.vmdk
- file format: vmdk
+     if 'generic' in supported:
- virtual size: 0.977 TiB (1073741824000 bytes)
+@@ -XXX,XX +XXX,XX @@ def verify_protocol(supported=(), unsupported=()):
-@@ -XXX,XX +XXX,XX @@ Format specific information:
+     if not_sup or (imgproto in unsupported):
- qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1
+         notrun('not suitable for this protocol: %s' % imgproto)
- === Testing truncated sparse ===
+-def verify_platform(supported=(), unsupported=()):
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse
++def _verify_platform(supported: Sequence[str] = (),
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400
++                     unsupported: Sequence[str] = ()) -> None:
- qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes
+     if any((sys.platform.startswith(x) for x in unsupported)):
+         notrun('not suitable for this OS: %s' % sys.platform)
- === Converting to streamOptimized from image with small cluster size===
-@@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240
+@@ -XXX,XX +XXX,XX @@ def verify_platform(supported=(), unsupported=()):
-bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+         if not any((sys.platform.startswith(x) for x in supported)):
+             notrun('not suitable for this OS: %s' % sys.platform)
- === Testing monolithicFlat with internally generated JSON file name ===
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat
+-def verify_cache_mode(supported_cache_modes=()):
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
++def _verify_cache_mode(supported_cache_modes: Sequence[str] = ()) -> None:
- qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
+     if supported_cache_modes and (cachemode not in supported_cache_modes):
+         notrun('not suitable for this cache mode: %s' % cachemode)
- === Testing version 3 ===
-@@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 64931328
+-def verify_aio_mode(supported_aio_modes=()):
-bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
++def _verify_aio_mode(supported_aio_modes: Sequence[str] = ()):
+     if supported_aio_modes and (aiomode not in supported_aio_modes):
- === Testing 4TB monolithicFlat creation and IO ===
+         notrun('not suitable for this aio mode: %s' % aiomode)
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104
+@@ -XXX,XX +XXX,XX @@ def execute_setup_common(supported_fmts: Sequence[str] = (),
- image: TEST_DIR/t.IMGFMT
+         sys.stderr.write('Please run this test via the "check" script\n')
- file format: IMGFMT
+         sys.exit(os.EX_USAGE)
- virtual size: 4 TiB (4398046511104 bytes)
-@@ -XXX,XX +XXX,XX @@ read 1024/1024 bytes at offset 966367641600
+-    verify_image_format(supported_fmts, unsupported_fmts)
-KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+-    verify_protocol(supported_protocols, unsupported_protocols)
+-    verify_platform(supported=supported_platforms)
- === Testing qemu-img map on extents ===
+-    verify_cache_mode(supported_cache_modes)
--Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
+-    verify_aio_mode(supported_aio_modes)
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544
++    _verify_image_format(supported_fmts, unsupported_fmts)
- wrote 1024/1024 bytes at offset 65024
++    _verify_protocol(supported_protocols, unsupported_protocols)
-KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
++    _verify_platform(supported=supported_platforms)
- wrote 1024/1024 bytes at offset 2147483136
++    _verify_cache_mode(supported_cache_modes)
-@@ -XXX,XX +XXX,XX @@ Offset          Length          Mapped to       File
++    _verify_aio_mode(supported_aio_modes)
-0x20000         0x3f0000        TEST_DIR/t.vmdk
-x7fff0000      0x20000         0x410000        TEST_DIR/t.vmdk
+     debug = '-d' in sys.argv
-x140000000     0x10000         0x430000        TEST_DIR/t.vmdk
+     if debug:
 -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
 +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544
  wrote 1024/1024 bytes at offset 65024
 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
  wrote 1024/1024 bytes at offset 2147483136
 diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/common.filter
 +++ b/tests/qemu-iotests/common.filter
@@ -XXX,XX +XXX,XX @@ _filter_img_create()
          -e "s# compat6=\\(on\\|off\\)##g" \
          -e "s# static=\\(on\\|off\\)##g" \
          -e "s# zeroed_grain=\\(on\\|off\\)##g" \
 -        -e "s# subformat='[^']*'##g" \
 -        -e "s# adapter_type='[^']*'##g" \
 +        -e "s# subformat=[^ ]*##g" \
 +        -e "s# adapter_type=[^ ]*##g" \
          -e "s# hwversion=[^ ]*##g" \
          -e "s# lazy_refcounts=\\(on\\|off\\)##g" \
          -e "s# block_size=[0-9]\\+##g" \
 --
-.21.0
+.26.2

-New patch
+[PULL 14/24] iotests: use python logging for iotests.log()
+From: John Snow <jsnow@redhat.com>
 We can turn logging on/off globally instead of per-function.
 Remove use_log from run_job, and use python logging to turn on
 diffable output when we run through a script entry point.
 iotest 245 changes output order due to buffering reasons.
 An extended note on python logging:
 A NullHandler is added to `qemu.iotests` to stop output from being
 generated if this code is used as a library without configuring logging.
 A NullHandler is only needed at the root, so a duplicate handler is not
 needed for `qemu.iotests.diff_io`.
 When logging is not configured, messages at the 'WARNING' levels or
 above are printed with default settings. The NullHandler stops this from
 occurring, which is considered good hygiene for code used as a library.
 See https://docs.python.org/3/howto/logging.html#library-config
 When logging is actually enabled (always at the behest of an explicit
 call by a client script), a root logger is implicitly created at the
 root, which allows messages to propagate upwards and be handled/emitted
 from the root logger with default settings.
 When we want iotest logging, we attach a handler to the
 qemu.iotests.diff_io logger and disable propagation to avoid possible
 double-printing.
 For more information on python logging infrastructure, I highly
 recommend downloading the pip package `logging_tree`, which provides
 convenient visualizations of the hierarchical logging configuration
 under different circumstances.
 See https://pypi.org/project/logging_tree/ for more information.
 Signed-off-by: John Snow <jsnow@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200331000014.11581-15-jsnow@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  tests/qemu-iotests/030        |  4 +--
  tests/qemu-iotests/155        |  2 +-
  tests/qemu-iotests/245        |  1 +
  tests/qemu-iotests/245.out    | 10 +++----
  tests/qemu-iotests/iotests.py | 53 ++++++++++++++++++++---------------
 files changed, 39 insertions(+), 31 deletions(-)
 diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/030
 +++ b/tests/qemu-iotests/030
@@ -XXX,XX +XXX,XX @@ class TestParallelOps(iotests.QMPTestCase):
          result = self.vm.qmp('block-job-set-speed', device='drive0', speed=0)
          self.assert_qmp(result, 'return', {})
 -        self.vm.run_job(job='drive0', auto_dismiss=True, use_log=False)
 -        self.vm.run_job(job='node4', auto_dismiss=True, use_log=False)
 +        self.vm.run_job(job='drive0', auto_dismiss=True)
 +        self.vm.run_job(job='node4', auto_dismiss=True)
          self.assert_no_active_block_jobs()
      # Test a block-stream and a block-commit job in parallel
 diff --git a/tests/qemu-iotests/155 b/tests/qemu-iotests/155
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/155
 +++ b/tests/qemu-iotests/155
@@ -XXX,XX +XXX,XX @@ class MirrorBaseClass(BaseClass):
          self.assert_qmp(result, 'return', {})
 -        self.vm.run_job('mirror-job', use_log=False, auto_finalize=False,
 +        self.vm.run_job('mirror-job', auto_finalize=False,
                          pre_finalize=self.openBacking, auto_dismiss=True)
      def testFull(self):
 diff --git a/tests/qemu-iotests/245 b/tests/qemu-iotests/245
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/245
 +++ b/tests/qemu-iotests/245
@@ -XXX,XX +XXX,XX @@ class TestBlockdevReopen(iotests.QMPTestCase):
          self.run_test_iothreads(None, 'iothread0')
  if __name__ == '__main__':
 +    iotests.activate_logging()
      iotests.main(supported_fmts=["qcow2"],
                   supported_protocols=["file"])
 diff --git a/tests/qemu-iotests/245.out b/tests/qemu-iotests/245.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/245.out
 +++ b/tests/qemu-iotests/245.out
@@ -XXX,XX +XXX,XX @@
 -.....................
 -----------------------------------------------------------------------
 -Ran 21 tests
 -
 -OK
  {"execute": "job-finalize", "arguments": {"id": "commit0"}}
  {"return": {}}
  {"data": {"id": "commit0", "type": "commit"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
@@ -XXX,XX +XXX,XX @@ OK
  {"return": {}}
  {"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
  {"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
 +.....................
 +----------------------------------------------------------------------
 +Ran 21 tests
 +
 +OK
 diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/iotests.py
 +++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ assert sys.version_info >= (3, 6)
  QMPResponse = Dict[str, Any]
 +# Use this logger for logging messages directly from the iotests module
 +logger = logging.getLogger('qemu.iotests')
 +logger.addHandler(logging.NullHandler())
 +
 +# Use this logger for messages that ought to be used for diff output.
 +test_logger = logging.getLogger('qemu.iotests.diff_io')
 +
 +
  faulthandler.enable()
  # This will not work if arguments contain spaces but is necessary if we
@@ -XXX,XX +XXX,XX @@ def log(msg: Msg,
      if isinstance(msg, (dict, list)):
          # Don't sort if it's already sorted
          do_sort = not isinstance(msg, OrderedDict)
 -        print(json.dumps(msg, sort_keys=do_sort, indent=indent))
 +        test_logger.info(json.dumps(msg, sort_keys=do_sort, indent=indent))
      else:
 -        print(msg)
 +        test_logger.info(msg)
  class Timeout:
      def __init__(self, seconds, errmsg="Timeout"):
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
      # Returns None on success, and an error string on failure
      def run_job(self, job, auto_finalize=True, auto_dismiss=False,
 -                pre_finalize=None, cancel=False, use_log=True, wait=60.0):
 +                pre_finalize=None, cancel=False, wait=60.0):
          """
          run_job moves a job from creation through to dismissal.
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
                               invoked prior to issuing job-finalize, if any.
          :param cancel: Bool. When true, cancels the job after the pre_finalize
                         callback.
 -        :param use_log: Bool. When false, does not log QMP messages.
          :param wait: Float. Timeout value specifying how long to wait for any
                       event, in seconds. Defaults to 60.0.
          """
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
          while True:
              ev = filter_qmp_event(self.events_wait(events, timeout=wait))
              if ev['event'] != 'JOB_STATUS_CHANGE':
 -                if use_log:
 -                    log(ev)
 +                log(ev)
                  continue
              status = ev['data']['status']
              if status == 'aborting':
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
                  for j in result['return']:
                      if j['id'] == job:
                          error = j['error']
 -                        if use_log:
 -                            log('Job failed: %s' % (j['error']))
 +                        log('Job failed: %s' % (j['error']))
              elif status == 'ready':
 -                if use_log:
 -                    self.qmp_log('job-complete', id=job)
 -                else:
 -                    self.qmp('job-complete', id=job)
 +                self.qmp_log('job-complete', id=job)
              elif status == 'pending' and not auto_finalize:
                  if pre_finalize:
                      pre_finalize()
 -                if cancel and use_log:
 +                if cancel:
                      self.qmp_log('job-cancel', id=job)
 -                elif cancel:
 -                    self.qmp('job-cancel', id=job)
 -                elif use_log:
 -                    self.qmp_log('job-finalize', id=job)
                  else:
 -                    self.qmp('job-finalize', id=job)
 +                    self.qmp_log('job-finalize', id=job)
              elif status == 'concluded' and not auto_dismiss:
 -                if use_log:
 -                    self.qmp_log('job-dismiss', id=job)
 -                else:
 -                    self.qmp('job-dismiss', id=job)
 +                self.qmp_log('job-dismiss', id=job)
              elif status == 'null':
                  return error
@@ -XXX,XX +XXX,XX @@ def notrun(reason):
      seq = os.path.basename(sys.argv[0])
      open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n')
 -    print('%s not run: %s' % (seq, reason))
 +    logger.warning("%s not run: %s", seq, reason)
      sys.exit(0)
  def case_notrun(reason):
@@ -XXX,XX +XXX,XX @@ def execute_setup_common(supported_fmts: Sequence[str] = (),
      if debug:
          sys.argv.remove('-d')
      logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN))
 +    logger.debug("iotests debugging messages active")
      return debug
@@ -XXX,XX +XXX,XX @@ def execute_test(*args, test_function=None, **kwargs):
      else:
          test_function()
 +def activate_logging():
 +    """Activate iotests.log() output to stdout for script-style tests."""
 +    handler = logging.StreamHandler(stream=sys.stdout)
 +    formatter = logging.Formatter('%(message)s')
 +    handler.setFormatter(formatter)
 +    test_logger.addHandler(handler)
 +    test_logger.setLevel(logging.INFO)
 +    test_logger.propagate = False
 +
  # This is called from script-style iotests without a single point of entry
  def script_initialize(*args, **kwargs):
      """Initialize script-style tests without running any tests."""
 +    activate_logging()
      execute_setup_common(*args, **kwargs)
  # This is called from script-style iotests with a single point of entry
  def script_main(test_function, *args, **kwargs):
      """Run script-style tests outside of the unittest framework"""
 +    activate_logging()
      execute_test(*args, test_function=test_function, **kwargs)
  # This is called from unittest style iotests
 --
 .26.2

-[Qemu-devel] [PULL 01/15] qemu-io: add pattern file for write command
+[PULL 15/24] block: Add blk_new_with_bs() helper
-From: Denis Plotnikov <dplotnikov@virtuozzo.com>
+From: Eric Blake <eblake@redhat.com>
-The patch allows to provide a pattern file for write
+There are several callers that need to create a new block backend from
-command. There was no similar ability before.
+an existing BDS; make the task slightly easier with a common helper
+routine.
-Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
-Message-id: 20190820164616.4072-1-dplotnikov@virtuozzo.com
+Suggested-by: Max Reitz <mreitz@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Signed-off-by: Eric Blake <eblake@redhat.com>
-[mreitz: Keep optstring in alphabetical order]
+Message-Id: <20200424190903.522087-2-eblake@redhat.com>
 [mreitz: Set @ret only in error paths, see
  https://lists.nongnu.org/archive/html/qemu-block/2020-04/msg01216.html]
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200428192648.749066-2-eblake@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- qemu-io-cmds.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++---
+ include/sysemu/block-backend.h |  2 ++
-file changed, 93 insertions(+), 6 deletions(-)
+ block/block-backend.c          | 23 +++++++++++++++++++++++
+ block/crypto.c                 |  9 ++++-----
-diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
+ block/parallels.c              |  8 ++++----
-index XXXXXXX..XXXXXXX 100644
+ block/qcow.c                   |  8 ++++----
---- a/qemu-io-cmds.c
+ block/qcow2.c                  | 18 ++++++++----------
-+++ b/qemu-io-cmds.c
+ block/qed.c                    |  8 ++++----
-@@ -XXX,XX +XXX,XX @@ static void qemu_io_free(void *p)
+ block/sheepdog.c               | 10 +++++-----
-     qemu_vfree(p);
+ block/vdi.c                    |  8 ++++----
  block/vhdx.c                   |  8 ++++----
  block/vmdk.c                   |  9 ++++-----
  block/vpc.c                    |  8 ++++----
  blockdev.c                     |  8 +++-----
  blockjob.c                     |  7 ++-----
 files changed, 75 insertions(+), 59 deletions(-)
 diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/sysemu/block-backend.h
 +++ b/include/sysemu/block-backend.h
@@ -XXX,XX +XXX,XX @@ typedef struct BlockBackendPublic {
  } BlockBackendPublic;
  BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm);
 +BlockBackend *blk_new_with_bs(BlockDriverState *bs, uint64_t perm,
 +                              uint64_t shared_perm, Error **errp);
  BlockBackend *blk_new_open(const char *filename, const char *reference,
                             QDict *options, int flags, Error **errp);
  int blk_get_refcnt(BlockBackend *blk);
 diff --git a/block/block-backend.c b/block/block-backend.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/block-backend.c
 +++ b/block/block-backend.c
@@ -XXX,XX +XXX,XX @@ BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm)
      return blk;
  }
 +/*
-+ * qemu_io_alloc_from_file()
++ * Create a new BlockBackend connected to an existing BlockDriverState.
 + *
-+ * Allocates the buffer and populates it with the content of the given file
++ * @perm is a bitmasks of BLK_PERM_* constants which describes the
-+ * up to @len bytes. If the file length is less than @len, then the buffer
++ * permissions to request for @bs that is attached to this
-+ * is populated with the file content cyclically.
++ * BlockBackend.  @shared_perm is a bitmask which describes which
 + * permissions may be granted to other users of the attached node.
 + * Both sets of permissions can be changed later using blk_set_perm().
 + *
-+ * @blk - the block backend where the buffer content is going to be written to
++ * Return the new BlockBackend on success, null on failure.
 + * @len - the buffer length
 + * @file_name - the file to read the content from
 + *
 + * Returns: the buffer pointer on success
 + *          NULL on error
 + */
-+static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
++BlockBackend *blk_new_with_bs(BlockDriverState *bs, uint64_t perm,
-+                                     const char *file_name)
++                              uint64_t shared_perm, Error **errp)
 +{
-+    char *buf, *buf_origin;
++    BlockBackend *blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
 +    FILE *f = fopen(file_name, "r");
 +    int pattern_len;
 +
-+    if (!f) {
++    if (blk_insert_bs(blk, bs, errp) < 0) {
-+        perror(file_name);
++        blk_unref(blk);
 +        return NULL;
 +    }
-+
++    return blk;
 +    if (qemuio_misalign) {
 +        len += MISALIGN_OFFSET;
 +    }
 +
 +    buf_origin = buf = blk_blockalign(blk, len);
 +
 +    if (qemuio_misalign) {
 +        buf_origin += MISALIGN_OFFSET;
 +        buf += MISALIGN_OFFSET;
 +        len -= MISALIGN_OFFSET;
 +    }
 +
 +    pattern_len = fread(buf_origin, 1, len, f);
 +
 +    if (ferror(f)) {
 +        perror(file_name);
 +        goto error;
 +    }
 +
 +    if (pattern_len == 0) {
 +        fprintf(stderr, "%s: file is empty\n", file_name);
 +        goto error;
 +    }
 +
 +    fclose(f);
 +
 +    if (len > pattern_len) {
 +        len -= pattern_len;
 +        buf += pattern_len;
 +
 +        while (len > 0) {
 +            size_t len_to_copy = MIN(pattern_len, len);
 +
 +            memcpy(buf, buf_origin, len_to_copy);
 +
 +            len -= len_to_copy;
 +            buf += len_to_copy;
 +        }
 +    }
 +
 +    return buf_origin;
 +
 +error:
 +    qemu_io_free(buf_origin);
 +    return NULL;
 +}
 +
- static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
+ /*
   * Creates a new BlockBackend, opens a new BlockDriverState, and connects both.
   * The new BlockBackend is in the main AioContext.
 diff --git a/block/crypto.c b/block/crypto.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/crypto.c
 +++ b/block/crypto.c
@@ -XXX,XX +XXX,XX @@ static int block_crypto_co_create_generic(BlockDriverState *bs,
      QCryptoBlock *crypto = NULL;
      struct BlockCryptoCreateData data;
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                          errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto cleanup;
      }
 diff --git a/block/parallels.c b/block/parallels.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/parallels.c
 +++ b/block/parallels.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn parallels_co_create(BlockdevCreateOptions* opts,
          return -EIO;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                          errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto out;
      }
      blk_set_allow_write_beyond_eof(blk, true);
 diff --git a/block/qcow.c b/block/qcow.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/qcow.c
 +++ b/block/qcow.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow_co_create(BlockdevCreateOptions *opts,
          return -EIO;
      }
 -    qcow_blk = blk_new(bdrv_get_aio_context(bs),
 -                       BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(qcow_blk, bs, errp);
 -    if (ret < 0) {
 +    qcow_blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE,
 +                               BLK_PERM_ALL, errp);
 +    if (!qcow_blk) {
 +        ret = -EPERM;
          goto exit;
      }
      blk_set_allow_write_beyond_eof(qcow_blk, true);
 diff --git a/block/qcow2.c b/block/qcow2.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/qcow2.c
 +++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
      }
      /* Create BlockBackend to write to the image */
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                          errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto out;
      }
      blk_set_allow_write_beyond_eof(blk, true);
@@ -XXX,XX +XXX,XX @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
      }
      if (new_size) {
 -        BlockBackend *blk = blk_new(bdrv_get_aio_context(bs),
 -                                    BLK_PERM_RESIZE, BLK_PERM_ALL);
 -        ret = blk_insert_bs(blk, bs, errp);
 -        if (ret < 0) {
 -            blk_unref(blk);
 -            return ret;
 +        BlockBackend *blk = blk_new_with_bs(bs, BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                                            errp);
 +        if (!blk) {
 +            return -EPERM;
          }
          /*
 diff --git a/block/qed.c b/block/qed.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/qed.c
 +++ b/block/qed.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_qed_co_create(BlockdevCreateOptions *opts,
          return -EIO;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                          errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto out;
      }
      blk_set_allow_write_beyond_eof(blk, true);
 diff --git a/block/sheepdog.c b/block/sheepdog.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/sheepdog.c
 +++ b/block/sheepdog.c
@@ -XXX,XX +XXX,XX @@ static int sd_prealloc(BlockDriverState *bs, int64_t old_size, int64_t new_size,
      void *buf = NULL;
      int ret;
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
 -                  BLK_PERM_ALL);
 +    blk = blk_new_with_bs(bs,
 +                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
 +                          BLK_PERM_ALL, errp);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    if (!blk) {
 +        ret = -EPERM;
          goto out_with_err_set;
      }
 diff --git a/block/vdi.c b/block/vdi.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/vdi.c
 +++ b/block/vdi.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn vdi_co_do_create(BlockdevCreateOptions *create_options,
          goto exit;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs_file),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs_file, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs_file, BLK_PERM_WRITE | BLK_PERM_RESIZE,
 +                          BLK_PERM_ALL, errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto exit;
      }
 diff --git a/block/vhdx.c b/block/vhdx.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/vhdx.c
 +++ b/block/vhdx.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn vhdx_co_create(BlockdevCreateOptions *opts,
          return -EIO;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                          errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto delete_and_exit;
      }
      blk_set_allow_write_beyond_eof(blk, true);
 diff --git a/block/vmdk.c b/block/vmdk.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/vmdk.c
 +++ b/block/vmdk.c
@@ -XXX,XX +XXX,XX @@ static BlockBackend *vmdk_co_create_cb(int64_t size, int idx,
      if (!bs) {
          return NULL;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
 -                  BLK_PERM_ALL);
 -    if (blk_insert_bs(blk, bs, errp)) {
 -        bdrv_unref(bs);
 +    blk = blk_new_with_bs(bs,
 +                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
 +                          BLK_PERM_ALL, errp);
 +    if (!blk) {
          return NULL;
      }
      blk_set_allow_write_beyond_eof(blk, true);
 diff --git a/block/vpc.c b/block/vpc.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/vpc.c
 +++ b/block/vpc.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn vpc_co_create(BlockdevCreateOptions *opts,
          return -EIO;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs),
 -                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                          errp);
 +    if (!blk) {
 +        ret = -EPERM;
          goto out;
      }
      blk_set_allow_write_beyond_eof(blk, true);
 diff --git a/blockdev.c b/blockdev.c
 index XXXXXXX..XXXXXXX 100644
 --- a/blockdev.c
 +++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ void qmp_block_resize(bool has_device, const char *device,
      BlockBackend *blk = NULL;
      BlockDriverState *bs;
      AioContext *aio_context;
 -    int ret;
      bs = bdrv_lookup_bs(has_device ? device : NULL,
                          has_node_name ? node_name : NULL,
@@ -XXX,XX +XXX,XX @@ void qmp_block_resize(bool has_device, const char *device,
          goto out;
      }
 -    blk = blk_new(bdrv_get_aio_context(bs), BLK_PERM_RESIZE, BLK_PERM_ALL);
 -    ret = blk_insert_bs(blk, bs, errp);
 -    if (ret < 0) {
 +    blk = blk_new_with_bs(bs, BLK_PERM_RESIZE, BLK_PERM_ALL, errp);
 +    if (!blk) {
          goto out;
      }
      bdrv_drained_begin(bs);
 -    ret = blk_truncate(blk, size, false, PREALLOC_MODE_OFF, 0, errp);
 +    blk_truncate(blk, size, false, PREALLOC_MODE_OFF, 0, errp);
      bdrv_drained_end(bs);
  out:
 diff --git a/blockjob.c b/blockjob.c
 index XXXXXXX..XXXXXXX 100644
 --- a/blockjob.c
 +++ b/blockjob.c
@@ -XXX,XX +XXX,XX @@ void *block_job_create(const char *job_id, const BlockJobDriver *driver,
  {
-     uint64_t i;
+     BlockBackend *blk;
-@@ -XXX,XX +XXX,XX @@ static void write_help(void)
+     BlockJob *job;
- " -n, -- with -z, don't allow slow fallback\n"
+-    int ret;
- " -p, -- ignored for backwards compatibility\n"
- " -P, -- use different pattern to fill file\n"
+     if (job_id == NULL && !(flags & JOB_INTERNAL)) {
-+" -s, -- use a pattern file to fill the write buffer\n"
+         job_id = bdrv_get_device_name(bs);
- " -C, -- report statistics in a machine parsable format\n"
+     }
- " -q, -- quiet mode, do not show I/O statistics\n"
- " -u, -- with -z, allow unmapping\n"
+-    blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
-@@ -XXX,XX +XXX,XX @@ static const cmdinfo_t write_cmd = {
+-    ret = blk_insert_bs(blk, bs, errp);
-     .perm       = BLK_PERM_WRITE,
+-    if (ret < 0) {
-     .argmin     = 2,
+-        blk_unref(blk);
-     .argmax     = -1,
++    blk = blk_new_with_bs(bs, perm, shared_perm, errp);
--    .args       = "[-bcCfnquz] [-P pattern] off len",
++    if (!blk) {
-+    .args       = "[-bcCfnquz] [-P pattern | -s source_file] off len",
+         return NULL;
-     .oneline    = "writes a number of bytes at a specified offset",
+     }
-     .help       = write_help,
  };
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
  {
      struct timespec t1, t2;
      bool Cflag = false, qflag = false, bflag = false;
 -    bool Pflag = false, zflag = false, cflag = false;
 +    bool Pflag = false, zflag = false, cflag = false, sflag = false;
      int flags = 0;
      int c, cnt, ret;
      char *buf = NULL;
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
      /* Some compilers get confused and warn if this is not initialized.  */
      int64_t total = 0;
      int pattern = 0xcd;
 +    const char *file_name = NULL;
 -    while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {
 +    while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) {
          switch (c) {
          case 'b':
              bflag = true;
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
          case 'q':
              qflag = true;
              break;
 +        case 's':
 +            sflag = true;
 +            file_name = optarg;
 +            break;
          case 'u':
              flags |= BDRV_REQ_MAY_UNMAP;
              break;
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
          return -EINVAL;
      }
 -    if (zflag && Pflag) {
 -        printf("-z and -P cannot be specified at the same time\n");
 +    if (zflag + Pflag + sflag > 1) {
 +        printf("Only one of -z, -P, and -s "
 +               "can be specified at the same time\n");
          return -EINVAL;
      }
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
      }
      if (!zflag) {
 -        buf = qemu_io_alloc(blk, count, pattern);
 +        if (sflag) {
 +            buf = qemu_io_alloc_from_file(blk, count, file_name);
 +            if (!buf) {
 +                return -EINVAL;
 +            }
 +        } else {
 +            buf = qemu_io_alloc(blk, count, pattern);
 +        }
      }
      clock_gettime(CLOCK_MONOTONIC, &t1);
 --
-.21.0
+.26.2

-New patch
+[PULL 16/24] qcow2: Allow resize of images with internal snapshots
+From: Eric Blake <eblake@redhat.com>
 We originally refused to allow resize of images with internal
 snapshots because the v2 image format did not require the tracking of
 snapshot size, making it impossible to safely revert to a snapshot
 with a different size than the current view of the image.  But the
 snapshot size tracking was rectified in v3, and our recent fixes to
 qemu-img amend (see 0a85af35) guarantee that we always have a valid
 snapshot size.  Thus, we no longer need to artificially limit image
 resizes, but it does become one more thing that would prevent a
 downgrade back to v2.  And now that we support different-sized
 snapshots, it's also easy to fix reverting to a snapshot to apply the
 new size.
 Upgrade iotest 61 to cover this (we previously had NO coverage of
 refusal to resize while snapshots exist).  Note that the amend process
 can fail but still have effects: in particular, since we break things
 into upgrade, resize, downgrade, a failure during resize does not roll
 back changes made during upgrade, nor does failure in downgrade roll
 back a resize.  But this situation is pre-existing even without this
 patch; and without journaling, the best we could do is minimize the
 chance of partial failure by collecting all changes prior to doing any
 writes - which adds a lot of complexity but could still fail with EIO.
 On the other hand, we are careful that even if we have partial
 modification but then fail, the image is left viable (that is, we are
 careful to sequence things so that after each successful cluster
 write, there may be transient leaked clusters but no corrupt
 metadata).  And complicating the code to make it more transaction-like
 is not worth the effort: a user can always request multiple 'qemu-img
 amend' changing one thing each, if they need finer-grained control
 over detecting the first failure than what they get by letting qemu
 decide how to sequence multiple changes.
 Signed-off-by: Eric Blake <eblake@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200428192648.749066-3-eblake@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  block/qcow2-snapshot.c     | 20 ++++++++++++++++----
  block/qcow2.c              | 25 ++++++++++++++++++++++---
  tests/qemu-iotests/061     | 35 +++++++++++++++++++++++++++++++++++
  tests/qemu-iotests/061.out | 28 ++++++++++++++++++++++++++++
 files changed, 101 insertions(+), 7 deletions(-)
 diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/qcow2-snapshot.c
 +++ b/block/qcow2-snapshot.c
@@ -XXX,XX +XXX,XX @@
   */
  #include "qemu/osdep.h"
 +#include "sysemu/block-backend.h"
  #include "qapi/error.h"
  #include "qcow2.h"
  #include "qemu/bswap.h"
@@ -XXX,XX +XXX,XX @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
      }
      if (sn->disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
 -        error_report("qcow2: Loading snapshots with different disk "
 -            "size is not implemented");
 -        ret = -ENOTSUP;
 -        goto fail;
 +        BlockBackend *blk = blk_new_with_bs(bs, BLK_PERM_RESIZE, BLK_PERM_ALL,
 +                                            &local_err);
 +        if (!blk) {
 +            error_report_err(local_err);
 +            ret = -ENOTSUP;
 +            goto fail;
 +        }
 +
 +        ret = blk_truncate(blk, sn->disk_size, true, PREALLOC_MODE_OFF, 0,
 +                           &local_err);
 +        blk_unref(blk);
 +        if (ret < 0) {
 +            error_report_err(local_err);
 +            goto fail;
 +        }
      }
      /*
 diff --git a/block/qcow2.c b/block/qcow2.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/qcow2.c
 +++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_co_truncate(BlockDriverState *bs, int64_t offset,
      qemu_co_mutex_lock(&s->lock);
 -    /* cannot proceed if image has snapshots */
 -    if (s->nb_snapshots) {
 -        error_setg(errp, "Can't resize an image which has snapshots");
 +    /*
 +     * Even though we store snapshot size for all images, it was not
 +     * required until v3, so it is not safe to proceed for v2.
 +     */
 +    if (s->nb_snapshots && s->qcow_version < 3) {
 +        error_setg(errp, "Can't resize a v2 image which has snapshots");
          ret = -ENOTSUP;
          goto fail;
      }
@@ -XXX,XX +XXX,XX @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
      BDRVQcow2State *s = bs->opaque;
      int current_version = s->qcow_version;
      int ret;
 +    int i;
      /* This is qcow2_downgrade(), not qcow2_upgrade() */
      assert(target_version < current_version);
@@ -XXX,XX +XXX,XX @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
          return -ENOTSUP;
      }
 +    /*
 +     * If any internal snapshot has a different size than the current
 +     * image size, or VM state size that exceeds 32 bits, downgrading
 +     * is unsafe.  Even though we would still use v3-compliant output
 +     * to preserve that data, other v2 programs might not realize
 +     * those optional fields are important.
 +     */
 +    for (i = 0; i < s->nb_snapshots; i++) {
 +        if (s->snapshots[i].vm_state_size > UINT32_MAX ||
 +            s->snapshots[i].disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
 +            error_setg(errp, "Internal snapshots prevent downgrade of image");
 +            return -ENOTSUP;
 +        }
 +    }
 +
      /* clear incompatible features */
      if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
          ret = qcow2_mark_clean(bs);
 diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/061
 +++ b/tests/qemu-iotests/061
@@ -XXX,XX +XXX,XX @@ $PYTHON qcow2.py "$TEST_IMG" dump-header
  $QEMU_IO -c "read -P 0x2a 42M 64k" "$TEST_IMG" | _filter_qemu_io
  _check_test_img
 +echo
 +echo "=== Testing resize with snapshots ==="
 +echo
 +_make_test_img -o "compat=0.10" 32M
 +$QEMU_IO -c "write -P 0x2a 24M 64k" "$TEST_IMG" | _filter_qemu_io
 +$QEMU_IMG snapshot -c foo "$TEST_IMG"
 +$QEMU_IMG resize "$TEST_IMG" 64M &&
 +    echo "unexpected pass"
 +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)'
 +
 +$QEMU_IMG amend -o "compat=1.1,size=128M" "$TEST_IMG" ||
 +    echo "unexpected fail"
 +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)'
 +
 +$QEMU_IMG snapshot -c bar "$TEST_IMG"
 +$QEMU_IMG resize --shrink "$TEST_IMG" 64M ||
 +    echo "unexpected fail"
 +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)'
 +
 +$QEMU_IMG amend -o "compat=0.10,size=32M" "$TEST_IMG" &&
 +    echo "unexpected pass"
 +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)'
 +
 +$QEMU_IMG snapshot -a bar "$TEST_IMG" ||
 +    echo "unexpected fail"
 +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)'
 +
 +$QEMU_IMG snapshot -d bar "$TEST_IMG"
 +$QEMU_IMG amend -o "compat=0.10,size=32M" "$TEST_IMG" ||
 +    echo "unexpected fail"
 +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)'
 +
 +_check_test_img
 +
 +
  echo
  echo "=== Testing dirty lazy_refcounts=off ==="
  echo
 diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/061.out
 +++ b/tests/qemu-iotests/061.out
@@ -XXX,XX +XXX,XX @@ read 65536/65536 bytes at offset 44040192
 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
  No errors were found on the image.
 +=== Testing resize with snapshots ===
 +
 +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33554432
 +wrote 65536/65536 bytes at offset 25165824
 +64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 +qemu-img: Can't resize a v2 image which has snapshots
 +version                   2
 +size                      33554432
 +nb_snapshots              1
 +version                   3
 +size                      134217728
 +nb_snapshots              1
 +Image resized.
 +version                   3
 +size                      67108864
 +nb_snapshots              2
 +qemu-img: Internal snapshots prevent downgrade of image
 +version                   3
 +size                      33554432
 +nb_snapshots              2
 +version                   3
 +size                      134217728
 +nb_snapshots              2
 +version                   2
 +size                      33554432
 +nb_snapshots              1
 +No errors were found on the image.
 +
  === Testing dirty lazy_refcounts=off ===
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 --
 .26.2

-[Qemu-devel] [PULL 12/15] file-posix: fix request_alignment typo
+[PULL 17/24] qcow2: Tweak comment about bitmaps vs. resize
-From: Stefan Hajnoczi <stefanha@redhat.com>
+From: Eric Blake <eblake@redhat.com>
-Fixes: a6b257a08e3d72219f03e461a52152672fec0612
+Our comment did not actually match the code.  Rewrite the comment to
-       ("file-posix: Handle undetectable alignment")
+be less sensitive to any future changes to qcow2-bitmap.c that might
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+implement scenarios that we currently reject.
-Message-id: 20190827101328.4062-1-stefanha@redhat.com
-Reviewed-by: Thomas Huth <thuth@redhat.com>
+Signed-off-by: Eric Blake <eblake@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200428192648.749066-4-eblake@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- block/file-posix.c | 2 +-
+ block/qcow2.c | 2 +-
 file changed, 1 insertion(+), 1 deletion(-)
-diff --git a/block/file-posix.c b/block/file-posix.c
+diff --git a/block/qcow2.c b/block/qcow2.c
 index XXXXXXX..XXXXXXX 100644
---- a/block/file-posix.c
+--- a/block/qcow2.c
-+++ b/block/file-posix.c
++++ b/block/qcow2.c
-@@ -XXX,XX +XXX,XX @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_co_truncate(BlockDriverState *bs, int64_t offset,
-         for (i = 0; i < ARRAY_SIZE(alignments); i++) {
+         goto fail;
-             align = alignments[i];
+     }
-             if (raw_is_io_aligned(fd, buf + align, max_align)) {
--                /* Fallback to request_aligment. */
+-    /* cannot proceed if image has bitmaps */
-+                /* Fallback to request_alignment. */
++    /* See qcow2-bitmap.c for which bitmap scenarios prevent a resize. */
-                 s->buf_align = (align != 1) ? align : bs->bl.request_alignment;
+     if (qcow2_truncate_bitmaps_check(bs, errp)) {
-                 break;
+         ret = -ENOTSUP;
-             }
+         goto fail;
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 15/15] iotests: Unify cache mode quoting
+[PULL 18/24] block: Comment cleanups
-From: Nir Soffer <nirsof@gmail.com>
+From: Eric Blake <eblake@redhat.com>
-Quoting cache mode is not needed, and most tests use unquoted values.
+It's been a while since we got rid of the sector-based bdrv_read and
-Unify all test to use the same style.
+bdrv_write (commit 2e11d756); let's finish the job on a few remaining
 comments.
-Message-id: 20190827173432.7656-1-nsoffer@redhat.com
+Signed-off-by: Eric Blake <eblake@redhat.com>
-Signed-off-by: Nir Soffer <nsoffer@redhat.com>
+Message-Id: <20200428213807.776655-1-eblake@redhat.com>
 Reviewed-by: Alberto Garcia <berto@igalia.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/026 | 4 ++--
+ block/io.c             |  3 ++-
- tests/qemu-iotests/039 | 4 ++--
+ block/qcow2-refcount.c |  2 +-
- tests/qemu-iotests/052 | 2 +-
+ block/vvfat.c          | 10 +++++-----
- tests/qemu-iotests/091 | 4 ++--
+ tests/qemu-iotests/001 |  2 +-
-files changed, 7 insertions(+), 7 deletions(-)
+ tests/qemu-iotests/052 |  2 +-
  tests/qemu-iotests/134 |  2 +-
  tests/qemu-iotests/188 |  2 +-
 files changed, 12 insertions(+), 11 deletions(-)
-diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026
+diff --git a/block/io.c b/block/io.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/io.c
 +++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
   * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
   * BDRV_REQ_FUA).
   *
 - * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
 + * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
   */
  int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
  {
@@ -XXX,XX +XXX,XX @@ int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
      }
  }
 +/* return < 0 if error. See bdrv_pwrite() for the return codes */
  int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
  {
      int ret;
 diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/qcow2-refcount.c
 +++ b/block/qcow2-refcount.c
@@ -XXX,XX +XXX,XX @@ fail:
   * - 0 if writing to this offset will not affect the mentioned metadata
   * - a positive QCow2MetadataOverlap value indicating one overlapping section
   * - a negative value (-errno) indicating an error while performing a check,
 - *   e.g. when bdrv_read failed on QCOW2_OL_INACTIVE_L2
 + *   e.g. when bdrv_pread failed on QCOW2_OL_INACTIVE_L2
   */
  int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                   int64_t size)
 diff --git a/block/vvfat.c b/block/vvfat.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/vvfat.c
 +++ b/block/vvfat.c
@@ -XXX,XX +XXX,XX @@ DLOG(checkpoint());
       * - get modified FAT
       * - compare the two FATs (TODO)
       * - get buffer for marking used clusters
 -     * - recurse direntries from root (using bs->bdrv_read to make
 +     * - recurse direntries from root (using bs->bdrv_pread to make
       *    sure to get the new data)
       *   - check that the FAT agrees with the size
       *   - count the number of clusters occupied by this directory and
@@ -XXX,XX +XXX,XX @@ static int handle_deletes(BDRVVVFATState* s)
  /*
   * synchronize mapping with new state:
   *
 - * - copy FAT (with bdrv_read)
 + * - copy FAT (with bdrv_pread)
   * - mark all filenames corresponding to mappings as deleted
 - * - recurse direntries from root (using bs->bdrv_read)
 + * - recurse direntries from root (using bs->bdrv_pread)
   * - delete files corresponding to mappings marked as deleted
   */
  static int do_commit(BDRVVVFATState* s)
@@ -XXX,XX +XXX,XX @@ static int do_commit(BDRVVVFATState* s)
          return ret;
      }
 -    /* copy FAT (with bdrv_read) */
 +    /* copy FAT (with bdrv_pread) */
      memcpy(s->fat.pointer, s->fat2, 0x200 * s->sectors_per_fat);
 -    /* recurse direntries from root (using bs->bdrv_read) */
 +    /* recurse direntries from root (using bs->bdrv_pread) */
      ret = commit_direntries(s, 0, -1);
      if (ret) {
          fprintf(stderr, "Fatal: error while committing (%d)\n", ret);
 diff --git a/tests/qemu-iotests/001 b/tests/qemu-iotests/001
 index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/026
+--- a/tests/qemu-iotests/001
-+++ b/tests/qemu-iotests/026
++++ b/tests/qemu-iotests/001
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+@@ -XXX,XX +XXX,XX @@
- # Currently only qcow2 supports rebasing
+ #!/usr/bin/env bash
- _supported_fmt qcow2
+ #
- _supported_proto file
+-# Test simple read/write using plain bdrv_read/bdrv_write
--_default_cache_mode "writethrough"
++# Test simple read/write using plain bdrv_pread/bdrv_pwrite
--_supported_cache_modes "writethrough" "none"
+ #
-+_default_cache_mode writethrough
+ # Copyright (C) 2009 Red Hat, Inc.
-+_supported_cache_modes writethrough none
+ #
  # The refcount table tests expect a certain minimum width for refcount entries
  # (so that the refcount table actually needs to grow); that minimum is 16 bits,
  # being the default refcount entry width.
 diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/039
 +++ b/tests/qemu-iotests/039
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
  _supported_fmt qcow2
  _supported_proto file
  _supported_os Linux
 -_default_cache_mode "writethrough"
 -_supported_cache_modes "writethrough"
 +_default_cache_mode writethrough
 +_supported_cache_modes writethrough
  size=128M
 diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/052
 +++ b/tests/qemu-iotests/052
-@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
+@@ -XXX,XX +XXX,XX @@
- _supported_proto file
+ #!/usr/bin/env bash
+ #
- # Don't do O_DIRECT on tmpfs
+-# Test bdrv_read/bdrv_write using BDRV_O_SNAPSHOT
--_supported_cache_modes "writeback" "writethrough" "unsafe"
++# Test bdrv_pread/bdrv_pwrite using BDRV_O_SNAPSHOT
-+_supported_cache_modes writeback writethrough unsafe
+ #
+ # Copyright (C) 2013 Red Hat, Inc.
- size=128M
+ #
- _make_test_img $size
+diff --git a/tests/qemu-iotests/134 b/tests/qemu-iotests/134
 diff --git a/tests/qemu-iotests/091 b/tests/qemu-iotests/091
 index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/091
+--- a/tests/qemu-iotests/134
-+++ b/tests/qemu-iotests/091
++++ b/tests/qemu-iotests/134
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+@@ -XXX,XX +XXX,XX @@
- _supported_fmt qcow2
+ #!/usr/bin/env bash
- _supported_proto file
+ #
- _supported_os Linux
+-# Test encrypted read/write using plain bdrv_read/bdrv_write
--_default_cache_mode "none"
++# Test encrypted read/write using plain bdrv_pread/bdrv_pwrite
--_supported_cache_modes "writethrough" "none" "writeback"
+ #
-+_default_cache_mode none
+ # Copyright (C) 2015 Red Hat, Inc.
-+_supported_cache_modes writethrough none writeback
+ #
+diff --git a/tests/qemu-iotests/188 b/tests/qemu-iotests/188
- size=1G
+index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/188
 +++ b/tests/qemu-iotests/188
@@ -XXX,XX +XXX,XX @@
  #!/usr/bin/env bash
  #
 -# Test encrypted read/write using plain bdrv_read/bdrv_write
 +# Test encrypted read/write using plain bdrv_pread/bdrv_pwrite
  #
  # Copyright (C) 2017 Red Hat, Inc.
  #
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 10/15] iotests: Disable 110 for vmdk.twoGbMaxExtentSparse
+[PULL 19/24] Fix iotest 153
-The error message for the test case where we have a quorum node for
+From: Maxim Levitsky <mlevitsk@redhat.com>
 which no directory name can be generated is different: For
 twoGbMaxExtentSparse, it complains that it cannot open the extent file.
 For other (sub)formats, it just notes that it cannot determine the
 backing file path.  Both are fine, but just disable twoGbMaxExtentSparse
 for simplicity's sake.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+Commit f62514b3def5fb2acbef64d0e053c0c31fa45aff made qemu-img reject -o "" but this test uses it.
-Reviewed-by: John Snow <jsnow@redhat.com>
+Since this test only tries to do a dry-run run of qemu-img amend,
-Message-id: 20190815153638.4600-7-mreitz@redhat.com
+replace the -o "" with dummy -o "size=$size".
-Reviewed-by: John Snow <jsnow@redhat.com>
 Fixes: f62514b3def5fb2acbef64d0e053c0c31fa45aff
 Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
 Message-Id: <20200504131959.9533-1-mlevitsk@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/110 | 3 ++-
+ tests/qemu-iotests/153     |  2 +-
-file changed, 2 insertions(+), 1 deletion(-)
+ tests/qemu-iotests/153.out | 12 ++++++------
 files changed, 7 insertions(+), 7 deletions(-)
-diff --git a/tests/qemu-iotests/110 b/tests/qemu-iotests/110
+diff --git a/tests/qemu-iotests/153 b/tests/qemu-iotests/153
 index XXXXXXX..XXXXXXX 100755
---- a/tests/qemu-iotests/110
+--- a/tests/qemu-iotests/153
-+++ b/tests/qemu-iotests/110
++++ b/tests/qemu-iotests/153
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
+@@ -XXX,XX +XXX,XX @@ for opts1 in "" "read-only=on" "read-only=on,force-share=on"; do
- # Any format supporting backing files
+         _run_cmd $QEMU_IMG check       $L "${TEST_IMG}"
- _supported_fmt qed qcow qcow2 vmdk
+         _run_cmd $QEMU_IMG compare     $L "${TEST_IMG}" "${TEST_IMG}"
- _supported_proto file
+         _run_cmd $QEMU_IMG map         $L "${TEST_IMG}"
--_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+-        _run_cmd $QEMU_IMG amend -o "" $L "${TEST_IMG}"
-+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
++        _run_cmd $QEMU_IMG amend -o "size=$size" $L "${TEST_IMG}"
-+                     "subformat=twoGbMaxExtentSparse"
+         _run_cmd $QEMU_IMG commit      $L "${TEST_IMG}"
+         _run_cmd $QEMU_IMG resize      $L "${TEST_IMG}" $size
- TEST_IMG_REL=$(basename "$TEST_IMG")
+         _run_cmd $QEMU_IMG rebase      $L "${TEST_IMG}" -b "${TEST_IMG}.base"
 diff --git a/tests/qemu-iotests/153.out b/tests/qemu-iotests/153.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/153.out
 +++ b/tests/qemu-iotests/153.out
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper map TEST_DIR/t.qcow2
  qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get shared "write" lock
  Is another process using the image [TEST_DIR/t.qcow2]?
 -_qemu_img_wrapper amend -o  TEST_DIR/t.qcow2
 +_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2
  qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get "write" lock
  Is another process using the image [TEST_DIR/t.qcow2]?
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
  _qemu_img_wrapper map -U TEST_DIR/t.qcow2
 -_qemu_img_wrapper amend -o  -U TEST_DIR/t.qcow2
 +_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2
  qemu-img: unrecognized option '-U'
  Try 'qemu-img --help' for more information
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
  _qemu_img_wrapper map TEST_DIR/t.qcow2
 -_qemu_img_wrapper amend -o  TEST_DIR/t.qcow2
 +_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2
  qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get "write" lock
  Is another process using the image [TEST_DIR/t.qcow2]?
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
  _qemu_img_wrapper map -U TEST_DIR/t.qcow2
 -_qemu_img_wrapper amend -o  -U TEST_DIR/t.qcow2
 +_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2
  qemu-img: unrecognized option '-U'
  Try 'qemu-img --help' for more information
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
  _qemu_img_wrapper map TEST_DIR/t.qcow2
 -_qemu_img_wrapper amend -o  TEST_DIR/t.qcow2
 +_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2
  _qemu_img_wrapper commit TEST_DIR/t.qcow2
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
  _qemu_img_wrapper map -U TEST_DIR/t.qcow2
 -_qemu_img_wrapper amend -o  -U TEST_DIR/t.qcow2
 +_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2
  qemu-img: unrecognized option '-U'
  Try 'qemu-img --help' for more information
 --
-.21.0
+.26.2

-New patch
+[PULL 20/24] block/block-copy: rename in-flight requests to tasks
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 We are going to use aio-task-pool API and extend in-flight request
 structure to be a successor of AioTask, so rename things appropriately.
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200429130847.28124-2-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  block/block-copy.c | 98 +++++++++++++++++++++++-----------------------
 file changed, 48 insertions(+), 50 deletions(-)
 diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/block-copy.c
 +++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
  #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
  #define BLOCK_COPY_MAX_MEM (128 * MiB)
 -typedef struct BlockCopyInFlightReq {
 +typedef struct BlockCopyTask {
      int64_t offset;
      int64_t bytes;
 -    QLIST_ENTRY(BlockCopyInFlightReq) list;
 -    CoQueue wait_queue; /* coroutines blocked on this request */
 -} BlockCopyInFlightReq;
 +    QLIST_ENTRY(BlockCopyTask) list;
 +    CoQueue wait_queue; /* coroutines blocked on this task */
 +} BlockCopyTask;
  typedef struct BlockCopyState {
      /*
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
      bool use_copy_range;
      int64_t copy_size;
      uint64_t len;
 -    QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs;
 +    QLIST_HEAD(, BlockCopyTask) tasks;
      BdrvRequestFlags write_flags;
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
      SharedResource *mem;
  } BlockCopyState;
 -static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
 -                                                           int64_t offset,
 -                                                           int64_t bytes)
 +static BlockCopyTask *find_conflicting_task(BlockCopyState *s,
 +                                            int64_t offset, int64_t bytes)
  {
 -    BlockCopyInFlightReq *req;
 +    BlockCopyTask *t;
 -    QLIST_FOREACH(req, &s->inflight_reqs, list) {
 -        if (offset + bytes > req->offset && offset < req->offset + req->bytes) {
 -            return req;
 +    QLIST_FOREACH(t, &s->tasks, list) {
 +        if (offset + bytes > t->offset && offset < t->offset + t->bytes) {
 +            return t;
          }
      }
@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
  }
  /*
 - * If there are no intersecting requests return false. Otherwise, wait for the
 - * first found intersecting request to finish and return true.
 + * If there are no intersecting tasks return false. Otherwise, wait for the
 + * first found intersecting tasks to finish and return true.
   */
  static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
                                               int64_t bytes)
  {
 -    BlockCopyInFlightReq *req = find_conflicting_inflight_req(s, offset, bytes);
 +    BlockCopyTask *task = find_conflicting_task(s, offset, bytes);
 -    if (!req) {
 +    if (!task) {
          return false;
      }
 -    qemu_co_queue_wait(&req->wait_queue, NULL);
 +    qemu_co_queue_wait(&task->wait_queue, NULL);
      return true;
  }
  /* Called only on full-dirty region */
 -static void block_copy_inflight_req_begin(BlockCopyState *s,
 -                                          BlockCopyInFlightReq *req,
 -                                          int64_t offset, int64_t bytes)
 +static void block_copy_task_begin(BlockCopyState *s, BlockCopyTask *task,
 +                                  int64_t offset, int64_t bytes)
  {
 -    assert(!find_conflicting_inflight_req(s, offset, bytes));
 +    assert(!find_conflicting_task(s, offset, bytes));
      bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
      s->in_flight_bytes += bytes;
 -    req->offset = offset;
 -    req->bytes = bytes;
 -    qemu_co_queue_init(&req->wait_queue);
 -    QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
 +    task->offset = offset;
 +    task->bytes = bytes;
 +    qemu_co_queue_init(&task->wait_queue);
 +    QLIST_INSERT_HEAD(&s->tasks, task, list);
  }
  /*
 - * block_copy_inflight_req_shrink
 + * block_copy_task_shrink
   *
 - * Drop the tail of the request to be handled later. Set dirty bits back and
 - * wake up all requests waiting for us (may be some of them are not intersecting
 - * with shrunk request)
 + * Drop the tail of the task to be handled later. Set dirty bits back and
 + * wake up all tasks waiting for us (may be some of them are not intersecting
 + * with shrunk task)
   */
 -static void coroutine_fn block_copy_inflight_req_shrink(BlockCopyState *s,
 -        BlockCopyInFlightReq *req, int64_t new_bytes)
 +static void coroutine_fn block_copy_task_shrink(BlockCopyState *s,
 +                                                BlockCopyTask *task,
 +                                                int64_t new_bytes)
  {
 -    if (new_bytes == req->bytes) {
 +    if (new_bytes == task->bytes) {
          return;
      }
 -    assert(new_bytes > 0 && new_bytes < req->bytes);
 +    assert(new_bytes > 0 && new_bytes < task->bytes);
 -    s->in_flight_bytes -= req->bytes - new_bytes;
 +    s->in_flight_bytes -= task->bytes - new_bytes;
      bdrv_set_dirty_bitmap(s->copy_bitmap,
 -                          req->offset + new_bytes, req->bytes - new_bytes);
 +                          task->offset + new_bytes, task->bytes - new_bytes);
 -    req->bytes = new_bytes;
 -    qemu_co_queue_restart_all(&req->wait_queue);
 +    task->bytes = new_bytes;
 +    qemu_co_queue_restart_all(&task->wait_queue);
  }
 -static void coroutine_fn block_copy_inflight_req_end(BlockCopyState *s,
 -                                                     BlockCopyInFlightReq *req,
 -                                                     int ret)
 +static void coroutine_fn block_copy_task_end(BlockCopyState *s,
 +                                             BlockCopyTask *task, int ret)
  {
 -    s->in_flight_bytes -= req->bytes;
 +    s->in_flight_bytes -= task->bytes;
      if (ret < 0) {
 -        bdrv_set_dirty_bitmap(s->copy_bitmap, req->offset, req->bytes);
 +        bdrv_set_dirty_bitmap(s->copy_bitmap, task->offset, task->bytes);
      }
 -    QLIST_REMOVE(req, list);
 -    qemu_co_queue_restart_all(&req->wait_queue);
 +    QLIST_REMOVE(task, list);
 +    qemu_co_queue_restart_all(&task->wait_queue);
  }
  void block_copy_state_free(BlockCopyState *s)
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
          s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
      }
 -    QLIST_INIT(&s->inflight_reqs);
 +    QLIST_INIT(&s->tasks);
      return s;
  }
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
      assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
      while (bytes) {
 -        BlockCopyInFlightReq req;
 +        BlockCopyTask task;
          int64_t next_zero, cur_bytes, status_bytes;
          if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
              assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
              cur_bytes = next_zero - offset;
          }
 -        block_copy_inflight_req_begin(s, &req, offset, cur_bytes);
 +        block_copy_task_begin(s, &task, offset, cur_bytes);
          ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
          assert(ret >= 0); /* never fail */
          cur_bytes = MIN(cur_bytes, status_bytes);
 -        block_copy_inflight_req_shrink(s, &req, cur_bytes);
 +        block_copy_task_shrink(s, &task, cur_bytes);
          if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
 -            block_copy_inflight_req_end(s, &req, 0);
 +            block_copy_task_end(s, &task, 0);
              progress_set_remaining(s->progress,
                                     bdrv_get_dirty_count(s->copy_bitmap) +
                                     s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
          ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                   error_is_read);
          co_put_to_shres(s->mem, cur_bytes);
 -        block_copy_inflight_req_end(s, &req, ret);
 +        block_copy_task_end(s, &task, ret);
          if (ret < 0) {
              return ret;
          }
 --
 .26.2

-[Qemu-devel] [PULL 04/15] iotests: Test allocate_first_block() with O_DIRECT
+[PULL 21/24] block/block-copy: alloc task on each iteration
-From: Nir Soffer <nirsof@gmail.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Using block_resize we can test allocate_first_block() with file
+We are going to use aio-task-pool API, so tasks will be handled in
-descriptor opened with O_DIRECT, ensuring that it works for any size
+parallel. We need therefore separate allocated task on each iteration.
-larger than 4096 bytes.
+Introduce this logic now.
-Testing smaller sizes is tricky as the result depends on the filesystem
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 used for testing. For example on NFS any size will work since O_DIRECT
 does not require any alignment.
 Signed-off-by: Nir Soffer <nsoffer@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
-Message-id: 20190827010528.8818-3-nsoffer@redhat.com
+Message-Id: <20200429130847.28124-3-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/175     | 28 ++++++++++++++++++++++++++++
+ block/block-copy.c | 18 +++++++++++-------
- tests/qemu-iotests/175.out |  8 ++++++++
+file changed, 11 insertions(+), 7 deletions(-)
 files changed, 36 insertions(+)
-diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175
+diff --git a/block/block-copy.c b/block/block-copy.c
-index XXXXXXX..XXXXXXX 100755
+index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/175
+--- a/block/block-copy.c
-+++ b/tests/qemu-iotests/175
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ _filter_blocks()
+@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
          -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/"
  }
-+# Resize image using block_resize.
+ /* Called only on full-dirty region */
-+# Parameter 1: image path
+-static void block_copy_task_begin(BlockCopyState *s, BlockCopyTask *task,
-+# Parameter 2: new size
+-                                  int64_t offset, int64_t bytes)
-+_block_resize()
++static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
-+{
++                                             int64_t offset, int64_t bytes)
-+    local path=$1
+ {
-+    local size=$2
++    BlockCopyTask *task = g_new(BlockCopyTask, 1);
 +
-+    $QEMU -qmp stdio -nographic -nodefaults \
+     assert(!find_conflicting_task(s, offset, bytes));
-+        -blockdev file,node-name=file,filename=$path,cache.direct=on \
-+        <<EOF
+     bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
-+{'execute': 'qmp_capabilities'}
+@@ -XXX,XX +XXX,XX @@ static void block_copy_task_begin(BlockCopyState *s, BlockCopyTask *task,
-+{'execute': 'block_resize', 'arguments': {'node-name': 'file', 'size': $size}}
+     task->bytes = bytes;
-+{'execute': 'quit'}
+     qemu_co_queue_init(&task->wait_queue);
-+EOF
+     QLIST_INSERT_HEAD(&s->tasks, task, list);
 +}
 +
- # get standard environment, filters and checks
++    return task;
- . ./common.rc
+ }
- . ./common.filter
-@@ -XXX,XX +XXX,XX @@ _supported_fmt raw
+ /*
- _supported_proto file
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
- _supported_os Linux
+     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
-+_default_cache_mode none
+     while (bytes) {
-+_supported_cache_modes none directsync
+-        BlockCopyTask task;
-+
++        g_autofree BlockCopyTask *task = NULL;
- size=$((1 * 1024 * 1024))
+         int64_t next_zero, cur_bytes, status_bytes;
- touch "$TEST_DIR/empty"
+         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
-@@ -XXX,XX +XXX,XX @@ for mode in off full falloc; do
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
-     stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
+             assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
- done
+             cur_bytes = next_zero - offset;
+         }
-+for new_size in 4096 1048576; do
+-        block_copy_task_begin(s, &task, offset, cur_bytes);
-+    echo
++        task = block_copy_task_create(s, offset, cur_bytes);
-+    echo "== resize empty image with block_resize =="
-+    _make_test_img 0 | _filter_imgfmt
+         ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
-+    _block_resize $TEST_IMG $new_size >/dev/null
+         assert(ret >= 0); /* never fail */
-+    stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $new_size
+         cur_bytes = MIN(cur_bytes, status_bytes);
-+done
+-        block_copy_task_shrink(s, &task, cur_bytes);
-+
++        block_copy_task_shrink(s, task, cur_bytes);
- # success, all done
+         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
- echo "*** done"
+-            block_copy_task_end(s, &task, 0);
- rm -f $seq.full
++            block_copy_task_end(s, task, 0);
-diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out
+             progress_set_remaining(s->progress,
-index XXXXXXX..XXXXXXX 100644
+                                    bdrv_get_dirty_count(s->copy_bitmap) +
---- a/tests/qemu-iotests/175.out
+                                    s->in_flight_bytes);
-+++ b/tests/qemu-iotests/175.out
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
-@@ -XXX,XX +XXX,XX @@ size=1048576, max allocation
+         ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
- == creating image with preallocation falloc ==
+                                  error_is_read);
- Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc
+         co_put_to_shres(s->mem, cur_bytes);
- size=1048576, max allocation
+-        block_copy_task_end(s, &task, ret);
-+
++        block_copy_task_end(s, task, ret);
-+== resize empty image with block_resize ==
+         if (ret < 0) {
-+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
+             return ret;
-+size=4096, min allocation
+         }
 +
 +== resize empty image with block_resize ==
 +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
 +size=1048576, min allocation
   *** done
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 02/15] block: fix permission update in bdrv_replace_node
+[PULL 22/24] block/block-copy: add state pointer to BlockCopyTask
 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-It's wrong to OR shared permissions. It may lead to crash on further
+We are going to use aio-task-pool API, so we'll need state pointer in
-permission updates.
+BlockCopyTask anyway. Add it now and use where possible.
 Also, no needs to consider previously calculated permissions, as at
 this point we already bind all new parents and bdrv_get_cumulative_perm
 result is enough. So fix the bug by just set permissions by
 bdrv_get_cumulative_perm result.
 Bug was introduced in long ago 234ac1a9025, in 2.9.
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Message-id: 20190824100740.61635-1-vsementsov@virtuozzo.com
+Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200429130847.28124-4-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- block.c | 5 ++---
+ block/block-copy.c | 28 +++++++++++++++-------------
-file changed, 2 insertions(+), 3 deletions(-)
+file changed, 15 insertions(+), 13 deletions(-)
-diff --git a/block.c b/block.c
+diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
---- a/block.c
+--- a/block/block-copy.c
-+++ b/block.c
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
+@@ -XXX,XX +XXX,XX @@
  #define BLOCK_COPY_MAX_MEM (128 * MiB)
  typedef struct BlockCopyTask {
 +    BlockCopyState *s;
      int64_t offset;
      int64_t bytes;
      QLIST_ENTRY(BlockCopyTask) list;
@@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
      bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
      s->in_flight_bytes += bytes;
 -    task->offset = offset;
 -    task->bytes = bytes;
 +    *task = (BlockCopyTask) {
 +        .s = s,
 +        .offset = offset,
 +        .bytes = bytes,
 +    };
      qemu_co_queue_init(&task->wait_queue);
      QLIST_INSERT_HEAD(&s->tasks, task, list);
@@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
   * wake up all tasks waiting for us (may be some of them are not intersecting
   * with shrunk task)
   */
 -static void coroutine_fn block_copy_task_shrink(BlockCopyState *s,
 -                                                BlockCopyTask *task,
 +static void coroutine_fn block_copy_task_shrink(BlockCopyTask *task,
                                                  int64_t new_bytes)
  {
-     BdrvChild *c, *next;
+     if (new_bytes == task->bytes) {
-     GSList *list = NULL, *p;
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn block_copy_task_shrink(BlockCopyState *s,
--    uint64_t old_perm, old_shared;
-     uint64_t perm = 0, shared = BLK_PERM_ALL;
+     assert(new_bytes > 0 && new_bytes < task->bytes);
-     int ret;
+-    s->in_flight_bytes -= task->bytes - new_bytes;
-@@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
+-    bdrv_set_dirty_bitmap(s->copy_bitmap,
-         bdrv_unref(from);
++    task->s->in_flight_bytes -= task->bytes - new_bytes;
 +    bdrv_set_dirty_bitmap(task->s->copy_bitmap,
                            task->offset + new_bytes, task->bytes - new_bytes);
      task->bytes = new_bytes;
      qemu_co_queue_restart_all(&task->wait_queue);
  }
 -static void coroutine_fn block_copy_task_end(BlockCopyState *s,
 -                                             BlockCopyTask *task, int ret)
 +static void coroutine_fn block_copy_task_end(BlockCopyTask *task, int ret)
  {
 -    s->in_flight_bytes -= task->bytes;
 +    task->s->in_flight_bytes -= task->bytes;
      if (ret < 0) {
 -        bdrv_set_dirty_bitmap(s->copy_bitmap, task->offset, task->bytes);
 +        bdrv_set_dirty_bitmap(task->s->copy_bitmap, task->offset, task->bytes);
      }
+     QLIST_REMOVE(task, list);
--    bdrv_get_cumulative_perm(to, &old_perm, &old_shared);
+     qemu_co_queue_restart_all(&task->wait_queue);
--    bdrv_set_perm(to, old_perm | perm, old_shared | shared);
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
-+    bdrv_get_cumulative_perm(to, &perm, &shared);
+         ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
-+    bdrv_set_perm(to, perm, shared);
+         assert(ret >= 0); /* never fail */
+         cur_bytes = MIN(cur_bytes, status_bytes);
- out:
+-        block_copy_task_shrink(s, task, cur_bytes);
-     g_slist_free(list);
++        block_copy_task_shrink(task, cur_bytes);
          if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
 -            block_copy_task_end(s, task, 0);
 +            block_copy_task_end(task, 0);
              progress_set_remaining(s->progress,
                                     bdrv_get_dirty_count(s->copy_bitmap) +
                                     s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
          ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                   error_is_read);
          co_put_to_shres(s->mem, cur_bytes);
 -        block_copy_task_end(s, task, ret);
 +        block_copy_task_end(task, ret);
          if (ret < 0) {
              return ret;
          }
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 06/15] vmdk: Use bdrv_dirname() for relative extent paths
+[PULL 23/24] block/block-copy: refactor task creation
-This makes iotest 033 pass with e.g. subformat=monolithicFlat.  It also
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 turns a former error in 059 into success.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+Instead of just relying on the comment "Called only on full-dirty
-Message-id: 20190815153638.4600-3-mreitz@redhat.com
+region" in block_copy_task_create() let's move initial dirty area
-Reviewed-by: John Snow <jsnow@redhat.com>
+search directly to block_copy_task_create(). Let's also use effective
 bdrv_dirty_bitmap_next_dirty_area instead of looping through all
 non-dirty clusters.
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200429130847.28124-5-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- block/vmdk.c               | 54 ++++++++++++++++++++++++--------------
+ block/block-copy.c | 80 ++++++++++++++++++++++++++--------------------
- tests/qemu-iotests/059     |  7 +++--
+file changed, 46 insertions(+), 34 deletions(-)
  tests/qemu-iotests/059.out |  4 ++-
 files changed, 42 insertions(+), 23 deletions(-)
-diff --git a/block/vmdk.c b/block/vmdk.c
+diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
---- a/block/vmdk.c
+--- a/block/block-copy.c
-+++ b/block/vmdk.c
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ static const char *next_line(const char *s)
+@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyTask {
      CoQueue wait_queue; /* coroutines blocked on this task */
  } BlockCopyTask;
 +static int64_t task_end(BlockCopyTask *task)
 +{
 +    return task->offset + task->bytes;
 +}
 +
  typedef struct BlockCopyState {
      /*
       * BdrvChild objects are not owned or managed by block-copy. They are
@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
      return true;
  }
- static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
+-/* Called only on full-dirty region */
--                              const char *desc_file_path, QDict *options,
++/*
--                              Error **errp)
++ * Search for the first dirty area in offset/bytes range and create task at
-+                              QDict *options, Error **errp)
++ * the beginning of it.
 + */
  static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
                                               int64_t offset, int64_t bytes)
  {
-     int ret;
+-    BlockCopyTask *task = g_new(BlockCopyTask, 1);
-     int matches;
++    BlockCopyTask *task;
-@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
-     const char *p, *np;
++    if (!bdrv_dirty_bitmap_next_dirty_area(s->copy_bitmap,
-     int64_t sectors = 0;
++                                           offset, offset + bytes,
-     int64_t flat_offset;
++                                           s->copy_size, &offset, &bytes))
-+    char *desc_file_dir = NULL;
++    {
-     char *extent_path;
++        return NULL;
-     BdrvChild *extent_file;
++    }
-     BDRVVmdkState *s = bs->opaque;
++
-@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
++    /* region is dirty, so no existent tasks possible in it */
      assert(!find_conflicting_task(s, offset, bytes));
      bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
      s->in_flight_bytes += bytes;
 +    task = g_new(BlockCopyTask, 1);
      *task = (BlockCopyTask) {
          .s = s,
          .offset = offset,
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
  {
      int ret = 0;
      bool found_dirty = false;
 +    int64_t end = offset + bytes;
      /*
       * block_copy() user is responsible for keeping source and target in same
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
      while (bytes) {
          g_autofree BlockCopyTask *task = NULL;
 -        int64_t next_zero, cur_bytes, status_bytes;
 +        int64_t status_bytes;
 -        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
 -            trace_block_copy_skip(s, offset);
 -            offset += s->cluster_size;
 -            bytes -= s->cluster_size;
 -            continue; /* already copied */
 +        task = block_copy_task_create(s, offset, bytes);
 +        if (!task) {
 +            /* No more dirty bits in the bitmap */
 +            trace_block_copy_skip_range(s, offset, bytes);
 +            break;
 +        }
 +        if (task->offset > offset) {
 +            trace_block_copy_skip_range(s, offset, task->offset - offset);
          }
          found_dirty = true;
 -        cur_bytes = MIN(bytes, s->copy_size);
 -
 -        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,
 -                                                cur_bytes);
 -        if (next_zero >= 0) {
 -            assert(next_zero > offset); /* offset is dirty */
 -            assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
 -            cur_bytes = next_zero - offset;
 -        }
 -        task = block_copy_task_create(s, offset, cur_bytes);
 -
 -        ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
 +        ret = block_copy_block_status(s, task->offset, task->bytes,
 +                                      &status_bytes);
          assert(ret >= 0); /* never fail */
 -        cur_bytes = MIN(cur_bytes, status_bytes);
 -        block_copy_task_shrink(task, cur_bytes);
 +        if (status_bytes < task->bytes) {
 +            block_copy_task_shrink(task, status_bytes);
 +        }
          if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
              block_copy_task_end(task, 0);
              progress_set_remaining(s->progress,
                                     bdrv_get_dirty_count(s->copy_bitmap) +
                                     s->in_flight_bytes);
 -            trace_block_copy_skip_range(s, offset, status_bytes);
 -            offset += status_bytes;
 -            bytes -= status_bytes;
 +            trace_block_copy_skip_range(s, task->offset, task->bytes);
 +            offset = task_end(task);
 +            bytes = end - offset;
              continue;
          }
--        if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
+-        trace_block_copy_process(s, offset);
--            !desc_file_path[0])
++        trace_block_copy_process(s, task->offset);
--        {
--            bdrv_refresh_filename(bs->file->bs);
+-        co_get_from_shres(s->mem, cur_bytes);
--            error_setg(errp, "Cannot use relative extent paths with VMDK "
+-        ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
--                       "descriptor file '%s'", bs->file->bs->filename);
+-                                 error_is_read);
--            return -EINVAL;
+-        co_put_to_shres(s->mem, cur_bytes);
--        }
++        co_get_from_shres(s->mem, task->bytes);
-+        if (path_is_absolute(fname)) {
++        ret = block_copy_do_copy(s, task->offset, task->bytes,
-+            extent_path = g_strdup(fname);
++                                 ret & BDRV_BLOCK_ZERO, error_is_read);
-+        } else {
++        co_put_to_shres(s->mem, task->bytes);
-+            if (!desc_file_dir) {
+         block_copy_task_end(task, ret);
-+                desc_file_dir = bdrv_dirname(bs->file->bs, errp);
+         if (ret < 0) {
-+                if (!desc_file_dir) {
+             return ret;
 +                    bdrv_refresh_filename(bs->file->bs);
 +                    error_prepend(errp, "Cannot use relative paths with VMDK "
 +                                  "descriptor file '%s': ",
 +                                  bs->file->bs->filename);
 +                    ret = -EINVAL;
 +                    goto out;
 +                }
 +            }
 -        extent_path = path_combine(desc_file_path, fname);
 +            extent_path = g_strconcat(desc_file_dir, fname, NULL);
 +        }
          ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
          assert(ret < 32);
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
          g_free(extent_path);
          if (local_err) {
              error_propagate(errp, local_err);
 -            return -EINVAL;
 +            ret = -EINVAL;
 +            goto out;
          }
-         /* save to extents array */
+-        progress_work_done(s->progress, cur_bytes);
-@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
+-        s->progress_bytes_callback(cur_bytes, s->progress_opaque);
-, 0, 0, 0, 0, &extent, errp);
+-        offset += cur_bytes;
-             if (ret < 0) {
+-        bytes -= cur_bytes;
-                 bdrv_unref_child(bs, extent_file);
++        progress_work_done(s->progress, task->bytes);
--                return ret;
++        s->progress_bytes_callback(task->bytes, s->progress_opaque);
-+                goto out;
++        offset = task_end(task);
-             }
++        bytes = end - offset;
              extent->flat_start_offset = flat_offset << 9;
          } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
              g_free(buf);
              if (ret) {
                  bdrv_unref_child(bs, extent_file);
 -                return ret;
 +                goto out;
              }
              extent = &s->extents[s->num_extents - 1];
          } else if (!strcmp(type, "SESPARSE")) {
              ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp);
              if (ret) {
                  bdrv_unref_child(bs, extent_file);
 -                return ret;
 +                goto out;
              }
              extent = &s->extents[s->num_extents - 1];
          } else {
              error_setg(errp, "Unsupported extent type '%s'", type);
              bdrv_unref_child(bs, extent_file);
 -            return -ENOTSUP;
 +            ret = -ENOTSUP;
 +            goto out;
          }
          extent->type = g_strdup(type);
      }
--    return 0;
-+
+     return found_dirty;
 +    ret = 0;
 +    goto out;
  invalid:
      np = next_line(p);
@@ -XXX,XX +XXX,XX @@ invalid:
          np--;
      }
      error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);
 -    return -EINVAL;
 +    ret = -EINVAL;
 +
 +out:
 +    g_free(desc_file_dir);
 +    return ret;
  }
  static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
@@ -XXX,XX +XXX,XX @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
      }
      s->create_type = g_strdup(ct);
      s->desc_offset = 0;
 -    ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
 -                             errp);
 +    ret = vmdk_parse_extents(buf, bs, options, errp);
  exit:
      return ret;
  }
 diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/059
 +++ b/tests/qemu-iotests/059
@@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2
  echo
  echo "=== Testing monolithicFlat with internally generated JSON file name ==="
 +# Should work, because bdrv_dirname() works fine with blkdebug
  IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
 -$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \
 -    | _filter_testdir | _filter_imgfmt
 +$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
 +         -c info \
 +    2>&1 \
 +    | _filter_testdir | _filter_imgfmt | _filter_img_info
  _cleanup_test_img
  echo
 diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/059.out
 +++ b/tests/qemu-iotests/059.out
@@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240
  === Testing monolithicFlat with internally generated JSON file name ===
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 -qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
 +format name: IMGFMT
 +cluster size: 0 bytes
 +vm state offset: 0 bytes
  === Testing version 3 ===
  image: TEST_DIR/iotest-version3.IMGFMT
 --
-.21.0
+.26.2

-[Qemu-devel] [PULL 03/15] block: posix: Always allocate the first block
+[PULL 24/24] block/block-copy: use aio-task-pool API
-From: Nir Soffer <nirsof@gmail.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-When creating an image with preallocation "off" or "falloc", the first
+Run block_copy iterations in parallel in aio tasks.
-block of the image is typically not allocated. When using Gluster
-storage backed by XFS filesystem, reading this block using direct I/O
+Changes:
-succeeds regardless of request length, fooling alignment detection.
+  - BlockCopyTask becomes aio task structure. Add zeroes field to pass
+    it to block_copy_do_copy
-In this case we fallback to a safe value (4096) instead of the optimal
+  - add call state - it's a state of one call of block_copy(), shared
-value (512), which may lead to unneeded data copying when aligning
+    between parallel tasks. For now used only to keep information about
-requests.  Allocating the first block avoids the fallback.
+    first error: is it read or not.
+  - convert block_copy_dirty_clusters to aio-task loop.
-Since we allocate the first block even with preallocation=off, we no
-longer create images with zero disk size:
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
+Message-Id: <20200429130847.28124-6-vsementsov@virtuozzo.com>
     $ ./qemu-img create -f raw test.raw 1g
     Formatting 'test.raw', fmt=raw size=1073741824
     $ ls -lhs test.raw
 .0K -rw-r--r--. 1 nsoffer nsoffer 1.0G Aug 16 23:48 test.raw
 And converting the image requires additional cluster:
     $ ./qemu-img measure -f raw -O qcow2 test.raw
     required size: 458752
     fully allocated size: 1074135040
 When using format like vmdk with multiple files per image, we allocate
 one block per file:
     $ ./qemu-img create -f vmdk -o subformat=twoGbMaxExtentFlat test.vmdk 4g
     Formatting 'test.vmdk', fmt=vmdk size=4294967296 compat6=off hwversion=undefined subformat=twoGbMaxExtentFlat
     $ ls -lhs test*.vmdk
 .0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f001.vmdk
 .0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f002.vmdk
 .0K -rw-r--r--. 1 nsoffer nsoffer  353 Aug 27 03:23 test.vmdk
 I did quick performance test for copying disks with qemu-img convert to
 new raw target image to Gluster storage with sector size of 512 bytes:
     for i in $(seq 10); do
         rm -f dst.raw
         sleep 10
         time ./qemu-img convert -f raw -O raw -t none -T none src.raw dst.raw
     done
 Here is a table comparing the total time spent:
 Type    Before(s)   After(s)    Diff(%)
 ---------------------------------------
 real      530.028    469.123      -11.4
 user       17.204     10.768      -37.4
 sys        17.881      7.011      -60.7
 We can see very clear improvement in CPU usage.
 Signed-off-by: Nir Soffer <nsoffer@redhat.com>
 Message-id: 20190827010528.8818-2-nsoffer@redhat.com
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- block/file-posix.c                            | 51 +++++++++++++++++++
+ block/block-copy.c | 119 ++++++++++++++++++++++++++++++++++++++++-----
- tests/qemu-iotests/059.out                    |  2 +-
+file changed, 106 insertions(+), 13 deletions(-)
- tests/qemu-iotests/{150.out => 150.out.qcow2} |  0
- tests/qemu-iotests/150.out.raw                | 12 +++++
+diff --git a/block/block-copy.c b/block/block-copy.c
  tests/qemu-iotests/175                        | 19 ++++---
  tests/qemu-iotests/175.out                    |  8 +--
  tests/qemu-iotests/178.out.qcow2              |  4 +-
  tests/qemu-iotests/221.out                    | 12 +++--
  tests/qemu-iotests/253.out                    | 12 +++--
 files changed, 99 insertions(+), 21 deletions(-)
  rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%)
  create mode 100644 tests/qemu-iotests/150.out.raw
 diff --git a/block/file-posix.c b/block/file-posix.c
 index XXXXXXX..XXXXXXX 100644
---- a/block/file-posix.c
+--- a/block/block-copy.c
-+++ b/block/file-posix.c
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ static int handle_aiocb_discard(void *opaque)
+@@ -XXX,XX +XXX,XX @@
  #include "block/block-copy.h"
  #include "sysemu/block-backend.h"
  #include "qemu/units.h"
 +#include "qemu/coroutine.h"
 +#include "block/aio_task.h"
  #define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
  #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
  #define BLOCK_COPY_MAX_MEM (128 * MiB)
 +#define BLOCK_COPY_MAX_WORKERS 64
 +
 +static coroutine_fn int block_copy_task_entry(AioTask *task);
 +
 +typedef struct BlockCopyCallState {
 +    bool failed;
 +    bool error_is_read;
 +} BlockCopyCallState;
  typedef struct BlockCopyTask {
 +    AioTask task;
 +
      BlockCopyState *s;
 +    BlockCopyCallState *call_state;
      int64_t offset;
      int64_t bytes;
 +    bool zeroes;
      QLIST_ENTRY(BlockCopyTask) list;
      CoQueue wait_queue; /* coroutines blocked on this task */
  } BlockCopyTask;
@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
   * the beginning of it.
   */
  static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
 +                                             BlockCopyCallState *call_state,
                                               int64_t offset, int64_t bytes)
  {
      BlockCopyTask *task;
@@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
      task = g_new(BlockCopyTask, 1);
      *task = (BlockCopyTask) {
 +        .task.func = block_copy_task_entry,
          .s = s,
 +        .call_state = call_state,
          .offset = offset,
          .bytes = bytes,
      };
@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
      s->progress = pm;
  }
 +/*
 + * Takes ownership of @task
 + *
 + * If pool is NULL directly run the task, otherwise schedule it into the pool.
 + *
 + * Returns: task.func return code if pool is NULL
 + *          otherwise -ECANCELED if pool status is bad
 + *          otherwise 0 (successfully scheduled)
 + */
 +static coroutine_fn int block_copy_task_run(AioTaskPool *pool,
 +                                            BlockCopyTask *task)
 +{
 +    if (!pool) {
 +        int ret = task->task.func(&task->task);
 +
 +        g_free(task);
 +        return ret;
 +    }
 +
 +    aio_task_pool_wait_slot(pool);
 +    if (aio_task_pool_status(pool) < 0) {
 +        co_put_to_shres(task->s->mem, task->bytes);
 +        block_copy_task_end(task, -ECANCELED);
 +        g_free(task);
 +        return -ECANCELED;
 +    }
 +
 +    aio_task_pool_start_task(pool, &task->task);
 +
 +    return 0;
 +}
 +
  /*
   * block_copy_do_copy
   *
@@ -XXX,XX +XXX,XX @@ out:
      return ret;
  }
-+/*
++static coroutine_fn int block_copy_task_entry(AioTask *task)
 + * Help alignment probing by allocating the first block.
 + *
 + * When reading with direct I/O from unallocated area on Gluster backed by XFS,
 + * reading succeeds regardless of request length. In this case we fallback to
 + * safe alignment which is not optimal. Allocating the first block avoids this
 + * fallback.
 + *
 + * fd may be opened with O_DIRECT, but we don't know the buffer alignment or
 + * request alignment, so we use safe values.
 + *
 + * Returns: 0 on success, -errno on failure. Since this is an optimization,
 + * caller may ignore failures.
 + */
 +static int allocate_first_block(int fd, size_t max_size)
 +{
-+    size_t write_size = (max_size < MAX_BLOCKSIZE)
++    BlockCopyTask *t = container_of(task, BlockCopyTask, task);
-+        ? BDRV_SECTOR_SIZE
++    bool error_is_read;
 +        : MAX_BLOCKSIZE;
 +    size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
 +    void *buf;
 +    ssize_t n;
 +    int ret;
 +
-+    buf = qemu_memalign(max_align, write_size);
++    ret = block_copy_do_copy(t->s, t->offset, t->bytes, t->zeroes,
-+    memset(buf, 0, write_size);
++                             &error_is_read);
-+
++    if (ret < 0 && !t->call_state->failed) {
-+    do {
++        t->call_state->failed = true;
-+        n = pwrite(fd, buf, write_size, 0);
++        t->call_state->error_is_read = error_is_read;
-+    } while (n == -1 && errno == EINTR);
++    } else {
-+
++        progress_work_done(t->s->progress, t->bytes);
-+    ret = (n == -1) ? -errno : 0;
++        t->s->progress_bytes_callback(t->bytes, t->s->progress_opaque);
-+
++    }
-+    qemu_vfree(buf);
++    co_put_to_shres(t->s->mem, t->bytes);
 +    block_copy_task_end(t, ret);
 +
 +    return ret;
 +}
 +
- static int handle_aiocb_truncate(void *opaque)
+ static int block_copy_block_status(BlockCopyState *s, int64_t offset,
                                     int64_t bytes, int64_t *pnum)
  {
-     RawPosixAIOData *aiocb = opaque;
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
-@@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque)
+     int ret = 0;
-                 /* posix_fallocate() doesn't set errno. */
+     bool found_dirty = false;
-                 error_setg_errno(errp, -result,
+     int64_t end = offset + bytes;
-                                  "Could not preallocate new data");
++    AioTaskPool *aio = NULL;
-+            } else if (current_length == 0) {
++    BlockCopyCallState call_state = {false, false};
-+                /*
-+                 * posix_fallocate() uses fallocate() if the filesystem
+     /*
-+                 * supports it, or fallback to manually writing zeroes. If
+      * block_copy() user is responsible for keeping source and target in same
-+                 * fallocate() was used, unaligned reads from the fallocated
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
-+                 * area in raw_probe_alignment() will succeed, hence we need to
+     assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
-+                 * allocate the first block.
+     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
-+                 *
-+                 * Optimize future alignment probing; ignore failures.
+-    while (bytes) {
-+                 */
+-        g_autofree BlockCopyTask *task = NULL;
-+                allocate_first_block(fd, offset);
++    while (bytes && aio_task_pool_status(aio) == 0) {
-             }
++        BlockCopyTask *task;
-         } else {
+         int64_t status_bytes;
-             result = 0;
-@@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque)
+-        task = block_copy_task_create(s, offset, bytes);
-         if (ftruncate(fd, offset) != 0) {
++        task = block_copy_task_create(s, &call_state, offset, bytes);
-             result = -errno;
+         if (!task) {
-             error_setg_errno(errp, -result, "Could not resize file");
+             /* No more dirty bits in the bitmap */
-+        } else if (current_length == 0 && offset > current_length) {
+             trace_block_copy_skip_range(s, offset, bytes);
-+            /* Optimize future alignment probing; ignore failures. */
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
 +            allocate_first_block(fd, offset);
          }
-         return result;
+         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
-     default:
+             block_copy_task_end(task, 0);
-diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
++            g_free(task);
-index XXXXXXX..XXXXXXX 100644
+             progress_set_remaining(s->progress,
---- a/tests/qemu-iotests/059.out
+                                    bdrv_get_dirty_count(s->copy_bitmap) +
-+++ b/tests/qemu-iotests/059.out
+                                    s->in_flight_bytes);
-@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMax
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
- image: TEST_DIR/t.vmdk
+             bytes = end - offset;
- file format: vmdk
+             continue;
- virtual size: 0.977 TiB (1073741824000 bytes)
+         }
--disk size: 16 KiB
++        task->zeroes = ret & BDRV_BLOCK_ZERO;
-+disk size: 1.97 MiB
- Format specific information:
+         trace_block_copy_process(s, task->offset);
-     cid: XXXXXXXX
-     parent cid: XXXXXXXX
+         co_get_from_shres(s->mem, task->bytes);
-diff --git a/tests/qemu-iotests/150.out b/tests/qemu-iotests/150.out.qcow2
+-        ret = block_copy_do_copy(s, task->offset, task->bytes,
-similarity index 100%
+-                                 ret & BDRV_BLOCK_ZERO, error_is_read);
-rename from tests/qemu-iotests/150.out
+-        co_put_to_shres(s->mem, task->bytes);
-rename to tests/qemu-iotests/150.out.qcow2
+-        block_copy_task_end(task, ret);
-diff --git a/tests/qemu-iotests/150.out.raw b/tests/qemu-iotests/150.out.raw
+-        if (ret < 0) {
-new file mode 100644
+-            return ret;
-index XXXXXXX..XXXXXXX
+-        }
---- /dev/null
-+++ b/tests/qemu-iotests/150.out.raw
+-        progress_work_done(s->progress, task->bytes);
-@@ -XXX,XX +XXX,XX @@
+-        s->progress_bytes_callback(task->bytes, s->progress_opaque);
-+QA output created by 150
+         offset = task_end(task);
-+
+         bytes = end - offset;
-+=== Mapping sparse conversion ===
++
-+
++        if (!aio && bytes) {
-+Offset          Length          File
++            aio = aio_task_pool_new(BLOCK_COPY_MAX_WORKERS);
-+0               0x1000          TEST_DIR/t.IMGFMT
++        }
 +
-+=== Mapping non-sparse conversion ===
++        ret = block_copy_task_run(aio, task);
-+
++        if (ret < 0) {
-+Offset          Length          File
++            goto out;
-+0               0x100000        TEST_DIR/t.IMGFMT
++        }
-+*** done
++    }
-diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175
++
-index XXXXXXX..XXXXXXX 100755
++out:
---- a/tests/qemu-iotests/175
++    if (aio) {
-+++ b/tests/qemu-iotests/175
++        aio_task_pool_wait_all(aio);
-@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
++
- # the file size.  This function hides the resulting difference in the
++        /*
- # stat -c '%b' output.
++         * We are not really interested in -ECANCELED returned from
- # Parameter 1: Number of blocks an empty file occupies
++         * block_copy_task_run. If it fails, it means some task already failed
--# Parameter 2: Image size in bytes
++         * for real reason, let's return first failure.
-+# Parameter 2: Minimal number of blocks in an image
++         * Still, assert that we don't rewrite failure by success.
-+# Parameter 3: Image size in bytes
++         */
- _filter_blocks()
++        assert(ret == 0 || aio_task_pool_status(aio) < 0);
- {
++        ret = aio_task_pool_status(aio);
-     extra_blocks=$1
++
--    img_size=$2
++        aio_task_pool_free(aio);
-+    min_blocks=$2
++    }
-+    img_size=$3
++    if (error_is_read && ret < 0) {
++        *error_is_read = call_state.error_is_read;
--    sed -e "s/blocks=$extra_blocks\\(\$\\|[^0-9]\\)/nothing allocated/" \
+     }
--        -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/everything allocated/"
-+    sed -e "s/blocks=$min_blocks\\(\$\\|[^0-9]\\)/min allocation/" \
+-    return found_dirty;
-+        -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/"
++    return ret < 0 ? ret : found_dirty;
  }
- # get standard environment, filters and checks
+ /*
@@ -XXX,XX +XXX,XX @@ size=$((1 * 1024 * 1024))
  touch "$TEST_DIR/empty"
  extra_blocks=$(stat -c '%b' "$TEST_DIR/empty")
 +# We always write the first byte; check how many blocks this filesystem
 +# allocates to match empty image alloation.
 +printf "\0" > "$TEST_DIR/empty"
 +min_blocks=$(stat -c '%b' "$TEST_DIR/empty")
 +
  echo
  echo "== creating image with default preallocation =="
  _make_test_img $size | _filter_imgfmt
 -stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size
 +stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
  for mode in off full falloc; do
      echo
      echo "== creating image with preallocation $mode =="
      IMGOPTS=preallocation=$mode _make_test_img $size | _filter_imgfmt
 -    stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size
 +    stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
  done
  # success, all done
 diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/175.out
 +++ b/tests/qemu-iotests/175.out
@@ -XXX,XX +XXX,XX @@ QA output created by 175
  == creating image with default preallocation ==
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
 -size=1048576, nothing allocated
 +size=1048576, min allocation
  == creating image with preallocation off ==
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=off
 -size=1048576, nothing allocated
 +size=1048576, min allocation
  == creating image with preallocation full ==
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=full
 -size=1048576, everything allocated
 +size=1048576, max allocation
  == creating image with preallocation falloc ==
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc
 -size=1048576, everything allocated
 +size=1048576, max allocation
   *** done
 diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/178.out.qcow2
 +++ b/tests/qemu-iotests/178.out.qcow2
@@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608
  == raw input image with data (human) ==
  Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824
 -required size: 393216
 +required size: 458752
  fully allocated size: 1074135040
  wrote 512/512 bytes at offset 512
 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608
  Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824
  {
 -    "required": 393216,
 +    "required": 458752,
      "fully-allocated": 1074135040
  }
  wrote 512/512 bytes at offset 512
 diff --git a/tests/qemu-iotests/221.out b/tests/qemu-iotests/221.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/221.out
 +++ b/tests/qemu-iotests/221.out
@@ -XXX,XX +XXX,XX @@ QA output created by 221
  === Check mapping of unaligned raw image ===
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65537
 -[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 -[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
  wrote 1/1 bytes at offset 65536
 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 -[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
  { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
  { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 -[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
  { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
  { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
  *** done
 diff --git a/tests/qemu-iotests/253.out b/tests/qemu-iotests/253.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/253.out
 +++ b/tests/qemu-iotests/253.out
@@ -XXX,XX +XXX,XX @@ QA output created by 253
  === Check mapping of unaligned raw image ===
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048575
 -[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 -[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
  wrote 65535/65535 bytes at offset 983040
 .999 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 -[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
  { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
 -[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 +{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
  { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
  *** done
 --
-.21.0
+.26.2

The following changes since commit 23919ddfd56135cad3cb468a8f54d5a595f024f4:

Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190827' into staging (2019-08-27 15:52:36 +0100)

are available in the Git repository at:

https://github.com/XanClic/qemu.git tags/pull-block-2019-08-27

for you to fetch changes up to bb043c056cffcc2f3ce88bfdaf2e76e455c09e2c:

iotests: Unify cache mode quoting (2019-08-27 19:48:44 +0200)

----------------------------------------------------------------
Block patches:
- qemu-io now accepts a file to read a write pattern from
- Ensure that raw files have their first block allocated so we can probe
  the O_DIRECT alignment if necessary
- Various fixes

----------------------------------------------------------------
Denis Plotnikov (1):
  qemu-io: add pattern file for write command

Max Reitz (7):
  iotests: Fix _filter_img_create()
  vmdk: Use bdrv_dirname() for relative extent paths
  iotests: Keep testing broken relative extent paths
  vmdk: Reject invalid compressed writes
  iotests: Disable broken streamOptimized tests
  iotests: Disable 110 for vmdk.twoGbMaxExtentSparse
  iotests: Disable 126 for flat vmdk subformats

Nir Soffer (3):
  block: posix: Always allocate the first block
  iotests: Test allocate_first_block() with O_DIRECT
  iotests: Unify cache mode quoting

Stefan Hajnoczi (1):
  file-posix: fix request_alignment typo

Thomas Huth (2):
  iotests: Check for enabled drivers before testing them
  tests/check-block: Skip iotests when sanitizers are enabled

Vladimir Sementsov-Ogievskiy (1):
  block: fix permission update in bdrv_replace_node

block.c                                       |  5 +-
 block/file-posix.c                            | 53 +++++++++-
 block/vmdk.c                                  | 64 ++++++++----
 qemu-io-cmds.c                                | 99 +++++++++++++++++--
 tests/check-block.sh                          |  5 +
 tests/qemu-iotests/002                        |  1 +
 tests/qemu-iotests/003                        |  1 +
 tests/qemu-iotests/005                        |  3 +-
 tests/qemu-iotests/009                        |  1 +
 tests/qemu-iotests/010                        |  1 +
 tests/qemu-iotests/011                        |  1 +
 tests/qemu-iotests/017                        |  3 +-
 tests/qemu-iotests/018                        |  3 +-
 tests/qemu-iotests/019                        |  3 +-
 tests/qemu-iotests/020                        |  3 +-
 tests/qemu-iotests/026                        |  4 +-
 tests/qemu-iotests/027                        |  1 +
 tests/qemu-iotests/032                        |  1 +
 tests/qemu-iotests/033                        |  1 +
 tests/qemu-iotests/034                        |  3 +-
 tests/qemu-iotests/037                        |  3 +-
 tests/qemu-iotests/039                        |  4 +-
 tests/qemu-iotests/052                        |  2 +-
 tests/qemu-iotests/059                        | 34 ++++++-
 tests/qemu-iotests/059.out                    | 26 +++--
 tests/qemu-iotests/063                        |  3 +-
 tests/qemu-iotests/071                        |  1 +
 tests/qemu-iotests/072                        |  1 +
 tests/qemu-iotests/081                        |  4 +-
 tests/qemu-iotests/091                        |  4 +-
 tests/qemu-iotests/099                        |  1 +
 tests/qemu-iotests/105                        |  3 +-
 tests/qemu-iotests/110                        |  3 +-
 tests/qemu-iotests/120                        |  1 +
 tests/qemu-iotests/126                        |  2 +
 tests/qemu-iotests/{150.out => 150.out.qcow2} |  0
 tests/qemu-iotests/150.out.raw                | 12 +++
 tests/qemu-iotests/162                        |  4 +-
 tests/qemu-iotests/175                        | 47 +++++++--
 tests/qemu-iotests/175.out                    | 16 ++-
 tests/qemu-iotests/178.out.qcow2              |  4 +-
 tests/qemu-iotests/184                        |  1 +
 tests/qemu-iotests/186                        |  1 +
 tests/qemu-iotests/197                        |  1 +
 tests/qemu-iotests/215                        |  1 +
 tests/qemu-iotests/221.out                    | 12 ++-
 tests/qemu-iotests/251                        |  1 +
 tests/qemu-iotests/253.out                    | 12 ++-
 tests/qemu-iotests/common.filter              |  4 +-
 tests/qemu-iotests/common.rc                  | 14 +++
 50 files changed, 391 insertions(+), 87 deletions(-)
 rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%)
 create mode 100644 tests/qemu-iotests/150.out.raw

-- 
2.21.0

From: Denis Plotnikov <dplotnikov@virtuozzo.com>

The patch allows to provide a pattern file for write
command. There was no similar ability before.

Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
Message-id: 20190820164616.4072-1-dplotnikov@virtuozzo.com
Reviewed-by: Eric Blake <eblake@redhat.com>
[mreitz: Keep optstring in alphabetical order]
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 qemu-io-cmds.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 93 insertions(+), 6 deletions(-)

diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index XXXXXXX..XXXXXXX 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -XXX,XX +XXX,XX @@ static void qemu_io_free(void *p)
     qemu_vfree(p);
 }
 
+/*
+ * qemu_io_alloc_from_file()
+ *
+ * Allocates the buffer and populates it with the content of the given file
+ * up to @len bytes. If the file length is less than @len, then the buffer
+ * is populated with the file content cyclically.
+ *
+ * @blk - the block backend where the buffer content is going to be written to
+ * @len - the buffer length
+ * @file_name - the file to read the content from
+ *
+ * Returns: the buffer pointer on success
+ *          NULL on error
+ */
+static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
+                                     const char *file_name)
+{
+    char *buf, *buf_origin;
+    FILE *f = fopen(file_name, "r");
+    int pattern_len;
+
+    if (!f) {
+        perror(file_name);
+        return NULL;
+    }
+
+    if (qemuio_misalign) {
+        len += MISALIGN_OFFSET;
+    }
+
+    buf_origin = buf = blk_blockalign(blk, len);
+
+    if (qemuio_misalign) {
+        buf_origin += MISALIGN_OFFSET;
+        buf += MISALIGN_OFFSET;
+        len -= MISALIGN_OFFSET;
+    }
+
+    pattern_len = fread(buf_origin, 1, len, f);
+
+    if (ferror(f)) {
+        perror(file_name);
+        goto error;
+    }
+
+    if (pattern_len == 0) {
+        fprintf(stderr, "%s: file is empty\n", file_name);
+        goto error;
+    }
+
+    fclose(f);
+
+    if (len > pattern_len) {
+        len -= pattern_len;
+        buf += pattern_len;
+
+        while (len > 0) {
+            size_t len_to_copy = MIN(pattern_len, len);
+
+            memcpy(buf, buf_origin, len_to_copy);
+
+            len -= len_to_copy;
+            buf += len_to_copy;
+        }
+    }
+
+    return buf_origin;
+
+error:
+    qemu_io_free(buf_origin);
+    return NULL;
+}
+
 static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
 {
     uint64_t i;
@@ -XXX,XX +XXX,XX @@ static void write_help(void)
 " -n, -- with -z, don't allow slow fallback\n"
 " -p, -- ignored for backwards compatibility\n"
 " -P, -- use different pattern to fill file\n"
+" -s, -- use a pattern file to fill the write buffer\n"
 " -C, -- report statistics in a machine parsable format\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
 " -u, -- with -z, allow unmapping\n"
@@ -XXX,XX +XXX,XX @@ static const cmdinfo_t write_cmd = {
     .perm       = BLK_PERM_WRITE,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-bcCfnquz] [-P pattern] off len",
+    .args       = "[-bcCfnquz] [-P pattern | -s source_file] off len",
     .oneline    = "writes a number of bytes at a specified offset",
     .help       = write_help,
 };
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timespec t1, t2;
     bool Cflag = false, qflag = false, bflag = false;
-    bool Pflag = false, zflag = false, cflag = false;
+    bool Pflag = false, zflag = false, cflag = false, sflag = false;
     int flags = 0;
     int c, cnt, ret;
     char *buf = NULL;
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     /* Some compilers get confused and warn if this is not initialized.  */
     int64_t total = 0;
     int pattern = 0xcd;
+    const char *file_name = NULL;
 
-    while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {
+    while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) {
         switch (c) {
         case 'b':
             bflag = true;
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         case 'q':
             qflag = true;
             break;
+        case 's':
+            sflag = true;
+            file_name = optarg;
+            break;
         case 'u':
             flags |= BDRV_REQ_MAY_UNMAP;
             break;
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         return -EINVAL;
     }
 
-    if (zflag && Pflag) {
-        printf("-z and -P cannot be specified at the same time\n");
+    if (zflag + Pflag + sflag > 1) {
+        printf("Only one of -z, -P, and -s "
+               "can be specified at the same time\n");
         return -EINVAL;
     }
 
@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     }
 
     if (!zflag) {
-        buf = qemu_io_alloc(blk, count, pattern);
+        if (sflag) {
+            buf = qemu_io_alloc_from_file(blk, count, file_name);
+            if (!buf) {
+                return -EINVAL;
+            }
+        } else {
+            buf = qemu_io_alloc(blk, count, pattern);
+        }
     }
 
     clock_gettime(CLOCK_MONOTONIC, &t1);
-- 
2.21.0

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

It's wrong to OR shared permissions. It may lead to crash on further
permission updates.
Also, no needs to consider previously calculated permissions, as at
this point we already bind all new parents and bdrv_get_cumulative_perm
result is enough. So fix the bug by just set permissions by
bdrv_get_cumulative_perm result.

Bug was introduced in long ago 234ac1a9025, in 2.9.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Message-id: 20190824100740.61635-1-vsementsov@virtuozzo.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
 {
     BdrvChild *c, *next;
     GSList *list = NULL, *p;
-    uint64_t old_perm, old_shared;
     uint64_t perm = 0, shared = BLK_PERM_ALL;
     int ret;
 
@@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
         bdrv_unref(from);
     }
 
-    bdrv_get_cumulative_perm(to, &old_perm, &old_shared);
-    bdrv_set_perm(to, old_perm | perm, old_shared | shared);
+    bdrv_get_cumulative_perm(to, &perm, &shared);
+    bdrv_set_perm(to, perm, shared);
 
 out:
     g_slist_free(list);
-- 
2.21.0

From: Nir Soffer <nirsof@gmail.com>

When creating an image with preallocation "off" or "falloc", the first
block of the image is typically not allocated. When using Gluster
storage backed by XFS filesystem, reading this block using direct I/O
succeeds regardless of request length, fooling alignment detection.

In this case we fallback to a safe value (4096) instead of the optimal
value (512), which may lead to unneeded data copying when aligning
requests.  Allocating the first block avoids the fallback.

Since we allocate the first block even with preallocation=off, we no
longer create images with zero disk size:

$ ./qemu-img create -f raw test.raw 1g
    Formatting 'test.raw', fmt=raw size=1073741824

$ ls -lhs test.raw
    4.0K -rw-r--r--. 1 nsoffer nsoffer 1.0G Aug 16 23:48 test.raw

And converting the image requires additional cluster:

$ ./qemu-img measure -f raw -O qcow2 test.raw
    required size: 458752
    fully allocated size: 1074135040

When using format like vmdk with multiple files per image, we allocate
one block per file:

$ ./qemu-img create -f vmdk -o subformat=twoGbMaxExtentFlat test.vmdk 4g
    Formatting 'test.vmdk', fmt=vmdk size=4294967296 compat6=off hwversion=undefined subformat=twoGbMaxExtentFlat

$ ls -lhs test*.vmdk
    4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f001.vmdk
    4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f002.vmdk
    4.0K -rw-r--r--. 1 nsoffer nsoffer  353 Aug 27 03:23 test.vmdk

I did quick performance test for copying disks with qemu-img convert to
new raw target image to Gluster storage with sector size of 512 bytes:

for i in $(seq 10); do
        rm -f dst.raw
        sleep 10
        time ./qemu-img convert -f raw -O raw -t none -T none src.raw dst.raw
    done

Here is a table comparing the total time spent:

Type    Before(s)   After(s)    Diff(%)
---------------------------------------
real      530.028    469.123      -11.4
user       17.204     10.768      -37.4
sys        17.881      7.011      -60.7

We can see very clear improvement in CPU usage.

Signed-off-by: Nir Soffer <nsoffer@redhat.com>
Message-id: 20190827010528.8818-2-nsoffer@redhat.com
Reviewed-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/file-posix.c                            | 51 +++++++++++++++++++
 tests/qemu-iotests/059.out                    |  2 +-
 tests/qemu-iotests/{150.out => 150.out.qcow2} |  0
 tests/qemu-iotests/150.out.raw                | 12 +++++
 tests/qemu-iotests/175                        | 19 ++++---
 tests/qemu-iotests/175.out                    |  8 +--
 tests/qemu-iotests/178.out.qcow2              |  4 +-
 tests/qemu-iotests/221.out                    | 12 +++--
 tests/qemu-iotests/253.out                    | 12 +++--
 9 files changed, 99 insertions(+), 21 deletions(-)
 rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%)
 create mode 100644 tests/qemu-iotests/150.out.raw

diff --git a/block/file-posix.c b/block/file-posix.c
index XXXXXXX..XXXXXXX 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -XXX,XX +XXX,XX @@ static int handle_aiocb_discard(void *opaque)
     return ret;
 }
 
+/*
+ * Help alignment probing by allocating the first block.
+ *
+ * When reading with direct I/O from unallocated area on Gluster backed by XFS,
+ * reading succeeds regardless of request length. In this case we fallback to
+ * safe alignment which is not optimal. Allocating the first block avoids this
+ * fallback.
+ *
+ * fd may be opened with O_DIRECT, but we don't know the buffer alignment or
+ * request alignment, so we use safe values.
+ *
+ * Returns: 0 on success, -errno on failure. Since this is an optimization,
+ * caller may ignore failures.
+ */
+static int allocate_first_block(int fd, size_t max_size)
+{
+    size_t write_size = (max_size < MAX_BLOCKSIZE)
+        ? BDRV_SECTOR_SIZE
+        : MAX_BLOCKSIZE;
+    size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
+    void *buf;
+    ssize_t n;
+    int ret;
+
+    buf = qemu_memalign(max_align, write_size);
+    memset(buf, 0, write_size);
+
+    do {
+        n = pwrite(fd, buf, write_size, 0);
+    } while (n == -1 && errno == EINTR);
+
+    ret = (n == -1) ? -errno : 0;
+
+    qemu_vfree(buf);
+    return ret;
+}
+
 static int handle_aiocb_truncate(void *opaque)
 {
     RawPosixAIOData *aiocb = opaque;
@@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque)
                 /* posix_fallocate() doesn't set errno. */
                 error_setg_errno(errp, -result,
                                  "Could not preallocate new data");
+            } else if (current_length == 0) {
+                /*
+                 * posix_fallocate() uses fallocate() if the filesystem
+                 * supports it, or fallback to manually writing zeroes. If
+                 * fallocate() was used, unaligned reads from the fallocated
+                 * area in raw_probe_alignment() will succeed, hence we need to
+                 * allocate the first block.
+                 *
+                 * Optimize future alignment probing; ignore failures.
+                 */
+                allocate_first_block(fd, offset);
             }
         } else {
             result = 0;
@@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque)
         if (ftruncate(fd, offset) != 0) {
             result = -errno;
             error_setg_errno(errp, -result, "Could not resize file");
+        } else if (current_length == 0 && offset > current_length) {
+            /* Optimize future alignment probing; ignore failures. */
+            allocate_first_block(fd, offset);
         }
         return result;
     default:
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMax
 image: TEST_DIR/t.vmdk
 file format: vmdk
 virtual size: 0.977 TiB (1073741824000 bytes)
-disk size: 16 KiB
+disk size: 1.97 MiB
 Format specific information:
     cid: XXXXXXXX
     parent cid: XXXXXXXX
diff --git a/tests/qemu-iotests/150.out b/tests/qemu-iotests/150.out.qcow2
similarity index 100%
rename from tests/qemu-iotests/150.out
rename to tests/qemu-iotests/150.out.qcow2
diff --git a/tests/qemu-iotests/150.out.raw b/tests/qemu-iotests/150.out.raw
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/150.out.raw
@@ -XXX,XX +XXX,XX @@
+QA output created by 150
+
+=== Mapping sparse conversion ===
+
+Offset          Length          File
+0               0x1000          TEST_DIR/t.IMGFMT
+
+=== Mapping non-sparse conversion ===
+
+Offset          Length          File
+0               0x100000        TEST_DIR/t.IMGFMT
+*** done
diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/175
+++ b/tests/qemu-iotests/175
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # the file size.  This function hides the resulting difference in the
 # stat -c '%b' output.
 # Parameter 1: Number of blocks an empty file occupies
-# Parameter 2: Image size in bytes
+# Parameter 2: Minimal number of blocks in an image
+# Parameter 3: Image size in bytes
 _filter_blocks()
 {
     extra_blocks=$1
-    img_size=$2
+    min_blocks=$2
+    img_size=$3
 
-    sed -e "s/blocks=$extra_blocks\$\$\\|[^0-9]\$/nothing allocated/" \
-        -e "s/blocks=$((extra_blocks + img_size / 512))\$\$\\|[^0-9]\$/everything allocated/"
+    sed -e "s/blocks=$min_blocks\$\$\\|[^0-9]\$/min allocation/" \
+        -e "s/blocks=$((extra_blocks + img_size / 512))\$\$\\|[^0-9]\$/max allocation/"
 }
 
 # get standard environment, filters and checks
@@ -XXX,XX +XXX,XX @@ size=$((1 * 1024 * 1024))
 touch "$TEST_DIR/empty"
 extra_blocks=$(stat -c '%b' "$TEST_DIR/empty")
 
+# We always write the first byte; check how many blocks this filesystem
+# allocates to match empty image alloation.
+printf "\0" > "$TEST_DIR/empty"
+min_blocks=$(stat -c '%b' "$TEST_DIR/empty")
+
 echo
 echo "== creating image with default preallocation =="
 _make_test_img $size | _filter_imgfmt
-stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size
+stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
 
 for mode in off full falloc; do
     echo
     echo "== creating image with preallocation $mode =="
     IMGOPTS=preallocation=$mode _make_test_img $size | _filter_imgfmt
-    stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size
+    stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
 done
 
 # success, all done
diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/175.out
+++ b/tests/qemu-iotests/175.out
@@ -XXX,XX +XXX,XX @@ QA output created by 175
 
 == creating image with default preallocation ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
-size=1048576, nothing allocated
+size=1048576, min allocation
 
 == creating image with preallocation off ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=off
-size=1048576, nothing allocated
+size=1048576, min allocation
 
 == creating image with preallocation full ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=full
-size=1048576, everything allocated
+size=1048576, max allocation
 
 == creating image with preallocation falloc ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc
-size=1048576, everything allocated
+size=1048576, max allocation
  *** done
diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/178.out.qcow2
+++ b/tests/qemu-iotests/178.out.qcow2
@@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608
 == raw input image with data (human) ==
 
 Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824
-required size: 393216
+required size: 458752
 fully allocated size: 1074135040
 wrote 512/512 bytes at offset 512
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608
 
 Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824
 {
-    "required": 393216,
+    "required": 458752,
     "fully-allocated": 1074135040
 }
 wrote 512/512 bytes at offset 512
diff --git a/tests/qemu-iotests/221.out b/tests/qemu-iotests/221.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/221.out
+++ b/tests/qemu-iotests/221.out
@@ -XXX,XX +XXX,XX @@ QA output created by 221
 === Check mapping of unaligned raw image ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65537
-[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
-[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 wrote 1/1 bytes at offset 65536
 1 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
-[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
 { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 *** done
diff --git a/tests/qemu-iotests/253.out b/tests/qemu-iotests/253.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/253.out
+++ b/tests/qemu-iotests/253.out
@@ -XXX,XX +XXX,XX @@ QA output created by 253
 === Check mapping of unaligned raw image ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048575
-[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
-[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
 wrote 65535/65535 bytes at offset 983040
 63.999 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
-[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
 { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
 *** done
-- 
2.21.0

From: Nir Soffer <nirsof@gmail.com>

Using block_resize we can test allocate_first_block() with file
descriptor opened with O_DIRECT, ensuring that it works for any size
larger than 4096 bytes.

Testing smaller sizes is tricky as the result depends on the filesystem
used for testing. For example on NFS any size will work since O_DIRECT
does not require any alignment.

Signed-off-by: Nir Soffer <nsoffer@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190827010528.8818-3-nsoffer@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/175     | 28 ++++++++++++++++++++++++++++
 tests/qemu-iotests/175.out |  8 ++++++++
 2 files changed, 36 insertions(+)

diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/175
+++ b/tests/qemu-iotests/175
@@ -XXX,XX +XXX,XX @@ _filter_blocks()
         -e "s/blocks=$((extra_blocks + img_size / 512))\$\$\\|[^0-9]\$/max allocation/"
 }
 
+# Resize image using block_resize.
+# Parameter 1: image path
+# Parameter 2: new size
+_block_resize()
+{
+    local path=$1
+    local size=$2
+
+    $QEMU -qmp stdio -nographic -nodefaults \
+        -blockdev file,node-name=file,filename=$path,cache.direct=on \
+        <<EOF
+{'execute': 'qmp_capabilities'}
+{'execute': 'block_resize', 'arguments': {'node-name': 'file', 'size': $size}}
+{'execute': 'quit'}
+EOF
+}
+
 # get standard environment, filters and checks
 . ./common.rc
 . ./common.filter
@@ -XXX,XX +XXX,XX @@ _supported_fmt raw
 _supported_proto file
 _supported_os Linux
 
+_default_cache_mode none
+_supported_cache_modes none directsync
+
 size=$((1 * 1024 * 1024))
 
 touch "$TEST_DIR/empty"
@@ -XXX,XX +XXX,XX @@ for mode in off full falloc; do
     stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
 done
 
+for new_size in 4096 1048576; do
+    echo
+    echo "== resize empty image with block_resize =="
+    _make_test_img 0 | _filter_imgfmt
+    _block_resize $TEST_IMG $new_size >/dev/null
+    stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $new_size
+done
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/175.out
+++ b/tests/qemu-iotests/175.out
@@ -XXX,XX +XXX,XX @@ size=1048576, max allocation
 == creating image with preallocation falloc ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc
 size=1048576, max allocation
+
+== resize empty image with block_resize ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
+size=4096, min allocation
+
+== resize empty image with block_resize ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
+size=1048576, min allocation
  *** done
-- 
2.21.0

fe646693acc changed qemu-img create's output so that it no longer prints
single quotes around parameter values.  The subformat and adapter_type
filters in _filter_img_create() have never been adapted to that change.

Fixes: fe646693acc13ac48b98435d14149ab04dc597bc
Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-2-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/059.out       | 16 ++++++++--------
 tests/qemu-iotests/common.filter |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
 
 === Testing monolithicFlat creation and opening ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 2 GiB (2147483648 bytes)
 
 === Testing monolithicFlat with zeroed_grain ===
 qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
 
 === Testing big twoGbMaxExtentFlat ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000
 image: TEST_DIR/t.vmdk
 file format: vmdk
 virtual size: 0.977 TiB (1073741824000 bytes)
@@ -XXX,XX +XXX,XX @@ Format specific information:
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1
 
 === Testing truncated sparse ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes
 
 === Converting to streamOptimized from image with small cluster size===
@@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing monolithicFlat with internally generated JSON file name ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
 
 === Testing version 3 ===
@@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 64931328
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing 4TB monolithicFlat creation and IO ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 4 TiB (4398046511104 bytes)
@@ -XXX,XX +XXX,XX @@ read 1024/1024 bytes at offset 966367641600
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing qemu-img map on extents ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544
 wrote 1024/1024 bytes at offset 65024
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 1024/1024 bytes at offset 2147483136
@@ -XXX,XX +XXX,XX @@ Offset          Length          Mapped to       File
 0               0x20000         0x3f0000        TEST_DIR/t.vmdk
 0x7fff0000      0x20000         0x410000        TEST_DIR/t.vmdk
 0x140000000     0x10000         0x430000        TEST_DIR/t.vmdk
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544
 wrote 1024/1024 bytes at offset 65024
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 1024/1024 bytes at offset 2147483136
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -XXX,XX +XXX,XX @@ _filter_img_create()
         -e "s# compat6=\$on\\|off\$##g" \
         -e "s# static=\$on\\|off\$##g" \
         -e "s# zeroed_grain=\$on\\|off\$##g" \
-        -e "s# subformat='[^']*'##g" \
-        -e "s# adapter_type='[^']*'##g" \
+        -e "s# subformat=[^ ]*##g" \
+        -e "s# adapter_type=[^ ]*##g" \
         -e "s# hwversion=[^ ]*##g" \
         -e "s# lazy_refcounts=\$on\\|off\$##g" \
         -e "s# block_size=[0-9]\\+##g" \
-- 
2.21.0

This makes iotest 033 pass with e.g. subformat=monolithicFlat.  It also
turns a former error in 059 into success.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190815153638.4600-3-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/vmdk.c               | 54 ++++++++++++++++++++++++--------------
 tests/qemu-iotests/059     |  7 +++--
 tests/qemu-iotests/059.out |  4 ++-
 3 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/block/vmdk.c b/block/vmdk.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -XXX,XX +XXX,XX @@ static const char *next_line(const char *s)
 }
 
 static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
-                              const char *desc_file_path, QDict *options,
-                              Error **errp)
+                              QDict *options, Error **errp)
 {
     int ret;
     int matches;
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
     const char *p, *np;
     int64_t sectors = 0;
     int64_t flat_offset;
+    char *desc_file_dir = NULL;
     char *extent_path;
     BdrvChild *extent_file;
     BDRVVmdkState *s = bs->opaque;
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             continue;
         }
 
-        if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
-            !desc_file_path[0])
-        {
-            bdrv_refresh_filename(bs->file->bs);
-            error_setg(errp, "Cannot use relative extent paths with VMDK "
-                       "descriptor file '%s'", bs->file->bs->filename);
-            return -EINVAL;
-        }
+        if (path_is_absolute(fname)) {
+            extent_path = g_strdup(fname);
+        } else {
+            if (!desc_file_dir) {
+                desc_file_dir = bdrv_dirname(bs->file->bs, errp);
+                if (!desc_file_dir) {
+                    bdrv_refresh_filename(bs->file->bs);
+                    error_prepend(errp, "Cannot use relative paths with VMDK "
+                                  "descriptor file '%s': ",
+                                  bs->file->bs->filename);
+                    ret = -EINVAL;
+                    goto out;
+                }
+            }
 
-        extent_path = path_combine(desc_file_path, fname);
+            extent_path = g_strconcat(desc_file_dir, fname, NULL);
+        }
 
         ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
         assert(ret < 32);
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
         g_free(extent_path);
         if (local_err) {
             error_propagate(errp, local_err);
-            return -EINVAL;
+            ret = -EINVAL;
+            goto out;
         }
 
         /* save to extents array */
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
                             0, 0, 0, 0, 0, &extent, errp);
             if (ret < 0) {
                 bdrv_unref_child(bs, extent_file);
-                return ret;
+                goto out;
             }
             extent->flat_start_offset = flat_offset << 9;
         } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
@@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             g_free(buf);
             if (ret) {
                 bdrv_unref_child(bs, extent_file);
-                return ret;
+                goto out;
             }
             extent = &s->extents[s->num_extents - 1];
         } else if (!strcmp(type, "SESPARSE")) {
             ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp);
             if (ret) {
                 bdrv_unref_child(bs, extent_file);
-                return ret;
+                goto out;
             }
             extent = &s->extents[s->num_extents - 1];
         } else {
             error_setg(errp, "Unsupported extent type '%s'", type);
             bdrv_unref_child(bs, extent_file);
-            return -ENOTSUP;
+            ret = -ENOTSUP;
+            goto out;
         }
         extent->type = g_strdup(type);
     }
-    return 0;
+
+    ret = 0;
+    goto out;
 
 invalid:
     np = next_line(p);
@@ -XXX,XX +XXX,XX @@ invalid:
         np--;
     }
     error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);
-    return -EINVAL;
+    ret = -EINVAL;
+
+out:
+    g_free(desc_file_dir);
+    return ret;
 }
 
 static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
@@ -XXX,XX +XXX,XX @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
     }
     s->create_type = g_strdup(ct);
     s->desc_offset = 0;
-    ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
-                             errp);
+    ret = vmdk_parse_extents(buf, bs, options, errp);
 exit:
     return ret;
 }
diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2
 
 echo
 echo "=== Testing monolithicFlat with internally generated JSON file name ==="
+# Should work, because bdrv_dirname() works fine with blkdebug
 IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
-$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \
-    | _filter_testdir | _filter_imgfmt
+$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
+         -c info \
+    2>&1 \
+    | _filter_testdir | _filter_imgfmt | _filter_img_info
 _cleanup_test_img
 
 echo
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240
 
 === Testing monolithicFlat with internally generated JSON file name ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
+format name: IMGFMT
+cluster size: 0 bytes
+vm state offset: 0 bytes
 
 === Testing version 3 ===
 image: TEST_DIR/iotest-version3.IMGFMT
-- 
2.21.0

We had a test for a case where relative extent paths did not work, but
unfortunately we just fixed the underlying problem, so it works now.
This patch adds a new test case that still fails.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-4-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/059     | 27 +++++++++++++++++++++++++++
 tests/qemu-iotests/059.out |  4 ++++
 2 files changed, 31 insertions(+)

diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2
 
 echo
 echo "=== Testing monolithicFlat with internally generated JSON file name ==="
+
+echo '--- blkdebug ---'
 # Should work, because bdrv_dirname() works fine with blkdebug
 IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
 $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
@@ -XXX,XX +XXX,XX @@ $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TE
     | _filter_testdir | _filter_imgfmt | _filter_img_info
 _cleanup_test_img
 
+echo '--- quorum ---'
+# Should not work, because bdrv_dirname() does not work with quorum
+IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
+cp "$TEST_IMG" "$TEST_IMG.orig"
+
+filename="json:{
+    \"driver\": \"$IMGFMT\",
+    \"file\": {
+        \"driver\": \"quorum\",
+        \"children\": [ {
+            \"driver\": \"file\",
+            \"filename\": \"$TEST_IMG\"
+        }, {
+            \"driver\": \"file\",
+            \"filename\": \"$TEST_IMG.orig\"
+        } ],
+        \"vote-threshold\": 1
+    } }"
+
+filename=$(echo "$filename" | tr '\n' ' ' | sed -e 's/\s\+/ /g')
+$QEMU_IMG info "$filename" 2>&1 \
+    | sed -e "s/'json:[^']*'/\$QUORUM_FILE/g" \
+    | _filter_testdir | _filter_imgfmt | _filter_img_info
+
+
 echo
 echo "=== Testing version 3 ==="
 _use_sample_img iotest-version3.vmdk.bz2
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing monolithicFlat with internally generated JSON file name ===
+--- blkdebug ---
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 format name: IMGFMT
 cluster size: 0 bytes
 vm state offset: 0 bytes
+--- quorum ---
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+qemu-img: Could not open $QUORUM_FILE: Cannot use relative paths with VMDK descriptor file $QUORUM_FILE: Cannot generate a base directory for quorum nodes
 
 === Testing version 3 ===
 image: TEST_DIR/iotest-version3.IMGFMT
-- 
2.21.0

Compressed writes generally have to write full clusters, not just in
theory but also in practice when it comes to vmdk's streamOptimized
subformat.  It currently is just silently broken for writes with
non-zero in-cluster offsets:

$ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M
$ qemu-io -c 'write 4k 4k' -c 'read 4k 4k' foo.vmdk
wrote 4096/4096 bytes at offset 4096
4 KiB, 1 ops; 00.01 sec (443.724 KiB/sec and 110.9309 ops/sec)
read failed: Invalid argument

(The technical reason is that vmdk_write_extent() just writes the
incomplete compressed data actually to offset 4k.  When reading the
data, vmdk_read_extent() looks at offset 0 and finds the compressed data
size to be 0, because that is what it reads from there.  This yields an
error.)

For incomplete writes with zero in-cluster offsets, the error path when
reading the rest of the cluster is a bit different, but the result is
the same:

$ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M
$ qemu-io -c 'write 0k 4k' -c 'read 4k 4k' foo.vmdk
wrote 4096/4096 bytes at offset 0
4 KiB, 1 ops; 00.01 sec (362.641 KiB/sec and 90.6603 ops/sec)
read failed: Invalid argument

(Here, vmdk_read_extent() finds the data and then sees that the
uncompressed data is short.)

It is better to reject invalid writes than to make the user believe they
might have succeeded and then fail when trying to read it back.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-5-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/vmdk.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/block/vmdk.c b/block/vmdk.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -XXX,XX +XXX,XX @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
     if (extent->compressed) {
         void *compressed_data;
 
+        /* Only whole clusters */
+        if (offset_in_cluster ||
+            n_bytes > (extent->cluster_sectors * SECTOR_SIZE) ||
+            (n_bytes < (extent->cluster_sectors * SECTOR_SIZE) &&
+             offset + n_bytes != extent->end_sector * SECTOR_SIZE))
+        {
+            ret = -EINVAL;
+            goto out;
+        }
+
         if (!extent->has_marker) {
             ret = -EINVAL;
             goto out;
-- 
2.21.0

streamOptimized does not support writes that do not span exactly one
cluster.  Furthermore, it cannot rewrite already allocated clusters.
As such, many iotests do not work with it.  Disable them.

diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/002
+++ b/tests/qemu-iotests/002
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 
 size=128M
diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/003
+++ b/tests/qemu-iotests/003
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 size=128M
 offset=67M
diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/005
+++ b/tests/qemu-iotests/005
@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
 _supported_proto generic
 _supported_os Linux
 _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 # vpc is limited to 127GB, so we can't test it here
 if [ "$IMGFMT" = "vpc" ]; then
diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/009
+++ b/tests/qemu-iotests/009
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 
 size=6G
diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/010
+++ b/tests/qemu-iotests/010
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 
 size=6G
diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/011
+++ b/tests/qemu-iotests/011
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 
 size=6G
diff --git a/tests/qemu-iotests/017 b/tests/qemu-iotests/017
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/017
+++ b/tests/qemu-iotests/017
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow qcow2 vmdk qed
 _supported_proto generic
 _unsupported_proto vxhs
-_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+                     "subformat=streamOptimized"
 
 TEST_OFFSETS="0 4294967296"
 
diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/018
+++ b/tests/qemu-iotests/018
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow qcow2 vmdk qed
 _supported_proto file
 _supported_os Linux
-_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+                     "streamOptimized"
 
 TEST_OFFSETS="0 4294967296"
 
diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/019
+++ b/tests/qemu-iotests/019
@@ -XXX,XX +XXX,XX @@ _supported_proto file
 _supported_os Linux
 _unsupported_imgopts "subformat=monolithicFlat" \
                      "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 TEST_OFFSETS="0 4294967296"
 CLUSTER_SIZE=65536
diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/020
+++ b/tests/qemu-iotests/020
@@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed
 _supported_proto file
 _unsupported_imgopts "subformat=monolithicFlat" \
                      "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 TEST_OFFSETS="0 4294967296"
 
diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/027
+++ b/tests/qemu-iotests/027
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt vmdk qcow qcow2 qed
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 
 size=128M
diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/032
+++ b/tests/qemu-iotests/032
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # This works for any image format (though unlikely to segfault for raw)
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 echo
 echo === Prepare image ===
diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/033
+++ b/tests/qemu-iotests/033
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt generic
 _supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
 
 
 size=128M
diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/034
+++ b/tests/qemu-iotests/034
@@ -XXX,XX +XXX,XX @@ _supported_proto file
 _supported_os Linux
 _unsupported_imgopts "subformat=monolithicFlat" \
                      "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 CLUSTER_SIZE=4k
 size=128M
diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/037
+++ b/tests/qemu-iotests/037
@@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed
 _supported_proto file
 _unsupported_imgopts "subformat=monolithicFlat" \
                      "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 CLUSTER_SIZE=4k
 size=128M
diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/063
+++ b/tests/qemu-iotests/063
@@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed raw
 _supported_proto file
 _unsupported_imgopts "subformat=monolithicFlat" \
                      "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 _make_test_img 4M
 
diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/072
+++ b/tests/qemu-iotests/072
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow
 _supported_proto file
+_unsupported_imgopts "subformat=streamOptimized"
 
 IMG_SIZE=64M
 
diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/105
+++ b/tests/qemu-iotests/105
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2 vmdk vhdx qed
 _supported_proto generic
 _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
-                     "subformat=twoGbMaxExtentSparse"
+                     "subformat=twoGbMaxExtentSparse" \
+                     "subformat=streamOptimized"
 
 echo
 echo "creating large image"
diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/197
+++ b/tests/qemu-iotests/197
@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
 _supported_proto generic
 # LUKS support may be possible, but it complicates things.
 _unsupported_fmt luks
+_unsupported_imgopts "subformat=streamOptimized"
 
 echo
 echo '=== Copy-on-read ==='
diff --git a/tests/qemu-iotests/215 b/tests/qemu-iotests/215
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/215
+++ b/tests/qemu-iotests/215
@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
 _supported_proto generic
 # LUKS support may be possible, but it complicates things.
 _unsupported_fmt luks
+_unsupported_imgopts "subformat=streamOptimized"
 
 echo
 echo '=== Copy-on-read ==='
diff --git a/tests/qemu-iotests/251 b/tests/qemu-iotests/251
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/251
+++ b/tests/qemu-iotests/251
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt generic
 _supported_proto file
 _supported_os Linux
+_unsupported_imgopts "subformat=streamOptimized"
 
 if [ "$IMGOPTSSYNTAX" = "true" ]; then
     # We use json:{} filenames here, so we cannot work with additional options.
-- 
2.21.0

The error message for the test case where we have a quorum node for
which no directory name can be generated is different: For
twoGbMaxExtentSparse, it complains that it cannot open the extent file.
For other (sub)formats, it just notes that it cannot determine the
backing file path.  Both are fine, but just disable twoGbMaxExtentSparse
for simplicity's sake.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-7-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/110 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/qemu-iotests/110 b/tests/qemu-iotests/110
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/110
+++ b/tests/qemu-iotests/110
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # Any format supporting backing files
 _supported_fmt qed qcow qcow2 vmdk
 _supported_proto file
-_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+                     "subformat=twoGbMaxExtentSparse"
 
 TEST_IMG_REL=$(basename "$TEST_IMG")
 
-- 
2.21.0

From: Stefan Hajnoczi <stefanha@redhat.com>

Fixes: a6b257a08e3d72219f03e461a52152672fec0612
       ("file-posix: Handle undetectable alignment")
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: 20190827101328.4062-1-stefanha@redhat.com
Reviewed-by: Thomas Huth <thuth@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/file-posix.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/block/file-posix.c b/block/file-posix.c
index XXXXXXX..XXXXXXX 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -XXX,XX +XXX,XX @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
         for (i = 0; i < ARRAY_SIZE(alignments); i++) {
             align = alignments[i];
             if (raw_is_io_aligned(fd, buf + align, max_align)) {
-                /* Fallback to request_aligment. */
+                /* Fallback to request_alignment. */
                 s->buf_align = (align != 1) ? align : bs->bl.request_alignment;
                 break;
             }
-- 
2.21.0

From: Thomas Huth <thuth@redhat.com>

It is possible to enable only a subset of the block drivers with the
"--block-drv-rw-whitelist" option of the "configure" script. All other
drivers are marked as unusable (or only included as read-only with the
"--block-drv-ro-whitelist" option). If an iotest is now using such a
disabled block driver, it is failing - which is bad, since at least the
tests in the "auto" group should be able to deal with this situation.
Thus let's introduce a "_require_drivers" function that can be used by
the shell tests to check for the availability of certain drivers first,
and marks the test as "not run" if one of the drivers is missing.

This patch mainly targets the test in the "auto" group which should
never fail in such a case, but also improves some of the other tests
along the way. Note that we also assume that the "qcow2" and "file"
drivers are always available - otherwise it does not make sense to
run "make check-block" at all (which only tests with qcow2 by default).

Signed-off-by: Thomas Huth <thuth@redhat.com>
Message-id: 20190823133552.11680-1-thuth@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/071       |  1 +
 tests/qemu-iotests/081       |  4 +---
 tests/qemu-iotests/099       |  1 +
 tests/qemu-iotests/120       |  1 +
 tests/qemu-iotests/162       |  4 +---
 tests/qemu-iotests/184       |  1 +
 tests/qemu-iotests/186       |  1 +
 tests/qemu-iotests/common.rc | 14 ++++++++++++++
 8 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/071
+++ b/tests/qemu-iotests/071
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt qcow2
 _supported_proto file
+_require_drivers blkdebug blkverify
 
 do_run_qemu()
 {
diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/081
+++ b/tests/qemu-iotests/081
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt raw
 _supported_proto file
 _supported_os Linux
+_require_drivers quorum
 
 do_run_qemu()
 {
@@ -XXX,XX +XXX,XX @@ run_qemu()
                           | _filter_qemu_io | _filter_generated_node_ids
 }
 
-test_quorum=$($QEMU_IMG --help|grep quorum)
-[ "$test_quorum" = "" ] && _supported_fmt quorum
-
 quorum="driver=raw,file.driver=quorum,file.vote-threshold=2"
 quorum="$quorum,file.children.0.file.filename=$TEST_DIR/1.raw"
 quorum="$quorum,file.children.1.file.filename=$TEST_DIR/2.raw"
diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/099
+++ b/tests/qemu-iotests/099
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc
 _supported_proto file
 _supported_os Linux
+_require_drivers blkdebug blkverify
 _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
     "subformat=twoGbMaxExtentSparse"
 
diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/120
+++ b/tests/qemu-iotests/120
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt generic
 _supported_proto file
 _unsupported_fmt luks
+_require_drivers raw
 
 _make_test_img 64M
 
diff --git a/tests/qemu-iotests/162 b/tests/qemu-iotests/162
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/162
+++ b/tests/qemu-iotests/162
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_fmt generic
-
-test_ssh=$($QEMU_IMG --help | grep '^Supported formats:.* ssh$ \|$$')
-[ "$test_ssh" = "" ] && _notrun "ssh support required"
+_require_drivers ssh
 
 echo
 echo '=== NBD ==='
diff --git a/tests/qemu-iotests/184 b/tests/qemu-iotests/184
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/184
+++ b/tests/qemu-iotests/184
@@ -XXX,XX +XXX,XX @@ trap "exit \$status" 0 1 2 3 15
 . ./common.filter
 
 _supported_os Linux
+_require_drivers throttle
 
 do_run_qemu()
 {
diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/186
+++ b/tests/qemu-iotests/186
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _supported_fmt qcow2
 _supported_proto file
+_require_drivers null-co
 
 if [ "$QEMU_DEFAULT_MACHINE" != "pc" ]; then
     _notrun "Requires a PC machine"
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -XXX,XX +XXX,XX @@ _require_command()
     [ -x "$c" ] || _notrun "$1 utility required, skipped this test"
 }
 
+# Check that a set of drivers has been whitelisted in the QEMU binary
+#
+_require_drivers()
+{
+    available=$($QEMU -drive format=help | \
+                sed -e '/Supported formats:/!d' -e 's/Supported formats://')
+    for driver
+    do
+        if ! echo "$available" | grep -q " $driver$ \|$$"; then
+            _notrun "$driver not available"
+        fi
+    done
+}
+
 # make sure this script returns success
 true
-- 
2.21.0

From: Thomas Huth <thuth@redhat.com>

The sanitizers (especially the address sanitizer from Clang) are
sometimes printing out warnings or false positives - this spoils
the output of the iotests, causing some of the tests to fail.
Thus let's skip the automatic iotests during "make check" when the
user configured QEMU with --enable-sanitizers.

Signed-off-by: Thomas Huth <thuth@redhat.com>
Message-id: 20190823084203.29734-1-thuth@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/check-block.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/check-block.sh b/tests/check-block.sh
index XXXXXXX..XXXXXXX 100755
--- a/tests/check-block.sh
+++ b/tests/check-block.sh
@@ -XXX,XX +XXX,XX @@ if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then
     exit 0
 fi
 
+if grep -q "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ; then
+    echo "Sanitizers are enabled ==> Not running the qemu-iotests."
+    exit 0
+fi
+
 if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then
     echo "No qemu-system binary available ==> Not running the qemu-iotests."
     exit 0
-- 
2.21.0

From: Nir Soffer <nirsof@gmail.com>

Quoting cache mode is not needed, and most tests use unquoted values.
Unify all test to use the same style.

Message-id: 20190827173432.7656-1-nsoffer@redhat.com
Signed-off-by: Nir Soffer <nsoffer@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/026 | 4 ++--
 tests/qemu-iotests/039 | 4 ++--
 tests/qemu-iotests/052 | 2 +-
 tests/qemu-iotests/091 | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/026
+++ b/tests/qemu-iotests/026
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 # Currently only qcow2 supports rebasing
 _supported_fmt qcow2
 _supported_proto file
-_default_cache_mode "writethrough"
-_supported_cache_modes "writethrough" "none"
+_default_cache_mode writethrough
+_supported_cache_modes writethrough none
 # The refcount table tests expect a certain minimum width for refcount entries
 # (so that the refcount table actually needs to grow); that minimum is 16 bits,
 # being the default refcount entry width.
diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
-_default_cache_mode "writethrough"
-_supported_cache_modes "writethrough"
+_default_cache_mode writethrough
+_supported_cache_modes writethrough
 
 size=128M
 
diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/052
+++ b/tests/qemu-iotests/052
@@ -XXX,XX +XXX,XX @@ _supported_fmt generic
 _supported_proto file
 
 # Don't do O_DIRECT on tmpfs
-_supported_cache_modes "writeback" "writethrough" "unsafe"
+_supported_cache_modes writeback writethrough unsafe
 
 size=128M
 _make_test_img $size
diff --git a/tests/qemu-iotests/091 b/tests/qemu-iotests/091
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/091
+++ b/tests/qemu-iotests/091
@@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
-_default_cache_mode "none"
-_supported_cache_modes "writethrough" "none" "writeback"
+_default_cache_mode none
+_supported_cache_modes writethrough none writeback
 
 size=1G
 
-- 
2.21.0

The following changes since commit 5375af3cd7b8adcc10c18d8083b7be63976c9645:

Merge remote-tracking branch 'remotes/mst/tags/for_upstream' into staging (2020-05-04 15:51:09 +0100)

are available in the Git repository at:

https://github.com/XanClic/qemu.git tags/pull-block-2020-05-05

for you to fetch changes up to 4ce5dd3e9b5ee0fac18625860eb3727399ee965e:

block/block-copy: use aio-task-pool API (2020-05-05 14:03:28 +0200)

----------------------------------------------------------------
Block patches:
- Asynchronous copying for block-copy (i.e., the backup job)
- Allow resizing of qcow2 images when they have internal snapshots
- iotests: Logging improvements for Python tests
- iotest 153 fix, and block comment cleanups

----------------------------------------------------------------
Eric Blake (4):
  block: Add blk_new_with_bs() helper
  qcow2: Allow resize of images with internal snapshots
  qcow2: Tweak comment about bitmaps vs. resize
  block: Comment cleanups

John Snow (14):
  iotests: do a light delinting
  iotests: don't use 'format' for drive_add
  iotests: ignore import warnings from pylint
  iotests: replace mutable list default args
  iotests: add pylintrc file
  iotests: alphabetize standard imports
  iotests: drop pre-Python 3.4 compatibility code
  iotests: touch up log function signature
  iotests: limit line length to 79 chars
  iotests: add hmp helper with logging
  iotests: add script_initialize
  iotest 258: use script_main
  iotests: Mark verify functions as private
  iotests: use python logging for iotests.log()

Maxim Levitsky (1):
  Fix iotest 153

Vladimir Sementsov-Ogievskiy (5):
  block/block-copy: rename in-flight requests to tasks
  block/block-copy: alloc task on each iteration
  block/block-copy: add state pointer to BlockCopyTask
  block/block-copy: refactor task creation
  block/block-copy: use aio-task-pool API

-- 
2.26.2

From: John Snow <jsnow@redhat.com>

This doesn't fix everything in here, but it does help clean up the
pylint report considerably.

This should be 100% style changes only; the intent is to make pylint
more useful by working on establishing a baseline for iotests that we
can gate against in the future.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-2-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 83 ++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 40 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import errno
 import os
 import re
 import subprocess
-import string
 import unittest
 import sys
 import struct
@@ -XXX,XX +XXX,XX @@ import faulthandler
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import qtest
 
-assert sys.version_info >= (3,6)
+assert sys.version_info >= (3, 6)
 
 faulthandler.enable()
 
@@ -XXX,XX +XXX,XX @@ def qemu_img_log(*args):
     return result
 
 def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]):
-    args = [ 'info' ]
+    args = ['info']
     if imgopts:
         args.append('--image-opts')
     else:
-        args += [ '-f', imgfmt ]
+        args += ['-f', imgfmt]
     args += extra_args
     args.append(filename)
 
@@ -XXX,XX +XXX,XX @@ class QemuIoInteractive:
         # quit command is in close(), '\n' is added automatically
         assert '\n' not in cmd
         cmd = cmd.strip()
-        assert cmd != 'q' and cmd != 'quit'
+        assert cmd not in ('q', 'quit')
         self._p.stdin.write(cmd + '\n')
         self._p.stdin.flush()
         return self._read_output()
@@ -XXX,XX +XXX,XX @@ def qemu_nbd_early_pipe(*args):
         sys.stderr.write('qemu-nbd received signal %i: %s\n' %
                          (-exitcode,
                           ' '.join(qemu_nbd_args + ['--fork'] + list(args))))
-    if exitcode == 0:
-        return exitcode, ''
-    else:
-        return exitcode, subp.communicate()[0]
+
+    return exitcode, subp.communicate()[0] if exitcode else ''
 
 def qemu_nbd_popen(*args):
     '''Run qemu-nbd in daemon mode and return the parent's exit code'''
@@ -XXX,XX +XXX,XX @@ def filter_qmp(qmsg, filter_fn):
         items = qmsg.items()
 
     for k, v in items:
-        if isinstance(v, list) or isinstance(v, dict):
+        if isinstance(v, (dict, list)):
             qmsg[k] = filter_qmp(v, filter_fn)
         else:
             qmsg[k] = filter_fn(k, v)
@@ -XXX,XX +XXX,XX @@ def filter_testfiles(msg):
     return msg.replace(prefix, 'TEST_DIR/PID-')
 
 def filter_qmp_testfiles(qmsg):
-    def _filter(key, value):
+    def _filter(_key, value):
         if is_str(value):
             return filter_testfiles(value)
         return value
@@ -XXX,XX +XXX,XX @@ def filter_imgfmt(msg):
     return msg.replace(imgfmt, 'IMGFMT')
 
 def filter_qmp_imgfmt(qmsg):
-    def _filter(key, value):
+    def _filter(_key, value):
         if is_str(value):
             return filter_imgfmt(value)
         return value
@@ -XXX,XX +XXX,XX @@ def log(msg, filters=[], indent=None):
     If indent is provided, JSON serializable messages are pretty-printed.'''
     for flt in filters:
         msg = flt(msg)
-    if isinstance(msg, dict) or isinstance(msg, list):
+    if isinstance(msg, (dict, list)):
         # Python < 3.4 needs to know not to add whitespace when pretty-printing:
         separators = (', ', ': ') if indent is None else (',', ': ')
         # Don't sort if it's already sorted
@@ -XXX,XX +XXX,XX @@ def log(msg, filters=[], indent=None):
         print(msg)
 
 class Timeout:
-    def __init__(self, seconds, errmsg = "Timeout"):
+    def __init__(self, seconds, errmsg="Timeout"):
         self.seconds = seconds
         self.errmsg = errmsg
     def __enter__(self):
         signal.signal(signal.SIGALRM, self.timeout)
         signal.setitimer(signal.ITIMER_REAL, self.seconds)
         return self
-    def __exit__(self, type, value, traceback):
+    def __exit__(self, exc_type, value, traceback):
         signal.setitimer(signal.ITIMER_REAL, 0)
         return False
     def timeout(self, signum, frame):
@@ -XXX,XX +XXX,XX @@ class Timeout:
 def file_pattern(name):
     return "{0}-{1}".format(os.getpid(), name)
 
-class FilePaths(object):
+class FilePaths:
     """
     FilePaths is an auto-generated filename that cleans itself up.
 
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
             self.pause_drive(drive, "write_aio")
             return
         self.qmp('human-monitor-command',
-                    command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive))
+                 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive))
 
     def resume_drive(self, drive):
         self.qmp('human-monitor-command',
-                    command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive))
+                 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive))
 
     def hmp_qemu_io(self, drive, cmd):
         '''Write to a given drive using an HMP command'''
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
         if output is None:
             output = dict()
         if isinstance(obj, list):
-            for i in range(len(obj)):
-                self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.')
+            for i, item in enumerate(obj):
+                self.flatten_qmp_object(item, output, basestr + str(i) + '.')
         elif isinstance(obj, dict):
             for key in obj:
                 self.flatten_qmp_object(obj[key], output, basestr + key + '.')
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
 
         for bitmap in bitmaps[node_name]:
             if bitmap.get('name', '') == bitmap_name:
-                if recording is None:
-                    return bitmap
-                elif bitmap.get('recording') == recording:
+                if recording is None or bitmap.get('recording') == recording:
                     return bitmap
         return None
 
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
             assert node is not None, 'Cannot follow path %s%s' % (root, path)
 
             try:
-                node_id = next(edge['child'] for edge in graph['edges'] \
-                                             if edge['parent'] == node['id'] and
-                                                edge['name'] == child_name)
+                node_id = next(edge['child'] for edge in graph['edges']
+                               if (edge['parent'] == node['id'] and
+                                   edge['name'] == child_name))
+
+                node = next(node for node in graph['nodes']
+                            if node['id'] == node_id)
 
-                node = next(node for node in graph['nodes'] \
-                                 if node['id'] == node_id)
             except StopIteration:
                 node = None
 
@@ -XXX,XX +XXX,XX @@ index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
 class QMPTestCase(unittest.TestCase):
     '''Abstract base class for QMP test cases'''
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Many users of this class set a VM property we rely on heavily
+        # in the methods below.
+        self.vm = None
+
     def dictpath(self, d, path):
         '''Traverse a path in a nested dict'''
         for component in path.split('/'):
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
         else:
             self.assertEqual(result, value,
                              '"%s" is "%s", expected "%s"'
-                                 % (path, str(result), str(value)))
+                             % (path, str(result), str(value)))
 
     def assert_no_active_block_jobs(self):
         result = self.vm.qmp('query-block-jobs')
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
         """Issue a query-named-block-nodes and assert node_name and/or
         file_name is present in the result"""
         def check_equal_or_none(a, b):
-            return a == None or b == None or a == b
+            return a is None or b is None or a == b
         assert node_name or file_name
         result = self.vm.qmp('query-named-block-nodes')
         for x in result["return"]:
             if check_equal_or_none(x.get("node-name"), node_name) and \
                     check_equal_or_none(x.get("file"), file_name):
                 return
-        self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \
-                (node_name, file_name, result))
+        self.fail("Cannot find %s %s in result:\n%s" %
+                  (node_name, file_name, result))
 
     def assert_json_filename_equal(self, json_filename, reference):
         '''Asserts that the given filename is a json: filename and that its
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
                         self.assert_qmp(event, 'data/error', error)
                     self.assert_no_active_block_jobs()
                     return event
-                elif event['event'] == 'JOB_STATUS_CHANGE':
+                if event['event'] == 'JOB_STATUS_CHANGE':
                     self.assert_qmp(event, 'data/id', drive)
 
     def wait_ready(self, drive='drive0'):
-        '''Wait until a block job BLOCK_JOB_READY event'''
-        f = {'data': {'type': 'mirror', 'device': drive } }
-        event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f)
+        """Wait until a BLOCK_JOB_READY event, and return the event."""
+        f = {'data': {'type': 'mirror', 'device': drive}}
+        return self.vm.event_wait(name='BLOCK_JOB_READY', match=f)
 
     def wait_ready_and_cancel(self, drive='drive0'):
         self.wait_ready(drive=drive)
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
                 for job in result['return']:
                     if job['device'] == job_id:
                         found = True
-                        if job['paused'] == True and job['busy'] == False:
+                        if job['paused'] and not job['busy']:
                             return job
                         break
                 assert found
@@ -XXX,XX +XXX,XX @@ def qemu_pipe(*args):
                             universal_newlines=True)
     exitcode = subp.wait()
     if exitcode < 0:
-        sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode,
-                         ' '.join(args)))
+        sys.stderr.write('qemu received signal %i: %s\n' %
+                         (-exitcode, ' '.join(args)))
     return subp.communicate()[0]
 
 def supported_formats(read_only=False):
@@ -XXX,XX +XXX,XX @@ def skip_if_unsupported(required_formats=[], read_only=False):
             if usf_list:
                 test_case.case_skip('{}: formats {} are not whitelisted'.format(
                     test_case, usf_list))
+                return None
             else:
                 return func(test_case, *args, **kwargs)
         return func_wrapper
@@ -XXX,XX +XXX,XX @@ def skip_if_user_is_root(func):
     def func_wrapper(*args, **kwargs):
         if os.getuid() == 0:
             case_notrun('{}: cannot be run as root'.format(args[0]))
+            return None
         else:
             return func(*args, **kwargs)
     return func_wrapper
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

It shadows (with a different type) the built-in format.
Use something else.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-3-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/055        | 3 ++-
 tests/qemu-iotests/iotests.py | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/055
+++ b/tests/qemu-iotests/055
@@ -XXX,XX +XXX,XX @@ class TestDriveCompression(iotests.QMPTestCase):
         qemu_img('create', '-f', fmt, blockdev_target_img,
                  str(TestDriveCompression.image_len), *args)
         if attach_target:
-            self.vm.add_drive(blockdev_target_img, format=fmt, interface="none")
+            self.vm.add_drive(blockdev_target_img,
+                              img_format=fmt, interface="none")
 
         self.vm.launch()
 
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
         self._args.append(opts)
         return self
 
-    def add_drive(self, path, opts='', interface='virtio', format=imgfmt):
+    def add_drive(self, path, opts='', interface='virtio', img_format=imgfmt):
         '''Add a virtio-blk drive to the VM'''
         options = ['if=%s' % interface,
                    'id=drive%d' % self._num_drives]
 
         if path is not None:
             options.append('file=%s' % path)
-            options.append('format=%s' % format)
+            options.append('format=%s' % img_format)
             options.append('cache=%s' % cachemode)
             options.append('aio=%s' % aiomode)
 
         if opts:
             options.append(opts)
 
-        if format == 'luks' and 'key-secret' not in opts:
+        if img_format == 'luks' and 'key-secret' not in opts:
             # default luks support
             if luks_default_secret_object not in self._args:
                 self.add_object(luks_default_secret_object)
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

The right way to solve this is to come up with a virtual environment
infrastructure that sets all the paths correctly, and/or to create
installable python modules that can be imported normally.

That's hard, so just silence this error for now.

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: John Snow <jsnow@redhat.com>
Message-Id: <20200331000014.11581-4-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ import io
 from collections import OrderedDict
 import faulthandler
 
+# pylint: disable=import-error, wrong-import-position
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import qtest
 
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

It's bad hygiene: if we modify this list, it will be modified across all
invocations.

(Remaining bad usages are fixed in a subsequent patch which changes the
function signature anyway.)

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-5-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ def qemu_img_log(*args):
     log(result, filters=[filter_testfiles])
     return result
 
-def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]):
+def img_info_log(filename, filter_path=None, imgopts=False, extra_args=()):
     args = ['info']
     if imgopts:
         args.append('--image-opts')
@@ -XXX,XX +XXX,XX @@ def filter_qmp_imgfmt(qmsg):
         return value
     return filter_qmp(qmsg, _filter)
 
-def log(msg, filters=[], indent=None):
+def log(msg, filters=(), indent=None):
     '''Logs either a string message or a JSON serializable message (like QMP).
     If indent is provided, JSON serializable messages are pretty-printed.'''
     for flt in filters:
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
             result.append(filter_qmp_event(ev))
         return result
 
-    def qmp_log(self, cmd, filters=[], indent=None, **kwargs):
+    def qmp_log(self, cmd, filters=(), indent=None, **kwargs):
         full_cmd = OrderedDict((
             ("execute", cmd),
             ("arguments", ordered_qmp(kwargs))
@@ -XXX,XX +XXX,XX @@ def case_notrun(reason):
     open('%s/%s.casenotrun' % (output_dir, seq), 'a').write(
         '    [case not run] ' + reason + '\n')
 
-def verify_image_format(supported_fmts=[], unsupported_fmts=[]):
+def verify_image_format(supported_fmts=(), unsupported_fmts=()):
     assert not (supported_fmts and unsupported_fmts)
 
     if 'generic' in supported_fmts and \
@@ -XXX,XX +XXX,XX @@ def verify_image_format(supported_fmts=[], unsupported_fmts=[]):
     if not_sup or (imgfmt in unsupported_fmts):
         notrun('not suitable for this image format: %s' % imgfmt)
 
-def verify_protocol(supported=[], unsupported=[]):
+def verify_protocol(supported=(), unsupported=()):
     assert not (supported and unsupported)
 
     if 'generic' in supported:
@@ -XXX,XX +XXX,XX @@ def verify_platform(supported=None, unsupported=None):
         if not any((sys.platform.startswith(x) for x in supported)):
             notrun('not suitable for this OS: %s' % sys.platform)
 
-def verify_cache_mode(supported_cache_modes=[]):
+def verify_cache_mode(supported_cache_modes=()):
     if supported_cache_modes and (cachemode not in supported_cache_modes):
         notrun('not suitable for this cache mode: %s' % cachemode)
 
-def verify_aio_mode(supported_aio_modes=[]):
+def verify_aio_mode(supported_aio_modes=()):
     if supported_aio_modes and (aiomode not in supported_aio_modes):
         notrun('not suitable for this aio mode: %s' % aiomode)
 
@@ -XXX,XX +XXX,XX @@ def supported_formats(read_only=False):
 
     return supported_formats.formats[read_only]
 
-def skip_if_unsupported(required_formats=[], read_only=False):
+def skip_if_unsupported(required_formats=(), read_only=False):
     '''Skip Test Decorator
        Runs the test if all the required formats are whitelisted'''
     def skip_test_decorator(func):
@@ -XXX,XX +XXX,XX @@ def execute_unittest(output, verbosity, debug):
             sys.stderr.write(out)
 
 def execute_test(test_function=None,
-                 supported_fmts=[],
+                 supported_fmts=(),
                  supported_platforms=None,
-                 supported_cache_modes=[], supported_aio_modes={},
-                 unsupported_fmts=[], supported_protocols=[],
-                 unsupported_protocols=[]):
+                 supported_cache_modes=(), supported_aio_modes=(),
+                 unsupported_fmts=(), supported_protocols=(),
+                 unsupported_protocols=()):
     """Run either unittest or script-style tests."""
 
     # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

This allows others to get repeatable results with pylint. If you run
`pylint iotests.py`, you should see a 100% pass.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-6-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/pylintrc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 tests/qemu-iotests/pylintrc

diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/pylintrc
@@ -XXX,XX +XXX,XX @@
+[MESSAGES CONTROL]
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=invalid-name,
+        no-else-return,
+        too-few-public-methods,
+        too-many-arguments,
+        too-many-branches,
+        too-many-lines,
+        too-many-locals,
+        too-many-public-methods,
+        # These are temporary, and should be removed:
+        line-too-long,
+        missing-docstring,
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

I had to fix a merge conflict, so do this tiny harmless thing while I'm
here.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-7-jsnow@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

From: John Snow <jsnow@redhat.com>

We no longer need to accommodate <3.4, drop this code.
(The lines were > 79 chars and it stood out.)

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-8-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ def log(msg, filters=(), indent=None):
     for flt in filters:
         msg = flt(msg)
     if isinstance(msg, (dict, list)):
-        # Python < 3.4 needs to know not to add whitespace when pretty-printing:
-        separators = (', ', ': ') if indent is None else (',', ': ')
         # Don't sort if it's already sorted
         do_sort = not isinstance(msg, OrderedDict)
-        print(json.dumps(msg, sort_keys=do_sort,
-                         indent=indent, separators=separators))
+        print(json.dumps(msg, sort_keys=do_sort, indent=indent))
     else:
         print(msg)
 
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

Representing nested, recursive data structures in mypy is notoriously
difficult; the best we can reliably do right now is denote the leaf
types as "Any" while describing the general shape of the data.

Regardless, this fully annotates the log() function.

Typing notes:

TypeVar is a Type variable that can optionally be constrained by a
sequence of possible types. This variable is bound to a specific type
per-invocation, like a Generic.

log() behaves as log<Msg>() now, where the incoming type informs the
signature it expects for any filter arguments passed in. If Msg is a
str, then filter should take and return a str.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-9-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ import signal
 import struct
 import subprocess
 import sys
+from typing import (Any, Callable, Dict, Iterable, List, Optional, TypeVar)
 import unittest
 
 # pylint: disable=import-error, wrong-import-position
@@ -XXX,XX +XXX,XX @@ def filter_qmp_imgfmt(qmsg):
         return value
     return filter_qmp(qmsg, _filter)
 
-def log(msg, filters=(), indent=None):
-    '''Logs either a string message or a JSON serializable message (like QMP).
-    If indent is provided, JSON serializable messages are pretty-printed.'''
+
+Msg = TypeVar('Msg', Dict[str, Any], List[Any], str)
+
+def log(msg: Msg,
+        filters: Iterable[Callable[[Msg], Msg]] = (),
+        indent: Optional[int] = None) -> None:
+    """
+    Logs either a string message or a JSON serializable message (like QMP).
+    If indent is provided, JSON serializable messages are pretty-printed.
+    """
     for flt in filters:
         msg = flt(msg)
     if isinstance(msg, (dict, list)):
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

79 is the PEP8 recommendation. This recommendation works well for
reading patch diffs in TUI email clients.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-10-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 64 +++++++++++++++++++++++------------
 tests/qemu-iotests/pylintrc   |  6 +++-
 2 files changed, 47 insertions(+), 23 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ luks_default_key_secret_opt = 'key-secret=keysec0'
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
-    exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    exitcode = subprocess.call(qemu_img_args + list(args),
+                               stdin=devnull, stdout=devnull)
     if exitcode < 0:
-        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+        sys.stderr.write('qemu-img received signal %i: %s\n'
+                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
     return exitcode
 
 def ordered_qmp(qmsg, conv_keys=True):
@@ -XXX,XX +XXX,XX @@ def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
     exitcode = subprocess.call(qemu_img_args + list(args))
     if exitcode < 0:
-        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+        sys.stderr.write('qemu-img received signal %i: %s\n'
+                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
     return exitcode
 
 def qemu_img_pipe(*args):
@@ -XXX,XX +XXX,XX @@ def qemu_img_pipe(*args):
                             universal_newlines=True)
     exitcode = subp.wait()
     if exitcode < 0:
-        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+        sys.stderr.write('qemu-img received signal %i: %s\n'
+                         % (-exitcode, ' '.join(qemu_img_args + list(args))))
     return subp.communicate()[0]
 
 def qemu_img_log(*args):
@@ -XXX,XX +XXX,XX @@ def qemu_io(*args):
                             universal_newlines=True)
     exitcode = subp.wait()
     if exitcode < 0:
-        sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args)))
+        sys.stderr.write('qemu-io received signal %i: %s\n'
+                         % (-exitcode, ' '.join(args)))
     return subp.communicate()[0]
 
 def qemu_io_log(*args):
@@ -XXX,XX +XXX,XX @@ win32_re = re.compile(r"\r")
 def filter_win32(msg):
     return win32_re.sub("", msg)
 
-qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* $[0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec$")
+qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* "
+                        r"$[0-9\/.inf]* [EPTGMKiBbytes]*\/sec "
+                        r"and [0-9\/.inf]* ops\/sec$")
 def filter_qemu_io(msg):
     msg = filter_win32(msg)
-    return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg)
+    return qemu_io_re.sub("X ops; XX:XX:XX.X "
+                          "(XXX YYY/sec and XXX ops/sec)", msg)
 
 chown_re = re.compile(r"chown [0-9]+:[0-9]+")
 def filter_chown(msg):
@@ -XXX,XX +XXX,XX @@ def filter_img_info(output, filename):
         line = filter_testfiles(line)
         line = line.replace(imgfmt, 'IMGFMT')
         line = re.sub('iters: [0-9]+', 'iters: XXX', line)
-        line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line)
+        line = re.sub('uuid: [-a-f0-9]+',
+                      'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',
+                      line)
         line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line)
         lines.append(line)
     return '\n'.join(lines)
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
             self.pause_drive(drive, "write_aio")
             return
         self.qmp('human-monitor-command',
-                 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive))
+                 command_line='qemu-io %s "break %s bp_%s"'
+                 % (drive, event, drive))
 
     def resume_drive(self, drive):
         self.qmp('human-monitor-command',
-                 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive))
+                 command_line='qemu-io %s "remove_break bp_%s"'
+                 % (drive, drive))
 
     def hmp_qemu_io(self, drive, cmd):
         '''Write to a given drive using an HMP command'''
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
                 idx = int(idx)
 
             if not isinstance(d, dict) or component not in d:
-                self.fail('failed path traversal for "%s" in "%s"' % (path, str(d)))
+                self.fail(f'failed path traversal for "{path}" in "{d}"')
             d = d[component]
 
             if m:
                 if not isinstance(d, list):
-                    self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d)))
+                    self.fail(f'path component "{component}" in "{path}" '
+                              f'is not a list in "{d}"')
                 try:
                     d = d[idx]
                 except IndexError:
-                    self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
+                    self.fail(f'invalid index "{idx}" in path "{path}" '
+                              f'in "{d}"')
         return d
 
     def assert_qmp_absent(self, d, path):
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
         '''Asserts that the given filename is a json: filename and that its
            content is equal to the given reference object'''
         self.assertEqual(json_filename[:5], 'json:')
-        self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])),
-                         self.vm.flatten_qmp_object(reference))
+        self.assertEqual(
+            self.vm.flatten_qmp_object(json.loads(json_filename[5:])),
+            self.vm.flatten_qmp_object(reference)
+        )
 
-    def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0):
+    def cancel_and_wait(self, drive='drive0', force=False,
+                        resume=False, wait=60.0):
         '''Cancel a block job and wait for it to finish, returning the event'''
         result = self.vm.qmp('block-job-cancel', device=drive, force=force)
         self.assert_qmp(result, 'return', {})
@@ -XXX,XX +XXX,XX @@ class QMPTestCase(unittest.TestCase):
         self.assert_no_active_block_jobs()
         return result
 
-    def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0,
-                             error=None):
+    def wait_until_completed(self, drive='drive0', check_offset=True,
+                             wait=60.0, error=None):
         '''Wait for a block job to finish, returning the event'''
         while True:
             for event in self.vm.get_qmp_events(wait=wait):
@@ -XXX,XX +XXX,XX @@ def verify_quorum():
         notrun('quorum support missing')
 
 def qemu_pipe(*args):
-    '''Run qemu with an option to print something and exit (e.g. a help option),
-    and return its output'''
+    """
+    Run qemu with an option to print something and exit (e.g. a help option).
+
+    :return: QEMU's stdout output.
+    """
     args = [qemu_prog] + qemu_opts + list(args)
     subp = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
@@ -XXX,XX +XXX,XX @@ def skip_if_unsupported(required_formats=(), read_only=False):
 
             usf_list = list(set(fmts) - set(supported_formats(read_only)))
             if usf_list:
-                test_case.case_skip('{}: formats {} are not whitelisted'.format(
-                    test_case, usf_list))
+                msg = f'{test_case}: formats {usf_list} are not whitelisted'
+                test_case.case_skip(msg)
                 return None
             else:
                 return func(test_case, *args, **kwargs)
diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/pylintrc
+++ b/tests/qemu-iotests/pylintrc
@@ -XXX,XX +XXX,XX @@ disable=invalid-name,
         too-many-locals,
         too-many-public-methods,
         # These are temporary, and should be removed:
-        line-too-long,
         missing-docstring,
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=79
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

Minor cleanup for HMP functions; helps with line length and consolidates
HMP helpers through one implementation function.

Although we are adding a universal toggle to turn QMP logging on or off,
many existing callers to hmp functions don't expect that output to be
logged, which causes quite a few changes in the test output.

For now, offer a use_log parameter.

Typing notes:

QMPResponse is just an alias for Dict[str, Any]. It holds no special
meanings and it is not a formal subtype of Dict[str, Any]. It is best
thought of as a lexical synonym.

We may well wish to add stricter subtypes in the future for certain
shapes of data that are not formalized as Python objects, at which point
we can simply retire the alias and allow mypy to more strictly check
usages of the name.

Signed-off-by: John Snow <jsnow@redhat.com>
Message-Id: <20200331000014.11581-11-jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 39 +++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ from qemu import qtest
 
 assert sys.version_info >= (3, 6)
 
+# Type Aliases
+QMPResponse = Dict[str, Any]
+
+
 faulthandler.enable()
 
 # This will not work if arguments contain spaces but is necessary if we
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
         self._args.append(addr)
         return self
 
-    def pause_drive(self, drive, event=None):
-        '''Pause drive r/w operations'''
+    def hmp(self, command_line: str, use_log: bool = False) -> QMPResponse:
+        cmd = 'human-monitor-command'
+        kwargs = {'command-line': command_line}
+        if use_log:
+            return self.qmp_log(cmd, **kwargs)
+        else:
+            return self.qmp(cmd, **kwargs)
+
+    def pause_drive(self, drive: str, event: Optional[str] = None) -> None:
+        """Pause drive r/w operations"""
         if not event:
             self.pause_drive(drive, "read_aio")
             self.pause_drive(drive, "write_aio")
             return
-        self.qmp('human-monitor-command',
-                 command_line='qemu-io %s "break %s bp_%s"'
-                 % (drive, event, drive))
-
-    def resume_drive(self, drive):
-        self.qmp('human-monitor-command',
-                 command_line='qemu-io %s "remove_break bp_%s"'
-                 % (drive, drive))
-
-    def hmp_qemu_io(self, drive, cmd):
-        '''Write to a given drive using an HMP command'''
-        return self.qmp('human-monitor-command',
-                        command_line='qemu-io %s "%s"' % (drive, cmd))
+        self.hmp(f'qemu-io {drive} "break {event} bp_{drive}"')
+
+    def resume_drive(self, drive: str) -> None:
+        """Resume drive r/w operations"""
+        self.hmp(f'qemu-io {drive} "remove_break bp_{drive}"')
+
+    def hmp_qemu_io(self, drive: str, cmd: str,
+                    use_log: bool = False) -> QMPResponse:
+        """Write to a given drive using an HMP command"""
+        return self.hmp(f'qemu-io {drive} "{cmd}"', use_log=use_log)
 
     def flatten_qmp_object(self, obj, output=None, basestr=''):
         if output is None:
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

Like script_main, but doesn't require a single point of entry.
Replace all existing initialization sections with this drop-in replacement.

This brings debug support to all existing script-style iotests.

diff --git a/tests/qemu-iotests/149 b/tests/qemu-iotests/149
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/149
+++ b/tests/qemu-iotests/149
@@ -XXX,XX +XXX,XX @@ def test_once(config, qemu_img=False):
 
 
 # Obviously we only work with the luks image format
-iotests.verify_image_format(supported_fmts=['luks'])
-iotests.verify_platform()
+iotests.script_initialize(supported_fmts=['luks'])
 
 # We need sudo in order to run cryptsetup to create
 # dm-crypt devices. This is safe to use on any
diff --git a/tests/qemu-iotests/194 b/tests/qemu-iotests/194
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/194
+++ b/tests/qemu-iotests/194
@@ -XXX,XX +XXX,XX @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['qcow2', 'qed', 'raw'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2', 'qed', 'raw'],
+                          supported_platforms=['linux'])
 
 with iotests.FilePath('source.img') as source_img_path, \
      iotests.FilePath('dest.img') as dest_img_path, \
diff --git a/tests/qemu-iotests/202 b/tests/qemu-iotests/202
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/202
+++ b/tests/qemu-iotests/202
@@ -XXX,XX +XXX,XX @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2'],
+                          supported_platforms=['linux'])
 
 with iotests.FilePath('disk0.img') as disk0_img_path, \
      iotests.FilePath('disk1.img') as disk1_img_path, \
diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/203
+++ b/tests/qemu-iotests/203
@@ -XXX,XX +XXX,XX @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2'],
+                          supported_platforms=['linux'])
 
 with iotests.FilePath('disk0.img') as disk0_img_path, \
      iotests.FilePath('disk1.img') as disk1_img_path, \
diff --git a/tests/qemu-iotests/206 b/tests/qemu-iotests/206
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/206
+++ b/tests/qemu-iotests/206
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 with iotests.FilePath('t.qcow2') as disk_path, \
      iotests.FilePath('t.qcow2.base') as backing_path, \
diff --git a/tests/qemu-iotests/207 b/tests/qemu-iotests/207
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/207
+++ b/tests/qemu-iotests/207
@@ -XXX,XX +XXX,XX @@ import iotests
 import subprocess
 import re
 
-iotests.verify_image_format(supported_fmts=['raw'])
-iotests.verify_protocol(supported=['ssh'])
+iotests.script_initialize(
+    supported_fmts=['raw'],
+    supported_protocols=['ssh'],
+)
 
 def filter_hash(qmsg):
     def _filter(key, value):
diff --git a/tests/qemu-iotests/208 b/tests/qemu-iotests/208
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/208
+++ b/tests/qemu-iotests/208
@@ -XXX,XX +XXX,XX @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['generic'])
+iotests.script_initialize(supported_fmts=['generic'])
 
 with iotests.FilePath('disk.img') as disk_img_path, \
      iotests.FilePath('disk-snapshot.img') as disk_snapshot_img_path, \
diff --git a/tests/qemu-iotests/209 b/tests/qemu-iotests/209
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/209
+++ b/tests/qemu-iotests/209
@@ -XXX,XX +XXX,XX @@ import iotests
 from iotests import qemu_img_create, qemu_io, qemu_img_verbose, qemu_nbd, \
                     file_path
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 disk = file_path('disk')
 nbd_sock = file_path('nbd-sock', base_dir=iotests.sock_dir)
diff --git a/tests/qemu-iotests/210 b/tests/qemu-iotests/210
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/210
+++ b/tests/qemu-iotests/210
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['luks'])
-iotests.verify_protocol(supported=['file'])
+iotests.script_initialize(
+    supported_fmts=['luks'],
+    supported_protocols=['file'],
+)
 
 with iotests.FilePath('t.luks') as disk_path, \
      iotests.VM() as vm:
diff --git a/tests/qemu-iotests/211 b/tests/qemu-iotests/211
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/211
+++ b/tests/qemu-iotests/211
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['vdi'])
-iotests.verify_protocol(supported=['file'])
+iotests.script_initialize(
+    supported_fmts=['vdi'],
+    supported_protocols=['file'],
+)
 
 def blockdev_create(vm, options):
     error = vm.blockdev_create(options)
diff --git a/tests/qemu-iotests/212 b/tests/qemu-iotests/212
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/212
+++ b/tests/qemu-iotests/212
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['parallels'])
-iotests.verify_protocol(supported=['file'])
+iotests.script_initialize(
+    supported_fmts=['parallels'],
+    supported_protocols=['file'],
+)
 
 with iotests.FilePath('t.parallels') as disk_path, \
      iotests.VM() as vm:
diff --git a/tests/qemu-iotests/213 b/tests/qemu-iotests/213
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/213
+++ b/tests/qemu-iotests/213
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['vhdx'])
-iotests.verify_protocol(supported=['file'])
+iotests.script_initialize(
+    supported_fmts=['vhdx'],
+    supported_protocols=['file'],
+)
 
 with iotests.FilePath('t.vhdx') as disk_path, \
      iotests.VM() as vm:
diff --git a/tests/qemu-iotests/216 b/tests/qemu-iotests/216
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/216
+++ b/tests/qemu-iotests/216
@@ -XXX,XX +XXX,XX @@ import iotests
 from iotests import log, qemu_img, qemu_io_silent
 
 # Need backing file support
-iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk'],
+                          supported_platforms=['linux'])
 
 log('')
 log('=== Copy-on-read across nodes ===')
diff --git a/tests/qemu-iotests/218 b/tests/qemu-iotests/218
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/218
+++ b/tests/qemu-iotests/218
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import log, qemu_img, qemu_io_silent
 
-iotests.verify_image_format(supported_fmts=['qcow2', 'raw'])
+iotests.script_initialize(supported_fmts=['qcow2', 'raw'])
 
 
 # Launches the VM, adds two null-co nodes (source and target), and
diff --git a/tests/qemu-iotests/219 b/tests/qemu-iotests/219
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/219
+++ b/tests/qemu-iotests/219
@@ -XXX,XX +XXX,XX @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 img_size = 4 * 1024 * 1024
 
diff --git a/tests/qemu-iotests/222 b/tests/qemu-iotests/222
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/222
+++ b/tests/qemu-iotests/222
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import log, qemu_img, qemu_io, qemu_io_silent
 
-iotests.verify_platform(['linux'])
-iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk',
-                                            'vhdx', 'raw'])
+iotests.script_initialize(
+    supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'],
+    supported_platforms=['linux'],
+)
 
 patterns = [("0x5d", "0",         "64k"),
             ("0xd5", "1M",        "64k"),
diff --git a/tests/qemu-iotests/224 b/tests/qemu-iotests/224
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/224
+++ b/tests/qemu-iotests/224
@@ -XXX,XX +XXX,XX @@ from iotests import log, qemu_img, qemu_io_silent, filter_qmp_testfiles, \
 import json
 
 # Need backing file support (for arbitrary backing formats)
-iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2', 'qcow', 'qed'],
+                          supported_platforms=['linux'])
 
 
 # There are two variations of this test:
diff --git a/tests/qemu-iotests/228 b/tests/qemu-iotests/228
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/228
+++ b/tests/qemu-iotests/228
@@ -XXX,XX +XXX,XX @@ from iotests import log, qemu_img, filter_testfiles, filter_imgfmt, \
         filter_qmp_testfiles, filter_qmp_imgfmt
 
 # Need backing file and change-backing-file support
-iotests.verify_image_format(supported_fmts=['qcow2', 'qed'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(
+    supported_fmts=['qcow2', 'qed'],
+    supported_platforms=['linux'],
+)
 
 
 def log_node_info(node):
diff --git a/tests/qemu-iotests/234 b/tests/qemu-iotests/234
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/234
+++ b/tests/qemu-iotests/234
@@ -XXX,XX +XXX,XX @@
 import iotests
 import os
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2'],
+                          supported_platforms=['linux'])
 
 with iotests.FilePath('img') as img_path, \
      iotests.FilePath('backing') as backing_path, \
diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/235
+++ b/tests/qemu-iotests/235
@@ -XXX,XX +XXX,XX @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 
 from qemu.machine import QEMUMachine
 
+iotests.script_initialize(supported_fmts=['qcow2'])
+
 # Note:
 # This test was added to check that mirror dead-lock was fixed (see previous
 # commit before this test addition).
@@ -XXX,XX +XXX,XX @@ from qemu.machine import QEMUMachine
 
 size = 1 * 1024 * 1024 * 1024
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-
 disk = file_path('disk')
 
 # prepare source image
diff --git a/tests/qemu-iotests/236 b/tests/qemu-iotests/236
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/236
+++ b/tests/qemu-iotests/236
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import log
 
-iotests.verify_image_format(supported_fmts=['generic'])
+iotests.script_initialize(supported_fmts=['generic'])
 size = 64 * 1024 * 1024
 granularity = 64 * 1024
 
diff --git a/tests/qemu-iotests/237 b/tests/qemu-iotests/237
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/237
+++ b/tests/qemu-iotests/237
@@ -XXX,XX +XXX,XX @@ import math
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['vmdk'])
+iotests.script_initialize(supported_fmts=['vmdk'])
 
 with iotests.FilePath('t.vmdk') as disk_path, \
      iotests.FilePath('t.vmdk.1') as extent1_path, \
diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/238
+++ b/tests/qemu-iotests/238
@@ -XXX,XX +XXX,XX @@ import os
 import iotests
 from iotests import log
 
+iotests.script_initialize()
+
 virtio_scsi_device = iotests.get_virtio_scsi_device()
 
 vm = iotests.VM()
diff --git a/tests/qemu-iotests/242 b/tests/qemu-iotests/242
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/242
+++ b/tests/qemu-iotests/242
@@ -XXX,XX +XXX,XX @@ import struct
 from iotests import qemu_img_create, qemu_io, qemu_img_pipe, \
     file_path, img_info_log, log, filter_qemu_io
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 disk = file_path('disk')
 chunk = 256 * 1024
diff --git a/tests/qemu-iotests/246 b/tests/qemu-iotests/246
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/246
+++ b/tests/qemu-iotests/246
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import log
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 size = 64 * 1024 * 1024 * 1024
 gran_small = 32 * 1024
 gran_large = 128 * 1024
diff --git a/tests/qemu-iotests/248 b/tests/qemu-iotests/248
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/248
+++ b/tests/qemu-iotests/248
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import qemu_img_create, qemu_io, file_path, filter_qmp_testfiles
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 source, target = file_path('source', 'target')
 size = 5 * 1024 * 1024
diff --git a/tests/qemu-iotests/254 b/tests/qemu-iotests/254
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/254
+++ b/tests/qemu-iotests/254
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import qemu_img_create, file_path, log
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 disk, top = file_path('disk', 'top')
 size = 1024 * 1024
diff --git a/tests/qemu-iotests/255 b/tests/qemu-iotests/255
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/255
+++ b/tests/qemu-iotests/255
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import imgfmt
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 
 iotests.log('Finishing a commit job with background reads')
 iotests.log('============================================')
diff --git a/tests/qemu-iotests/256 b/tests/qemu-iotests/256
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/256
+++ b/tests/qemu-iotests/256
@@ -XXX,XX +XXX,XX @@ import os
 import iotests
 from iotests import log
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(supported_fmts=['qcow2'])
 size = 64 * 1024 * 1024
 
 with iotests.FilePath('img0') as img0_path, \
diff --git a/tests/qemu-iotests/258 b/tests/qemu-iotests/258
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/258
+++ b/tests/qemu-iotests/258
@@ -XXX,XX +XXX,XX @@ from iotests import log, qemu_img, qemu_io_silent, \
         filter_qmp_testfiles, filter_qmp_imgfmt
 
 # Need backing file and change-backing-file support
-iotests.verify_image_format(supported_fmts=['qcow2', 'qed'])
-iotests.verify_platform(['linux'])
-
+iotests.script_initialize(
+    supported_fmts=['qcow2', 'qed'],
+    supported_platforms=['linux'],
+)
 
 # Returns a node for blockdev-add
 def node(node_name, path, backing=None, fmt=None, throttle=None):
diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/260
+++ b/tests/qemu-iotests/260
@@ -XXX,XX +XXX,XX @@
 import iotests
 from iotests import qemu_img_create, file_path, log, filter_qmp_event
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(
+    supported_fmts=['qcow2']
+)
 
 base, top = file_path('base', 'top')
 size = 64 * 1024 * 3
diff --git a/tests/qemu-iotests/262 b/tests/qemu-iotests/262
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/262
+++ b/tests/qemu-iotests/262
@@ -XXX,XX +XXX,XX @@
 import iotests
 import os
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2'],
+                          supported_platforms=['linux'])
 
 with iotests.FilePath('img') as img_path, \
      iotests.FilePath('mig_fifo') as fifo, \
diff --git a/tests/qemu-iotests/264 b/tests/qemu-iotests/264
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/264
+++ b/tests/qemu-iotests/264
@@ -XXX,XX +XXX,XX @@ import iotests
 from iotests import qemu_img_create, qemu_io_silent_check, file_path, \
         qemu_nbd_popen, log
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(
+    supported_fmts=['qcow2'],
+)
 
 disk_a, disk_b, nbd_sock = file_path('disk_a', 'disk_b', 'nbd-sock')
 nbd_uri = 'nbd+unix:///?socket=' + nbd_sock
diff --git a/tests/qemu-iotests/274 b/tests/qemu-iotests/274
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/274
+++ b/tests/qemu-iotests/274
@@ -XXX,XX +XXX,XX @@
 
 import iotests
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(supported_fmts=['qcow2'],
+                          supported_platforms=['linux'])
 
 size_short = 1 * 1024 * 1024
 size_long = 2 * 1024 * 1024
diff --git a/tests/qemu-iotests/277 b/tests/qemu-iotests/277
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/277
+++ b/tests/qemu-iotests/277
@@ -XXX,XX +XXX,XX @@ import subprocess
 import iotests
 from iotests import file_path, log
 
+iotests.script_initialize()
+
 
 nbd_sock, conf_file = file_path('nbd-sock', 'nbd-fault-injector.conf')
 
diff --git a/tests/qemu-iotests/280 b/tests/qemu-iotests/280
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/280
+++ b/tests/qemu-iotests/280
@@ -XXX,XX +XXX,XX @@
 import iotests
 import os
 
-iotests.verify_image_format(supported_fmts=['qcow2'])
-iotests.verify_protocol(supported=['file'])
-iotests.verify_platform(['linux'])
+iotests.script_initialize(
+    supported_fmts=['qcow2'],
+    supported_protocols=['file'],
+    supported_platforms=['linux'],
+)
 
 with iotests.FilePath('base') as base_path , \
      iotests.FilePath('top') as top_path, \
diff --git a/tests/qemu-iotests/283 b/tests/qemu-iotests/283
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/283
+++ b/tests/qemu-iotests/283
@@ -XXX,XX +XXX,XX @@
 import iotests
 
 # The test is unrelated to formats, restrict it to qcow2 to avoid extra runs
-iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.script_initialize(
+    supported_fmts=['qcow2'],
+)
 
 size = 1024 * 1024
 
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ import signal
 import struct
 import subprocess
 import sys
-from typing import (Any, Callable, Dict, Iterable, List, Optional, TypeVar)
+from typing import (Any, Callable, Dict, Iterable,
+                    List, Optional, Sequence, TypeVar)
 import unittest
 
 # pylint: disable=import-error, wrong-import-position
@@ -XXX,XX +XXX,XX @@ def verify_protocol(supported=(), unsupported=()):
     if not_sup or (imgproto in unsupported):
         notrun('not suitable for this protocol: %s' % imgproto)
 
-def verify_platform(supported=None, unsupported=None):
-    if unsupported is not None:
-        if any((sys.platform.startswith(x) for x in unsupported)):
-            notrun('not suitable for this OS: %s' % sys.platform)
+def verify_platform(supported=(), unsupported=()):
+    if any((sys.platform.startswith(x) for x in unsupported)):
+        notrun('not suitable for this OS: %s' % sys.platform)
 
-    if supported is not None:
+    if supported:
         if not any((sys.platform.startswith(x) for x in supported)):
             notrun('not suitable for this OS: %s' % sys.platform)
 
@@ -XXX,XX +XXX,XX @@ def skip_if_user_is_root(func):
             return func(*args, **kwargs)
     return func_wrapper
 
-def execute_unittest(output, verbosity, debug):
+def execute_unittest(debug=False):
+    """Executes unittests within the calling module."""
+
+    verbosity = 2 if debug else 1
+
+    if debug:
+        output = sys.stdout
+    else:
+        # We need to filter out the time taken from the output so that
+        # qemu-iotest can reliably diff the results against master output.
+        output = io.StringIO()
+
     runner = unittest.TextTestRunner(stream=output, descriptions=True,
                                      verbosity=verbosity)
     try:
@@ -XXX,XX +XXX,XX @@ def execute_unittest(output, verbosity, debug):
         # exception
         unittest.main(testRunner=runner)
     finally:
+        # We need to filter out the time taken from the output so that
+        # qemu-iotest can reliably diff the results against master output.
         if not debug:
             out = output.getvalue()
             out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out)
@@ -XXX,XX +XXX,XX @@ def execute_unittest(output, verbosity, debug):
 
             sys.stderr.write(out)
 
-def execute_test(test_function=None,
-                 supported_fmts=(),
-                 supported_platforms=None,
-                 supported_cache_modes=(), supported_aio_modes=(),
-                 unsupported_fmts=(), supported_protocols=(),
-                 unsupported_protocols=()):
-    """Run either unittest or script-style tests."""
+def execute_setup_common(supported_fmts: Sequence[str] = (),
+                         supported_platforms: Sequence[str] = (),
+                         supported_cache_modes: Sequence[str] = (),
+                         supported_aio_modes: Sequence[str] = (),
+                         unsupported_fmts: Sequence[str] = (),
+                         supported_protocols: Sequence[str] = (),
+                         unsupported_protocols: Sequence[str] = ()) -> bool:
+    """
+    Perform necessary setup for either script-style or unittest-style tests.
+
+    :return: Bool; Whether or not debug mode has been requested via the CLI.
+    """
+    # Note: Python 3.6 and pylint do not like 'Collection' so use 'Sequence'.
 
     # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
     # indicate that we're not being run via "check". There may be
@@ -XXX,XX +XXX,XX @@ def execute_test(test_function=None,
         sys.stderr.write('Please run this test via the "check" script\n')
         sys.exit(os.EX_USAGE)
 
-    debug = '-d' in sys.argv
-    verbosity = 1
     verify_image_format(supported_fmts, unsupported_fmts)
     verify_protocol(supported_protocols, unsupported_protocols)
     verify_platform(supported=supported_platforms)
     verify_cache_mode(supported_cache_modes)
     verify_aio_mode(supported_aio_modes)
 
+    debug = '-d' in sys.argv
     if debug:
-        output = sys.stdout
-        verbosity = 2
         sys.argv.remove('-d')
-    else:
-        # We need to filter out the time taken from the output so that
-        # qemu-iotest can reliably diff the results against master output.
-        output = io.StringIO()
-
     logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN))
 
+    return debug
+
+def execute_test(*args, test_function=None, **kwargs):
+    """Run either unittest or script-style tests."""
+
+    debug = execute_setup_common(*args, **kwargs)
     if not test_function:
-        execute_unittest(output, verbosity, debug)
+        execute_unittest(debug)
     else:
         test_function()
 
+# This is called from script-style iotests without a single point of entry
+def script_initialize(*args, **kwargs):
+    """Initialize script-style tests without running any tests."""
+    execute_setup_common(*args, **kwargs)
+
+# This is called from script-style iotests with a single point of entry
 def script_main(test_function, *args, **kwargs):
     """Run script-style tests outside of the unittest framework"""
-    execute_test(test_function, *args, **kwargs)
+    execute_test(*args, test_function=test_function, **kwargs)
 
+# This is called from unittest style iotests
 def main(*args, **kwargs):
     """Run tests using the unittest framework"""
-    execute_test(None, *args, **kwargs)
+    execute_test(*args, **kwargs)
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

Since this one is nicely factored to use a single entry point,
use script_main to run the tests.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-13-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/258 | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/tests/qemu-iotests/258 b/tests/qemu-iotests/258
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/258
+++ b/tests/qemu-iotests/258
@@ -XXX,XX +XXX,XX @@ import iotests
 from iotests import log, qemu_img, qemu_io_silent, \
         filter_qmp_testfiles, filter_qmp_imgfmt
 
-# Need backing file and change-backing-file support
-iotests.script_initialize(
-    supported_fmts=['qcow2', 'qed'],
-    supported_platforms=['linux'],
-)
-
 # Returns a node for blockdev-add
 def node(node_name, path, backing=None, fmt=None, throttle=None):
     if fmt is None:
@@ -XXX,XX +XXX,XX @@ def main():
     test_concurrent_finish(False)
 
 if __name__ == '__main__':
-    main()
+    # Need backing file and change-backing-file support
+    iotests.script_main(main,
+                        supported_fmts=['qcow2', 'qed'],
+                        supported_platforms=['linux'])
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

Mark the verify functions as "private" with a leading underscore, to
discourage their use. Update type signatures while we're here.

(Also, make pending patches not yet using the new entry points fail in a
very obvious way.)

Signed-off-by: John Snow <jsnow@redhat.com>
Message-Id: <20200331000014.11581-14-jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/iotests.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ def case_notrun(reason):
     open('%s/%s.casenotrun' % (output_dir, seq), 'a').write(
         '    [case not run] ' + reason + '\n')
 
-def verify_image_format(supported_fmts=(), unsupported_fmts=()):
+def _verify_image_format(supported_fmts: Sequence[str] = (),
+                         unsupported_fmts: Sequence[str] = ()) -> None:
     assert not (supported_fmts and unsupported_fmts)
 
     if 'generic' in supported_fmts and \
@@ -XXX,XX +XXX,XX @@ def verify_image_format(supported_fmts=(), unsupported_fmts=()):
     if not_sup or (imgfmt in unsupported_fmts):
         notrun('not suitable for this image format: %s' % imgfmt)
 
-def verify_protocol(supported=(), unsupported=()):
+def _verify_protocol(supported: Sequence[str] = (),
+                     unsupported: Sequence[str] = ()) -> None:
     assert not (supported and unsupported)
 
     if 'generic' in supported:
@@ -XXX,XX +XXX,XX @@ def verify_protocol(supported=(), unsupported=()):
     if not_sup or (imgproto in unsupported):
         notrun('not suitable for this protocol: %s' % imgproto)
 
-def verify_platform(supported=(), unsupported=()):
+def _verify_platform(supported: Sequence[str] = (),
+                     unsupported: Sequence[str] = ()) -> None:
     if any((sys.platform.startswith(x) for x in unsupported)):
         notrun('not suitable for this OS: %s' % sys.platform)
 
@@ -XXX,XX +XXX,XX @@ def verify_platform(supported=(), unsupported=()):
         if not any((sys.platform.startswith(x) for x in supported)):
             notrun('not suitable for this OS: %s' % sys.platform)
 
-def verify_cache_mode(supported_cache_modes=()):
+def _verify_cache_mode(supported_cache_modes: Sequence[str] = ()) -> None:
     if supported_cache_modes and (cachemode not in supported_cache_modes):
         notrun('not suitable for this cache mode: %s' % cachemode)
 
-def verify_aio_mode(supported_aio_modes=()):
+def _verify_aio_mode(supported_aio_modes: Sequence[str] = ()):
     if supported_aio_modes and (aiomode not in supported_aio_modes):
         notrun('not suitable for this aio mode: %s' % aiomode)
 
@@ -XXX,XX +XXX,XX @@ def execute_setup_common(supported_fmts: Sequence[str] = (),
         sys.stderr.write('Please run this test via the "check" script\n')
         sys.exit(os.EX_USAGE)
 
-    verify_image_format(supported_fmts, unsupported_fmts)
-    verify_protocol(supported_protocols, unsupported_protocols)
-    verify_platform(supported=supported_platforms)
-    verify_cache_mode(supported_cache_modes)
-    verify_aio_mode(supported_aio_modes)
+    _verify_image_format(supported_fmts, unsupported_fmts)
+    _verify_protocol(supported_protocols, unsupported_protocols)
+    _verify_platform(supported=supported_platforms)
+    _verify_cache_mode(supported_cache_modes)
+    _verify_aio_mode(supported_aio_modes)
 
     debug = '-d' in sys.argv
     if debug:
-- 
2.26.2

From: John Snow <jsnow@redhat.com>

We can turn logging on/off globally instead of per-function.

Remove use_log from run_job, and use python logging to turn on
diffable output when we run through a script entry point.

iotest 245 changes output order due to buffering reasons.

An extended note on python logging:

A NullHandler is added to `qemu.iotests` to stop output from being
generated if this code is used as a library without configuring logging.
A NullHandler is only needed at the root, so a duplicate handler is not
needed for `qemu.iotests.diff_io`.

When logging is not configured, messages at the 'WARNING' levels or
above are printed with default settings. The NullHandler stops this from
occurring, which is considered good hygiene for code used as a library.

See https://docs.python.org/3/howto/logging.html#library-config

When logging is actually enabled (always at the behest of an explicit
call by a client script), a root logger is implicitly created at the
root, which allows messages to propagate upwards and be handled/emitted
from the root logger with default settings.

When we want iotest logging, we attach a handler to the
qemu.iotests.diff_io logger and disable propagation to avoid possible
double-printing.

For more information on python logging infrastructure, I highly
recommend downloading the pip package `logging_tree`, which provides
convenient visualizations of the hierarchical logging configuration
under different circumstances.

See https://pypi.org/project/logging_tree/ for more information.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200331000014.11581-15-jsnow@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/030        |  4 +--
 tests/qemu-iotests/155        |  2 +-
 tests/qemu-iotests/245        |  1 +
 tests/qemu-iotests/245.out    | 10 +++----
 tests/qemu-iotests/iotests.py | 53 ++++++++++++++++++++---------------
 5 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -XXX,XX +XXX,XX @@ class TestParallelOps(iotests.QMPTestCase):
         result = self.vm.qmp('block-job-set-speed', device='drive0', speed=0)
         self.assert_qmp(result, 'return', {})
 
-        self.vm.run_job(job='drive0', auto_dismiss=True, use_log=False)
-        self.vm.run_job(job='node4', auto_dismiss=True, use_log=False)
+        self.vm.run_job(job='drive0', auto_dismiss=True)
+        self.vm.run_job(job='node4', auto_dismiss=True)
         self.assert_no_active_block_jobs()
 
     # Test a block-stream and a block-commit job in parallel
diff --git a/tests/qemu-iotests/155 b/tests/qemu-iotests/155
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/155
+++ b/tests/qemu-iotests/155
@@ -XXX,XX +XXX,XX @@ class MirrorBaseClass(BaseClass):
 
         self.assert_qmp(result, 'return', {})
 
-        self.vm.run_job('mirror-job', use_log=False, auto_finalize=False,
+        self.vm.run_job('mirror-job', auto_finalize=False,
                         pre_finalize=self.openBacking, auto_dismiss=True)
 
     def testFull(self):
diff --git a/tests/qemu-iotests/245 b/tests/qemu-iotests/245
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/245
+++ b/tests/qemu-iotests/245
@@ -XXX,XX +XXX,XX @@ class TestBlockdevReopen(iotests.QMPTestCase):
         self.run_test_iothreads(None, 'iothread0')
 
 if __name__ == '__main__':
+    iotests.activate_logging()
     iotests.main(supported_fmts=["qcow2"],
                  supported_protocols=["file"])
diff --git a/tests/qemu-iotests/245.out b/tests/qemu-iotests/245.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/245.out
+++ b/tests/qemu-iotests/245.out
@@ -XXX,XX +XXX,XX @@
-.....................
-----------------------------------------------------------------------
-Ran 21 tests
-
-OK
 {"execute": "job-finalize", "arguments": {"id": "commit0"}}
 {"return": {}}
 {"data": {"id": "commit0", "type": "commit"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
@@ -XXX,XX +XXX,XX @@ OK
 {"return": {}}
 {"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
 {"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
+.....................
+----------------------------------------------------------------------
+Ran 21 tests
+
+OK
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -XXX,XX +XXX,XX @@ assert sys.version_info >= (3, 6)
 QMPResponse = Dict[str, Any]
 
 
+# Use this logger for logging messages directly from the iotests module
+logger = logging.getLogger('qemu.iotests')
+logger.addHandler(logging.NullHandler())
+
+# Use this logger for messages that ought to be used for diff output.
+test_logger = logging.getLogger('qemu.iotests.diff_io')
+
+
 faulthandler.enable()
 
 # This will not work if arguments contain spaces but is necessary if we
@@ -XXX,XX +XXX,XX @@ def log(msg: Msg,
     if isinstance(msg, (dict, list)):
         # Don't sort if it's already sorted
         do_sort = not isinstance(msg, OrderedDict)
-        print(json.dumps(msg, sort_keys=do_sort, indent=indent))
+        test_logger.info(json.dumps(msg, sort_keys=do_sort, indent=indent))
     else:
-        print(msg)
+        test_logger.info(msg)
 
 class Timeout:
     def __init__(self, seconds, errmsg="Timeout"):
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
 
     # Returns None on success, and an error string on failure
     def run_job(self, job, auto_finalize=True, auto_dismiss=False,
-                pre_finalize=None, cancel=False, use_log=True, wait=60.0):
+                pre_finalize=None, cancel=False, wait=60.0):
         """
         run_job moves a job from creation through to dismissal.
 
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
                              invoked prior to issuing job-finalize, if any.
         :param cancel: Bool. When true, cancels the job after the pre_finalize
                        callback.
-        :param use_log: Bool. When false, does not log QMP messages.
         :param wait: Float. Timeout value specifying how long to wait for any
                      event, in seconds. Defaults to 60.0.
         """
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
         while True:
             ev = filter_qmp_event(self.events_wait(events, timeout=wait))
             if ev['event'] != 'JOB_STATUS_CHANGE':
-                if use_log:
-                    log(ev)
+                log(ev)
                 continue
             status = ev['data']['status']
             if status == 'aborting':
@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
                 for j in result['return']:
                     if j['id'] == job:
                         error = j['error']
-                        if use_log:
-                            log('Job failed: %s' % (j['error']))
+                        log('Job failed: %s' % (j['error']))
             elif status == 'ready':
-                if use_log:
-                    self.qmp_log('job-complete', id=job)
-                else:
-                    self.qmp('job-complete', id=job)
+                self.qmp_log('job-complete', id=job)
             elif status == 'pending' and not auto_finalize:
                 if pre_finalize:
                     pre_finalize()
-                if cancel and use_log:
+                if cancel:
                     self.qmp_log('job-cancel', id=job)
-                elif cancel:
-                    self.qmp('job-cancel', id=job)
-                elif use_log:
-                    self.qmp_log('job-finalize', id=job)
                 else:
-                    self.qmp('job-finalize', id=job)
+                    self.qmp_log('job-finalize', id=job)
             elif status == 'concluded' and not auto_dismiss:
-                if use_log:
-                    self.qmp_log('job-dismiss', id=job)
-                else:
-                    self.qmp('job-dismiss', id=job)
+                self.qmp_log('job-dismiss', id=job)
             elif status == 'null':
                 return error
 
@@ -XXX,XX +XXX,XX @@ def notrun(reason):
     seq = os.path.basename(sys.argv[0])
 
     open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n')
-    print('%s not run: %s' % (seq, reason))
+    logger.warning("%s not run: %s", seq, reason)
     sys.exit(0)
 
 def case_notrun(reason):
@@ -XXX,XX +XXX,XX @@ def execute_setup_common(supported_fmts: Sequence[str] = (),
     if debug:
         sys.argv.remove('-d')
     logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN))
+    logger.debug("iotests debugging messages active")
 
     return debug
 
@@ -XXX,XX +XXX,XX @@ def execute_test(*args, test_function=None, **kwargs):
     else:
         test_function()
 
+def activate_logging():
+    """Activate iotests.log() output to stdout for script-style tests."""
+    handler = logging.StreamHandler(stream=sys.stdout)
+    formatter = logging.Formatter('%(message)s')
+    handler.setFormatter(formatter)
+    test_logger.addHandler(handler)
+    test_logger.setLevel(logging.INFO)
+    test_logger.propagate = False
+
 # This is called from script-style iotests without a single point of entry
 def script_initialize(*args, **kwargs):
     """Initialize script-style tests without running any tests."""
+    activate_logging()
     execute_setup_common(*args, **kwargs)
 
 # This is called from script-style iotests with a single point of entry
 def script_main(test_function, *args, **kwargs):
     """Run script-style tests outside of the unittest framework"""
+    activate_logging()
     execute_test(*args, test_function=test_function, **kwargs)
 
 # This is called from unittest style iotests
-- 
2.26.2

From: Eric Blake <eblake@redhat.com>

There are several callers that need to create a new block backend from
an existing BDS; make the task slightly easier with a common helper
routine.

Suggested-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Eric Blake <eblake@redhat.com>
Message-Id: <20200424190903.522087-2-eblake@redhat.com>
[mreitz: Set @ret only in error paths, see
 https://lists.nongnu.org/archive/html/qemu-block/2020-04/msg01216.html]
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200428192648.749066-2-eblake@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 include/sysemu/block-backend.h |  2 ++
 block/block-backend.c          | 23 +++++++++++++++++++++++
 block/crypto.c                 |  9 ++++-----
 block/parallels.c              |  8 ++++----
 block/qcow.c                   |  8 ++++----
 block/qcow2.c                  | 18 ++++++++----------
 block/qed.c                    |  8 ++++----
 block/sheepdog.c               | 10 +++++-----
 block/vdi.c                    |  8 ++++----
 block/vhdx.c                   |  8 ++++----
 block/vmdk.c                   |  9 ++++-----
 block/vpc.c                    |  8 ++++----
 blockdev.c                     |  8 +++-----
 blockjob.c                     |  7 ++-----
 14 files changed, 75 insertions(+), 59 deletions(-)

diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
index XXXXXXX..XXXXXXX 100644
--- a/include/sysemu/block-backend.h
+++ b/include/sysemu/block-backend.h
@@ -XXX,XX +XXX,XX @@ typedef struct BlockBackendPublic {
 } BlockBackendPublic;
 
 BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm);
+BlockBackend *blk_new_with_bs(BlockDriverState *bs, uint64_t perm,
+                              uint64_t shared_perm, Error **errp);
 BlockBackend *blk_new_open(const char *filename, const char *reference,
                            QDict *options, int flags, Error **errp);
 int blk_get_refcnt(BlockBackend *blk);
diff --git a/block/block-backend.c b/block/block-backend.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-backend.c
+++ b/block/block-backend.c
@@ -XXX,XX +XXX,XX @@ BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm)
     return blk;
 }
 
+/*
+ * Create a new BlockBackend connected to an existing BlockDriverState.
+ *
+ * @perm is a bitmasks of BLK_PERM_* constants which describes the
+ * permissions to request for @bs that is attached to this
+ * BlockBackend.  @shared_perm is a bitmask which describes which
+ * permissions may be granted to other users of the attached node.
+ * Both sets of permissions can be changed later using blk_set_perm().
+ *
+ * Return the new BlockBackend on success, null on failure.
+ */
+BlockBackend *blk_new_with_bs(BlockDriverState *bs, uint64_t perm,
+                              uint64_t shared_perm, Error **errp)
+{
+    BlockBackend *blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
+
+    if (blk_insert_bs(blk, bs, errp) < 0) {
+        blk_unref(blk);
+        return NULL;
+    }
+    return blk;
+}
+
 /*
  * Creates a new BlockBackend, opens a new BlockDriverState, and connects both.
  * The new BlockBackend is in the main AioContext.
diff --git a/block/crypto.c b/block/crypto.c
index XXXXXXX..XXXXXXX 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -XXX,XX +XXX,XX @@ static int block_crypto_co_create_generic(BlockDriverState *bs,
     QCryptoBlock *crypto = NULL;
     struct BlockCryptoCreateData data;
 
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
+                          errp);
+    if (!blk) {
+        ret = -EPERM;
         goto cleanup;
     }
 
diff --git a/block/parallels.c b/block/parallels.c
index XXXXXXX..XXXXXXX 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn parallels_co_create(BlockdevCreateOptions* opts,
         return -EIO;
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
+                          errp);
+    if (!blk) {
+        ret = -EPERM;
         goto out;
     }
     blk_set_allow_write_beyond_eof(blk, true);
diff --git a/block/qcow.c b/block/qcow.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow_co_create(BlockdevCreateOptions *opts,
         return -EIO;
     }
 
-    qcow_blk = blk_new(bdrv_get_aio_context(bs),
-                       BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(qcow_blk, bs, errp);
-    if (ret < 0) {
+    qcow_blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE,
+                               BLK_PERM_ALL, errp);
+    if (!qcow_blk) {
+        ret = -EPERM;
         goto exit;
     }
     blk_set_allow_write_beyond_eof(qcow_blk, true);
diff --git a/block/qcow2.c b/block/qcow2.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
     }
 
     /* Create BlockBackend to write to the image */
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
+                          errp);
+    if (!blk) {
+        ret = -EPERM;
         goto out;
     }
     blk_set_allow_write_beyond_eof(blk, true);
@@ -XXX,XX +XXX,XX @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
     }
 
     if (new_size) {
-        BlockBackend *blk = blk_new(bdrv_get_aio_context(bs),
-                                    BLK_PERM_RESIZE, BLK_PERM_ALL);
-        ret = blk_insert_bs(blk, bs, errp);
-        if (ret < 0) {
-            blk_unref(blk);
-            return ret;
+        BlockBackend *blk = blk_new_with_bs(bs, BLK_PERM_RESIZE, BLK_PERM_ALL,
+                                            errp);
+        if (!blk) {
+            return -EPERM;
         }
 
         /*
diff --git a/block/qed.c b/block/qed.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_qed_co_create(BlockdevCreateOptions *opts,
         return -EIO;
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
+                          errp);
+    if (!blk) {
+        ret = -EPERM;
         goto out;
     }
     blk_set_allow_write_beyond_eof(blk, true);
diff --git a/block/sheepdog.c b/block/sheepdog.c
index XXXXXXX..XXXXXXX 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -XXX,XX +XXX,XX @@ static int sd_prealloc(BlockDriverState *bs, int64_t old_size, int64_t new_size,
     void *buf = NULL;
     int ret;
 
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
-                  BLK_PERM_ALL);
+    blk = blk_new_with_bs(bs,
+                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
+                          BLK_PERM_ALL, errp);
 
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    if (!blk) {
+        ret = -EPERM;
         goto out_with_err_set;
     }
 
diff --git a/block/vdi.c b/block/vdi.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn vdi_co_do_create(BlockdevCreateOptions *create_options,
         goto exit;
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs_file),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs_file, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs_file, BLK_PERM_WRITE | BLK_PERM_RESIZE,
+                          BLK_PERM_ALL, errp);
+    if (!blk) {
+        ret = -EPERM;
         goto exit;
     }
 
diff --git a/block/vhdx.c b/block/vhdx.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn vhdx_co_create(BlockdevCreateOptions *opts,
         return -EIO;
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
+                          errp);
+    if (!blk) {
+        ret = -EPERM;
         goto delete_and_exit;
     }
     blk_set_allow_write_beyond_eof(blk, true);
diff --git a/block/vmdk.c b/block/vmdk.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -XXX,XX +XXX,XX @@ static BlockBackend *vmdk_co_create_cb(int64_t size, int idx,
     if (!bs) {
         return NULL;
     }
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
-                  BLK_PERM_ALL);
-    if (blk_insert_bs(blk, bs, errp)) {
-        bdrv_unref(bs);
+    blk = blk_new_with_bs(bs,
+                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE | BLK_PERM_RESIZE,
+                          BLK_PERM_ALL, errp);
+    if (!blk) {
         return NULL;
     }
     blk_set_allow_write_beyond_eof(blk, true);
diff --git a/block/vpc.c b/block/vpc.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn vpc_co_create(BlockdevCreateOptions *opts,
         return -EIO;
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs),
-                  BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_WRITE | BLK_PERM_RESIZE, BLK_PERM_ALL,
+                          errp);
+    if (!blk) {
+        ret = -EPERM;
         goto out;
     }
     blk_set_allow_write_beyond_eof(blk, true);
diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ void qmp_block_resize(bool has_device, const char *device,
     BlockBackend *blk = NULL;
     BlockDriverState *bs;
     AioContext *aio_context;
-    int ret;
 
     bs = bdrv_lookup_bs(has_device ? device : NULL,
                         has_node_name ? node_name : NULL,
@@ -XXX,XX +XXX,XX @@ void qmp_block_resize(bool has_device, const char *device,
         goto out;
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs), BLK_PERM_RESIZE, BLK_PERM_ALL);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
+    blk = blk_new_with_bs(bs, BLK_PERM_RESIZE, BLK_PERM_ALL, errp);
+    if (!blk) {
         goto out;
     }
 
     bdrv_drained_begin(bs);
-    ret = blk_truncate(blk, size, false, PREALLOC_MODE_OFF, 0, errp);
+    blk_truncate(blk, size, false, PREALLOC_MODE_OFF, 0, errp);
     bdrv_drained_end(bs);
 
 out:
diff --git a/blockjob.c b/blockjob.c
index XXXXXXX..XXXXXXX 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -XXX,XX +XXX,XX @@ void *block_job_create(const char *job_id, const BlockJobDriver *driver,
 {
     BlockBackend *blk;
     BlockJob *job;
-    int ret;
 
     if (job_id == NULL && !(flags & JOB_INTERNAL)) {
         job_id = bdrv_get_device_name(bs);
     }
 
-    blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
-    ret = blk_insert_bs(blk, bs, errp);
-    if (ret < 0) {
-        blk_unref(blk);
+    blk = blk_new_with_bs(bs, perm, shared_perm, errp);
+    if (!blk) {
         return NULL;
     }
 
-- 
2.26.2

From: Eric Blake <eblake@redhat.com>

We originally refused to allow resize of images with internal
snapshots because the v2 image format did not require the tracking of
snapshot size, making it impossible to safely revert to a snapshot
with a different size than the current view of the image.  But the
snapshot size tracking was rectified in v3, and our recent fixes to
qemu-img amend (see 0a85af35) guarantee that we always have a valid
snapshot size.  Thus, we no longer need to artificially limit image
resizes, but it does become one more thing that would prevent a
downgrade back to v2.  And now that we support different-sized
snapshots, it's also easy to fix reverting to a snapshot to apply the
new size.

Upgrade iotest 61 to cover this (we previously had NO coverage of
refusal to resize while snapshots exist).  Note that the amend process
can fail but still have effects: in particular, since we break things
into upgrade, resize, downgrade, a failure during resize does not roll
back changes made during upgrade, nor does failure in downgrade roll
back a resize.  But this situation is pre-existing even without this
patch; and without journaling, the best we could do is minimize the
chance of partial failure by collecting all changes prior to doing any
writes - which adds a lot of complexity but could still fail with EIO.
On the other hand, we are careful that even if we have partial
modification but then fail, the image is left viable (that is, we are
careful to sequence things so that after each successful cluster
write, there may be transient leaked clusters but no corrupt
metadata).  And complicating the code to make it more transaction-like
is not worth the effort: a user can always request multiple 'qemu-img
amend' changing one thing each, if they need finer-grained control
over detecting the first failure than what they get by letting qemu
decide how to sequence multiple changes.

Signed-off-by: Eric Blake <eblake@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200428192648.749066-3-eblake@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/qcow2-snapshot.c     | 20 ++++++++++++++++----
 block/qcow2.c              | 25 ++++++++++++++++++++++---
 tests/qemu-iotests/061     | 35 +++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/061.out | 28 ++++++++++++++++++++++++++++
 4 files changed, 101 insertions(+), 7 deletions(-)

diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -XXX,XX +XXX,XX @@
  */
 
 #include "qemu/osdep.h"
+#include "sysemu/block-backend.h"
 #include "qapi/error.h"
 #include "qcow2.h"
 #include "qemu/bswap.h"
@@ -XXX,XX +XXX,XX @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
     }
 
     if (sn->disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
-        error_report("qcow2: Loading snapshots with different disk "
-            "size is not implemented");
-        ret = -ENOTSUP;
-        goto fail;
+        BlockBackend *blk = blk_new_with_bs(bs, BLK_PERM_RESIZE, BLK_PERM_ALL,
+                                            &local_err);
+        if (!blk) {
+            error_report_err(local_err);
+            ret = -ENOTSUP;
+            goto fail;
+        }
+
+        ret = blk_truncate(blk, sn->disk_size, true, PREALLOC_MODE_OFF, 0,
+                           &local_err);
+        blk_unref(blk);
+        if (ret < 0) {
+            error_report_err(local_err);
+            goto fail;
+        }
     }
 
     /*
diff --git a/block/qcow2.c b/block/qcow2.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn qcow2_co_truncate(BlockDriverState *bs, int64_t offset,
 
     qemu_co_mutex_lock(&s->lock);
 
-    /* cannot proceed if image has snapshots */
-    if (s->nb_snapshots) {
-        error_setg(errp, "Can't resize an image which has snapshots");
+    /*
+     * Even though we store snapshot size for all images, it was not
+     * required until v3, so it is not safe to proceed for v2.
+     */
+    if (s->nb_snapshots && s->qcow_version < 3) {
+        error_setg(errp, "Can't resize a v2 image which has snapshots");
         ret = -ENOTSUP;
         goto fail;
     }
@@ -XXX,XX +XXX,XX @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
     BDRVQcow2State *s = bs->opaque;
     int current_version = s->qcow_version;
     int ret;
+    int i;
 
     /* This is qcow2_downgrade(), not qcow2_upgrade() */
     assert(target_version < current_version);
@@ -XXX,XX +XXX,XX @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
         return -ENOTSUP;
     }
 
+    /*
+     * If any internal snapshot has a different size than the current
+     * image size, or VM state size that exceeds 32 bits, downgrading
+     * is unsafe.  Even though we would still use v3-compliant output
+     * to preserve that data, other v2 programs might not realize
+     * those optional fields are important.
+     */
+    for (i = 0; i < s->nb_snapshots; i++) {
+        if (s->snapshots[i].vm_state_size > UINT32_MAX ||
+            s->snapshots[i].disk_size != bs->total_sectors * BDRV_SECTOR_SIZE) {
+            error_setg(errp, "Internal snapshots prevent downgrade of image");
+            return -ENOTSUP;
+        }
+    }
+
     /* clear incompatible features */
     if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
         ret = qcow2_mark_clean(bs);
diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/061
+++ b/tests/qemu-iotests/061
@@ -XXX,XX +XXX,XX @@ $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IO -c "read -P 0x2a 42M 64k" "$TEST_IMG" | _filter_qemu_io
 _check_test_img
 
+echo
+echo "=== Testing resize with snapshots ==="
+echo
+_make_test_img -o "compat=0.10" 32M
+$QEMU_IO -c "write -P 0x2a 24M 64k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IMG resize "$TEST_IMG" 64M &&
+    echo "unexpected pass"
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^$version\|size\|nb_snap$'
+
+$QEMU_IMG amend -o "compat=1.1,size=128M" "$TEST_IMG" ||
+    echo "unexpected fail"
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^$version\|size\|nb_snap$'
+
+$QEMU_IMG snapshot -c bar "$TEST_IMG"
+$QEMU_IMG resize --shrink "$TEST_IMG" 64M ||
+    echo "unexpected fail"
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^$version\|size\|nb_snap$'
+
+$QEMU_IMG amend -o "compat=0.10,size=32M" "$TEST_IMG" &&
+    echo "unexpected pass"
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^$version\|size\|nb_snap$'
+
+$QEMU_IMG snapshot -a bar "$TEST_IMG" ||
+    echo "unexpected fail"
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^$version\|size\|nb_snap$'
+
+$QEMU_IMG snapshot -d bar "$TEST_IMG"
+$QEMU_IMG amend -o "compat=0.10,size=32M" "$TEST_IMG" ||
+    echo "unexpected fail"
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^$version\|size\|nb_snap$'
+
+_check_test_img
+
+
 echo
 echo "=== Testing dirty lazy_refcounts=off ==="
 echo
diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/061.out
+++ b/tests/qemu-iotests/061.out
@@ -XXX,XX +XXX,XX @@ read 65536/65536 bytes at offset 44040192
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
 
+=== Testing resize with snapshots ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33554432
+wrote 65536/65536 bytes at offset 25165824
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Can't resize a v2 image which has snapshots
+version                   2
+size                      33554432
+nb_snapshots              1
+version                   3
+size                      134217728
+nb_snapshots              1
+Image resized.
+version                   3
+size                      67108864
+nb_snapshots              2
+qemu-img: Internal snapshots prevent downgrade of image
+version                   3
+size                      33554432
+nb_snapshots              2
+version                   3
+size                      134217728
+nb_snapshots              2
+version                   2
+size                      33554432
+nb_snapshots              1
+No errors were found on the image.
+
 === Testing dirty lazy_refcounts=off ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-- 
2.26.2

From: Eric Blake <eblake@redhat.com>

It's been a while since we got rid of the sector-based bdrv_read and
bdrv_write (commit 2e11d756); let's finish the job on a few remaining
comments.

Signed-off-by: Eric Blake <eblake@redhat.com>
Message-Id: <20200428213807.776655-1-eblake@redhat.com>
Reviewed-by: Alberto Garcia <berto@igalia.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/io.c             |  3 ++-
 block/qcow2-refcount.c |  2 +-
 block/vvfat.c          | 10 +++++-----
 tests/qemu-iotests/001 |  2 +-
 tests/qemu-iotests/052 |  2 +-
 tests/qemu-iotests/134 |  2 +-
 tests/qemu-iotests/188 |  2 +-
 7 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
  * BDRV_REQ_FUA).
  *
- * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
+ * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
  */
 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
 {
@@ -XXX,XX +XXX,XX @@ int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
     }
 }
 
+/* return < 0 if error. See bdrv_pwrite() for the return codes */
 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
 {
     int ret;
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -XXX,XX +XXX,XX @@ fail:
  * - 0 if writing to this offset will not affect the mentioned metadata
  * - a positive QCow2MetadataOverlap value indicating one overlapping section
  * - a negative value (-errno) indicating an error while performing a check,
- *   e.g. when bdrv_read failed on QCOW2_OL_INACTIVE_L2
+ *   e.g. when bdrv_pread failed on QCOW2_OL_INACTIVE_L2
  */
 int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                  int64_t size)
diff --git a/block/vvfat.c b/block/vvfat.c
index XXXXXXX..XXXXXXX 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -XXX,XX +XXX,XX @@ DLOG(checkpoint());
      * - get modified FAT
      * - compare the two FATs (TODO)
      * - get buffer for marking used clusters
-     * - recurse direntries from root (using bs->bdrv_read to make
+     * - recurse direntries from root (using bs->bdrv_pread to make
      *    sure to get the new data)
      *   - check that the FAT agrees with the size
      *   - count the number of clusters occupied by this directory and
@@ -XXX,XX +XXX,XX @@ static int handle_deletes(BDRVVVFATState* s)
 /*
  * synchronize mapping with new state:
  *
- * - copy FAT (with bdrv_read)
+ * - copy FAT (with bdrv_pread)
  * - mark all filenames corresponding to mappings as deleted
- * - recurse direntries from root (using bs->bdrv_read)
+ * - recurse direntries from root (using bs->bdrv_pread)
  * - delete files corresponding to mappings marked as deleted
  */
 static int do_commit(BDRVVVFATState* s)
@@ -XXX,XX +XXX,XX @@ static int do_commit(BDRVVVFATState* s)
         return ret;
     }
 
-    /* copy FAT (with bdrv_read) */
+    /* copy FAT (with bdrv_pread) */
     memcpy(s->fat.pointer, s->fat2, 0x200 * s->sectors_per_fat);
 
-    /* recurse direntries from root (using bs->bdrv_read) */
+    /* recurse direntries from root (using bs->bdrv_pread) */
     ret = commit_direntries(s, 0, -1);
     if (ret) {
         fprintf(stderr, "Fatal: error while committing (%d)\n", ret);
diff --git a/tests/qemu-iotests/001 b/tests/qemu-iotests/001
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/001
+++ b/tests/qemu-iotests/001
@@ -XXX,XX +XXX,XX @@
 #!/usr/bin/env bash
 #
-# Test simple read/write using plain bdrv_read/bdrv_write
+# Test simple read/write using plain bdrv_pread/bdrv_pwrite
 #
 # Copyright (C) 2009 Red Hat, Inc.
 #
diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/052
+++ b/tests/qemu-iotests/052
@@ -XXX,XX +XXX,XX @@
 #!/usr/bin/env bash
 #
-# Test bdrv_read/bdrv_write using BDRV_O_SNAPSHOT
+# Test bdrv_pread/bdrv_pwrite using BDRV_O_SNAPSHOT
 #
 # Copyright (C) 2013 Red Hat, Inc.
 #
diff --git a/tests/qemu-iotests/134 b/tests/qemu-iotests/134
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/134
+++ b/tests/qemu-iotests/134
@@ -XXX,XX +XXX,XX @@
 #!/usr/bin/env bash
 #
-# Test encrypted read/write using plain bdrv_read/bdrv_write
+# Test encrypted read/write using plain bdrv_pread/bdrv_pwrite
 #
 # Copyright (C) 2015 Red Hat, Inc.
 #
diff --git a/tests/qemu-iotests/188 b/tests/qemu-iotests/188
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/188
+++ b/tests/qemu-iotests/188
@@ -XXX,XX +XXX,XX @@
 #!/usr/bin/env bash
 #
-# Test encrypted read/write using plain bdrv_read/bdrv_write
+# Test encrypted read/write using plain bdrv_pread/bdrv_pwrite
 #
 # Copyright (C) 2017 Red Hat, Inc.
 #
-- 
2.26.2

From: Maxim Levitsky <mlevitsk@redhat.com>

Commit f62514b3def5fb2acbef64d0e053c0c31fa45aff made qemu-img reject -o "" but this test uses it.
Since this test only tries to do a dry-run run of qemu-img amend,
replace the -o "" with dummy -o "size=$size".

Fixes: f62514b3def5fb2acbef64d0e053c0c31fa45aff

Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20200504131959.9533-1-mlevitsk@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/153     |  2 +-
 tests/qemu-iotests/153.out | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/qemu-iotests/153 b/tests/qemu-iotests/153
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/153
+++ b/tests/qemu-iotests/153
@@ -XXX,XX +XXX,XX @@ for opts1 in "" "read-only=on" "read-only=on,force-share=on"; do
         _run_cmd $QEMU_IMG check       $L "${TEST_IMG}"
         _run_cmd $QEMU_IMG compare     $L "${TEST_IMG}" "${TEST_IMG}"
         _run_cmd $QEMU_IMG map         $L "${TEST_IMG}"
-        _run_cmd $QEMU_IMG amend -o "" $L "${TEST_IMG}"
+        _run_cmd $QEMU_IMG amend -o "size=$size" $L "${TEST_IMG}"
         _run_cmd $QEMU_IMG commit      $L "${TEST_IMG}"
         _run_cmd $QEMU_IMG resize      $L "${TEST_IMG}" $size
         _run_cmd $QEMU_IMG rebase      $L "${TEST_IMG}" -b "${TEST_IMG}.base"
diff --git a/tests/qemu-iotests/153.out b/tests/qemu-iotests/153.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/153.out
+++ b/tests/qemu-iotests/153.out
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper map TEST_DIR/t.qcow2
 qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get shared "write" lock
 Is another process using the image [TEST_DIR/t.qcow2]?
 
-_qemu_img_wrapper amend -o  TEST_DIR/t.qcow2
+_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2
 qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get "write" lock
 Is another process using the image [TEST_DIR/t.qcow2]?
 
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
 
 _qemu_img_wrapper map -U TEST_DIR/t.qcow2
 
-_qemu_img_wrapper amend -o  -U TEST_DIR/t.qcow2
+_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2
 qemu-img: unrecognized option '-U'
 Try 'qemu-img --help' for more information
 
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
 
 _qemu_img_wrapper map TEST_DIR/t.qcow2
 
-_qemu_img_wrapper amend -o  TEST_DIR/t.qcow2
+_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2
 qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get "write" lock
 Is another process using the image [TEST_DIR/t.qcow2]?
 
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
 
 _qemu_img_wrapper map -U TEST_DIR/t.qcow2
 
-_qemu_img_wrapper amend -o  -U TEST_DIR/t.qcow2
+_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2
 qemu-img: unrecognized option '-U'
 Try 'qemu-img --help' for more information
 
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
 
 _qemu_img_wrapper map TEST_DIR/t.qcow2
 
-_qemu_img_wrapper amend -o  TEST_DIR/t.qcow2
+_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2
 
 _qemu_img_wrapper commit TEST_DIR/t.qcow2
 
@@ -XXX,XX +XXX,XX @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2
 
 _qemu_img_wrapper map -U TEST_DIR/t.qcow2
 
-_qemu_img_wrapper amend -o  -U TEST_DIR/t.qcow2
+_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2
 qemu-img: unrecognized option '-U'
 Try 'qemu-img --help' for more information
 
-- 
2.26.2

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

We are going to use aio-task-pool API and extend in-flight request
structure to be a successor of AioTask, so rename things appropriately.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200429130847.28124-2-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 98 +++++++++++++++++++++++-----------------------
 1 file changed, 48 insertions(+), 50 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
 #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
 #define BLOCK_COPY_MAX_MEM (128 * MiB)
 
-typedef struct BlockCopyInFlightReq {
+typedef struct BlockCopyTask {
     int64_t offset;
     int64_t bytes;
-    QLIST_ENTRY(BlockCopyInFlightReq) list;
-    CoQueue wait_queue; /* coroutines blocked on this request */
-} BlockCopyInFlightReq;
+    QLIST_ENTRY(BlockCopyTask) list;
+    CoQueue wait_queue; /* coroutines blocked on this task */
+} BlockCopyTask;
 
 typedef struct BlockCopyState {
     /*
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
     bool use_copy_range;
     int64_t copy_size;
     uint64_t len;
-    QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs;
+    QLIST_HEAD(, BlockCopyTask) tasks;
 
     BdrvRequestFlags write_flags;
 
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
     SharedResource *mem;
 } BlockCopyState;
 
-static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
-                                                           int64_t offset,
-                                                           int64_t bytes)
+static BlockCopyTask *find_conflicting_task(BlockCopyState *s,
+                                            int64_t offset, int64_t bytes)
 {
-    BlockCopyInFlightReq *req;
+    BlockCopyTask *t;
 
-    QLIST_FOREACH(req, &s->inflight_reqs, list) {
-        if (offset + bytes > req->offset && offset < req->offset + req->bytes) {
-            return req;
+    QLIST_FOREACH(t, &s->tasks, list) {
+        if (offset + bytes > t->offset && offset < t->offset + t->bytes) {
+            return t;
         }
     }
 
@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
 }
 
 /*
- * If there are no intersecting requests return false. Otherwise, wait for the
- * first found intersecting request to finish and return true.
+ * If there are no intersecting tasks return false. Otherwise, wait for the
+ * first found intersecting tasks to finish and return true.
  */
 static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
                                              int64_t bytes)
 {
-    BlockCopyInFlightReq *req = find_conflicting_inflight_req(s, offset, bytes);
+    BlockCopyTask *task = find_conflicting_task(s, offset, bytes);
 
-    if (!req) {
+    if (!task) {
         return false;
     }
 
-    qemu_co_queue_wait(&req->wait_queue, NULL);
+    qemu_co_queue_wait(&task->wait_queue, NULL);
 
     return true;
 }
 
 /* Called only on full-dirty region */
-static void block_copy_inflight_req_begin(BlockCopyState *s,
-                                          BlockCopyInFlightReq *req,
-                                          int64_t offset, int64_t bytes)
+static void block_copy_task_begin(BlockCopyState *s, BlockCopyTask *task,
+                                  int64_t offset, int64_t bytes)
 {
-    assert(!find_conflicting_inflight_req(s, offset, bytes));
+    assert(!find_conflicting_task(s, offset, bytes));
 
     bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
     s->in_flight_bytes += bytes;
 
-    req->offset = offset;
-    req->bytes = bytes;
-    qemu_co_queue_init(&req->wait_queue);
-    QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
+    task->offset = offset;
+    task->bytes = bytes;
+    qemu_co_queue_init(&task->wait_queue);
+    QLIST_INSERT_HEAD(&s->tasks, task, list);
 }
 
 /*
- * block_copy_inflight_req_shrink
+ * block_copy_task_shrink
  *
- * Drop the tail of the request to be handled later. Set dirty bits back and
- * wake up all requests waiting for us (may be some of them are not intersecting
- * with shrunk request)
+ * Drop the tail of the task to be handled later. Set dirty bits back and
+ * wake up all tasks waiting for us (may be some of them are not intersecting
+ * with shrunk task)
  */
-static void coroutine_fn block_copy_inflight_req_shrink(BlockCopyState *s,
-        BlockCopyInFlightReq *req, int64_t new_bytes)
+static void coroutine_fn block_copy_task_shrink(BlockCopyState *s,
+                                                BlockCopyTask *task,
+                                                int64_t new_bytes)
 {
-    if (new_bytes == req->bytes) {
+    if (new_bytes == task->bytes) {
         return;
     }
 
-    assert(new_bytes > 0 && new_bytes < req->bytes);
+    assert(new_bytes > 0 && new_bytes < task->bytes);
 
-    s->in_flight_bytes -= req->bytes - new_bytes;
+    s->in_flight_bytes -= task->bytes - new_bytes;
     bdrv_set_dirty_bitmap(s->copy_bitmap,
-                          req->offset + new_bytes, req->bytes - new_bytes);
+                          task->offset + new_bytes, task->bytes - new_bytes);
 
-    req->bytes = new_bytes;
-    qemu_co_queue_restart_all(&req->wait_queue);
+    task->bytes = new_bytes;
+    qemu_co_queue_restart_all(&task->wait_queue);
 }
 
-static void coroutine_fn block_copy_inflight_req_end(BlockCopyState *s,
-                                                     BlockCopyInFlightReq *req,
-                                                     int ret)
+static void coroutine_fn block_copy_task_end(BlockCopyState *s,
+                                             BlockCopyTask *task, int ret)
 {
-    s->in_flight_bytes -= req->bytes;
+    s->in_flight_bytes -= task->bytes;
     if (ret < 0) {
-        bdrv_set_dirty_bitmap(s->copy_bitmap, req->offset, req->bytes);
+        bdrv_set_dirty_bitmap(s->copy_bitmap, task->offset, task->bytes);
     }
-    QLIST_REMOVE(req, list);
-    qemu_co_queue_restart_all(&req->wait_queue);
+    QLIST_REMOVE(task, list);
+    qemu_co_queue_restart_all(&task->wait_queue);
 }
 
 void block_copy_state_free(BlockCopyState *s)
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
         s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
     }
 
-    QLIST_INIT(&s->inflight_reqs);
+    QLIST_INIT(&s->tasks);
 
     return s;
 }
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 
     while (bytes) {
-        BlockCopyInFlightReq req;
+        BlockCopyTask task;
         int64_t next_zero, cur_bytes, status_bytes;
 
         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
             assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
             cur_bytes = next_zero - offset;
         }
-        block_copy_inflight_req_begin(s, &req, offset, cur_bytes);
+        block_copy_task_begin(s, &task, offset, cur_bytes);
 
         ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
         assert(ret >= 0); /* never fail */
         cur_bytes = MIN(cur_bytes, status_bytes);
-        block_copy_inflight_req_shrink(s, &req, cur_bytes);
+        block_copy_task_shrink(s, &task, cur_bytes);
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
-            block_copy_inflight_req_end(s, &req, 0);
+            block_copy_task_end(s, &task, 0);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
         ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                  error_is_read);
         co_put_to_shres(s->mem, cur_bytes);
-        block_copy_inflight_req_end(s, &req, ret);
+        block_copy_task_end(s, &task, ret);
         if (ret < 0) {
             return ret;
         }
-- 
2.26.2

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

We are going to use aio-task-pool API, so tasks will be handled in
parallel. We need therefore separate allocated task on each iteration.
Introduce this logic now.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200429130847.28124-3-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
 }
 
 /* Called only on full-dirty region */
-static void block_copy_task_begin(BlockCopyState *s, BlockCopyTask *task,
-                                  int64_t offset, int64_t bytes)
+static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
+                                             int64_t offset, int64_t bytes)
 {
+    BlockCopyTask *task = g_new(BlockCopyTask, 1);
+
     assert(!find_conflicting_task(s, offset, bytes));
 
     bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
@@ -XXX,XX +XXX,XX @@ static void block_copy_task_begin(BlockCopyState *s, BlockCopyTask *task,
     task->bytes = bytes;
     qemu_co_queue_init(&task->wait_queue);
     QLIST_INSERT_HEAD(&s->tasks, task, list);
+
+    return task;
 }
 
 /*
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 
     while (bytes) {
-        BlockCopyTask task;
+        g_autofree BlockCopyTask *task = NULL;
         int64_t next_zero, cur_bytes, status_bytes;
 
         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
             assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
             cur_bytes = next_zero - offset;
         }
-        block_copy_task_begin(s, &task, offset, cur_bytes);
+        task = block_copy_task_create(s, offset, cur_bytes);
 
         ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
         assert(ret >= 0); /* never fail */
         cur_bytes = MIN(cur_bytes, status_bytes);
-        block_copy_task_shrink(s, &task, cur_bytes);
+        block_copy_task_shrink(s, task, cur_bytes);
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
-            block_copy_task_end(s, &task, 0);
+            block_copy_task_end(s, task, 0);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
         ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                  error_is_read);
         co_put_to_shres(s->mem, cur_bytes);
-        block_copy_task_end(s, &task, ret);
+        block_copy_task_end(s, task, ret);
         if (ret < 0) {
             return ret;
         }
-- 
2.26.2

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

We are going to use aio-task-pool API, so we'll need state pointer in
BlockCopyTask anyway. Add it now and use where possible.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200429130847.28124-4-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
 #define BLOCK_COPY_MAX_MEM (128 * MiB)
 
 typedef struct BlockCopyTask {
+    BlockCopyState *s;
     int64_t offset;
     int64_t bytes;
     QLIST_ENTRY(BlockCopyTask) list;
@@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
     bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
     s->in_flight_bytes += bytes;
 
-    task->offset = offset;
-    task->bytes = bytes;
+    *task = (BlockCopyTask) {
+        .s = s,
+        .offset = offset,
+        .bytes = bytes,
+    };
     qemu_co_queue_init(&task->wait_queue);
     QLIST_INSERT_HEAD(&s->tasks, task, list);
 
@@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
  * wake up all tasks waiting for us (may be some of them are not intersecting
  * with shrunk task)
  */
-static void coroutine_fn block_copy_task_shrink(BlockCopyState *s,
-                                                BlockCopyTask *task,
+static void coroutine_fn block_copy_task_shrink(BlockCopyTask *task,
                                                 int64_t new_bytes)
 {
     if (new_bytes == task->bytes) {
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn block_copy_task_shrink(BlockCopyState *s,
 
     assert(new_bytes > 0 && new_bytes < task->bytes);
 
-    s->in_flight_bytes -= task->bytes - new_bytes;
-    bdrv_set_dirty_bitmap(s->copy_bitmap,
+    task->s->in_flight_bytes -= task->bytes - new_bytes;
+    bdrv_set_dirty_bitmap(task->s->copy_bitmap,
                           task->offset + new_bytes, task->bytes - new_bytes);
 
     task->bytes = new_bytes;
     qemu_co_queue_restart_all(&task->wait_queue);
 }
 
-static void coroutine_fn block_copy_task_end(BlockCopyState *s,
-                                             BlockCopyTask *task, int ret)
+static void coroutine_fn block_copy_task_end(BlockCopyTask *task, int ret)
 {
-    s->in_flight_bytes -= task->bytes;
+    task->s->in_flight_bytes -= task->bytes;
     if (ret < 0) {
-        bdrv_set_dirty_bitmap(s->copy_bitmap, task->offset, task->bytes);
+        bdrv_set_dirty_bitmap(task->s->copy_bitmap, task->offset, task->bytes);
     }
     QLIST_REMOVE(task, list);
     qemu_co_queue_restart_all(&task->wait_queue);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
         ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
         assert(ret >= 0); /* never fail */
         cur_bytes = MIN(cur_bytes, status_bytes);
-        block_copy_task_shrink(s, task, cur_bytes);
+        block_copy_task_shrink(task, cur_bytes);
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
-            block_copy_task_end(s, task, 0);
+            block_copy_task_end(task, 0);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
         ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                  error_is_read);
         co_put_to_shres(s->mem, cur_bytes);
-        block_copy_task_end(s, task, ret);
+        block_copy_task_end(task, ret);
         if (ret < 0) {
             return ret;
         }
-- 
2.26.2

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Instead of just relying on the comment "Called only on full-dirty
region" in block_copy_task_create() let's move initial dirty area
search directly to block_copy_task_create(). Let's also use effective
bdrv_dirty_bitmap_next_dirty_area instead of looping through all
non-dirty clusters.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200429130847.28124-5-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 80 ++++++++++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 34 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyTask {
     CoQueue wait_queue; /* coroutines blocked on this task */
 } BlockCopyTask;
 
+static int64_t task_end(BlockCopyTask *task)
+{
+    return task->offset + task->bytes;
+}
+
 typedef struct BlockCopyState {
     /*
      * BdrvChild objects are not owned or managed by block-copy. They are
@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
     return true;
 }
 
-/* Called only on full-dirty region */
+/*
+ * Search for the first dirty area in offset/bytes range and create task at
+ * the beginning of it.
+ */
 static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
                                              int64_t offset, int64_t bytes)
 {
-    BlockCopyTask *task = g_new(BlockCopyTask, 1);
+    BlockCopyTask *task;
 
+    if (!bdrv_dirty_bitmap_next_dirty_area(s->copy_bitmap,
+                                           offset, offset + bytes,
+                                           s->copy_size, &offset, &bytes))
+    {
+        return NULL;
+    }
+
+    /* region is dirty, so no existent tasks possible in it */
     assert(!find_conflicting_task(s, offset, bytes));
 
     bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
     s->in_flight_bytes += bytes;
 
+    task = g_new(BlockCopyTask, 1);
     *task = (BlockCopyTask) {
         .s = s,
         .offset = offset,
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
 {
     int ret = 0;
     bool found_dirty = false;
+    int64_t end = offset + bytes;
 
     /*
      * block_copy() user is responsible for keeping source and target in same
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
 
     while (bytes) {
         g_autofree BlockCopyTask *task = NULL;
-        int64_t next_zero, cur_bytes, status_bytes;
+        int64_t status_bytes;
 
-        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
-            trace_block_copy_skip(s, offset);
-            offset += s->cluster_size;
-            bytes -= s->cluster_size;
-            continue; /* already copied */
+        task = block_copy_task_create(s, offset, bytes);
+        if (!task) {
+            /* No more dirty bits in the bitmap */
+            trace_block_copy_skip_range(s, offset, bytes);
+            break;
+        }
+        if (task->offset > offset) {
+            trace_block_copy_skip_range(s, offset, task->offset - offset);
         }
 
         found_dirty = true;
 
-        cur_bytes = MIN(bytes, s->copy_size);
-
-        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,
-                                                cur_bytes);
-        if (next_zero >= 0) {
-            assert(next_zero > offset); /* offset is dirty */
-            assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
-            cur_bytes = next_zero - offset;
-        }
-        task = block_copy_task_create(s, offset, cur_bytes);
-
-        ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
+        ret = block_copy_block_status(s, task->offset, task->bytes,
+                                      &status_bytes);
         assert(ret >= 0); /* never fail */
-        cur_bytes = MIN(cur_bytes, status_bytes);
-        block_copy_task_shrink(task, cur_bytes);
+        if (status_bytes < task->bytes) {
+            block_copy_task_shrink(task, status_bytes);
+        }
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
             block_copy_task_end(task, 0);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
-            trace_block_copy_skip_range(s, offset, status_bytes);
-            offset += status_bytes;
-            bytes -= status_bytes;
+            trace_block_copy_skip_range(s, task->offset, task->bytes);
+            offset = task_end(task);
+            bytes = end - offset;
             continue;
         }
 
-        trace_block_copy_process(s, offset);
+        trace_block_copy_process(s, task->offset);
 
-        co_get_from_shres(s->mem, cur_bytes);
-        ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
-                                 error_is_read);
-        co_put_to_shres(s->mem, cur_bytes);
+        co_get_from_shres(s->mem, task->bytes);
+        ret = block_copy_do_copy(s, task->offset, task->bytes,
+                                 ret & BDRV_BLOCK_ZERO, error_is_read);
+        co_put_to_shres(s->mem, task->bytes);
         block_copy_task_end(task, ret);
         if (ret < 0) {
             return ret;
         }
 
-        progress_work_done(s->progress, cur_bytes);
-        s->progress_bytes_callback(cur_bytes, s->progress_opaque);
-        offset += cur_bytes;
-        bytes -= cur_bytes;
+        progress_work_done(s->progress, task->bytes);
+        s->progress_bytes_callback(task->bytes, s->progress_opaque);
+        offset = task_end(task);
+        bytes = end - offset;
     }
 
     return found_dirty;
-- 
2.26.2

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Run block_copy iterations in parallel in aio tasks.

Changes:
  - BlockCopyTask becomes aio task structure. Add zeroes field to pass
    it to block_copy_do_copy
  - add call state - it's a state of one call of block_copy(), shared
    between parallel tasks. For now used only to keep information about
    first error: is it read or not.
  - convert block_copy_dirty_clusters to aio-task loop.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Message-Id: <20200429130847.28124-6-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 119 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 106 insertions(+), 13 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
 #include "block/block-copy.h"
 #include "sysemu/block-backend.h"
 #include "qemu/units.h"
+#include "qemu/coroutine.h"
+#include "block/aio_task.h"
 
 #define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
 #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
 #define BLOCK_COPY_MAX_MEM (128 * MiB)
+#define BLOCK_COPY_MAX_WORKERS 64
+
+static coroutine_fn int block_copy_task_entry(AioTask *task);
+
+typedef struct BlockCopyCallState {
+    bool failed;
+    bool error_is_read;
+} BlockCopyCallState;
 
 typedef struct BlockCopyTask {
+    AioTask task;
+
     BlockCopyState *s;
+    BlockCopyCallState *call_state;
     int64_t offset;
     int64_t bytes;
+    bool zeroes;
     QLIST_ENTRY(BlockCopyTask) list;
     CoQueue wait_queue; /* coroutines blocked on this task */
 } BlockCopyTask;
@@ -XXX,XX +XXX,XX @@ static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
  * the beginning of it.
  */
 static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
+                                             BlockCopyCallState *call_state,
                                              int64_t offset, int64_t bytes)
 {
     BlockCopyTask *task;
@@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s,
 
     task = g_new(BlockCopyTask, 1);
     *task = (BlockCopyTask) {
+        .task.func = block_copy_task_entry,
         .s = s,
+        .call_state = call_state,
         .offset = offset,
         .bytes = bytes,
     };
@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
     s->progress = pm;
 }
 
+/*
+ * Takes ownership of @task
+ *
+ * If pool is NULL directly run the task, otherwise schedule it into the pool.
+ *
+ * Returns: task.func return code if pool is NULL
+ *          otherwise -ECANCELED if pool status is bad
+ *          otherwise 0 (successfully scheduled)
+ */
+static coroutine_fn int block_copy_task_run(AioTaskPool *pool,
+                                            BlockCopyTask *task)
+{
+    if (!pool) {
+        int ret = task->task.func(&task->task);
+
+        g_free(task);
+        return ret;
+    }
+
+    aio_task_pool_wait_slot(pool);
+    if (aio_task_pool_status(pool) < 0) {
+        co_put_to_shres(task->s->mem, task->bytes);
+        block_copy_task_end(task, -ECANCELED);
+        g_free(task);
+        return -ECANCELED;
+    }
+
+    aio_task_pool_start_task(pool, &task->task);
+
+    return 0;
+}
+
 /*
  * block_copy_do_copy
  *
@@ -XXX,XX +XXX,XX @@ out:
     return ret;
 }
 
+static coroutine_fn int block_copy_task_entry(AioTask *task)
+{
+    BlockCopyTask *t = container_of(task, BlockCopyTask, task);
+    bool error_is_read;
+    int ret;
+
+    ret = block_copy_do_copy(t->s, t->offset, t->bytes, t->zeroes,
+                             &error_is_read);
+    if (ret < 0 && !t->call_state->failed) {
+        t->call_state->failed = true;
+        t->call_state->error_is_read = error_is_read;
+    } else {
+        progress_work_done(t->s->progress, t->bytes);
+        t->s->progress_bytes_callback(t->bytes, t->s->progress_opaque);
+    }
+    co_put_to_shres(t->s->mem, t->bytes);
+    block_copy_task_end(t, ret);
+
+    return ret;
+}
+
 static int block_copy_block_status(BlockCopyState *s, int64_t offset,
                                    int64_t bytes, int64_t *pnum)
 {
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
     int ret = 0;
     bool found_dirty = false;
     int64_t end = offset + bytes;
+    AioTaskPool *aio = NULL;
+    BlockCopyCallState call_state = {false, false};
 
     /*
      * block_copy() user is responsible for keeping source and target in same
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
     assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 
-    while (bytes) {
-        g_autofree BlockCopyTask *task = NULL;
+    while (bytes && aio_task_pool_status(aio) == 0) {
+        BlockCopyTask *task;
         int64_t status_bytes;
 
-        task = block_copy_task_create(s, offset, bytes);
+        task = block_copy_task_create(s, &call_state, offset, bytes);
         if (!task) {
             /* No more dirty bits in the bitmap */
             trace_block_copy_skip_range(s, offset, bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
         }
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
             block_copy_task_end(task, 0);
+            g_free(task);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
             bytes = end - offset;
             continue;
         }
+        task->zeroes = ret & BDRV_BLOCK_ZERO;
 
         trace_block_copy_process(s, task->offset);
 
         co_get_from_shres(s->mem, task->bytes);
-        ret = block_copy_do_copy(s, task->offset, task->bytes,
-                                 ret & BDRV_BLOCK_ZERO, error_is_read);
-        co_put_to_shres(s->mem, task->bytes);
-        block_copy_task_end(task, ret);
-        if (ret < 0) {
-            return ret;
-        }
 
-        progress_work_done(s->progress, task->bytes);
-        s->progress_bytes_callback(task->bytes, s->progress_opaque);
         offset = task_end(task);
         bytes = end - offset;
+
+        if (!aio && bytes) {
+            aio = aio_task_pool_new(BLOCK_COPY_MAX_WORKERS);
+        }
+
+        ret = block_copy_task_run(aio, task);
+        if (ret < 0) {
+            goto out;
+        }
+    }
+
+out:
+    if (aio) {
+        aio_task_pool_wait_all(aio);
+
+        /*
+         * We are not really interested in -ECANCELED returned from
+         * block_copy_task_run. If it fails, it means some task already failed
+         * for real reason, let's return first failure.
+         * Still, assert that we don't rewrite failure by success.
+         */
+        assert(ret == 0 || aio_task_pool_status(aio) < 0);
+        ret = aio_task_pool_status(aio);
+
+        aio_task_pool_free(aio);
+    }
+    if (error_is_read && ret < 0) {
+        *error_is_read = call_state.error_is_read;
     }
 
-    return found_dirty;
+    return ret < 0 ? ret : found_dirty;
 }
 
 /*
-- 
2.26.2