The following changes since commit 36609b4fa36f0ac934874371874416f7533a5408:

Merge remote-tracking branch 'remotes/palmer/tags/palmer-for-master-4.2-sf1' into staging (2019-11-02 17:59:03 +0000)

are available in the Git repository at:

https://github.com/stefanha/qemu.git tags/block-pull-request

for you to fetch changes up to 9fdd7860adec188ed50d2530e9a819e8d953f9bb:

image-fuzzer: Use OSerror.strerror instead of tuple subscript (2019-11-05 16:36:11 +0100)

Pull request

Let's get the image fuzzer Python 3 changes merged in QEMU 4.2.

Eduardo Habkost (11):

image-fuzzer: Open image files in binary mode

image-fuzzer: Write bytes instead of string to image file

image-fuzzer: Explicitly use integer division operator

image-fuzzer: Use io.StringIO

image-fuzzer: Use %r for all fiels at Field.__repr__()

image-fuzzer: Return bytes objects on string fuzzing functions

image-fuzzer: Use bytes constant for field values

image-fuzzer: Encode file name and file format to bytes

image-fuzzer: Run using python3

image-fuzzer: Use errors parameter of subprocess.Popen()

image-fuzzer: Use OSerror.strerror instead of tuple subscript

tests/image-fuzzer/qcow2/__init__.py | 1 -

tests/image-fuzzer/qcow2/fuzz.py | 54 +++++++++++++-------------

tests/image-fuzzer/qcow2/layout.py | 57 ++++++++++++++--------------

tests/image-fuzzer/runner.py | 16 ++++----

4 files changed, 63 insertions(+), 65 deletions(-)

2.23.0

From: Eduardo Habkost <ehabkost@redhat.com>

Most of the division expressions in image-fuzzer assume integer

division. Use the // operator to keep the same behavior when we

move to Python 3.

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>

Reviewed-by: John Snow <jsnow@redhat.com>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Message-id: 20191016192430.25098-4-ehabkost@redhat.com

Message-Id: <20191016192430.25098-4-ehabkost@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

tests/image-fuzzer/qcow2/fuzz.py | 12 ++++-----

tests/image-fuzzer/qcow2/layout.py | 40 +++++++++++++++---------------

2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/tests/image-fuzzer/qcow2/fuzz.py b/tests/image-fuzzer/qcow2/fuzz.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/image-fuzzer/qcow2/fuzz.py

+++ b/tests/image-fuzzer/qcow2/fuzz.py

@@ -XXX,XX +XXX,XX @@ UINT64 = 0xffffffffffffffff

UINT32_M = 31

UINT64_M = 63

# Fuzz vectors

-UINT8_V = [0, 0x10, UINT8/4, UINT8/2 - 1, UINT8/2, UINT8/2 + 1, UINT8 - 1,

+UINT8_V = [0, 0x10, UINT8//4, UINT8//2 - 1, UINT8//2, UINT8//2 + 1, UINT8 - 1,

UINT8]

-UINT16_V = [0, 0x100, 0x1000, UINT16/4, UINT16/2 - 1, UINT16/2, UINT16/2 + 1,

+UINT16_V = [0, 0x100, 0x1000, UINT16//4, UINT16//2 - 1, UINT16//2, UINT16//2 + 1,

UINT16 - 1, UINT16]

-UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32/4, UINT32/2 - 1,

- UINT32/2, UINT32/2 + 1, UINT32 - 1, UINT32]

-UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64/4,

- UINT64/2 - 1, UINT64/2, UINT64/2 + 1, UINT64 - 1,

+UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32//4, UINT32//2 - 1,

+ UINT32//2, UINT32//2 + 1, UINT32 - 1, UINT32]

+UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64//4,

+ UINT64//2 - 1, UINT64//2, UINT64//2 + 1, UINT64 - 1,

UINT64]

STRING_V = ['%s%p%x%d', '.1024d', '%.2049d', '%p%p%p%p', '%x%x%x%x',

'%d%d%d%d', '%s%s%s%s', '%99999999999s', '%08x', '%%20d', '%%20n',

diff --git a/tests/image-fuzzer/qcow2/layout.py b/tests/image-fuzzer/qcow2/layout.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/image-fuzzer/qcow2/layout.py

+++ b/tests/image-fuzzer/qcow2/layout.py

@@ -XXX,XX +XXX,XX @@ class Image(object):

['>I', self.ext_offset, 0x6803f857, 'ext_magic'],

# One feature table contains 3 fields and takes 48 bytes

['>I', self.ext_offset + UINT32_S,

- len(feature_tables) / 3 * 48, 'ext_length']

+ len(feature_tables) // 3 * 48, 'ext_length']

] + feature_tables)

self.ext_offset = inner_offset

@@ -XXX,XX +XXX,XX @@ class Image(object):

def create_l2_entry(host, guest, l2_cluster):

"""Generate one L2 entry."""

offset = l2_cluster * self.cluster_size

- l2_size = self.cluster_size / UINT64_S

+ l2_size = self.cluster_size // UINT64_S

entry_offset = offset + UINT64_S * (guest % l2_size)

cluster_descriptor = host * self.cluster_size

if not self.header['version'][0].value == 2:

@@ -XXX,XX +XXX,XX @@ class Image(object):

def create_l1_entry(l2_cluster, l1_offset, guest):

"""Generate one L1 entry."""

- l2_size = self.cluster_size / UINT64_S

- entry_offset = l1_offset + UINT64_S * (guest / l2_size)

+ l2_size = self.cluster_size // UINT64_S

+ entry_offset = l1_offset + UINT64_S * (guest // l2_size)

# While snapshots are not supported bit #63 = 1

entry_val = (1 << 63) + l2_cluster * self.cluster_size

return ['>Q', entry_offset, entry_val, 'l1_entry']

@@ -XXX,XX +XXX,XX @@ class Image(object):

l2 = []

else:

meta_data = self._get_metadata()

- guest_clusters = random.sample(range(self.image_size /

+ guest_clusters = random.sample(range(self.image_size //

self.cluster_size),

len(self.data_clusters))

# Number of entries in a L1/L2 table

- l_size = self.cluster_size / UINT64_S

+ l_size = self.cluster_size // UINT64_S

# Number of clusters necessary for L1 table

l1_size = int(ceil((max(guest_clusters) + 1) / float(l_size**2)))

l1_start = self._get_adjacent_clusters(self.data_clusters |

@@ -XXX,XX +XXX,XX @@ class Image(object):

# L2 entries

l2 = []

for host, guest in zip(self.data_clusters, guest_clusters):

- l2_id = guest / l_size

+ l2_id = guest // l_size

if l2_id not in l2_ids:

l2_ids.append(l2_id)

l2_clusters.append(self._get_adjacent_clusters(

@@ -XXX,XX +XXX,XX @@ class Image(object):

def allocate_rfc_blocks(data, size):

"""Return indices of clusters allocated for refcount blocks."""

cluster_ids = set()

- diff = block_ids = set([x / size for x in data])

+ diff = block_ids = set([x // size for x in data])

while len(diff) != 0:

# Allocate all yet not allocated clusters

new = self._get_available_clusters(data | cluster_ids,

len(diff))

# Indices of new refcount blocks necessary to cover clusters

# in 'new'

- diff = set([x / size for x in new]) - block_ids

+ diff = set([x // size for x in new]) - block_ids

cluster_ids |= new

block_ids |= diff

return cluster_ids, block_ids

@@ -XXX,XX +XXX,XX @@ class Image(object):

blocks = set(init_blocks)

clusters = set()

# Number of entries in one cluster of the refcount table

- size = self.cluster_size / UINT64_S

+ size = self.cluster_size // UINT64_S

# Number of clusters necessary for the refcount table based on

# the current number of refcount blocks

table_size = int(ceil((max(blocks) + 1) / float(size)))

@@ -XXX,XX +XXX,XX @@ class Image(object):

table_size + 1))

# New refcount blocks necessary for clusters occupied by the

# refcount table

- diff = set([c / block_size for c in table_clusters]) - blocks

+ diff = set([c // block_size for c in table_clusters]) - blocks

blocks |= diff

while len(diff) != 0:

# Allocate clusters for new refcount blocks

@@ -XXX,XX +XXX,XX @@ class Image(object):

len(diff))

# Indices of new refcount blocks necessary to cover

# clusters in 'new'

- diff = set([x / block_size for x in new]) - blocks

+ diff = set([x // block_size for x in new]) - blocks

clusters |= new

blocks |= diff

# Check if the refcount table needs one more cluster

if int(ceil((max(blocks) + 1) / float(size))) > table_size:

- new_block_id = (table_start + table_size) / block_size

+ new_block_id = (table_start + table_size) // block_size

# Check if the additional table cluster needs

# one more refcount block

if new_block_id not in blocks:

@@ -XXX,XX +XXX,XX @@ class Image(object):

def create_table_entry(table_offset, block_cluster, block_size,

cluster):

"""Generate a refcount table entry."""

- offset = table_offset + UINT64_S * (cluster / block_size)

+ offset = table_offset + UINT64_S * (cluster // block_size)

return ['>Q', offset, block_cluster * self.cluster_size,

'refcount_table_entry']

def create_block_entry(block_cluster, block_size, cluster):

"""Generate a list of entries for the current block."""

- entry_size = self.cluster_size / block_size

+ entry_size = self.cluster_size // block_size

offset = block_cluster * self.cluster_size

entry_offset = offset + entry_size * (cluster % block_size)

# While snapshots are not supported all refcounts are set to 1

@@ -XXX,XX +XXX,XX @@ class Image(object):

# Number of refcount entries per refcount block

# Convert self.cluster_size from bytes to bits to have the same

# base for the numerator and denominator

- block_size = self.cluster_size * 8 / refcount_bits

+ block_size = self.cluster_size * 8 // refcount_bits

meta_data = self._get_metadata()

if len(self.data_clusters) == 0:

# All metadata for an empty guest image needs 4 clusters:

@@ -XXX,XX +XXX,XX @@ class Image(object):

rfc_blocks = []

for cluster in sorted(self.data_clusters | meta_data):

- if cluster / block_size != block_id:

- block_id = cluster / block_size

+ if cluster // block_size != block_id:

+ block_id = cluster // block_size

block_cluster = block_clusters[block_ids.index(block_id)]

rfc_table.append(create_table_entry(table_offset,

block_cluster,

@@ -XXX,XX +XXX,XX @@ class Image(object):

def _alloc_data(img_size, cluster_size):

"""Return a set of random indices of clusters allocated for guest data.

"""

- num_of_cls = img_size/cluster_size

+ num_of_cls = img_size // cluster_size

return set(random.sample(range(1, num_of_cls + 1),

random.randint(0, num_of_cls)))

@@ -XXX,XX +XXX,XX @@ class Image(object):

"""Return indices of clusters allocated for image metadata."""

ids = set()

for x in self:

- ids.add(x.offset/self.cluster_size)

+ ids.add(x.offset // self.cluster_size)

return ids

--

2.23.0

From: Eduardo Habkost <ehabkost@redhat.com>

Instead of manually encoding stderr and stdout output, use

`errors` parameter of subprocess.Popen(). This will make

process.communicate() return unicode strings instead of bytes

objects.

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>

Reviewed-by: John Snow <jsnow@redhat.com>

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Message-id: 20191016192430.25098-11-ehabkost@redhat.com

Message-Id: <20191016192430.25098-11-ehabkost@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

tests/image-fuzzer/runner.py | 11 ++++-------

1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/tests/image-fuzzer/runner.py b/tests/image-fuzzer/runner.py

--- a/tests/image-fuzzer/runner.py

+++ b/tests/image-fuzzer/runner.py

@@ -XXX,XX +XXX,XX @@ def run_app(fd, q_args):

devnull = open('/dev/null', 'r+')

process = subprocess.Popen(q_args, stdin=devnull,

stdout=subprocess.PIPE,

- stderr=subprocess.PIPE)

+ stderr=subprocess.PIPE,

+ errors='replace')

try:

out, err = process.communicate()

signal.alarm(0)

- # fd is a text file, so we need to decode the process output before

- # writing to it.

- # We could be simply using the `errors` parameter of subprocess.Popen(),

- # but this will be possible only after migrating to Python 3

- fd.write(out.decode(errors='replace'))

- fd.write(err.decode(errors='replace'))

+ fd.write(out)

+ fd.write(err)

fd.flush()

return process.returncode

2.23.0

From: Eduardo Habkost <ehabkost@redhat.com>

OSError can't be used like a tuple on Python 3, so change the

code to use `e.sterror` instead of `e[1]`.

Reported-by: John Snow <jsnow@redhat.com>

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>

Reviewed-by: John Snow <jsnow@redhat.com>

Message-id: 20191021214117.18091-1-ehabkost@redhat.com

Message-Id: <20191021214117.18091-1-ehabkost@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

---

tests/image-fuzzer/runner.py | 4 ++--

1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/image-fuzzer/runner.py b/tests/image-fuzzer/runner.py

index XXXXXXX..XXXXXXX 100755

--- a/tests/image-fuzzer/runner.py

+++ b/tests/image-fuzzer/runner.py

@@ -XXX,XX +XXX,XX @@ class TestEnv(object):

os.makedirs(self.current_dir)

except OSError as e:

print("Error: The working directory '%s' cannot be used. Reason: %s"\

- % (self.work_dir, e[1]), file=sys.stderr)

+ % (self.work_dir, e.strerror), file=sys.stderr)

raise TestException

self.log = open(os.path.join(self.current_dir, "test.log"), "w")

self.parent_log = open(run_log, "a")

@@ -XXX,XX +XXX,XX @@ class TestEnv(object):

except OSError as e:

multilog("%sError: Start of '%s' failed. Reason: %s\n\n"

% (test_summary, os.path.basename(current_cmd[0]),

- e[1]),

+ e.strerror),

sys.stderr, self.log, self.parent_log)

raise TestException

2.23.0