Add TestEnv class, which will handle test environment in a new python
iotests running framework.
Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
1 file changed, 278 insertions(+)
create mode 100644 tests/qemu-iotests/testenv.py
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
new file mode 100644
index 0000000000..348af593e9
--- /dev/null
+++ b/tests/qemu-iotests/testenv.py
@@ -0,0 +1,278 @@
+# TestEnv class to manage test environment variables.
+#
+# Copyright (c) 2020-2021 Virtuozzo International GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+import tempfile
+from pathlib import Path
+import shutil
+import collections
+import random
+import subprocess
+import glob
+from contextlib import AbstractContextManager
+from typing import Dict, Any, Optional
+
+
+def get_default_machine(qemu_prog: str) -> str:
+ outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
+ universal_newlines=True,
+ stdout=subprocess.PIPE).stdout
+
+ machines = outp.split('\n')
+ default_machine = next(m for m in machines if m.endswith(' (default)'))
+ default_machine = default_machine.split(' ', 1)[0]
+
+ alias_suf = ' (alias of {})'.format(default_machine)
+ alias = next((m for m in machines if m.endswith(alias_suf)), None)
+ if alias is not None:
+ default_machine = alias.split(' ', 1)[0]
+
+ return default_machine
+
+
+class TestEnv(AbstractContextManager['TestEnv']):
+ """
+ Manage system environment for running tests
+
+ The following variables are supported/provided. They are represented by
+ lower-cased TestEnv attributes.
+ """
+
+ # We store environment variables as instance attributes, and there are a
+ # lot of them. Silence pylint:
+ # pylint: disable=too-many-instance-attributes
+
+ env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
+ 'OUTPUT_DIR', 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG',
+ 'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG',
+ 'SOCKET_SCM_HELPER', 'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS',
+ 'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT',
+ 'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO',
+ 'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU',
+ 'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',
+ 'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_']
+
+ def get_env(self) -> Dict[str, str]:
+ env = {}
+ for v in self.env_variables:
+ val = getattr(self, v.lower(), None)
+ if val is not None:
+ env[v] = val
+
+ return env
+
+ def init_directories(self):
+ """Init directory variables:
+ PYTHONPATH
+ TEST_DIR
+ SOCK_DIR
+ SAMPLE_IMG_DIR
+ OUTPUT_DIR
+ """
+ self.pythonpath = os.getenv('PYTHONPATH')
+ if self.pythonpath:
+ self.pythonpath = self.source_iotests + os.pathsep + \
+ self.pythonpath
+ else:
+ self.pythonpath = self.source_iotests
+
+ self.test_dir = os.getenv('TEST_DIR',
+ os.path.join(os.getcwd(), 'scratch'))
+ Path(self.test_dir).mkdir(parents=True, exist_ok=True)
+
+ self.sock_dir = os.getenv('SOCK_DIR')
+ self.tmp_sock_dir = False
+ if self.sock_dir:
+ Path(self.test_dir).mkdir(parents=True, exist_ok=True)
+ else:
+ self.sock_dir = tempfile.mkdtemp()
+ self.tmp_sock_dir = True
+
+ self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR',
+ os.path.join(self.source_iotests,
+ 'sample_images'))
+
+ self.output_dir = os.getcwd() # OUTPUT_DIR
+
+ def init_binaries(self):
+ """Init binary path variables:
+ PYTHON (for bash tests)
+ QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
+ SOCKET_SCM_HELPER
+ """
+ self.python = sys.executable
+
+ def root(*names):
+ return os.path.join(self.build_root, *names)
+
+ arch = os.uname().machine
+ if 'ppc64' in arch:
+ arch = 'ppc64'
+
+ self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}'))
+ if not os.path.exists(self.qemu_prog):
+ pattern = root('qemu-system-*')
+ progs = glob.glob(pattern)
+ if not progs:
+ sys.exit(f"Not found any Qemu binary by pattern '{pattern}'")
+ if len(progs) > 1:
+ progs_list = ', '.join(progs)
+ sys.exit(f"Several non '{arch}' qemu binaries found: "
+ f"{progs_list}, please set QEMU_PROG environment "
+ "variable")
+ self.qemu_prog = progs[0]
+
+ self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img'))
+ self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io'))
+ self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd'))
+ self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon',
+ 'qemu-storage-daemon'))
+
+ for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog,
+ self.qemu_prog, self.qsd_prog]:
+ if not os.path.exists(b):
+ sys.exit('No such file: ' + b)
+ if not os.access(b, os.X_OK):
+ sys.exit('Not executable: ' + b)
+
+ helper_path = os.path.join(self.build_iotests, 'socket_scm_helper')
+ if os.access(helper_path, os.X_OK):
+ self.socket_scm_helper = helper_path # SOCKET_SCM_HELPER
+
+ def __init__(self, imgfmt: str, imgproto: str, aiomode: str,
+ cachemode: Optional[str] = None,
+ imgopts: Optional[str] = None,
+ misalign: bool = False,
+ debug: bool = False,
+ valgrind: bool = False) -> None:
+ self.imgfmt = imgfmt
+ self.imgproto = imgproto
+ self.aiomode = aiomode
+ self.imgopts = imgopts
+ self.misalign = misalign
+ self.debug = debug
+
+ if valgrind:
+ self.valgrind_qemu = 'y'
+
+ if cachemode is None:
+ self.cachemode_is_default = 'true'
+ self.cachemode = 'writeback'
+ else:
+ self.cachemode_is_default = 'false'
+ self.cachemode = cachemode
+
+ # Initialize generic paths: build_root, build_iotests, source_iotests,
+ # which are needed to initialize some environment variables. They are
+ # used by init_*() functions as well.
+
+ if os.path.islink(sys.argv[0]):
+ # called from the build tree
+ self.source_iotests = os.path.dirname(os.readlink(sys.argv[0]))
+ self.build_iotests = os.path.dirname(os.path.abspath(sys.argv[0]))
+ else:
+ # called from the source tree
+ self.source_iotests = os.getcwd()
+ self.build_iotests = self.source_iotests
+
+ self.build_root = os.path.join(self.build_iotests, '..', '..')
+
+ self.init_directories()
+ self.init_binaries()
+
+ self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_',
+ str(random.randrange(1, 255)))
+
+ # QEMU_OPTIONS
+ self.qemu_options = '-nodefaults -display none -accel qtest'
+ machine_map = (
+ ('arm', 'virt'),
+ ('aarch64', 'virt'),
+ ('avr', 'mega2560'),
+ ('rx', 'gdbsim-r5f562n8'),
+ ('tricore', 'tricore_testboard')
+ )
+ for suffix, machine in machine_map:
+ if self.qemu_prog.endswith(f'qemu-system-{suffix}'):
+ self.qemu_options += f' -machine {machine}'
+
+ # QEMU_DEFAULT_MACHINE
+ self.qemu_default_machine = get_default_machine(self.qemu_prog)
+
+ self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS')
+ self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS')
+
+ is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg']
+ self.imgfmt_generic = 'true' if is_generic else 'false'
+
+ self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}'
+ if self.misalign:
+ self.qemu_io_options += ' --misalign'
+
+ self.qemu_io_options_no_fmt = self.qemu_io_options
+
+ if self.imgfmt == 'luks':
+ self.imgoptssyntax = 'true'
+ self.imgkeysecret = '123456'
+ if not self.imgopts:
+ self.imgopts = 'iter-time=10'
+ elif 'iter-time=' not in self.imgopts:
+ self.imgopts += ',iter-time=10'
+ else:
+ self.imgoptssyntax = 'false'
+ self.qemu_io_options += ' -f ' + self.imgfmt
+
+ if self.imgfmt == 'vmdk':
+ if not self.imgopts:
+ self.imgopts = 'zeroed_grain=on'
+ elif 'zeroed_grain=' not in self.imgopts:
+ self.imgopts += ',zeroed_grain=on'
+
+ def close(self) -> None:
+ if self.tmp_sock_dir:
+ shutil.rmtree(self.sock_dir)
+
+ def __enter__(self) -> 'TestEnv':
+ return self
+
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+ self.close()
+
+ def print_env(self) -> None:
+ template = """\
+QEMU -- "{QEMU_PROG}" {QEMU_OPTIONS}
+QEMU_IMG -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS}
+QEMU_IO -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS}
+QEMU_NBD -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS}
+IMGFMT -- {IMGFMT}{imgopts}
+IMGPROTO -- {IMGPROTO}
+PLATFORM -- {platform}
+TEST_DIR -- {TEST_DIR}
+SOCK_DIR -- {SOCK_DIR}
+SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}"""
+
+ args = collections.defaultdict(str, self.get_env())
+
+ if 'IMGOPTS' in args:
+ args['imgopts'] = f" ({args['IMGOPTS']})"
+
+ u = os.uname()
+ args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}'
+
+ print(template.format_map(args))
--
2.29.2
24.01.2021 00:04, Vladimir Sementsov-Ogievskiy wrote:
> Add TestEnv class, which will handle test environment in a new python
> iotests running framework.
>
> Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy<vsementsov@virtuozzo.com>
> ---
> tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
> 1 file changed, 278 insertions(+)
> create mode 100644 tests/qemu-iotests/testenv.py
>
> diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
> new file mode 100644
> index 0000000000..348af593e9
[..]
> + def init_binaries(self):
> + """Init binary path variables:
> + PYTHON (for bash tests)
> + QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
> + SOCKET_SCM_HELPER
> + """
> + self.python = sys.executable
> +
> + def root(*names):
> + return os.path.join(self.build_root, *names)
> +
> + arch = os.uname().machine
> + if 'ppc64' in arch:
> + arch = 'ppc64'
> +
> + self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}'))
> + if not os.path.exists(self.qemu_prog):
> + pattern = root('qemu-system-*')
> + progs = glob.glob(pattern)
> + if not progs:
> + sys.exit(f"Not found any Qemu binary by pattern '{pattern}'")
> + if len(progs) > 1:
> + progs_list = ', '.join(progs)
> + sys.exit(f"Several non '{arch}' qemu binaries found: "
> + f"{progs_list}, please set QEMU_PROG environment "
> + "variable")
> + self.qemu_prog = progs[0]
squash-in:
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
index 348af593e9..1633510caf 100644
--- a/tests/qemu-iotests/testenv.py
+++ b/tests/qemu-iotests/testenv.py
@@ -129,14 +129,14 @@ class TestEnv(AbstractContextManager['TestEnv']):
if not os.path.exists(self.qemu_prog):
pattern = root('qemu-system-*')
progs = glob.glob(pattern)
- if not progs:
- sys.exit(f"Not found any Qemu binary by pattern '{pattern}'")
- if len(progs) > 1:
- progs_list = ', '.join(progs)
- sys.exit(f"Several non '{arch}' qemu binaries found: "
- f"{progs_list}, please set QEMU_PROG environment "
- "variable")
- self.qemu_prog = progs[0]
+ found = False
+ for p in progs:
+ if os.access(p, os.X_OK):
+ self.qemu_prog = p
+ found = True
+ if not found:
+ sys.exit("Not found any Qemu executable binary by pattern "
+ f"'{pattern}'")
--
Best regards,
Vladimir
Am 23.01.2021 um 22:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
> Add TestEnv class, which will handle test environment in a new python
> iotests running framework.
>
> Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---
> tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
> 1 file changed, 278 insertions(+)
> create mode 100644 tests/qemu-iotests/testenv.py
>
> diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
> new file mode 100644
> index 0000000000..348af593e9
> --- /dev/null
> +++ b/tests/qemu-iotests/testenv.py
> @@ -0,0 +1,278 @@
> +# TestEnv class to manage test environment variables.
> +#
> +# Copyright (c) 2020-2021 Virtuozzo International GmbH
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +#
> +
> +import os
> +import sys
> +import tempfile
> +from pathlib import Path
> +import shutil
> +import collections
> +import random
> +import subprocess
> +import glob
> +from contextlib import AbstractContextManager
> +from typing import Dict, Any, Optional
> +
> +
> +def get_default_machine(qemu_prog: str) -> str:
> + outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
> + universal_newlines=True,
> + stdout=subprocess.PIPE).stdout
> +
> + machines = outp.split('\n')
> + default_machine = next(m for m in machines if m.endswith(' (default)'))
> + default_machine = default_machine.split(' ', 1)[0]
> +
> + alias_suf = ' (alias of {})'.format(default_machine)
> + alias = next((m for m in machines if m.endswith(alias_suf)), None)
> + if alias is not None:
> + default_machine = alias.split(' ', 1)[0]
> +
> + return default_machine
> +
> +
> +class TestEnv(AbstractContextManager['TestEnv']):
I'm getting CI failures here:
Traceback (most recent call last):
File "./check", line 23, in <module>
from testenv import TestEnv
File "/builds/.../qemu/tests/qemu-iotests/testenv.py", line 49, in <module>
class TestEnv(AbstractContextManager['TestEnv']):
TypeError: 'ABCMeta' object is not subscriptable
On the other hand, if I make it just AbstractContextManager without
giving the type parameter, mypy complains:
testenv.py:49: error: Missing type parameters for generic type "ContextManager"
I guess I need to have another look into this tomorrow.
By the way, mypy --strict still finds a few errors. I think we want to
address at least the warnings about missing type annotatings and calling
untyped functions.
Kevin
26.01.2021 01:05, Kevin Wolf wrote:
> Am 23.01.2021 um 22:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
>> Add TestEnv class, which will handle test environment in a new python
>> iotests running framework.
>>
>> Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
>>
>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>> ---
>> tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
>> 1 file changed, 278 insertions(+)
>> create mode 100644 tests/qemu-iotests/testenv.py
>>
>> diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
>> new file mode 100644
>> index 0000000000..348af593e9
>> --- /dev/null
>> +++ b/tests/qemu-iotests/testenv.py
>> @@ -0,0 +1,278 @@
>> +# TestEnv class to manage test environment variables.
>> +#
>> +# Copyright (c) 2020-2021 Virtuozzo International GmbH
>> +#
>> +# This program is free software; you can redistribute it and/or modify
>> +# it under the terms of the GNU General Public License as published by
>> +# the Free Software Foundation; either version 2 of the License, or
>> +# (at your option) any later version.
>> +#
>> +# This program is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
>> +#
>> +
>> +import os
>> +import sys
>> +import tempfile
>> +from pathlib import Path
>> +import shutil
>> +import collections
>> +import random
>> +import subprocess
>> +import glob
>> +from contextlib import AbstractContextManager
>> +from typing import Dict, Any, Optional
>> +
>> +
>> +def get_default_machine(qemu_prog: str) -> str:
>> + outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
>> + universal_newlines=True,
>> + stdout=subprocess.PIPE).stdout
>> +
>> + machines = outp.split('\n')
>> + default_machine = next(m for m in machines if m.endswith(' (default)'))
>> + default_machine = default_machine.split(' ', 1)[0]
>> +
>> + alias_suf = ' (alias of {})'.format(default_machine)
>> + alias = next((m for m in machines if m.endswith(alias_suf)), None)
>> + if alias is not None:
>> + default_machine = alias.split(' ', 1)[0]
>> +
>> + return default_machine
>> +
>> +
>> +class TestEnv(AbstractContextManager['TestEnv']):
>
> I'm getting CI failures here:
>
> Traceback (most recent call last):
> File "./check", line 23, in <module>
> from testenv import TestEnv
> File "/builds/.../qemu/tests/qemu-iotests/testenv.py", line 49, in <module>
> class TestEnv(AbstractContextManager['TestEnv']):
> TypeError: 'ABCMeta' object is not subscriptable
>
> On the other hand, if I make it just AbstractContextManager without
> giving the type parameter, mypy complains:
>
> testenv.py:49: error: Missing type parameters for generic type "ContextManager"
>
> I guess I need to have another look into this tomorrow.
It may help to use typing.ContextManager instead of AbstractContextManager. mypy is OK with it, probably CI will be OK too..
>
> By the way, mypy --strict still finds a few errors. I think we want to
> address at least the warnings about missing type annotatings and calling
> untyped functions.
>
OK
--
Best regards,
Vladimir
Am 26.01.2021 um 09:28 hat Vladimir Sementsov-Ogievskiy geschrieben:
> 26.01.2021 01:05, Kevin Wolf wrote:
> > Am 23.01.2021 um 22:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
> > > Add TestEnv class, which will handle test environment in a new python
> > > iotests running framework.
> > >
> > > Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
> > >
> > > Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> > > ---
> > > tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
> > > 1 file changed, 278 insertions(+)
> > > create mode 100644 tests/qemu-iotests/testenv.py
> > >
> > > diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
> > > new file mode 100644
> > > index 0000000000..348af593e9
> > > --- /dev/null
> > > +++ b/tests/qemu-iotests/testenv.py
> > > @@ -0,0 +1,278 @@
> > > +# TestEnv class to manage test environment variables.
> > > +#
> > > +# Copyright (c) 2020-2021 Virtuozzo International GmbH
> > > +#
> > > +# This program is free software; you can redistribute it and/or modify
> > > +# it under the terms of the GNU General Public License as published by
> > > +# the Free Software Foundation; either version 2 of the License, or
> > > +# (at your option) any later version.
> > > +#
> > > +# This program is distributed in the hope that it will be useful,
> > > +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> > > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> > > +# GNU General Public License for more details.
> > > +#
> > > +# You should have received a copy of the GNU General Public License
> > > +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> > > +#
> > > +
> > > +import os
> > > +import sys
> > > +import tempfile
> > > +from pathlib import Path
> > > +import shutil
> > > +import collections
> > > +import random
> > > +import subprocess
> > > +import glob
> > > +from contextlib import AbstractContextManager
> > > +from typing import Dict, Any, Optional
> > > +
> > > +
> > > +def get_default_machine(qemu_prog: str) -> str:
> > > + outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
> > > + universal_newlines=True,
> > > + stdout=subprocess.PIPE).stdout
> > > +
> > > + machines = outp.split('\n')
> > > + default_machine = next(m for m in machines if m.endswith(' (default)'))
> > > + default_machine = default_machine.split(' ', 1)[0]
> > > +
> > > + alias_suf = ' (alias of {})'.format(default_machine)
> > > + alias = next((m for m in machines if m.endswith(alias_suf)), None)
> > > + if alias is not None:
> > > + default_machine = alias.split(' ', 1)[0]
> > > +
> > > + return default_machine
> > > +
> > > +
> > > +class TestEnv(AbstractContextManager['TestEnv']):
> >
> > I'm getting CI failures here:
> >
> > Traceback (most recent call last):
> > File "./check", line 23, in <module>
> > from testenv import TestEnv
> > File "/builds/.../qemu/tests/qemu-iotests/testenv.py", line 49, in <module>
> > class TestEnv(AbstractContextManager['TestEnv']):
> > TypeError: 'ABCMeta' object is not subscriptable
> >
> > On the other hand, if I make it just AbstractContextManager without
> > giving the type parameter, mypy complains:
> >
> > testenv.py:49: error: Missing type parameters for generic type "ContextManager"
> >
> > I guess I need to have another look into this tomorrow.
>
> It may help to use typing.ContextManager instead of
> AbstractContextManager. mypy is OK with it, probably CI will be OK
> too..
Okay, I'm trying now if this change works (on top of v9, of course). If
it does, I'll just squash it in. I also silenced some of the mypy
warnings, so that I'm not testing with the following patch squashed in.
Kevin
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
index ca9cab531b..becea1bb7d 100644
--- a/tests/qemu-iotests/testenv.py
+++ b/tests/qemu-iotests/testenv.py
@@ -25,8 +25,7 @@ import collections
import random
import subprocess
import glob
-from contextlib import AbstractContextManager
-from typing import Dict, Any, Optional
+from typing import ContextManager, Dict, Any, Optional
def isxfile(path: str) -> bool:
@@ -50,7 +49,7 @@ def get_default_machine(qemu_prog: str) -> str:
return default_machine
-class TestEnv(AbstractContextManager['TestEnv']):
+class TestEnv(ContextManager['TestEnv']):
"""
Manage system environment for running tests
@@ -81,7 +80,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
return env
- def init_directories(self):
+ def init_directories(self) -> None:
"""Init directory variables:
PYTHONPATH
TEST_DIR
@@ -114,7 +113,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
self.output_dir = os.getcwd() # OUTPUT_DIR
- def init_binaries(self):
+ def init_binaries(self) -> None:
"""Init binary path variables:
PYTHON (for bash tests)
QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
@@ -122,7 +121,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
"""
self.python = sys.executable
- def root(*names):
+ def root(*names: str) -> str:
return os.path.join(self.build_root, *names)
arch = os.uname().machine
26.01.2021 12:45, Kevin Wolf wrote:
> Am 26.01.2021 um 09:28 hat Vladimir Sementsov-Ogievskiy geschrieben:
>> 26.01.2021 01:05, Kevin Wolf wrote:
>>> Am 23.01.2021 um 22:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
>>>> Add TestEnv class, which will handle test environment in a new python
>>>> iotests running framework.
>>>>
>>>> Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
>>>>
>>>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>>>> ---
>>>> tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
>>>> 1 file changed, 278 insertions(+)
>>>> create mode 100644 tests/qemu-iotests/testenv.py
>>>>
>>>> diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
>>>> new file mode 100644
>>>> index 0000000000..348af593e9
>>>> --- /dev/null
>>>> +++ b/tests/qemu-iotests/testenv.py
>>>> @@ -0,0 +1,278 @@
>>>> +# TestEnv class to manage test environment variables.
>>>> +#
>>>> +# Copyright (c) 2020-2021 Virtuozzo International GmbH
>>>> +#
>>>> +# This program is free software; you can redistribute it and/or modify
>>>> +# it under the terms of the GNU General Public License as published by
>>>> +# the Free Software Foundation; either version 2 of the License, or
>>>> +# (at your option) any later version.
>>>> +#
>>>> +# This program is distributed in the hope that it will be useful,
>>>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>>>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>>>> +# GNU General Public License for more details.
>>>> +#
>>>> +# You should have received a copy of the GNU General Public License
>>>> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
>>>> +#
>>>> +
>>>> +import os
>>>> +import sys
>>>> +import tempfile
>>>> +from pathlib import Path
>>>> +import shutil
>>>> +import collections
>>>> +import random
>>>> +import subprocess
>>>> +import glob
>>>> +from contextlib import AbstractContextManager
>>>> +from typing import Dict, Any, Optional
>>>> +
>>>> +
>>>> +def get_default_machine(qemu_prog: str) -> str:
>>>> + outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
>>>> + universal_newlines=True,
>>>> + stdout=subprocess.PIPE).stdout
>>>> +
>>>> + machines = outp.split('\n')
>>>> + default_machine = next(m for m in machines if m.endswith(' (default)'))
>>>> + default_machine = default_machine.split(' ', 1)[0]
>>>> +
>>>> + alias_suf = ' (alias of {})'.format(default_machine)
>>>> + alias = next((m for m in machines if m.endswith(alias_suf)), None)
>>>> + if alias is not None:
>>>> + default_machine = alias.split(' ', 1)[0]
>>>> +
>>>> + return default_machine
>>>> +
>>>> +
>>>> +class TestEnv(AbstractContextManager['TestEnv']):
>>>
>>> I'm getting CI failures here:
>>>
>>> Traceback (most recent call last):
>>> File "./check", line 23, in <module>
>>> from testenv import TestEnv
>>> File "/builds/.../qemu/tests/qemu-iotests/testenv.py", line 49, in <module>
>>> class TestEnv(AbstractContextManager['TestEnv']):
>>> TypeError: 'ABCMeta' object is not subscriptable
>>>
>>> On the other hand, if I make it just AbstractContextManager without
>>> giving the type parameter, mypy complains:
>>>
>>> testenv.py:49: error: Missing type parameters for generic type "ContextManager"
>>>
>>> I guess I need to have another look into this tomorrow.
>>
>> It may help to use typing.ContextManager instead of
>> AbstractContextManager. mypy is OK with it, probably CI will be OK
>> too..
>
> Okay, I'm trying now if this change works (on top of v9, of course). If
> it does, I'll just squash it in. I also silenced some of the mypy
> warnings, so that I'm not testing with the following patch squashed in.
>
> Kevin
>
>
> diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
> index ca9cab531b..becea1bb7d 100644
> --- a/tests/qemu-iotests/testenv.py
> +++ b/tests/qemu-iotests/testenv.py
> @@ -25,8 +25,7 @@ import collections
> import random
> import subprocess
> import glob
> -from contextlib import AbstractContextManager
> -from typing import Dict, Any, Optional
> +from typing import ContextManager, Dict, Any, Optional
>
>
> def isxfile(path: str) -> bool:
> @@ -50,7 +49,7 @@ def get_default_machine(qemu_prog: str) -> str:
> return default_machine
>
>
> -class TestEnv(AbstractContextManager['TestEnv']):
> +class TestEnv(ContextManager['TestEnv']):
> """
> Manage system environment for running tests
>
> @@ -81,7 +80,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
>
> return env
>
> - def init_directories(self):
> + def init_directories(self) -> None:
> """Init directory variables:
> PYTHONPATH
> TEST_DIR
> @@ -114,7 +113,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
>
> self.output_dir = os.getcwd() # OUTPUT_DIR
>
> - def init_binaries(self):
> + def init_binaries(self) -> None:
> """Init binary path variables:
> PYTHON (for bash tests)
> QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
> @@ -122,7 +121,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
> """
> self.python = sys.executable
>
> - def root(*names):
> + def root(*names: str) -> str:
> return os.path.join(self.build_root, *names)
>
> arch = os.uname().machine
>
Strange, that CI doesn't complain AbstractContextManager['...'] in testrunner.py.. Anyway, I think we should consistently use typing.ContextManager, if it works.
I've fixed a bit more mypy complains, I'll post squash-ins in v9 thread
--
Best regards,
Vladimir
Am 26.01.2021 um 11:08 hat Vladimir Sementsov-Ogievskiy geschrieben:
> 26.01.2021 12:45, Kevin Wolf wrote:
> > Am 26.01.2021 um 09:28 hat Vladimir Sementsov-Ogievskiy geschrieben:
> > > 26.01.2021 01:05, Kevin Wolf wrote:
> > > > Am 23.01.2021 um 22:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
> > > > > Add TestEnv class, which will handle test environment in a new python
> > > > > iotests running framework.
> > > > >
> > > > > Don't add compat=1.1 for qcow2 IMGOPTS, as v3 is default anyway.
> > > > >
> > > > > Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> > > > > ---
> > > > > tests/qemu-iotests/testenv.py | 278 ++++++++++++++++++++++++++++++++++
> > > > > 1 file changed, 278 insertions(+)
> > > > > create mode 100644 tests/qemu-iotests/testenv.py
> > > > >
> > > > > diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
> > > > > new file mode 100644
> > > > > index 0000000000..348af593e9
> > > > > --- /dev/null
> > > > > +++ b/tests/qemu-iotests/testenv.py
> > > > > @@ -0,0 +1,278 @@
> > > > > +# TestEnv class to manage test environment variables.
> > > > > +#
> > > > > +# Copyright (c) 2020-2021 Virtuozzo International GmbH
> > > > > +#
> > > > > +# This program is free software; you can redistribute it and/or modify
> > > > > +# it under the terms of the GNU General Public License as published by
> > > > > +# the Free Software Foundation; either version 2 of the License, or
> > > > > +# (at your option) any later version.
> > > > > +#
> > > > > +# This program is distributed in the hope that it will be useful,
> > > > > +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> > > > > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> > > > > +# GNU General Public License for more details.
> > > > > +#
> > > > > +# You should have received a copy of the GNU General Public License
> > > > > +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> > > > > +#
> > > > > +
> > > > > +import os
> > > > > +import sys
> > > > > +import tempfile
> > > > > +from pathlib import Path
> > > > > +import shutil
> > > > > +import collections
> > > > > +import random
> > > > > +import subprocess
> > > > > +import glob
> > > > > +from contextlib import AbstractContextManager
> > > > > +from typing import Dict, Any, Optional
> > > > > +
> > > > > +
> > > > > +def get_default_machine(qemu_prog: str) -> str:
> > > > > + outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
> > > > > + universal_newlines=True,
> > > > > + stdout=subprocess.PIPE).stdout
> > > > > +
> > > > > + machines = outp.split('\n')
> > > > > + default_machine = next(m for m in machines if m.endswith(' (default)'))
> > > > > + default_machine = default_machine.split(' ', 1)[0]
> > > > > +
> > > > > + alias_suf = ' (alias of {})'.format(default_machine)
> > > > > + alias = next((m for m in machines if m.endswith(alias_suf)), None)
> > > > > + if alias is not None:
> > > > > + default_machine = alias.split(' ', 1)[0]
> > > > > +
> > > > > + return default_machine
> > > > > +
> > > > > +
> > > > > +class TestEnv(AbstractContextManager['TestEnv']):
> > > >
> > > > I'm getting CI failures here:
> > > >
> > > > Traceback (most recent call last):
> > > > File "./check", line 23, in <module>
> > > > from testenv import TestEnv
> > > > File "/builds/.../qemu/tests/qemu-iotests/testenv.py", line 49, in <module>
> > > > class TestEnv(AbstractContextManager['TestEnv']):
> > > > TypeError: 'ABCMeta' object is not subscriptable
> > > >
> > > > On the other hand, if I make it just AbstractContextManager without
> > > > giving the type parameter, mypy complains:
> > > >
> > > > testenv.py:49: error: Missing type parameters for generic type "ContextManager"
> > > >
> > > > I guess I need to have another look into this tomorrow.
> > >
> > > It may help to use typing.ContextManager instead of
> > > AbstractContextManager. mypy is OK with it, probably CI will be OK
> > > too..
> >
> > Okay, I'm trying now if this change works (on top of v9, of course). If
> > it does, I'll just squash it in. I also silenced some of the mypy
> > warnings, so that I'm not testing with the following patch squashed in.
> >
> > Kevin
> >
> >
> > diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
> > index ca9cab531b..becea1bb7d 100644
> > --- a/tests/qemu-iotests/testenv.py
> > +++ b/tests/qemu-iotests/testenv.py
> > @@ -25,8 +25,7 @@ import collections
> > import random
> > import subprocess
> > import glob
> > -from contextlib import AbstractContextManager
> > -from typing import Dict, Any, Optional
> > +from typing import ContextManager, Dict, Any, Optional
> >
> >
> > def isxfile(path: str) -> bool:
> > @@ -50,7 +49,7 @@ def get_default_machine(qemu_prog: str) -> str:
> > return default_machine
> >
> >
> > -class TestEnv(AbstractContextManager['TestEnv']):
> > +class TestEnv(ContextManager['TestEnv']):
> > """
> > Manage system environment for running tests
> >
> > @@ -81,7 +80,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
> >
> > return env
> >
> > - def init_directories(self):
> > + def init_directories(self) -> None:
> > """Init directory variables:
> > PYTHONPATH
> > TEST_DIR
> > @@ -114,7 +113,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
> >
> > self.output_dir = os.getcwd() # OUTPUT_DIR
> >
> > - def init_binaries(self):
> > + def init_binaries(self) -> None:
> > """Init binary path variables:
> > PYTHON (for bash tests)
> > QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
> > @@ -122,7 +121,7 @@ class TestEnv(AbstractContextManager['TestEnv']):
> > """
> > self.python = sys.executable
> >
> > - def root(*names):
> > + def root(*names: str) -> str:
> > return os.path.join(self.build_root, *names)
> >
> > arch = os.uname().machine
>
> Strange, that CI doesn't complain AbstractContextManager['...'] in
> testrunner.py.. Anyway, I think we should consistently use
> typing.ContextManager, if it works.
Ah, it probably does. The runs just take so long that I haven't got
results yet. So I'll probably have to start another run.
Kevin
© 2016 - 2026 Red Hat, Inc.