Integrate the new python-based test framework with the meson build
system. Since these tests now require the pycotap module, make
sure that it gets installed in the venv.
The changes to the meson.build files are partly based on an earlier
patch by Ani Sinha (but heavily modified by Thomas Huth e.g. to use
pycotap for running the tests instead).
Signed-off-by: Thomas Huth <thuth@redhat.com>
---
pythondeps.toml | 3 +-
tests/Makefile.include | 18 ++++++++-
tests/functional/meson.build | 75 ++++++++++++++++++++++++++++++++++++
tests/meson.build | 1 +
4 files changed, 95 insertions(+), 2 deletions(-)
create mode 100644 tests/functional/meson.build
diff --git a/pythondeps.toml b/pythondeps.toml
index f6e590fdd8..c018b4d74a 100644
--- a/pythondeps.toml
+++ b/pythondeps.toml
@@ -26,9 +26,10 @@ meson = { accepted = ">=1.1.0", installed = "1.2.3", canary = "meson" }
sphinx = { accepted = ">=3.4.3", installed = "5.3.0", canary = "sphinx-build" }
sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" }
-[avocado]
+[tests]
# Note that qemu.git/python/ is always implicitly installed.
# Prefer an LTS version when updating the accepted versions of
# avocado-framework, for example right now the limit is 92.x.
avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" }
pycdlib = { accepted = ">=1.11.0" }
+pycotap = { accepted = ">=1.1.0" }
diff --git a/tests/Makefile.include b/tests/Makefile.include
index d39d5dd6a4..2bdf607977 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -9,6 +9,8 @@ check-help:
@echo "Individual test suites:"
@echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
@echo " $(MAKE) check-qtest Run qtest tests"
+ @echo " $(MAKE) check-functional Run python-based functional tests"
+ @echo " $(MAKE) check-functional-TARG Run functional tests for a given target"
@echo " $(MAKE) check-unit Run qobject tests"
@echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
@echo " $(MAKE) check-block Run block tests"
@@ -111,7 +113,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
$(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml
$(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
- $(MKVENV_ENSUREGROUP) $< avocado
+ $(MKVENV_ENSUREGROUP) $< tests
$(call quiet-command, touch $@)
$(TESTS_RESULTS_DIR):
@@ -152,6 +154,20 @@ check-acceptance-deprecated-warning:
check-acceptance: check-acceptance-deprecated-warning | check-avocado
+# Make sure that pycotap is installed before running any functional tests:
+ifneq ($(filter check-func%,$(MAKECMDGOALS))$(filter check,$(MAKECMDGOALS)),)
+do-meson-check: check-venv
+endif
+
+FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS)))
+.PHONY: $(FUNCTIONAL_TARGETS)
+$(FUNCTIONAL_TARGETS):
+ @make SPEED=thorough $(subst -functional,-func,$@)
+
+.PHONY: check-functional
+check-functional:
+ @make SPEED=thorough check-func check-func-quick
+
# Consolidated targets
.PHONY: check check-clean get-vm-images
diff --git a/tests/functional/meson.build b/tests/functional/meson.build
new file mode 100644
index 0000000000..11352b5bb5
--- /dev/null
+++ b/tests/functional/meson.build
@@ -0,0 +1,75 @@
+# QEMU functional tests:
+# Tests that are put in the 'quick' category are run by default during
+# 'make check'. Everything that should not be run during 'make check'
+# (e.g. tests that fetch assets from the internet) should be put into
+# the 'thorough' category instead.
+
+# Most tests run too slow with TCI enabled, so skip the functional tests there
+if get_option('tcg_interpreter')
+ subdir_done()
+endif
+
+# Timeouts for individual tests that can be slow e.g. with debugging enabled
+test_timeouts = {
+ 'ppc_74xx' : 90,
+}
+
+tests_generic = [
+ 'empty_cpu_model',
+ 'info_usernet',
+ 'version',
+]
+
+tests_ppc_quick = [
+ 'ppc_74xx',
+]
+
+tests_x86_64_quick = [
+ 'cpu_queries',
+ 'mem_addr_space',
+ 'pc_cpu_hotplug_props',
+ 'virtio_version',
+]
+
+foreach speed : ['quick', 'thorough']
+ foreach dir : target_dirs
+ if not dir.endswith('-softmmu')
+ continue
+ endif
+
+ target_base = dir.split('-')[0]
+ test_emulator = emulators['qemu-system-' + target_base]
+
+ if speed == 'quick'
+ suites = ['func-quick', 'func-' + target_base]
+ target_tests = get_variable('tests_' + target_base + '_quick', []) + tests_generic
+ else
+ suites = ['func-' + speed, 'func-' + target_base + '-' + speed, speed]
+ target_tests = get_variable('tests_' + target_base + '_' + speed, [])
+ endif
+
+ test_deps = roms
+ test_env = environment()
+ if have_tools
+ test_env.set('QEMU_TEST_QEMU_IMG', meson.global_build_root() / 'qemu-img')
+ test_deps += [qemu_img]
+ endif
+ test_env.set('QEMU_TEST_QEMU_BINARY',
+ meson.global_build_root() / 'qemu-system-' + target_base)
+ test_env.set('QEMU_BUILD_ROOT', meson.project_build_root())
+ test_env.set('PYTHONPATH', meson.project_source_root() / 'python:' +
+ meson.current_source_dir())
+
+ foreach test : target_tests
+ test('func-@0@/@1@'.format(target_base, test),
+ python,
+ depends: [test_deps, test_emulator, emulator_modules],
+ env: test_env,
+ args: [meson.current_source_dir() / 'test_' + test + '.py'],
+ protocol: 'tap',
+ timeout: test_timeouts.get(test, 60),
+ priority: test_timeouts.get(test, 60),
+ suite: suites)
+ endforeach
+ endforeach
+endforeach
diff --git a/tests/meson.build b/tests/meson.build
index acb6807094..3345ad2098 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -85,3 +85,4 @@ subdir('unit')
subdir('qapi-schema')
subdir('qtest')
subdir('migration')
+subdir('functional')
--
2.45.2On Tue, Jul 16, 2024 at 01:26:07PM +0200, Thomas Huth wrote:
> Integrate the new python-based test framework with the meson build
> system. Since these tests now require the pycotap module, make
> sure that it gets installed in the venv.
>
> The changes to the meson.build files are partly based on an earlier
> patch by Ani Sinha (but heavily modified by Thomas Huth e.g. to use
> pycotap for running the tests instead).
>
> Signed-off-by: Thomas Huth <thuth@redhat.com>
> ---
> pythondeps.toml | 3 +-
> tests/Makefile.include | 18 ++++++++-
> tests/functional/meson.build | 75 ++++++++++++++++++++++++++++++++++++
> tests/meson.build | 1 +
> 4 files changed, 95 insertions(+), 2 deletions(-)
> create mode 100644 tests/functional/meson.build
Strictly speaking this patch probably ought to be #2, otherwise we
have a bisection window where we've converted some tests but not
run them.
>
> diff --git a/pythondeps.toml b/pythondeps.toml
> index f6e590fdd8..c018b4d74a 100644
> --- a/pythondeps.toml
> +++ b/pythondeps.toml
> @@ -26,9 +26,10 @@ meson = { accepted = ">=1.1.0", installed = "1.2.3", canary = "meson" }
> sphinx = { accepted = ">=3.4.3", installed = "5.3.0", canary = "sphinx-build" }
> sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" }
>
> -[avocado]
> +[tests]
> # Note that qemu.git/python/ is always implicitly installed.
> # Prefer an LTS version when updating the accepted versions of
> # avocado-framework, for example right now the limit is 92.x.
> avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" }
> pycdlib = { accepted = ">=1.11.0" }
> +pycotap = { accepted = ">=1.1.0" }
> diff --git a/tests/Makefile.include b/tests/Makefile.include
> index d39d5dd6a4..2bdf607977 100644
> --- a/tests/Makefile.include
> +++ b/tests/Makefile.include
> @@ -9,6 +9,8 @@ check-help:
> @echo "Individual test suites:"
> @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
> @echo " $(MAKE) check-qtest Run qtest tests"
> + @echo " $(MAKE) check-functional Run python-based functional tests"
> + @echo " $(MAKE) check-functional-TARG Run functional tests for a given target"
We could increase whitespace by 2 to fit TARGET, or shorten all
cases to TGT ?
> @echo " $(MAKE) check-unit Run qobject tests"
> @echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
> @echo " $(MAKE) check-block Run block tests"
> @@ -111,7 +113,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
>
The above is a minor non-functional point though so
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Thomas Huth <thuth@redhat.com> writes:
> Integrate the new python-based test framework with the meson build
> system. Since these tests now require the pycotap module, make
> sure that it gets installed in the venv.
>
> The changes to the meson.build files are partly based on an earlier
> patch by Ani Sinha (but heavily modified by Thomas Huth e.g. to use
> pycotap for running the tests instead).
>
> Signed-off-by: Thomas Huth <thuth@redhat.com>
Just one touch-up below.
Reviewed-by: Fabiano Rosas <farosas@suse.de>
> ---
> pythondeps.toml | 3 +-
> tests/Makefile.include | 18 ++++++++-
> tests/functional/meson.build | 75 ++++++++++++++++++++++++++++++++++++
> tests/meson.build | 1 +
> 4 files changed, 95 insertions(+), 2 deletions(-)
> create mode 100644 tests/functional/meson.build
>
> diff --git a/pythondeps.toml b/pythondeps.toml
> index f6e590fdd8..c018b4d74a 100644
> --- a/pythondeps.toml
> +++ b/pythondeps.toml
> @@ -26,9 +26,10 @@ meson = { accepted = ">=1.1.0", installed = "1.2.3", canary = "meson" }
> sphinx = { accepted = ">=3.4.3", installed = "5.3.0", canary = "sphinx-build" }
> sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" }
>
> -[avocado]
> +[tests]
> # Note that qemu.git/python/ is always implicitly installed.
> # Prefer an LTS version when updating the accepted versions of
> # avocado-framework, for example right now the limit is 92.x.
> avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" }
> pycdlib = { accepted = ">=1.11.0" }
> +pycotap = { accepted = ">=1.1.0" }
> diff --git a/tests/Makefile.include b/tests/Makefile.include
> index d39d5dd6a4..2bdf607977 100644
> --- a/tests/Makefile.include
> +++ b/tests/Makefile.include
> @@ -9,6 +9,8 @@ check-help:
> @echo "Individual test suites:"
> @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
> @echo " $(MAKE) check-qtest Run qtest tests"
> + @echo " $(MAKE) check-functional Run python-based functional tests"
> + @echo " $(MAKE) check-functional-TARG Run functional tests for
> a given target"
> @echo " $(MAKE) check-unit Run qobject tests"
> @echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
> @echo " $(MAKE) check-block Run block tests"
> @@ -111,7 +113,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
>
> $(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml
> $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
> - $(MKVENV_ENSUREGROUP) $< avocado
> + $(MKVENV_ENSUREGROUP) $< tests
> $(call quiet-command, touch $@)
>
> $(TESTS_RESULTS_DIR):
> @@ -152,6 +154,20 @@ check-acceptance-deprecated-warning:
>
> check-acceptance: check-acceptance-deprecated-warning | check-avocado
>
> +# Make sure that pycotap is installed before running any functional tests:
> +ifneq ($(filter check-func%,$(MAKECMDGOALS))$(filter check,$(MAKECMDGOALS)),)
> +do-meson-check: check-venv
> +endif
> +
> +FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS)))
> +.PHONY: $(FUNCTIONAL_TARGETS)
> +$(FUNCTIONAL_TARGETS):
> + @make SPEED=thorough $(subst -functional,-func,$@)
> +
> +.PHONY: check-functional
> +check-functional:
> + @make SPEED=thorough check-func check-func-quick
I think these^ two should use $(MAKE) instead:
make[1]: warning: jobserver unavailable: using -j1. Add '+' to parent
make rule.
> +
> # Consolidated targets
>
> .PHONY: check check-clean get-vm-images
> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
> new file mode 100644
> index 0000000000..11352b5bb5
> --- /dev/null
> +++ b/tests/functional/meson.build
> @@ -0,0 +1,75 @@
> +# QEMU functional tests:
> +# Tests that are put in the 'quick' category are run by default during
> +# 'make check'. Everything that should not be run during 'make check'
> +# (e.g. tests that fetch assets from the internet) should be put into
> +# the 'thorough' category instead.
> +
> +# Most tests run too slow with TCI enabled, so skip the functional tests there
> +if get_option('tcg_interpreter')
> + subdir_done()
> +endif
> +
> +# Timeouts for individual tests that can be slow e.g. with debugging enabled
> +test_timeouts = {
> + 'ppc_74xx' : 90,
> +}
> +
> +tests_generic = [
> + 'empty_cpu_model',
> + 'info_usernet',
> + 'version',
> +]
> +
> +tests_ppc_quick = [
> + 'ppc_74xx',
> +]
> +
> +tests_x86_64_quick = [
> + 'cpu_queries',
> + 'mem_addr_space',
> + 'pc_cpu_hotplug_props',
> + 'virtio_version',
> +]
> +
> +foreach speed : ['quick', 'thorough']
> + foreach dir : target_dirs
> + if not dir.endswith('-softmmu')
> + continue
> + endif
> +
> + target_base = dir.split('-')[0]
> + test_emulator = emulators['qemu-system-' + target_base]
> +
> + if speed == 'quick'
> + suites = ['func-quick', 'func-' + target_base]
> + target_tests = get_variable('tests_' + target_base + '_quick', []) + tests_generic
> + else
> + suites = ['func-' + speed, 'func-' + target_base + '-' + speed, speed]
> + target_tests = get_variable('tests_' + target_base + '_' + speed, [])
> + endif
> +
> + test_deps = roms
> + test_env = environment()
> + if have_tools
> + test_env.set('QEMU_TEST_QEMU_IMG', meson.global_build_root() / 'qemu-img')
> + test_deps += [qemu_img]
> + endif
> + test_env.set('QEMU_TEST_QEMU_BINARY',
> + meson.global_build_root() / 'qemu-system-' + target_base)
> + test_env.set('QEMU_BUILD_ROOT', meson.project_build_root())
> + test_env.set('PYTHONPATH', meson.project_source_root() / 'python:' +
> + meson.current_source_dir())
> +
> + foreach test : target_tests
> + test('func-@0@/@1@'.format(target_base, test),
> + python,
> + depends: [test_deps, test_emulator, emulator_modules],
> + env: test_env,
> + args: [meson.current_source_dir() / 'test_' + test + '.py'],
> + protocol: 'tap',
> + timeout: test_timeouts.get(test, 60),
> + priority: test_timeouts.get(test, 60),
> + suite: suites)
> + endforeach
2/19 qemu:func-thorough+func-s390x-thorough+thorough / func-s390x/s390x_ccw_virtio OK 48.82s 2 subtests passe
func, thorough, func, thorough, thorough, func
s390x, s390x, s390x
=)
I know, not much we can do...
> + endforeach
> +endforeach
> diff --git a/tests/meson.build b/tests/meson.build
> index acb6807094..3345ad2098 100644
> --- a/tests/meson.build
> +++ b/tests/meson.build
> @@ -85,3 +85,4 @@ subdir('unit')
> subdir('qapi-schema')
> subdir('qtest')
> subdir('migration')
> +subdir('functional')
On 16/07/2024 17.15, Fabiano Rosas wrote:
> Thomas Huth <thuth@redhat.com> writes:
>
>> Integrate the new python-based test framework with the meson build
>> system. Since these tests now require the pycotap module, make
>> sure that it gets installed in the venv.
>>
>> The changes to the meson.build files are partly based on an earlier
>> patch by Ani Sinha (but heavily modified by Thomas Huth e.g. to use
>> pycotap for running the tests instead).
>>
>> Signed-off-by: Thomas Huth <thuth@redhat.com>
>
> Just one touch-up below.
>
> Reviewed-by: Fabiano Rosas <farosas@suse.de>
>
>> ---
>> pythondeps.toml | 3 +-
>> tests/Makefile.include | 18 ++++++++-
>> tests/functional/meson.build | 75 ++++++++++++++++++++++++++++++++++++
>> tests/meson.build | 1 +
>> 4 files changed, 95 insertions(+), 2 deletions(-)
>> create mode 100644 tests/functional/meson.build
>>
>> diff --git a/pythondeps.toml b/pythondeps.toml
>> index f6e590fdd8..c018b4d74a 100644
>> --- a/pythondeps.toml
>> +++ b/pythondeps.toml
>> @@ -26,9 +26,10 @@ meson = { accepted = ">=1.1.0", installed = "1.2.3", canary = "meson" }
>> sphinx = { accepted = ">=3.4.3", installed = "5.3.0", canary = "sphinx-build" }
>> sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" }
>>
>> -[avocado]
>> +[tests]
>> # Note that qemu.git/python/ is always implicitly installed.
>> # Prefer an LTS version when updating the accepted versions of
>> # avocado-framework, for example right now the limit is 92.x.
>> avocado-framework = { accepted = "(>=88.1, <93.0)", installed = "88.1", canary = "avocado" }
>> pycdlib = { accepted = ">=1.11.0" }
>> +pycotap = { accepted = ">=1.1.0" }
>> diff --git a/tests/Makefile.include b/tests/Makefile.include
>> index d39d5dd6a4..2bdf607977 100644
>> --- a/tests/Makefile.include
>> +++ b/tests/Makefile.include
>> @@ -9,6 +9,8 @@ check-help:
>> @echo "Individual test suites:"
>> @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
>> @echo " $(MAKE) check-qtest Run qtest tests"
>> + @echo " $(MAKE) check-functional Run python-based functional tests"
>> + @echo " $(MAKE) check-functional-TARG Run functional tests for
>> a given target"
>> @echo " $(MAKE) check-unit Run qobject tests"
>> @echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
>> @echo " $(MAKE) check-block Run block tests"
>> @@ -111,7 +113,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
>>
>> $(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml
>> $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
>> - $(MKVENV_ENSUREGROUP) $< avocado
>> + $(MKVENV_ENSUREGROUP) $< tests
>> $(call quiet-command, touch $@)
>>
>> $(TESTS_RESULTS_DIR):
>> @@ -152,6 +154,20 @@ check-acceptance-deprecated-warning:
>>
>> check-acceptance: check-acceptance-deprecated-warning | check-avocado
>>
>> +# Make sure that pycotap is installed before running any functional tests:
>> +ifneq ($(filter check-func%,$(MAKECMDGOALS))$(filter check,$(MAKECMDGOALS)),)
>> +do-meson-check: check-venv
>> +endif
>> +
>> +FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS)))
>> +.PHONY: $(FUNCTIONAL_TARGETS)
>> +$(FUNCTIONAL_TARGETS):
>> + @make SPEED=thorough $(subst -functional,-func,$@)
>> +
>> +.PHONY: check-functional
>> +check-functional:
>> + @make SPEED=thorough check-func check-func-quick
>
> I think these^ two should use $(MAKE) instead:
>
> make[1]: warning: jobserver unavailable: using -j1. Add '+' to parent
> make rule.
Right, thanks! I'll change it in the next version.
Thomas
© 2016 - 2024 Red Hat, Inc.