Avocado: start multi-arch tests, add a Travis job

[Qemu-devel] [RFC PATCH v2 1/6] avocado: Add a Test.arch property

Posted by Philippe Mathieu-Daudé 7 years, 7 months ago

Tests can change this property to run tests in other
architectures than the host one.

Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
---
 tests/acceptance/avocado_qemu/__init__.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
index 1e54fd5932..6e9601f5e9 100644
--- a/tests/acceptance/avocado_qemu/__init__.py
+++ b/tests/acceptance/avocado_qemu/__init__.py
@@ -13,6 +13,7 @@ import sys
 
 import avocado
 
+HOST_ARCH = os.uname()[4]
 SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
 sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
@@ -23,12 +24,11 @@ def is_readable_executable_file(path):
     return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
 
 
-def pick_default_qemu_bin():
+def pick_default_qemu_bin(arch):
     """
     Picks the path of a QEMU binary, starting either in the current working
     directory or in the source tree root directory.
     """
-    arch = os.uname()[4]
     qemu_bin_relative_path = os.path.join("%s-softmmu" % arch,
                                           "qemu-system-%s" % arch)
     if is_readable_executable_file(qemu_bin_relative_path):
@@ -41,10 +41,19 @@ def pick_default_qemu_bin():
 
 
 class Test(avocado.Test):
+    _arch = HOST_ARCH
+
+    @property
+    def arch(self):
+        """
+        Returns the architecture required to run the current test
+        """
+        return self._arch
+
     def setUp(self):
         self.vm = None
-        self.qemu_bin = self.params.get('qemu_bin',
-                                        default=pick_default_qemu_bin())
+        qemu_bin = pick_default_qemu_bin(self.arch)
+        self.qemu_bin = self.params.get('qemu_bin', default=qemu_bin)
         if self.qemu_bin is None:
             self.cancel("No QEMU binary defined or found in the source tree")
         self.vm = QEMUMachine(self.qemu_bin)
-- 
2.18.0.rc2

Re: [Qemu-devel] [RFC PATCH v2 1/6] avocado: Add a Test.arch property

Posted by Alex Bennée 7 years, 7 months ago

Philippe Mathieu-Daudé <f4bug@amsat.org> writes:

> Tests can change this property to run tests in other
> architectures than the host one.
>
> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
> ---
>  tests/acceptance/avocado_qemu/__init__.py | 17 +++++++++++++----
>  1 file changed, 13 insertions(+), 4 deletions(-)
>
> diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
> index 1e54fd5932..6e9601f5e9 100644
> --- a/tests/acceptance/avocado_qemu/__init__.py
> +++ b/tests/acceptance/avocado_qemu/__init__.py
> @@ -13,6 +13,7 @@ import sys
>
>  import avocado
>
> +HOST_ARCH = os.uname()[4]

The python docs seem to point to platform as a more stable way of
querying this stuff:

  platform.machine() => 'x86_64' or 'aarch64'

>  SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
>  SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
>  sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
> @@ -23,12 +24,11 @@ def is_readable_executable_file(path):
>      return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
>
>
> -def pick_default_qemu_bin():
> +def pick_default_qemu_bin(arch):
>      """
>      Picks the path of a QEMU binary, starting either in the current working
>      directory or in the source tree root directory.
>      """
> -    arch = os.uname()[4]
>      qemu_bin_relative_path = os.path.join("%s-softmmu" % arch,
>                                            "qemu-system-%s" % arch)
>      if is_readable_executable_file(qemu_bin_relative_path):
> @@ -41,10 +41,19 @@ def pick_default_qemu_bin():
>
>
>  class Test(avocado.Test):
> +    _arch = HOST_ARCH
> +
> +    @property
> +    def arch(self):
> +        """
> +        Returns the architecture required to run the current test
> +        """
> +        return self._arch
> +
>      def setUp(self):
>          self.vm = None
> -        self.qemu_bin = self.params.get('qemu_bin',
> -                                        default=pick_default_qemu_bin())
> +        qemu_bin = pick_default_qemu_bin(self.arch)
> +        self.qemu_bin = self.params.get('qemu_bin', default=qemu_bin)
>          if self.qemu_bin is None:
>              self.cancel("No QEMU binary defined or found in the source tree")
>          self.vm = QEMUMachine(self.qemu_bin)


--
Alex Bennée

Re: [Qemu-devel] [RFC PATCH v2 1/6] avocado: Add a Test.arch property

Posted by Alex Bennée 7 years, 7 months ago

Alex Bennée <alex.bennee@linaro.org> writes:

> Philippe Mathieu-Daudé <f4bug@amsat.org> writes:
>
>> Tests can change this property to run tests in other
>> architectures than the host one.
>>
>> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
>> ---
>>  tests/acceptance/avocado_qemu/__init__.py | 17 +++++++++++++----
>>  1 file changed, 13 insertions(+), 4 deletions(-)
>>
>> diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
>> index 1e54fd5932..6e9601f5e9 100644
>> --- a/tests/acceptance/avocado_qemu/__init__.py
>> +++ b/tests/acceptance/avocado_qemu/__init__.py
>> @@ -13,6 +13,7 @@ import sys
>>
>>  import avocado
>>
>> +HOST_ARCH = os.uname()[4]
>
> The python docs seem to point to platform as a more stable way of
> querying this stuff:
>
>   platform.machine() => 'x86_64' or 'aarch64'
>
>>  SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
>>  SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
>>  sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
>> @@ -23,12 +24,11 @@ def is_readable_executable_file(path):
>>      return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
>>
>>
>> -def pick_default_qemu_bin():
>> +def pick_default_qemu_bin(arch):
>>      """
>>      Picks the path of a QEMU binary, starting either in the current working
>>      directory or in the source tree root directory.
>>      """
>> -    arch = os.uname()[4]
>>      qemu_bin_relative_path = os.path.join("%s-softmmu" % arch,
>>                                            "qemu-system-%s" % arch)
>>      if is_readable_executable_file(qemu_bin_relative_path):
>> @@ -41,10 +41,19 @@ def pick_default_qemu_bin():
>>
>>
>>  class Test(avocado.Test):
>> +    _arch = HOST_ARCH

But actually this is wrong - because the host arch may not boot machines
defined in the tests. I suspect each superclass needs to explicitly set
it's arch and we should assert it has done so here.

>> +
>> +    @property
>> +    def arch(self):
>> +        """
>> +        Returns the architecture required to run the current test
>> +        """
>> +        return self._arch
>> +
>>      def setUp(self):
>>          self.vm = None
>> -        self.qemu_bin = self.params.get('qemu_bin',
>> -                                        default=pick_default_qemu_bin())
>> +        qemu_bin = pick_default_qemu_bin(self.arch)
>> +        self.qemu_bin = self.params.get('qemu_bin', default=qemu_bin)
>>          if self.qemu_bin is None:
>>              self.cancel("No QEMU binary defined or found in the source tree")
>>          self.vm = QEMUMachine(self.qemu_bin)


--
Alex Bennée

Re: [Qemu-devel] [RFC PATCH v2 1/6] avocado: Add a Test.arch property

Posted by Philippe Mathieu-Daudé 7 years, 7 months ago

On 06/28/2018 06:54 PM, Alex Bennée wrote:
> Alex Bennée <alex.bennee@linaro.org> writes:
>> Philippe Mathieu-Daudé <f4bug@amsat.org> writes:
>>
>>> Tests can change this property to run tests in other
>>> architectures than the host one.
>>>
>>> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
>>> ---
>>>  tests/acceptance/avocado_qemu/__init__.py | 17 +++++++++++++----
>>>  1 file changed, 13 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
>>> index 1e54fd5932..6e9601f5e9 100644
>>> --- a/tests/acceptance/avocado_qemu/__init__.py
>>> +++ b/tests/acceptance/avocado_qemu/__init__.py
>>> @@ -13,6 +13,7 @@ import sys
>>>
>>>  import avocado
>>>
>>> +HOST_ARCH = os.uname()[4]
>>
>> The python docs seem to point to platform as a more stable way of
>> querying this stuff:
>>
>>   platform.machine() => 'x86_64' or 'aarch64'
>>
>>>  SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
>>>  SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
>>>  sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
>>> @@ -23,12 +24,11 @@ def is_readable_executable_file(path):
>>>      return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
>>>
>>>
>>> -def pick_default_qemu_bin():
>>> +def pick_default_qemu_bin(arch):
>>>      """
>>>      Picks the path of a QEMU binary, starting either in the current working
>>>      directory or in the source tree root directory.
>>>      """
>>> -    arch = os.uname()[4]
>>>      qemu_bin_relative_path = os.path.join("%s-softmmu" % arch,
>>>                                            "qemu-system-%s" % arch)
>>>      if is_readable_executable_file(qemu_bin_relative_path):
>>> @@ -41,10 +41,19 @@ def pick_default_qemu_bin():
>>>
>>>
>>>  class Test(avocado.Test):
>>> +    _arch = HOST_ARCH
> 
> But actually this is wrong - because the host arch may not boot machines
> defined in the tests. I suspect each superclass needs to explicitly set
> it's arch and we should assert it has done so here.

Hmm but arch-specific tests are protected by the 'arch' tag:

class BootLinuxConsoleMips(Test):
    """
    :avocado: enable
    :avocado: tags=endian:big
    :avocado: tags=arch:mips
    :avocado: tags=board:malta
    """

Oh no they aren't, it is just a way to filter which selection of tests
to run :|

Cleber can you help us here?

>>> +
>>> +    @property
>>> +    def arch(self):
>>> +        """
>>> +        Returns the architecture required to run the current test
>>> +        """
>>> +        return self._arch
>>> +> 
> 
> --
> Alex Bennée
> 
>>>      def setUp(self):
>>>          self.vm = None
>>> -        self.qemu_bin = self.params.get('qemu_bin',
>>> -                                        default=pick_default_qemu_bin())
>>> +        qemu_bin = pick_default_qemu_bin(self.arch)
>>> +        self.qemu_bin = self.params.get('qemu_bin', default=qemu_bin)
>>>          if self.qemu_bin is None:
>>>              self.cancel("No QEMU binary defined or found in the source tree")
>>>          self.vm = QEMUMachine(self.qemu_bin)

Re: [Qemu-devel] [RFC PATCH v2 1/6] avocado: Add a Test.arch property

Posted by Philippe Mathieu-Daudé 7 years, 6 months ago

Hi Cleber,

On 06/28/2018 07:03 PM, Philippe Mathieu-Daudé wrote:
> On 06/28/2018 06:54 PM, Alex Bennée wrote:
>> Alex Bennée <alex.bennee@linaro.org> writes:
>>> Philippe Mathieu-Daudé <f4bug@amsat.org> writes:
>>>
>>>> Tests can change this property to run tests in other
>>>> architectures than the host one.
>>>>
>>>> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
>>>> ---
>>>>  tests/acceptance/avocado_qemu/__init__.py | 17 +++++++++++++----
>>>>  1 file changed, 13 insertions(+), 4 deletions(-)
>>>>
>>>> diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
>>>> index 1e54fd5932..6e9601f5e9 100644
>>>> --- a/tests/acceptance/avocado_qemu/__init__.py
>>>> +++ b/tests/acceptance/avocado_qemu/__init__.py
>>>> @@ -13,6 +13,7 @@ import sys
>>>>
>>>>  import avocado
>>>>
>>>> +HOST_ARCH = os.uname()[4]
>>>
>>> The python docs seem to point to platform as a more stable way of
>>> querying this stuff:
>>>
>>>   platform.machine() => 'x86_64' or 'aarch64'
>>>
>>>>  SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
>>>>  SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR))
>>>>  sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts'))
>>>> @@ -23,12 +24,11 @@ def is_readable_executable_file(path):
>>>>      return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
>>>>
>>>>
>>>> -def pick_default_qemu_bin():
>>>> +def pick_default_qemu_bin(arch):
>>>>      """
>>>>      Picks the path of a QEMU binary, starting either in the current working
>>>>      directory or in the source tree root directory.
>>>>      """
>>>> -    arch = os.uname()[4]
>>>>      qemu_bin_relative_path = os.path.join("%s-softmmu" % arch,
>>>>                                            "qemu-system-%s" % arch)
>>>>      if is_readable_executable_file(qemu_bin_relative_path):
>>>> @@ -41,10 +41,19 @@ def pick_default_qemu_bin():
>>>>
>>>>
>>>>  class Test(avocado.Test):
>>>> +    _arch = HOST_ARCH
>>
>> But actually this is wrong - because the host arch may not boot machines
>> defined in the tests. I suspect each superclass needs to explicitly set
>> it's arch and we should assert it has done so here.

Do you have any suggestion on how we could clear this?

> 
> Hmm but arch-specific tests are protected by the 'arch' tag:
> 
> class BootLinuxConsoleMips(Test):
>     """
>     :avocado: enable
>     :avocado: tags=endian:big
>     :avocado: tags=arch:mips
>     :avocado: tags=board:malta
>     """
> 
> Oh no they aren't, it is just a way to filter which selection of tests
> to run :|

>>>> +
>>>> +    @property
>>>> +    def arch(self):
>>>> +        """
>>>> +        Returns the architecture required to run the current test
>>>> +        """
>>>> +        return self._arch
>>>> +> 
>>
>> --
>> Alex Bennée
>>
>>>>      def setUp(self):
>>>>          self.vm = None
>>>> -        self.qemu_bin = self.params.get('qemu_bin',
>>>> -                                        default=pick_default_qemu_bin())
>>>> +        qemu_bin = pick_default_qemu_bin(self.arch)
>>>> +        self.qemu_bin = self.params.get('qemu_bin', default=qemu_bin)
>>>>          if self.qemu_bin is None:
>>>>              self.cancel("No QEMU binary defined or found in the source tree")
>>>>          self.vm = QEMUMachine(self.qemu_bin)

[Qemu-devel] [RFC PATCH v2 1/6] avocado: Add a Test.arch property
[Qemu-devel] [PATCH v2 2/6] tests/acceptance: Rename the x86-64 specific BootLinuxConsole test
[Qemu-devel] [PATCH v2 3/6] tests/acceptance: Improve the Avocado tags
[Qemu-devel] [PATCH v2 4/6] tests/acceptance: Add a BootLinuxConsoleMips test
[Qemu-devel] [RFC PATCH v2 5/6] tests/acceptance: Add a kludge to not use the default console
[Qemu-devel] [PATCH v2 6/6] travis: Add Avocado tests