This allows building with "./configure --python=python3", where
the python3 program is at least version 3.6. It preserves
compatibility with Python 2. The changes include:
- Avoiding "print" usage
- Using bytes with files opened in binary mode
- Switching .iteritems() to .items()
- Adding fallback imports for functions moved to other modules
Signed-off-by: David Michael <fedora.dm0@gmail.com>
---
Hi,
I've been applying these changes when building on Fedora 26, which does
not include any Python 2 packages by default. It was tested with Python
2.7 and 3.6.
I just saw the list of scripts that need updating on the mailing list,
and this doesn't cover all of them, but it is enough to build a binary
for running virtual machines with KVM. Maybe it is still useful as a
starting point.
Thanks.
David
configure | 6 ++++--
scripts/qapi.py | 31 ++++++++++++++++++++-----------
scripts/qapi2texi.py | 10 +++++-----
scripts/signrom.py | 4 ++--
4 files changed, 31 insertions(+), 20 deletions(-)
diff --git a/configure b/configure
index dd73cce..09f4d68 100755
--- a/configure
+++ b/configure
@@ -1548,9 +1548,11 @@ fi
# Note that if the Python conditional here evaluates True we will exit
# with status 1 which is a shell 'false' value.
-if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then
+if ! $python -c 'import sys; sys.exit(sys.version_info >= (3,) and sys.version_info < (3,6))'; then
+ error_exit "Cannot use '$python', Python 3.6 or later is required." \
+ "Use --python=/path/to/python3 to specify a supported Python 3."
+elif ! $python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then
error_exit "Cannot use '$python', Python 2.6 or later is required." \
- "Note that Python 3 or later is not yet supported." \
"Use --python=/path/to/python to specify a supported Python."
fi
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 8aa2775..6450998 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -15,9 +15,11 @@ import errno
import getopt
import os
import re
-import string
import sys
-from ordereddict import OrderedDict
+try:
+ from collections import OrderedDict
+except ImportError:
+ from ordereddict import OrderedDict
builtin_types = {
'null': 'QTYPE_QNULL',
@@ -252,7 +254,7 @@ class QAPIDoc(object):
"'Returns:' is only valid for commands")
def check(self):
- bogus = [name for name, section in self.args.iteritems()
+ bogus = [name for name, section in self.args.items()
if not section.member]
if bogus:
raise QAPISemError(
@@ -308,7 +310,7 @@ class QAPISchemaParser(object):
if not isinstance(pragma, dict):
raise QAPISemError(
info, "Value of 'pragma' must be a dictionary")
- for name, value in pragma.iteritems():
+ for name, value in pragma.items():
self._pragma(name, value, info)
else:
expr_elem = {'expr': expr,
@@ -1574,7 +1576,7 @@ class QAPISchema(object):
def _make_members(self, data, info):
return [self._make_member(key, value, info)
- for (key, value) in data.iteritems()]
+ for (key, value) in data.items()]
def _def_struct_type(self, expr, info, doc):
name = expr['struct']
@@ -1606,11 +1608,11 @@ class QAPISchema(object):
name, info, doc, 'base', self._make_members(base, info)))
if tag_name:
variants = [self._make_variant(key, value)
- for (key, value) in data.iteritems()]
+ for (key, value) in data.items()]
members = []
else:
variants = [self._make_simple_variant(key, value, info)
- for (key, value) in data.iteritems()]
+ for (key, value) in data.items()]
typ = self._make_implicit_enum_type(name, info,
[v.name for v in variants])
tag_member = QAPISchemaObjectTypeMember('type', typ, False)
@@ -1625,7 +1627,7 @@ class QAPISchema(object):
name = expr['alternate']
data = expr['data']
variants = [self._make_variant(key, value)
- for (key, value) in data.iteritems()]
+ for (key, value) in data.items()]
tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
self._def_entity(
QAPISchemaAlternateType(name, info, doc,
@@ -1735,7 +1737,11 @@ def c_enum_const(type_name, const_name, prefix=None):
type_name = prefix
return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
-c_name_trans = string.maketrans('.-', '__')
+try:
+ c_name_trans = str.maketrans('.-', '__')
+except AttributeError:
+ import string
+ c_name_trans = string.maketrans('.-', '__')
# Map @name to a valid C identifier.
@@ -1997,8 +2003,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
if really:
return open(name, opt)
else:
- import StringIO
- return StringIO.StringIO()
+ try:
+ from StringIO import StringIO
+ except ImportError:
+ from io import StringIO
+ return StringIO()
fdef = maybe_open(do_c, c_file, 'w')
fdecl = maybe_open(do_h, h_file, 'w')
diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
index a317526..9a6fed9 100755
--- a/scripts/qapi2texi.py
+++ b/scripts/qapi2texi.py
@@ -147,7 +147,7 @@ def texi_member(member, suffix=''):
def texi_members(doc, what, base, variants, member_func):
"""Format the table of members"""
items = ''
- for section in doc.args.itervalues():
+ for section in doc.args.values():
# TODO Drop fallbacks when undocumented members are outlawed
if section.content:
desc = texi_format(str(section))
@@ -285,15 +285,15 @@ def texi_schema(schema):
def main(argv):
"""Takes schema argument, prints result to stdout"""
if len(argv) != 2:
- print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0]
+ sys.stderr.write("%s: need exactly 1 argument: SCHEMA\n" % argv[0])
sys.exit(1)
schema = qapi.QAPISchema(argv[1])
if not qapi.doc_required:
- print >>sys.stderr, ("%s: need pragma 'doc-required' "
- "to generate documentation" % argv[0])
+ sys.stderr.write("%s: need pragma 'doc-required' "
+ "to generate documentation\n" % argv[0])
sys.exit(1)
- print texi_schema(schema)
+ sys.stdout.write(texi_schema(schema) + "\n")
if __name__ == '__main__':
diff --git a/scripts/signrom.py b/scripts/signrom.py
index d1dabe0..0497a1c 100644
--- a/scripts/signrom.py
+++ b/scripts/signrom.py
@@ -18,7 +18,7 @@ fin = open(sys.argv[1], 'rb')
fout = open(sys.argv[2], 'wb')
magic = fin.read(2)
-if magic != '\x55\xaa':
+if magic != b'\x55\xaa':
sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1])
size_byte = ord(fin.read(1))
@@ -33,7 +33,7 @@ elif len(data) < size:
# Add padding if necessary, rounding the whole input to a multiple of
# 512 bytes according to the third byte of the input.
# size-1 because a final byte is added below to store the checksum.
- data = data.ljust(size-1, '\0')
+ data = data.ljust(size-1, b'\0')
else:
if ord(data[-1:]) != 0:
sys.stderr.write('WARNING: ROM includes nonzero checksum\n')
--
2.13.5
David Michael <fedora.dm0@gmail.com> writes:
> This allows building with "./configure --python=python3", where
> the python3 program is at least version 3.6. It preserves
> compatibility with Python 2. The changes include:
>
> - Avoiding "print" usage
> - Using bytes with files opened in binary mode
> - Switching .iteritems() to .items()
> - Adding fallback imports for functions moved to other modules
>
> Signed-off-by: David Michael <fedora.dm0@gmail.com>
> ---
>
> Hi,
>
> I've been applying these changes when building on Fedora 26, which does
> not include any Python 2 packages by default. It was tested with Python
> 2.7 and 3.6.
>
> I just saw the list of scripts that need updating on the mailing list,
> and this doesn't cover all of them, but it is enough to build a binary
> for running virtual machines with KVM. Maybe it is still useful as a
> starting point.
>
> Thanks.
>
> David
>
> configure | 6 ++++--
> scripts/qapi.py | 31 ++++++++++++++++++++-----------
> scripts/qapi2texi.py | 10 +++++-----
> scripts/signrom.py | 4 ++--
> 4 files changed, 31 insertions(+), 20 deletions(-)
>
> diff --git a/configure b/configure
> index dd73cce..09f4d68 100755
> --- a/configure
> +++ b/configure
> @@ -1548,9 +1548,11 @@ fi
>
> # Note that if the Python conditional here evaluates True we will exit
> # with status 1 which is a shell 'false' value.
> -if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then
> +if ! $python -c 'import sys; sys.exit(sys.version_info >= (3,) and sys.version_info < (3,6))'; then
> + error_exit "Cannot use '$python', Python 3.6 or later is required." \
> + "Use --python=/path/to/python3 to specify a supported Python 3."
> +elif ! $python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then
> error_exit "Cannot use '$python', Python 2.6 or later is required." \
> - "Note that Python 3 or later is not yet supported." \
> "Use --python=/path/to/python to specify a supported Python."
> fi
>
Hmm. If we detect a Python 2 that is too old, we ask the user to
provide a sufficiently new Python 2. We don't ask him for a
sufficiently new Python 3, even though that would work, too. Should we
tweak the error message a bit?
> diff --git a/scripts/qapi.py b/scripts/qapi.py
> index 8aa2775..6450998 100644
> --- a/scripts/qapi.py
> +++ b/scripts/qapi.py
> @@ -15,9 +15,11 @@ import errno
> import getopt
> import os
> import re
> -import string
> import sys
> -from ordereddict import OrderedDict
> +try:
> + from collections import OrderedDict
> +except ImportError:
> + from ordereddict import OrderedDict
>
> builtin_types = {
> 'null': 'QTYPE_QNULL',
> @@ -252,7 +254,7 @@ class QAPIDoc(object):
> "'Returns:' is only valid for commands")
>
> def check(self):
> - bogus = [name for name, section in self.args.iteritems()
> + bogus = [name for name, section in self.args.items()
> if not section.member]
> if bogus:
> raise QAPISemError(
Efficiency loss with Python 2. Not critical, just sad.
> @@ -308,7 +310,7 @@ class QAPISchemaParser(object):
> if not isinstance(pragma, dict):
> raise QAPISemError(
> info, "Value of 'pragma' must be a dictionary")
> - for name, value in pragma.iteritems():
> + for name, value in pragma.items():
> self._pragma(name, value, info)
> else:
> expr_elem = {'expr': expr,
> @@ -1574,7 +1576,7 @@ class QAPISchema(object):
>
> def _make_members(self, data, info):
> return [self._make_member(key, value, info)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
>
> def _def_struct_type(self, expr, info, doc):
> name = expr['struct']
> @@ -1606,11 +1608,11 @@ class QAPISchema(object):
> name, info, doc, 'base', self._make_members(base, info)))
> if tag_name:
> variants = [self._make_variant(key, value)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
> members = []
> else:
> variants = [self._make_simple_variant(key, value, info)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
> typ = self._make_implicit_enum_type(name, info,
> [v.name for v in variants])
> tag_member = QAPISchemaObjectTypeMember('type', typ, False)
> @@ -1625,7 +1627,7 @@ class QAPISchema(object):
> name = expr['alternate']
> data = expr['data']
> variants = [self._make_variant(key, value)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
> tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
> self._def_entity(
> QAPISchemaAlternateType(name, info, doc,
> @@ -1735,7 +1737,11 @@ def c_enum_const(type_name, const_name, prefix=None):
> type_name = prefix
> return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
>
> -c_name_trans = string.maketrans('.-', '__')
> +try:
> + c_name_trans = str.maketrans('.-', '__')
> +except AttributeError:
> + import string
> + c_name_trans = string.maketrans('.-', '__')
>
>
> # Map @name to a valid C identifier.
> @@ -1997,8 +2003,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
> if really:
> return open(name, opt)
> else:
> - import StringIO
> - return StringIO.StringIO()
> + try:
> + from StringIO import StringIO
> + except ImportError:
> + from io import StringIO
> + return StringIO()
>
> fdef = maybe_open(do_c, c_file, 'w')
> fdecl = maybe_open(do_h, h_file, 'w')
> diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
> index a317526..9a6fed9 100755
> --- a/scripts/qapi2texi.py
> +++ b/scripts/qapi2texi.py
> @@ -147,7 +147,7 @@ def texi_member(member, suffix=''):
> def texi_members(doc, what, base, variants, member_func):
> """Format the table of members"""
> items = ''
> - for section in doc.args.itervalues():
> + for section in doc.args.values():
> # TODO Drop fallbacks when undocumented members are outlawed
> if section.content:
> desc = texi_format(str(section))
> @@ -285,15 +285,15 @@ def texi_schema(schema):
> def main(argv):
> """Takes schema argument, prints result to stdout"""
> if len(argv) != 2:
> - print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0]
> + sys.stderr.write("%s: need exactly 1 argument: SCHEMA\n" % argv[0])
> sys.exit(1)
>
> schema = qapi.QAPISchema(argv[1])
> if not qapi.doc_required:
> - print >>sys.stderr, ("%s: need pragma 'doc-required' "
> - "to generate documentation" % argv[0])
> + sys.stderr.write("%s: need pragma 'doc-required' "
> + "to generate documentation\n" % argv[0])
> sys.exit(1)
> - print texi_schema(schema)
> + sys.stdout.write(texi_schema(schema) + "\n")
>
>
> if __name__ == '__main__':
> diff --git a/scripts/signrom.py b/scripts/signrom.py
> index d1dabe0..0497a1c 100644
> --- a/scripts/signrom.py
> +++ b/scripts/signrom.py
> @@ -18,7 +18,7 @@ fin = open(sys.argv[1], 'rb')
> fout = open(sys.argv[2], 'wb')
>
> magic = fin.read(2)
> -if magic != '\x55\xaa':
> +if magic != b'\x55\xaa':
> sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1])
>
> size_byte = ord(fin.read(1))
> @@ -33,7 +33,7 @@ elif len(data) < size:
> # Add padding if necessary, rounding the whole input to a multiple of
> # 512 bytes according to the third byte of the input.
> # size-1 because a final byte is added below to store the checksum.
> - data = data.ljust(size-1, '\0')
> + data = data.ljust(size-1, b'\0')
> else:
> if ord(data[-1:]) != 0:
> sys.stderr.write('WARNING: ROM includes nonzero checksum\n')
What is our Python 2 -> 3 migration strategy?
Don't support Python 3 until a flag day, then flip and don't support
Python 2?
Keep the source in Python 2 and support 3 via conversion with 2to3?
Port the source to Python 3 and support 2 via conversion with 3to2?
Port to Python 3, but keep it working with Python 2 with help of the six
module?
Support both with ad hoc hackery (like this patch does)?
I think we should make up our minds before we hack up things too much.
On Mon, Aug 21, 2017 at 04:29:44PM +0200, Markus Armbruster wrote: > What is our Python 2 -> 3 migration strategy? > > Don't support Python 3 until a flag day, then flip and don't support > Python 2? Add support for Python 3 so that both Python 2.6+ and Python 3 are supported. When Python 2 is EOL it will need to be dropped and the code becomes Python 3-only. > Keep the source in Python 2 and support 3 via conversion with 2to3? > > Port the source to Python 3 and support 2 via conversion with 3to2? > > Port to Python 3, but keep it working with Python 2 with help of the six > module? > > Support both with ad hoc hackery (like this patch does)? Yes, please see for details: https://lists.gnu.org/archive/html/qemu-devel/2017-08/msg01758.html Stefan
Stefan Hajnoczi <stefanha@gmail.com> writes: > On Mon, Aug 21, 2017 at 04:29:44PM +0200, Markus Armbruster wrote: >> What is our Python 2 -> 3 migration strategy? >> >> Don't support Python 3 until a flag day, then flip and don't support >> Python 2? > > Add support for Python 3 so that both Python 2.6+ and Python 3 are > supported. Python 2.7 is the end of the Python 2 line of development (PEP 404). It'll be maintained until 2020. Downstream support for Python 2 may or may not last longer. > When Python 2 is EOL it will need to be dropped and the code becomes > Python 3-only. > >> Keep the source in Python 2 and support 3 via conversion with 2to3? >> >> Port the source to Python 3 and support 2 via conversion with 3to2? >> >> Port to Python 3, but keep it working with Python 2 with help of the six >> module? >> >> Support both with ad hoc hackery (like this patch does)? > > Yes, please see for details: > https://lists.gnu.org/archive/html/qemu-devel/2017-08/msg01758.html So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense. I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all. Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
On 31 August 2017 at 07:35, Markus Armbruster <armbru@redhat.com> wrote: > So, first we'll invest in work-arounds to make both 2 and 3 work. Once > 2 is gone, we can invest some more to clean them up. Which probably > won't happen, so we'll continue to carry work-arounds that no longer > make sense. > > I maintain roughly one fourth of all Python code in qemu, and I'm not > looking forward to this hoop-jumping at all. > > Are we really, really sure we want to go this way? What exactly are we > hoping to accomplish by it? My take is that we have the following goals we want to achieve: (1) We need to continue to build and run on older (long-term-support) distros that still ship only Python 2.x (alas even back to 2.6) (2) We need to be able to build and run on newer distros that have dropped Python 2 altogether in favour of Python 3 (I don't know if there are any such today, but presumably by 2020 there will be) Unless we can confidently say that either (1) or (2) is the empty set, we need to handle both 2 and 3 in the same codebase. This is a pain, but unfortunately Python upstream have forced us into it by breaking source code compatibility. I think (1) is pretty clearly not (yet) an empty set, so the only alternative I see to "support 2 and 3 now" is "keep supporting only 2 for the moment and hope that no distro drops 2 support before all the LTS 2-only distro versions vanish into history". thanks -- PMM
On Thu, Aug 31, 2017 at 11:27:15AM +0100, Peter Maydell wrote: > On 31 August 2017 at 07:35, Markus Armbruster <armbru@redhat.com> wrote: > > So, first we'll invest in work-arounds to make both 2 and 3 work. Once > > 2 is gone, we can invest some more to clean them up. Which probably > > won't happen, so we'll continue to carry work-arounds that no longer > > make sense. > > > > I maintain roughly one fourth of all Python code in qemu, and I'm not > > looking forward to this hoop-jumping at all. > > > > Are we really, really sure we want to go this way? What exactly are we > > hoping to accomplish by it? > > My take is that we have the following goals we want to achieve: > > (1) We need to continue to build and run on older (long-term-support) > distros that still ship only Python 2.x (alas even back to 2.6) > (2) We need to be able to build and run on newer distros that > have dropped Python 2 altogether in favour of Python 3 > (I don't know if there are any such today, but presumably by > 2020 there will be) Fedora has dropped Python 2 in the default install now, so needs to be manually pulled in. While python 2 won't be dropped entirely in the near future, its days are definitely numbered and so once upstream officially dedclares it dead, I'd expect Fedora to follow. > Unless we can confidently say that either (1) or (2) is the > empty set, we need to handle both 2 and 3 in the same codebase. > This is a pain, but unfortunately Python upstream have forced > us into it by breaking source code compatibility. > > I think (1) is pretty clearly not (yet) an empty set, so the > only alternative I see to "support 2 and 3 now" is "keep supporting > only 2 for the moment and hope that no distro drops 2 support > before all the LTS 2-only distro versions vanish into history". If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard. From QEMU pov, I don't think supporting py2 and py3 at the same time is actually as bad as you might expect from reading about the pain in the broader community. We're lucky that pretty much all our python code is self-contained not relying on 3rd party modules, and it doesn't use particularly advanced features of python either. I worked on OpenStack Nova during the conversion to support py2 & py3 in parallel, and the resulting code was not really any harder to maintain than the original code. The hard bit was Nova's use of 3rd party modules whose maintainers hadn't converted. So, IMHO, provided we make sure our CI system is giving us test coverage across the various py2 & py3 versions, the burden on us will not be significant. Writing code that's py2 & py3 compatible is not hard as long as you have good testing. So we just have a lump for the initial conversion, then CI will take care of avoiding regressions going forward. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 31 August 2017 at 11:47, Daniel P. Berrange <berrange@redhat.com> wrote: > If we can update to python 2.7 as our minimum, then supporting py2 > and py3 gets simpler, avoiding some of the nastier hacks, even > without that though it isn't too hard. Unfortunately RHEL6 is what's holding us to retaining 2.6 support, and googling suggests that doesn't go EOL until 2020... thanks -- PMM
On Thu, Aug 31, 2017 at 11:55:18AM +0100, Peter Maydell wrote: > On 31 August 2017 at 11:47, Daniel P. Berrange <berrange@redhat.com> wrote: > > If we can update to python 2.7 as our minimum, then supporting py2 > > and py3 gets simpler, avoiding some of the nastier hacks, even > > without that though it isn't too hard. > > Unfortunately RHEL6 is what's holding us to retaining 2.6 > support, and googling suggests that doesn't go EOL until 2020... Who is actually requiring that we support RHEL6 until EOL though ? AFAIK, from Red Hat side, we don't need QEMU git master to be buildable on RHEL-6 machines anymore, since QEMU there is long since in bug-fix only mode, no rebases or feature backports. I assume there's probably some community interest in building git master on RHEL-6, but I think it is reasonable for us to say no at some point, rather than waiting until final EOL (2020 for normal subscribers, 2024 for people who paid for extended lifetime). Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
"Daniel P. Berrange" <berrange@redhat.com> writes: > On Thu, Aug 31, 2017 at 11:55:18AM +0100, Peter Maydell wrote: >> On 31 August 2017 at 11:47, Daniel P. Berrange <berrange@redhat.com> wrote: >> > If we can update to python 2.7 as our minimum, then supporting py2 >> > and py3 gets simpler, avoiding some of the nastier hacks, even >> > without that though it isn't too hard. >> >> Unfortunately RHEL6 is what's holding us to retaining 2.6 >> support, and googling suggests that doesn't go EOL until 2020... > > Who is actually requiring that we support RHEL6 until EOL though ? > > AFAIK, from Red Hat side, we don't need QEMU git master to be buildable > on RHEL-6 machines anymore, since QEMU there is long since in bug-fix > only mode, no rebases or feature backports. Even in the unlikely case that a bug fix involves backporting some Python, having to port just those bits to RHEL-6's version of Python would be less work than making *all* upstream Python code work with both Python dialects. > I assume there's probably some community interest in building git master > on RHEL-6, but I think it is reasonable for us to say no at some point, > rather than waiting until final EOL (2020 for normal subscribers, 2024 > for people who paid for extended lifetime). "Interest" by itself doesn't pay bills or write code. If you have to have upstream QEMU on RHEL-6, and you're fine with getting it by building it yourself (there being no other way), then you should also be fine with getting its build requirements. Python 3.4 is in EPEL.
On Thu, Aug 31, 2017 at 02:44:58PM +0200, Markus Armbruster wrote: > "Daniel P. Berrange" <berrange@redhat.com> writes: > > > On Thu, Aug 31, 2017 at 11:55:18AM +0100, Peter Maydell wrote: > >> On 31 August 2017 at 11:47, Daniel P. Berrange <berrange@redhat.com> wrote: > >> > If we can update to python 2.7 as our minimum, then supporting py2 > >> > and py3 gets simpler, avoiding some of the nastier hacks, even > >> > without that though it isn't too hard. > >> > >> Unfortunately RHEL6 is what's holding us to retaining 2.6 > >> support, and googling suggests that doesn't go EOL until 2020... > > > > Who is actually requiring that we support RHEL6 until EOL though ? > > > > AFAIK, from Red Hat side, we don't need QEMU git master to be buildable > > on RHEL-6 machines anymore, since QEMU there is long since in bug-fix > > only mode, no rebases or feature backports. > > Even in the unlikely case that a bug fix involves backporting some > Python, having to port just those bits to RHEL-6's version of Python > would be less work than making *all* upstream Python code work with both > Python dialects. > > > I assume there's probably some community interest in building git master > > on RHEL-6, but I think it is reasonable for us to say no at some point, > > rather than waiting until final EOL (2020 for normal subscribers, 2024 > > for people who paid for extended lifetime). > > "Interest" by itself doesn't pay bills or write code. > > If you have to have upstream QEMU on RHEL-6, and you're fine with > getting it by building it yourself (there being no other way), then you > should also be fine with getting its build requirements. Python 3.4 is > in EPEL. That sounds like a good argument (to ditch Python 2.6) to me -- EL-6 users getting it from EPEL[*], one of its purposes is to facilate cases exactly like the one above. [*] https://fedoraproject.org/wiki/EPEL -- /kashyap
On 31 August 2017 at 12:02, Daniel P. Berrange <berrange@redhat.com> wrote: > On Thu, Aug 31, 2017 at 11:55:18AM +0100, Peter Maydell wrote: >> On 31 August 2017 at 11:47, Daniel P. Berrange <berrange@redhat.com> wrote: >> > If we can update to python 2.7 as our minimum, then supporting py2 >> > and py3 gets simpler, avoiding some of the nastier hacks, even >> > without that though it isn't too hard. >> >> Unfortunately RHEL6 is what's holding us to retaining 2.6 >> support, and googling suggests that doesn't go EOL until 2020... > > Who is actually requiring that we support RHEL6 until EOL though ? OK, if we don't need to retain RHEL6 support that would help (we should in that case not bother pulling in that big lump of argument parsing code that just went past in a patchset purely for the benefit of 2.6 support). How are the other LTS distros for Python versions? thanks -- PMM
Peter Maydell <peter.maydell@linaro.org> writes: > On 31 August 2017 at 07:35, Markus Armbruster <armbru@redhat.com> wrote: >> So, first we'll invest in work-arounds to make both 2 and 3 work. Once >> 2 is gone, we can invest some more to clean them up. Which probably >> won't happen, so we'll continue to carry work-arounds that no longer >> make sense. >> >> I maintain roughly one fourth of all Python code in qemu, and I'm not >> looking forward to this hoop-jumping at all. >> >> Are we really, really sure we want to go this way? What exactly are we >> hoping to accomplish by it? > > My take is that we have the following goals we want to achieve: > > (1) We need to continue to build and run on older (long-term-support) > distros that still ship only Python 2.x (alas even back to 2.6) That's an assertion, not an answer to my question :) What exactly are we hoping to accomplish by supporting what distros exactly? > (2) We need to be able to build and run on newer distros that > have dropped Python 2 altogether in favour of Python 3 > (I don't know if there are any such today, but presumably by > 2020 there will be) Yup. > Unless we can confidently say that either (1) or (2) is the > empty set, we need to handle both 2 and 3 in the same codebase. > This is a pain, but unfortunately Python upstream have forced > us into it by breaking source code compatibility. > > I think (1) is pretty clearly not (yet) an empty set, so the > only alternative I see to "support 2 and 3 now" is "keep supporting > only 2 for the moment and hope that no distro drops 2 support > before all the LTS 2-only distro versions vanish into history". I don't buy the "clearly" in "pretty clearly not (yet) an empty set", because I don't understand *why* we need to support "older (long-term-support) distros". If we can leave Python 2 behind, I'm all for porting to Python 3 now. If we can't yet, then I don't see why I should do work *now* just because "some" distro might drop support for Python 2 "before all the LTS 2-only distro versions vanish into history". If it ain't broke, don't fix it!
On Thu, Aug 31, 2017 at 02:58:24PM +0200, Markus Armbruster wrote: > Peter Maydell <peter.maydell@linaro.org> writes: > > > On 31 August 2017 at 07:35, Markus Armbruster <armbru@redhat.com> wrote: > >> So, first we'll invest in work-arounds to make both 2 and 3 work. Once > >> 2 is gone, we can invest some more to clean them up. Which probably > >> won't happen, so we'll continue to carry work-arounds that no longer > >> make sense. > >> > >> I maintain roughly one fourth of all Python code in qemu, and I'm not > >> looking forward to this hoop-jumping at all. > >> > >> Are we really, really sure we want to go this way? What exactly are we > >> hoping to accomplish by it? > > > > My take is that we have the following goals we want to achieve: > > > > (1) We need to continue to build and run on older (long-term-support) > > distros that still ship only Python 2.x (alas even back to 2.6) > > That's an assertion, not an answer to my question :) > > What exactly are we hoping to accomplish by supporting what distros > exactly? > > > (2) We need to be able to build and run on newer distros that > > have dropped Python 2 altogether in favour of Python 3 > > (I don't know if there are any such today, but presumably by > > 2020 there will be) > > Yup. > > > Unless we can confidently say that either (1) or (2) is the > > empty set, we need to handle both 2 and 3 in the same codebase. > > This is a pain, but unfortunately Python upstream have forced > > us into it by breaking source code compatibility. > > > > I think (1) is pretty clearly not (yet) an empty set, so the > > only alternative I see to "support 2 and 3 now" is "keep supporting > > only 2 for the moment and hope that no distro drops 2 support > > before all the LTS 2-only distro versions vanish into history". > > I don't buy the "clearly" in "pretty clearly not (yet) an empty set", > because I don't understand *why* we need to support "older > (long-term-support) distros". > > If we can leave Python 2 behind, I'm all for porting to Python 3 now. > > If we can't yet, then I don't see why I should do work *now* just > because "some" distro might drop support for Python 2 "before all the > LTS 2-only distro versions vanish into history". As mentioned before, while for many projects, maintaining py2 & py3 in parallel is hard, I don't think that is going to be the case for QEMU due to the simplicity of much of our python usage. IOW, for QEMU I tend to view this as more like the work we do to fix up compiler warnings when new GCC comes out. There's some cleanup needed, but the result is rarely harder to maintain. It is fine if you don't want to do the work to convert QAPI code to support py3, but if some other contributor submits patches for parallel py2 & py3 support that don't appreciably harm maintainability I see little reason to reject them. > If it ain't broke, don't fix it! NB, some distros have already changed /usr/bin/python to point to py3 so all the files with #!/usr/bin/python in QEMU are already broken on those distros. In theory you could change the shebang to #!/usr/bin/python2, but the /usr/bin/python2 file symlink is not standard in py2.6 - though certain distros have backported support for it. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 31 August 2017 at 14:14, Daniel P. Berrange <berrange@redhat.com> wrote: > NB, some distros have already changed /usr/bin/python to point to py3 > so all the files with #!/usr/bin/python in QEMU are already broken on > those distros. This is a bug in those distros and they should fix it. Python 3 is not Python 2 and making /usr/bin/python point to the wrong interpreter will have predictably bad results for their users... thanks -- PMM
On Thu, Aug 31, 2017 at 02:19:05PM +0100, Peter Maydell wrote:
> On 31 August 2017 at 14:14, Daniel P. Berrange <berrange@redhat.com> wrote:
> > NB, some distros have already changed /usr/bin/python to point to py3
> > so all the files with #!/usr/bin/python in QEMU are already broken on
> > those distros.
>
> This is a bug in those distros and they should fix it. Python 3
> is not Python 2 and making /usr/bin/python point to the wrong
> interpreter will have predictably bad results for their users...
Having /usr/bin/python point to py3 is what upstream Python will
eventually recommend. Some distros jumped a bit early, but the
others will eventually follow, and I don't see those early distros
moving back.
https://www.python.org/dev/peps/pep-0394/
"* python2 will refer to some version of Python 2.x.
* python3 will refer to some version of Python 3.x.
* for the time being, all distributions should ensure
that python refers to the same target as python2.
* however, end users should be aware that python refers
to python3 on at least Arch Linux (that change is what
prompted the creation of this PEP), so python should
be used in the shebang line only for scripts that are
source compatible with both Python 2 and 3.
* in preparation for an eventual change in the default
version of Python, Python 2 only scripts should either
be updated to be source compatible with Python 3 or
else to use python2 in the shebang line."
Regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 31 August 2017 at 14:25, Daniel P. Berrange <berrange@redhat.com> wrote: > On Thu, Aug 31, 2017 at 02:19:05PM +0100, Peter Maydell wrote: >> On 31 August 2017 at 14:14, Daniel P. Berrange <berrange@redhat.com> wrote: >> > NB, some distros have already changed /usr/bin/python to point to py3 >> > so all the files with #!/usr/bin/python in QEMU are already broken on >> > those distros. >> >> This is a bug in those distros and they should fix it. Python 3 >> is not Python 2 and making /usr/bin/python point to the wrong >> interpreter will have predictably bad results for their users... > > Having /usr/bin/python point to py3 is what upstream Python will > eventually recommend. Some distros jumped a bit early, but the > others will eventually follow, and I don't see those early distros > moving back. > > https://www.python.org/dev/peps/pep-0394/ That appears to be "Arch Linux broke this, and rather than just telling them to fix this we decided to make everybody else deal with the breakage". Water under the bridge, I guess :-( thanks -- PMM
On 2017-08-31 15:14, Daniel P. Berrange wrote: [...] > NB, some distros have already changed /usr/bin/python to point to py3 > so all the files with #!/usr/bin/python in QEMU are already broken on > those distros. For what it's worth, I (at least used to) build qemu regularly on Arch Linux and the only issue I encountered with this is that the iotests wouldn't work. As a workaround, I made the check script detect that interpreter line and just run the script through the interpreter configured through --python, so it works now... So from what I could tell, everything worked when you specify --python=python2. Max
On 31 August 2017 at 13:58, Markus Armbruster <armbru@redhat.com> wrote: > Peter Maydell <peter.maydell@linaro.org> writes: > >> On 31 August 2017 at 07:35, Markus Armbruster <armbru@redhat.com> wrote: >>> So, first we'll invest in work-arounds to make both 2 and 3 work. Once >>> 2 is gone, we can invest some more to clean them up. Which probably >>> won't happen, so we'll continue to carry work-arounds that no longer >>> make sense. >>> >>> I maintain roughly one fourth of all Python code in qemu, and I'm not >>> looking forward to this hoop-jumping at all. >>> >>> Are we really, really sure we want to go this way? What exactly are we >>> hoping to accomplish by it? >> >> My take is that we have the following goals we want to achieve: >> >> (1) We need to continue to build and run on older (long-term-support) >> distros that still ship only Python 2.x (alas even back to 2.6) > > That's an assertion, not an answer to my question :) > > What exactly are we hoping to accomplish by supporting what distros > exactly? Of my current build-test machines: OSX is python 2.7 Ubuntu 16.04 (s390) has both 2.7 and 3.5 FreeBSD 11.0 has python 2.7 but I could install 3.3 OpenBSD 6.0 has python 2.7 but I could install 3.5 NetBSD 7.1 has python 2.7 but I could install 3.6 The ppc64 box is Centos 7 which has python 2.7 but I could ask the admins to install 3.4 The aarch64 box is Ubuntu 14.04 and has 2.7 and 3.4 So OSX is the major laggard in having no official Python 3 at all, but the majority still seem to have their stock python be a version 2, even if a version 3 is optionally available. Last time we discussed Python versions (in 2015) the three cited reasons for 2.6 were RHEL6, SLES 11, and OSX 10.6. We can maybe move forward to 2.7 (RHEL6 apparently has a 2.7 backport, I think our two or three OSX 10.6 users can cope with installing a 2.7 from homebrew, and I dunno what the SLES11 situation is). I don't think we should move further forward than that yet, I don't see the need. >> I think (1) is pretty clearly not (yet) an empty set, so the >> only alternative I see to "support 2 and 3 now" is "keep supporting >> only 2 for the moment and hope that no distro drops 2 support >> before all the LTS 2-only distro versions vanish into history". > > I don't buy the "clearly" in "pretty clearly not (yet) an empty set", > because I don't understand *why* we need to support "older > (long-term-support) distros". I like that you can build QEMU on a lot of distros without having to install from source a whole pile of dependent libraries and tools. I think that's a service to our users (and to the distros that we run on) which we shouldn't give up without a good reason. thanks -- PMM
On Sat, Aug 19, 2017 at 10:58:44PM -0700, David Michael wrote: > This allows building with "./configure --python=python3", where > the python3 program is at least version 3.6. It preserves > compatibility with Python 2. The changes include: > > - Avoiding "print" usage > - Using bytes with files opened in binary mode > - Switching .iteritems() to .items() > - Adding fallback imports for functions moved to other modules > > Signed-off-by: David Michael <fedora.dm0@gmail.com> > --- > > Hi, > > I've been applying these changes when building on Fedora 26, which does > not include any Python 2 packages by default. It was tested with Python > 2.7 and 3.6. > > I just saw the list of scripts that need updating on the mailing list, > and this doesn't cover all of them, but it is enough to build a binary > for running virtual machines with KVM. Maybe it is still useful as a > starting point. > > Thanks. > > David > > configure | 6 ++++-- > scripts/qapi.py | 31 ++++++++++++++++++++----------- > scripts/qapi2texi.py | 10 +++++----- > scripts/signrom.py | 4 ++-- > 4 files changed, 31 insertions(+), 20 deletions(-) Cool, thanks for contributing this! Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
On Sat, Aug 19, 2017 at 10:58:44PM -0700, David Michael wrote:
> This allows building with "./configure --python=python3", where
> the python3 program is at least version 3.6. It preserves
> compatibility with Python 2. The changes include:
>
> - Avoiding "print" usage
> - Using bytes with files opened in binary mode
> - Switching .iteritems() to .items()
> - Adding fallback imports for functions moved to other modules
>
> Signed-off-by: David Michael <fedora.dm0@gmail.com>
> ---
>
> Hi,
>
> I've been applying these changes when building on Fedora 26, which does
> not include any Python 2 packages by default. It was tested with Python
> 2.7 and 3.6.
>
> I just saw the list of scripts that need updating on the mailing list,
> and this doesn't cover all of them, but it is enough to build a binary
> for running virtual machines with KVM. Maybe it is still useful as a
> starting point.
>
> Thanks.
Markus,
You wanted to discuss the general direction of Python version support
before applying this.
Has consensus been reached?
Stefan
>
> David
>
> configure | 6 ++++--
> scripts/qapi.py | 31 ++++++++++++++++++++-----------
> scripts/qapi2texi.py | 10 +++++-----
> scripts/signrom.py | 4 ++--
> 4 files changed, 31 insertions(+), 20 deletions(-)
>
> diff --git a/configure b/configure
> index dd73cce..09f4d68 100755
> --- a/configure
> +++ b/configure
> @@ -1548,9 +1548,11 @@ fi
>
> # Note that if the Python conditional here evaluates True we will exit
> # with status 1 which is a shell 'false' value.
> -if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then
> +if ! $python -c 'import sys; sys.exit(sys.version_info >= (3,) and sys.version_info < (3,6))'; then
> + error_exit "Cannot use '$python', Python 3.6 or later is required." \
> + "Use --python=/path/to/python3 to specify a supported Python 3."
> +elif ! $python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then
> error_exit "Cannot use '$python', Python 2.6 or later is required." \
> - "Note that Python 3 or later is not yet supported." \
> "Use --python=/path/to/python to specify a supported Python."
> fi
>
> diff --git a/scripts/qapi.py b/scripts/qapi.py
> index 8aa2775..6450998 100644
> --- a/scripts/qapi.py
> +++ b/scripts/qapi.py
> @@ -15,9 +15,11 @@ import errno
> import getopt
> import os
> import re
> -import string
> import sys
> -from ordereddict import OrderedDict
> +try:
> + from collections import OrderedDict
> +except ImportError:
> + from ordereddict import OrderedDict
>
> builtin_types = {
> 'null': 'QTYPE_QNULL',
> @@ -252,7 +254,7 @@ class QAPIDoc(object):
> "'Returns:' is only valid for commands")
>
> def check(self):
> - bogus = [name for name, section in self.args.iteritems()
> + bogus = [name for name, section in self.args.items()
> if not section.member]
> if bogus:
> raise QAPISemError(
> @@ -308,7 +310,7 @@ class QAPISchemaParser(object):
> if not isinstance(pragma, dict):
> raise QAPISemError(
> info, "Value of 'pragma' must be a dictionary")
> - for name, value in pragma.iteritems():
> + for name, value in pragma.items():
> self._pragma(name, value, info)
> else:
> expr_elem = {'expr': expr,
> @@ -1574,7 +1576,7 @@ class QAPISchema(object):
>
> def _make_members(self, data, info):
> return [self._make_member(key, value, info)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
>
> def _def_struct_type(self, expr, info, doc):
> name = expr['struct']
> @@ -1606,11 +1608,11 @@ class QAPISchema(object):
> name, info, doc, 'base', self._make_members(base, info)))
> if tag_name:
> variants = [self._make_variant(key, value)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
> members = []
> else:
> variants = [self._make_simple_variant(key, value, info)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
> typ = self._make_implicit_enum_type(name, info,
> [v.name for v in variants])
> tag_member = QAPISchemaObjectTypeMember('type', typ, False)
> @@ -1625,7 +1627,7 @@ class QAPISchema(object):
> name = expr['alternate']
> data = expr['data']
> variants = [self._make_variant(key, value)
> - for (key, value) in data.iteritems()]
> + for (key, value) in data.items()]
> tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
> self._def_entity(
> QAPISchemaAlternateType(name, info, doc,
> @@ -1735,7 +1737,11 @@ def c_enum_const(type_name, const_name, prefix=None):
> type_name = prefix
> return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
>
> -c_name_trans = string.maketrans('.-', '__')
> +try:
> + c_name_trans = str.maketrans('.-', '__')
> +except AttributeError:
> + import string
> + c_name_trans = string.maketrans('.-', '__')
>
>
> # Map @name to a valid C identifier.
> @@ -1997,8 +2003,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
> if really:
> return open(name, opt)
> else:
> - import StringIO
> - return StringIO.StringIO()
> + try:
> + from StringIO import StringIO
> + except ImportError:
> + from io import StringIO
> + return StringIO()
>
> fdef = maybe_open(do_c, c_file, 'w')
> fdecl = maybe_open(do_h, h_file, 'w')
> diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
> index a317526..9a6fed9 100755
> --- a/scripts/qapi2texi.py
> +++ b/scripts/qapi2texi.py
> @@ -147,7 +147,7 @@ def texi_member(member, suffix=''):
> def texi_members(doc, what, base, variants, member_func):
> """Format the table of members"""
> items = ''
> - for section in doc.args.itervalues():
> + for section in doc.args.values():
> # TODO Drop fallbacks when undocumented members are outlawed
> if section.content:
> desc = texi_format(str(section))
> @@ -285,15 +285,15 @@ def texi_schema(schema):
> def main(argv):
> """Takes schema argument, prints result to stdout"""
> if len(argv) != 2:
> - print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0]
> + sys.stderr.write("%s: need exactly 1 argument: SCHEMA\n" % argv[0])
> sys.exit(1)
>
> schema = qapi.QAPISchema(argv[1])
> if not qapi.doc_required:
> - print >>sys.stderr, ("%s: need pragma 'doc-required' "
> - "to generate documentation" % argv[0])
> + sys.stderr.write("%s: need pragma 'doc-required' "
> + "to generate documentation\n" % argv[0])
> sys.exit(1)
> - print texi_schema(schema)
> + sys.stdout.write(texi_schema(schema) + "\n")
>
>
> if __name__ == '__main__':
> diff --git a/scripts/signrom.py b/scripts/signrom.py
> index d1dabe0..0497a1c 100644
> --- a/scripts/signrom.py
> +++ b/scripts/signrom.py
> @@ -18,7 +18,7 @@ fin = open(sys.argv[1], 'rb')
> fout = open(sys.argv[2], 'wb')
>
> magic = fin.read(2)
> -if magic != '\x55\xaa':
> +if magic != b'\x55\xaa':
> sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1])
>
> size_byte = ord(fin.read(1))
> @@ -33,7 +33,7 @@ elif len(data) < size:
> # Add padding if necessary, rounding the whole input to a multiple of
> # 512 bytes according to the third byte of the input.
> # size-1 because a final byte is added below to store the checksum.
> - data = data.ljust(size-1, '\0')
> + data = data.ljust(size-1, b'\0')
> else:
> if ord(data[-1:]) != 0:
> sys.stderr.write('WARNING: ROM includes nonzero checksum\n')
> --
> 2.13.5
>
>
Stefan Hajnoczi <stefanha@gmail.com> writes: > On Sat, Aug 19, 2017 at 10:58:44PM -0700, David Michael wrote: >> This allows building with "./configure --python=python3", where >> the python3 program is at least version 3.6. It preserves >> compatibility with Python 2. The changes include: >> >> - Avoiding "print" usage >> - Using bytes with files opened in binary mode >> - Switching .iteritems() to .items() >> - Adding fallback imports for functions moved to other modules >> >> Signed-off-by: David Michael <fedora.dm0@gmail.com> >> --- >> >> Hi, >> >> I've been applying these changes when building on Fedora 26, which does >> not include any Python 2 packages by default. It was tested with Python >> 2.7 and 3.6. >> >> I just saw the list of scripts that need updating on the mailing list, >> and this doesn't cover all of them, but it is enough to build a binary >> for running virtual machines with KVM. Maybe it is still useful as a >> starting point. >> >> Thanks. > > > Markus, > You wanted to discuss the general direction of Python version support > before applying this. Dan has since posted competing patches. > Has consensus been reached? I need to decide how much I hate supporting both Python 2 and Python 3, and that involves looking at both patch sets in more detail. QAPI is a rather busy subsystem, and right now my review queue is in a state where I have to decide whom to annoy and disappoint with overly long delays. Given the choice between addressing a problem that doesn't really exist, yet (Python 2 going away) and addressing problems that are hurting us *now* (such as QAPI's inability to express compile-time conditionals), I have to prioritize the latter. I wish I wouldn't have to make such choices, and I apologize for the inconvenience.
© 2016 - 2026 Red Hat, Inc.