From: "Sv. Lockal" <lockalsash@gmail.com>

Fix compilation with pip-25.2 due to missing distlib.version

Bug: https://gitlab.com/qemu-project/qemu/-/issues/3062

Signed-off-by: Sv. Lockal <lockalsash@gmail.com>
[Edits: Type "safety" whackamole --js]
Signed-off-by: John Snow <jsnow@redhat.com>
---
 python/scripts/mkvenv.py | 64 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 60 insertions(+), 4 deletions(-)

diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py
index 8ac5b0b2a05..f102527c4de 100644
--- a/python/scripts/mkvenv.py
+++ b/python/scripts/mkvenv.py
@@ -84,6 +84,7 @@
     Sequence,
     Tuple,
     Union,
+    cast,
 )
 import venv
 
@@ -94,17 +95,39 @@
 HAVE_DISTLIB = True
 try:
     import distlib.scripts
-    import distlib.version
 except ImportError:
     try:
         # Reach into pip's cookie jar.  pylint and flake8 don't understand
         # that these imports will be used via distlib.xxx.
         from pip._vendor import distlib
         import pip._vendor.distlib.scripts  # noqa, pylint: disable=unused-import
-        import pip._vendor.distlib.version  # noqa, pylint: disable=unused-import
     except ImportError:
         HAVE_DISTLIB = False
 
+# pip 25.2 does not vendor distlib.version, but it uses vendored
+# packaging.version
+HAVE_DISTLIB_VERSION = True
+try:
+    import distlib.version  # pylint: disable=ungrouped-imports
+except ImportError:
+    try:
+        # pylint: disable=unused-import,ungrouped-imports
+        import pip._vendor.distlib.version  # noqa
+    except ImportError:
+        HAVE_DISTLIB_VERSION = False
+
+HAVE_PACKAGING_VERSION = True
+try:
+    # Do not bother importing non-vendored packaging, because it is not
+    # in stdlib.
+    from pip._vendor import packaging
+    # pylint: disable=unused-import
+    import pip._vendor.packaging.requirements  # noqa
+    import pip._vendor.packaging.version  # noqa
+except ImportError:
+    HAVE_PACKAGING_VERSION = False
+
+
 # Try to load tomllib, with a fallback to tomli.
 # HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail
 # outside the venv or before a potential call to ensurepip in checkpip().
@@ -133,6 +156,39 @@ class Ouch(RuntimeError):
     """An Exception class we can't confuse with a builtin."""
 
 
+class Matcher:
+    """Compatibility appliance for version/requirement string parsing."""
+    def __init__(self, name_and_constraint: str):
+        """Create a matcher from a requirement-like string."""
+        if HAVE_DISTLIB_VERSION:
+            self._m = distlib.version.LegacyMatcher(name_and_constraint)
+        elif HAVE_PACKAGING_VERSION:
+            self._m = packaging.requirements.Requirement(name_and_constraint)
+        else:
+            raise Ouch("found neither distlib.version nor packaging.version")
+        self.name = self._m.name
+
+    def match(self, version_str: str) -> bool:
+        """Return True if `version` satisfies the stored constraint."""
+        if HAVE_DISTLIB_VERSION:
+            return cast(
+                bool,
+                self._m.match(distlib.version.LegacyVersion(version_str))
+            )
+
+        assert HAVE_PACKAGING_VERSION
+        return cast(
+            bool,
+            self._m.specifier.contains(
+                packaging.version.Version(version_str), prereleases=True
+            )
+        )
+
+    def __repr__(self) -> str:
+        """Stable debug representation delegated to the backend."""
+        return repr(self._m)
+
+
 class QemuEnvBuilder(venv.EnvBuilder):
     """
     An extension of venv.EnvBuilder for building QEMU's configure-time venv.
@@ -669,7 +725,7 @@ def _do_ensure(
     canary = None
     for name, info in group.items():
         constraint = _make_version_constraint(info, False)
-        matcher = distlib.version.LegacyMatcher(name + constraint)
+        matcher = Matcher(name + constraint)
         print(f"mkvenv: checking for {matcher}", file=sys.stderr)
 
         dist: Optional[Distribution] = None
@@ -683,7 +739,7 @@ def _do_ensure(
             # Always pass installed package to pip, so that they can be
             # updated if the requested version changes
             or not _is_system_package(dist)
-            or not matcher.match(distlib.version.LegacyVersion(dist.version))
+            or not matcher.match(dist.version)
         ):
             absent.append(name + _make_version_constraint(info, True))
             if len(absent) == 1:
-- 
2.50.1

On Mon, Aug 11, 2025 at 3:03 PM John Snow <jsnow@redhat.com> wrote:
>
> From: "Sv. Lockal" <lockalsash@gmail.com>
>
> Fix compilation with pip-25.2 due to missing distlib.version
>
> Bug: https://gitlab.com/qemu-project/qemu/-/issues/3062
>
> Signed-off-by: Sv. Lockal <lockalsash@gmail.com>
> [Edits: Type "safety" whackamole --js]
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
>  python/scripts/mkvenv.py | 64 +++++++++++++++++++++++++++++++++++++---
>  1 file changed, 60 insertions(+), 4 deletions(-)

Applied for QEMU v10.1.0-rc3. Thanks!

Stefan

> diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py
> index 8ac5b0b2a05..f102527c4de 100644
> --- a/python/scripts/mkvenv.py
> +++ b/python/scripts/mkvenv.py
> @@ -84,6 +84,7 @@
>      Sequence,
>      Tuple,
>      Union,
> +    cast,
>  )
>  import venv
>
> @@ -94,17 +95,39 @@
>  HAVE_DISTLIB = True
>  try:
>      import distlib.scripts
> -    import distlib.version
>  except ImportError:
>      try:
>          # Reach into pip's cookie jar.  pylint and flake8 don't understand
>          # that these imports will be used via distlib.xxx.
>          from pip._vendor import distlib
>          import pip._vendor.distlib.scripts  # noqa, pylint: disable=unused-import
> -        import pip._vendor.distlib.version  # noqa, pylint: disable=unused-import
>      except ImportError:
>          HAVE_DISTLIB = False
>
> +# pip 25.2 does not vendor distlib.version, but it uses vendored
> +# packaging.version
> +HAVE_DISTLIB_VERSION = True
> +try:
> +    import distlib.version  # pylint: disable=ungrouped-imports
> +except ImportError:
> +    try:
> +        # pylint: disable=unused-import,ungrouped-imports
> +        import pip._vendor.distlib.version  # noqa
> +    except ImportError:
> +        HAVE_DISTLIB_VERSION = False
> +
> +HAVE_PACKAGING_VERSION = True
> +try:
> +    # Do not bother importing non-vendored packaging, because it is not
> +    # in stdlib.
> +    from pip._vendor import packaging
> +    # pylint: disable=unused-import
> +    import pip._vendor.packaging.requirements  # noqa
> +    import pip._vendor.packaging.version  # noqa
> +except ImportError:
> +    HAVE_PACKAGING_VERSION = False
> +
> +
>  # Try to load tomllib, with a fallback to tomli.
>  # HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail
>  # outside the venv or before a potential call to ensurepip in checkpip().
> @@ -133,6 +156,39 @@ class Ouch(RuntimeError):
>      """An Exception class we can't confuse with a builtin."""
>
>
> +class Matcher:
> +    """Compatibility appliance for version/requirement string parsing."""
> +    def __init__(self, name_and_constraint: str):
> +        """Create a matcher from a requirement-like string."""
> +        if HAVE_DISTLIB_VERSION:
> +            self._m = distlib.version.LegacyMatcher(name_and_constraint)
> +        elif HAVE_PACKAGING_VERSION:
> +            self._m = packaging.requirements.Requirement(name_and_constraint)
> +        else:
> +            raise Ouch("found neither distlib.version nor packaging.version")
> +        self.name = self._m.name
> +
> +    def match(self, version_str: str) -> bool:
> +        """Return True if `version` satisfies the stored constraint."""
> +        if HAVE_DISTLIB_VERSION:
> +            return cast(
> +                bool,
> +                self._m.match(distlib.version.LegacyVersion(version_str))
> +            )
> +
> +        assert HAVE_PACKAGING_VERSION
> +        return cast(
> +            bool,
> +            self._m.specifier.contains(
> +                packaging.version.Version(version_str), prereleases=True
> +            )
> +        )
> +
> +    def __repr__(self) -> str:
> +        """Stable debug representation delegated to the backend."""
> +        return repr(self._m)
> +
> +
>  class QemuEnvBuilder(venv.EnvBuilder):
>      """
>      An extension of venv.EnvBuilder for building QEMU's configure-time venv.
> @@ -669,7 +725,7 @@ def _do_ensure(
>      canary = None
>      for name, info in group.items():
>          constraint = _make_version_constraint(info, False)
> -        matcher = distlib.version.LegacyMatcher(name + constraint)
> +        matcher = Matcher(name + constraint)
>          print(f"mkvenv: checking for {matcher}", file=sys.stderr)
>
>          dist: Optional[Distribution] = None
> @@ -683,7 +739,7 @@ def _do_ensure(
>              # Always pass installed package to pip, so that they can be
>              # updated if the requested version changes
>              or not _is_system_package(dist)
> -            or not matcher.match(distlib.version.LegacyVersion(dist.version))
> +            or not matcher.match(dist.version)
>          ):
>              absent.append(name + _make_version_constraint(info, True))
>              if len(absent) == 1:
> --
> 2.50.1
>
>

Il lun 11 ago 2025, 21:02 John Snow <jsnow@redhat.com> ha scritto:

> From: "Sv. Lockal" <lockalsash@gmail.com>
>
> Fix compilation with pip-25.2 due to missing distlib.version
>

Stupid question: can we just say that, starting with 10.1, people have to
install distlib to build QEMU?

We're bending over backwards just because the Python developers on one hand
keep shipping pip while saying that "no, pip is absolutely not special";
and on the other hand keep *not* shipping packaging and distlib while
complaining that people don't use them.

Paolo

>

On 12.08.2025 00:59, Paolo Bonzini wrote:
> 
> 
> Il lun 11 ago 2025, 21:02 John Snow <jsnow@redhat.com 
> <mailto:jsnow@redhat.com>> ha scritto:
> 
>     From: "Sv. Lockal" <lockalsash@gmail.com <mailto:lockalsash@gmail.com>>
> 
>     Fix compilation with pip-25.2 due to missing distlib.version
> 
> 
> Stupid question: can we just say that, starting with 10.1, people have 
> to install distlib to build QEMU?

There's nothing changed in this regard on the qemu side, - distlib has
always been required.  The change is on the python side, not on qemu.
The same fix is needed for all previous qemu releases.  Or to install
distlib.

> We're bending over backwards just because the Python developers on one 
> hand keep shipping pip while saying that "no, pip is absolutely not 
> special"; and on the other hand keep *not* shipping packaging and 
> distlib while complaining that people don't use them.

This is a bit backwards, it looks like, and a bit too gross.

First, pip vendors a few crate^Wlibs for internal use, but these libs
had never been public, - that's why distlib is imported from
pip._vendor.  We used this hack in qemu to simplify bootstrap, but
it was never the intended usage in pip, and hence there's no backwards
compatibility break here from the pip side.

Second, which is a continuation of the first, strictly speaking it
is our fault that we use distlib without depending on it.

And 3rd, it seems like Current Way (tm) is to use "packaging" library
instead of distlib.  And packaging library is shipped within pip for
a decade or so already - the same way as distlib.  This is why I
wrote "backwards" above - by requiring distlib, we seems to work
backwards, because current python way - which's been this way for
years - is to use packaging, not distlib.  But I'm not someone
involved in python, - I don't even know python language enough to
write a minimal hello,world! program :)  -- so I might be
misunderstanding something here.

In debian, I just added a build dependency on distlib for qemu -
in addition to venv, so it's definitely not a big deal.

But ok.  To sum it up:

right now, with introduction of pip 25.2, qemu can't be built anymore
without installing additional deps.  So far, qemu had worked around
missing python deps automatically, shipping everyhing it needs (except
of the python itself) in the source tarball.   We can continue to try
to be friendly to the target environment and require nothing, - if we
do, we have to adjust.

If we require some python library to be present, we as well can just
list everything we need and stop jumping through hoops entirely in
mkvenv.py (and elsewhere), and just use things directly, assuming
they're installed.  This simplifies our bootstrap, - but I don't
think this is actually necessary, bootstrap is not the most complicated
thing in qemu.

And whenever to embed/require distlib or to switch to packaging is an
orthogonal question, - it looks like "packaging" is the way to go now,
with switching other distlib usage(s) to it.  But I'm definitely not
the one to judge here.

Either way, we're at rc3 now, and have to do some quick fix, - this fix
(or similar one which prefers packaging over distlib) seems to be the
best quick solution.

Thanks,

/mjt

Il mar 12 ago 2025, 07:47 Michael Tokarev <mjt@tls.msk.ru> ha scritto:

> > We're bending over backwards just because the Python developers on one
> > hand keep shipping pip while saying that "no, pip is absolutely not
> > special"; and on the other hand keep *not* shipping packaging and
> > distlib while complaining that people don't use them.
>
> This is a bit backwards, it looks like, and a bit too gross.
>
> First, pip vendors a few crate^Wlibs for internal use, but these libs
> had never been public, - that's why distlib is imported from
> pip._vendor.  We used this hack in qemu to simplify bootstrap, but
> it was never the intended usage in pip, and hence there's no backwards
> compatibility break here from the pip side.
>

Yes, absolutely. We were doing something gross as a convenience, but never
documented that it's just that, a convenience.

Second, which is a continuation of the first, strictly speaking it
> is our fault that we use distlib without depending on it.
>

Agreed, hence my suggestion to just require distlib.

And 3rd, it seems like Current Way (tm) is to use "packaging" library
> instead of distlib.  And packaging library is shipped within pip for
> a decade or so already - the same way as distlib.  This is why I
> wrote "backwards" above - by requiring distlib, we seems to work
> backwards, because current python way - which's been this way for
> years - is to use packaging, not distlib.


They are different libraries and both have their role. pip used
distlib.scripts but not distlib.version, and therefore decided to stop
shipping what it doesn't use

https://github.com/pypa/pip/pull/13337
https://github.com/pypa/pip/pull/13342

Using distlib is not going backwards, especially considering that we have
no other use for "packaging" in QEMU. There's no replacement for
distlib.scripts in the "packaging" library, and no official deprecation of
distlib.version. All that we would get from doing version matching with
"packaging" is requiring two dependencies instead of one (which is why John
kept the possibility to use an installed distlib).

Paolo

On 11.08.2025 22:01, John Snow wrote:
> From: "Sv. Lockal" <lockalsash@gmail.com>
> 
> Fix compilation with pip-25.2 due to missing distlib.version
> 
> Bug: https://gitlab.com/qemu-project/qemu/-/issues/3062
> 
> Signed-off-by: Sv. Lockal <lockalsash@gmail.com>
> [Edits: Type "safety" whackamole --js]
> Signed-off-by: John Snow <jsnow@redhat.com>
> ---
>   python/scripts/mkvenv.py | 64 +++++++++++++++++++++++++++++++++++++---
>   1 file changed, 60 insertions(+), 4 deletions(-)

I think with this in place, there's no need to do any fallbacks here, --
replace distlib.version.LegacyMatcher with 
packaging.requirements.Requirement
directly in the code, and import it from pip directly without any "try"
and fallbacks.

Oldest pip we support (21.3 iirc) has "packaging" available.  There's
just no need for all this hackery/complexity.

When I first saw the patch I though it's the way to go.  But now when
I thought about it, I think it's better to just throw away all this
complexity.

It's sort of risky because we didn't verify how it works on all
supported systems, but it "should" be the same :)

/mjt

> diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py
> index 8ac5b0b2a05..f102527c4de 100644
> --- a/python/scripts/mkvenv.py
> +++ b/python/scripts/mkvenv.py
> @@ -84,6 +84,7 @@
>       Sequence,
>       Tuple,
>       Union,
> +    cast,
>   )
>   import venv
>   
> @@ -94,17 +95,39 @@
>   HAVE_DISTLIB = True
>   try:
>       import distlib.scripts
> -    import distlib.version
>   except ImportError:
>       try:
>           # Reach into pip's cookie jar.  pylint and flake8 don't understand
>           # that these imports will be used via distlib.xxx.
>           from pip._vendor import distlib
>           import pip._vendor.distlib.scripts  # noqa, pylint: disable=unused-import
> -        import pip._vendor.distlib.version  # noqa, pylint: disable=unused-import
>       except ImportError:
>           HAVE_DISTLIB = False
>   
> +# pip 25.2 does not vendor distlib.version, but it uses vendored
> +# packaging.version
> +HAVE_DISTLIB_VERSION = True
> +try:
> +    import distlib.version  # pylint: disable=ungrouped-imports
> +except ImportError:
> +    try:
> +        # pylint: disable=unused-import,ungrouped-imports
> +        import pip._vendor.distlib.version  # noqa
> +    except ImportError:
> +        HAVE_DISTLIB_VERSION = False
> +
> +HAVE_PACKAGING_VERSION = True
> +try:
> +    # Do not bother importing non-vendored packaging, because it is not
> +    # in stdlib.
> +    from pip._vendor import packaging
> +    # pylint: disable=unused-import
> +    import pip._vendor.packaging.requirements  # noqa
> +    import pip._vendor.packaging.version  # noqa
> +except ImportError:
> +    HAVE_PACKAGING_VERSION = False
> +
> +
>   # Try to load tomllib, with a fallback to tomli.
>   # HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail
>   # outside the venv or before a potential call to ensurepip in checkpip().
> @@ -133,6 +156,39 @@ class Ouch(RuntimeError):
>       """An Exception class we can't confuse with a builtin."""
>   
>   
> +class Matcher:
> +    """Compatibility appliance for version/requirement string parsing."""
> +    def __init__(self, name_and_constraint: str):
> +        """Create a matcher from a requirement-like string."""
> +        if HAVE_DISTLIB_VERSION:
> +            self._m = distlib.version.LegacyMatcher(name_and_constraint)
> +        elif HAVE_PACKAGING_VERSION:
> +            self._m = packaging.requirements.Requirement(name_and_constraint)
> +        else:
> +            raise Ouch("found neither distlib.version nor packaging.version")
> +        self.name = self._m.name
> +
> +    def match(self, version_str: str) -> bool:
> +        """Return True if `version` satisfies the stored constraint."""
> +        if HAVE_DISTLIB_VERSION:
> +            return cast(
> +                bool,
> +                self._m.match(distlib.version.LegacyVersion(version_str))
> +            )
> +
> +        assert HAVE_PACKAGING_VERSION
> +        return cast(
> +            bool,
> +            self._m.specifier.contains(
> +                packaging.version.Version(version_str), prereleases=True
> +            )
> +        )
> +
> +    def __repr__(self) -> str:
> +        """Stable debug representation delegated to the backend."""
> +        return repr(self._m)
> +
> +
>   class QemuEnvBuilder(venv.EnvBuilder):
>       """
>       An extension of venv.EnvBuilder for building QEMU's configure-time venv.
> @@ -669,7 +725,7 @@ def _do_ensure(
>       canary = None
>       for name, info in group.items():
>           constraint = _make_version_constraint(info, False)
> -        matcher = distlib.version.LegacyMatcher(name + constraint)
> +        matcher = Matcher(name + constraint)
>           print(f"mkvenv: checking for {matcher}", file=sys.stderr)
>   
>           dist: Optional[Distribution] = None
> @@ -683,7 +739,7 @@ def _do_ensure(
>               # Always pass installed package to pip, so that they can be
>               # updated if the requested version changes
>               or not _is_system_package(dist)
> -            or not matcher.match(distlib.version.LegacyVersion(dist.version))
> +            or not matcher.match(dist.version)
>           ):
>               absent.append(name + _make_version_constraint(info, True))
>               if len(absent) == 1: