Support PEP 600 tags

Author: mattip

packaging/tags.py

@@ -13,6 +13,7 @@
 
     EXTENSION_SUFFIXES = [x[0] for x in imp.get_suffixes()]
     del imp
+import collections
 import logging
 import os
 import platform
@@ -57,6 +58,24 @@
 _32_BIT_INTERPRETER = sys.maxsize <= 2 ** 32
 
 
+_LEGACY_MANYLINUX_MAP = {
+    # CentOS 7 w/ glibc 2.17 (PEP 599)
+    (2, 17): "manylinux2014",
+    # CentOS 6 w/ glibc 2.12 (PEP 571)
+    (2, 12): "manylinux2010",
+    # CentOS 5 w/ glibc 2.5 (PEP 513)
+    (2, 5): "manylinux1",
+}
+
+# If glibc ever changes its major version, we need to know what the last
+# minor version was, so we can build the complete list of all versions.
+# For now, guess what the highest minor version might be, assume it will
+# be 50 for testing. Once this actually happens, update the dictionary
+# with the actual value.
+_LAST_GLIBC_MINOR = collections.defaultdict(lambda: 50)  # type: Dict[int, int]
+glibcVersion = collections.namedtuple("Version", ["major", "minor"])
+
+
 class Tag(object):
     """
     A representation of the tag triple for a wheel.
@@ -494,9 +513,16 @@ def _glibc_version_string_ctypes():
 
 
 # Separated out from have_compatible_glibc for easier unit testing.
-def _check_glibc_version(version_str, required_major, minimum_minor):
+def _check_glibc_version(version_str, tag_major, tag_minor):
     # type: (str, int, int) -> bool
-    # Parse string and check against requested version.
+    # Check against requested version.
+    sys_major, sys_minor = _parse_glibc_version(version_str)
+    return (sys_major, sys_minor) >= (tag_major, tag_minor)
+
+
+def _parse_glibc_version(version_str):
+    # type: (str) -> Tuple[int, int]
+    # Parse glibc version
     #
     # We use a regexp instead of str.split because we want to discard any
     # random junk that might come after the minor version -- this might happen
@@ -509,19 +535,24 @@ def _check_glibc_version(version_str, required_major, minimum_minor):
             " got: %s" % version_str,
             RuntimeWarning,
         )
-        return False
-    return (
-        int(m.group("major")) == required_major
-        and int(m.group("minor")) >= minimum_minor
-    )
+        return -1, -1
+    return (int(m.group("major")), int(m.group("minor")))
 
 
-def _have_compatible_glibc(required_major, minimum_minor):
+def _have_compatible_glibc(tag_major, tag_minor):
     # type: (int, int) -> bool
     version_str = _glibc_version_string()
     if version_str is None:
         return False
-    return _check_glibc_version(version_str, required_major, minimum_minor)
+    return _check_glibc_version(version_str, tag_major, tag_minor)
+
+
+def _get_glibc_version():
+    # type: () -> Tuple[int, int]
+    version_str = _glibc_version_string()
+    if version_str is None:
+        return -1, -1
+    return _parse_glibc_version(version_str)
 
 
 # Python does not provide platform information at sufficient granularity to
@@ -639,7 +670,49 @@ def _have_compatible_manylinux_abi(arch):
         return _is_linux_armhf()
     if arch == "i686":
         return _is_linux_i686()
-    return True
+    return arch in {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x"}
+
+
+def _manylinux_tags(linux, arch):
+    # type: (str, str) -> Iterator[str]
+    # Oldest glibc to be supported is (2, 17).
+    too_old_glibc2 = glibcVersion(2, 16)
+    if arch in {"x86_64", "i686"}:
+        # On x86/i686 also oldest glibc to be supported is (2, 5).
+        too_old_glibc2 = glibcVersion(2, 4)
+    current_glibc = glibcVersion(*_get_glibc_version())
+    is_compat = False
+    glibc_max_list = [current_glibc]
+    # We can assume compatibility across glibc major versions.
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=24636
+    #
+    # Build a list of maximum glibc versions so that we can
+    # output the canonical list of all glibc from current_glibc
+    # down to too_old_glibc2, including all intermediary versions
+    for glibc_major in range(current_glibc.major - 1, 1, -1):
+        glibc_max_list.append(glibcVersion(glibc_major, _LAST_GLIBC_MINOR[glibc_major]))
+    for glibc_max in glibc_max_list:
+        if glibc_max.major == too_old_glibc2.major:
+            min_minor = too_old_glibc2.minor
+        else:
+            # For other glibc major versions oldest supported is (x, 0).
+            min_minor = -1
+        for glibc_minor in range(glibc_max.minor, min_minor, -1):
+            glibc_version = (glibc_max.major, glibc_minor)
+            tag = "manylinux_{}_{}".format(*glibc_version)
+            # Once _is_manylinux_compatible() is True, it is True for any
+            # lower manylinux tag for this glibc major version.
+            is_compat = is_compat or _is_manylinux_compatible(tag, glibc_version)
+            if is_compat:
+                yield linux.replace("linux", tag)
+            # Handle the manylinux1, manylinux2010, manylinux2014 tags.
+            if glibc_version in _LEGACY_MANYLINUX_MAP:
+                legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version]
+                is_compat = is_compat or _is_manylinux_compatible(
+                    legacy_tag, glibc_version
+                )
+                if is_compat:
+                    yield linux.replace("linux", legacy_tag)
 
 
 def _linux_platforms(is_32bit=_32_BIT_INTERPRETER):
@@ -650,28 +723,10 @@ def _linux_platforms(is_32bit=_32_BIT_INTERPRETER):
             linux = "linux_i686"
         elif linux == "linux_aarch64":
             linux = "linux_armv7l"
-    manylinux_support = []
     _, arch = linux.split("_", 1)
     if _have_compatible_manylinux_abi(arch):
-        if arch in {"x86_64", "i686", "aarch64", "armv7l", "ppc64", "ppc64le", "s390x"}:
-            manylinux_support.append(
-                ("manylinux2014", (2, 17))
-            )  # CentOS 7 w/ glibc 2.17 (PEP 599)
-        if arch in {"x86_64", "i686"}:
-            manylinux_support.append(
-                ("manylinux2010", (2, 12))
-            )  # CentOS 6 w/ glibc 2.12 (PEP 571)
-            manylinux_support.append(
-                ("manylinux1", (2, 5))
-            )  # CentOS 5 w/ glibc 2.5 (PEP 513)
-    manylinux_support_iter = iter(manylinux_support)
-    for name, glibc_version in manylinux_support_iter:
-        if _is_manylinux_compatible(name, glibc_version):
-            yield linux.replace("linux", name)
-            break
-    # Support for a later manylinux implies support for an earlier version.
-    for name, _ in manylinux_support_iter:
-        yield linux.replace("linux", name)
+        for tag in _manylinux_tags(linux, arch):
+            yield tag
     yield linux
 
 

tests/test_tags.py

@@ -326,7 +326,7 @@ def test_is_manylinux_compatible_glibc_support(
             ("2.4", 2, 4, True),
             ("2.4", 2, 5, False),
             ("2.4", 2, 3, True),
-            ("3.4", 2, 4, False),
+            ("3.4", 2, 4, True),
         ],
     )
     def test_check_glibc_version(self, version_str, major, minor, expected):
@@ -373,6 +373,13 @@ def test_glibc_version_string_confstr(self, monkeypatch):
         monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         assert tags._glibc_version_string_confstr() == "2.20"
 
+    def test_glibc_version_string_fail(self, monkeypatch):
+        monkeypatch.setattr(os, "confstr", lambda x: None, raising=False)
+        monkeypatch.setitem(sys.modules, "ctypes", None)
+        assert tags._glibc_version_string() is None
+        assert tags._have_compatible_glibc(2, 5) is False
+        assert tags._get_glibc_version() == (-1, -1)
+
     @pytest.mark.parametrize(
         "failure",
         [pretend.raiser(ValueError), pretend.raiser(OSError), lambda x: "XXX"],
@@ -433,12 +440,14 @@ def test_linux_platforms_32_64bit_on_64bit_os(
         self, arch, is_32bit, expected, monkeypatch
     ):
         monkeypatch.setattr(distutils.util, "get_platform", lambda: arch)
+        monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         monkeypatch.setattr(tags, "_is_manylinux_compatible", lambda *args: False)
         linux_platform = list(tags._linux_platforms(is_32bit=is_32bit))[-1]
         assert linux_platform == expected
 
     def test_linux_platforms_manylinux_unsupported(self, monkeypatch):
         monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_x86_64")
+        monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         monkeypatch.setattr(tags, "_is_manylinux_compatible", lambda *args: False)
         linux_platform = list(tags._linux_platforms(is_32bit=False))
         assert linux_platform == ["linux_x86_64"]
@@ -450,6 +459,7 @@ def test_linux_platforms_manylinux1(self, is_x86, monkeypatch):
         if platform.system() != "Linux" or not is_x86:
             monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_x86_64")
             monkeypatch.setattr(platform, "machine", lambda: "x86_64")
+            monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         platforms = list(tags._linux_platforms(is_32bit=False))
         arch = platform.machine()
         assert platforms == ["manylinux1_" + arch, "linux_" + arch]
@@ -461,9 +471,21 @@ def test_linux_platforms_manylinux2010(self, is_x86, monkeypatch):
         if platform.system() != "Linux" or not is_x86:
             monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_x86_64")
             monkeypatch.setattr(platform, "machine", lambda: "x86_64")
+            monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         platforms = list(tags._linux_platforms(is_32bit=False))
         arch = platform.machine()
-        expected = ["manylinux2010_" + arch, "manylinux1_" + arch, "linux_" + arch]
+        expected = [
+            "manylinux2010_" + arch,
+            "manylinux_2_11_" + arch,
+            "manylinux_2_10_" + arch,
+            "manylinux_2_9_" + arch,
+            "manylinux_2_8_" + arch,
+            "manylinux_2_7_" + arch,
+            "manylinux_2_6_" + arch,
+            "manylinux_2_5_" + arch,
+            "manylinux1_" + arch,
+            "linux_" + arch,
+        ]
         assert platforms == expected
 
     def test_linux_platforms_manylinux2014(self, is_x86, monkeypatch):
@@ -473,17 +495,31 @@ def test_linux_platforms_manylinux2014(self, is_x86, monkeypatch):
         if platform.system() != "Linux" or not is_x86:
             monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_x86_64")
             monkeypatch.setattr(platform, "machine", lambda: "x86_64")
+            monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         platforms = list(tags._linux_platforms(is_32bit=False))
         arch = platform.machine()
         expected = [
             "manylinux2014_" + arch,
+            "manylinux_2_16_" + arch,
+            "manylinux_2_15_" + arch,
+            "manylinux_2_14_" + arch,
+            "manylinux_2_13_" + arch,
+            "manylinux_2_12_" + arch,
             "manylinux2010_" + arch,
+            "manylinux_2_11_" + arch,
+            "manylinux_2_10_" + arch,
+            "manylinux_2_9_" + arch,
+            "manylinux_2_8_" + arch,
+            "manylinux_2_7_" + arch,
+            "manylinux_2_6_" + arch,
+            "manylinux_2_5_" + arch,
             "manylinux1_" + arch,
             "linux_" + arch,
         ]
         assert platforms == expected
 
     def test_linux_platforms_manylinux2014_armhf_abi(self, monkeypatch):
+        monkeypatch.setattr(tags, "_glibc_version_string", lambda: "2.30")
         monkeypatch.setattr(
             tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"
         )
@@ -498,6 +534,7 @@ def test_linux_platforms_manylinux2014_armhf_abi(self, monkeypatch):
         assert platforms == expected
 
     def test_linux_platforms_manylinux2014_i386_abi(self, monkeypatch):
+        monkeypatch.setattr(tags, "_glibc_version_string", lambda: "2.20")
         monkeypatch.setattr(
             tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"
         )
@@ -509,18 +546,50 @@ def test_linux_platforms_manylinux2014_i386_abi(self, monkeypatch):
         )
         platforms = list(tags._linux_platforms(is_32bit=True))
         expected = [
+            # "manylinux_2_17_i686",  # rejected since it comes before manylinux2014
             "manylinux2014_i686",
+            "manylinux_2_16_i686",
+            "manylinux_2_15_i686",
+            "manylinux_2_14_i686",
+            "manylinux_2_13_i686",
+            "manylinux_2_12_i686",
             "manylinux2010_i686",
+            "manylinux_2_11_i686",
+            "manylinux_2_10_i686",
+            "manylinux_2_9_i686",
+            "manylinux_2_8_i686",
+            "manylinux_2_7_i686",
+            "manylinux_2_6_i686",
+            "manylinux_2_5_i686",
             "manylinux1_i686",
             "linux_i686",
         ]
         assert platforms == expected
 
+    def test_linux_platforms_manylinux_glibc3(self, monkeypatch):
+        # test for a future glic 3.x version
+        monkeypatch.setattr(tags, "_glibc_version_string", lambda: "3.2")
+        monkeypatch.setattr(tags, "_is_manylinux_compatible", lambda name, _: True)
+        monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_aarch64")
+        monkeypatch.setattr(
+            sys,
+            "executable",
+            os.path.join(os.path.dirname(__file__), "hello-world-aarch64"),
+        )
+        platforms = list(tags._linux_platforms())
+        expected = (
+            ["manylinux_3_2_aarch64", "manylinux_3_1_aarch64", "manylinux_3_0_aarch64"]
+            + ["manylinux_2_{}_aarch64".format(i) for i in range(50, 16, -1)]
+            + ["manylinux2014_aarch64", "linux_aarch64"]
+        )
+        assert platforms == expected
+
     def test_linux_platforms_manylinux2014_armv6l(self, monkeypatch):
         monkeypatch.setattr(
             tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"
         )
         monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_armv6l")
+        monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False)
         platforms = list(tags._linux_platforms(is_32bit=True))
         expected = ["linux_armv6l"]
         assert platforms == expected