Fix Python 2.7 related CI issues (#4232)

* CI: Remove some Py2.7 specifics

* Add pip to PyPI skeleton's run_setuppy env

* Add limited support for platform_system/sys_platform env markers in PyPI skeleton

* lint

* CI: Travis: install pip

Author: mbargull

.travis.yml

@@ -31,7 +31,7 @@ install:
   - conda info
   # avoids a python 3.7 problem
   - conda install -q cytoolz
-  - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf ripgrep pyflakes beautifulsoup4 chardet pycrypto glob2 psutil pytz tqdm conda-package-handling py-lief python-libarchive-c
+  - conda install -q anaconda-client requests filelock contextlib2 jinja2 patchelf ripgrep pyflakes beautifulsoup4 chardet pycrypto glob2 psutil pytz tqdm conda-package-handling py-lief python-libarchive-c pip
   - pip install pkginfo
   - if [[ "$FLAKE8" == "true" ]]; then
       conda install -q flake8;

azure-pipelines.yml

@@ -38,12 +38,7 @@ jobs:
       ps -ef | grep $$
       conda list
       grep '^#' "${CONDA_PREFIX}/conda-meta/history"
-      if [ "$PYTHON_VERSION" = "2.7" ]; then
-        # No Python 2.7 builds after conda=4.8.4 available.
-        conda install python=$PYTHON_VERSION conda\<=4.8.4 -y
-      else
-        conda install python=$PYTHON_VERSION -y
-      fi
+      conda install python=$PYTHON_VERSION -y
       mkdir -p /usr/share/miniconda/locks
       mkdir -p /usr/share/miniconda/bin
       chmod a-w /usr/share/miniconda/locks
@@ -89,7 +84,7 @@ jobs:
 
   - publish: $(Build.ArtifactStagingDirectory)/pytest-replay
     artifact: 'Linux-$(CONDA_VERSION)-Py$(PYTHON_VERSION)'
-    condition: and(always(), ne(variables['PYTHON_VERSION'], '2.7'))
+    condition: always()
 
 
 - job: 'macOS'
@@ -153,12 +148,7 @@ jobs:
       conda info
       conda list
       grep '^#' "${CONDA_PREFIX}/conda-meta/history"
-      if [ "$PYTHON_VERSION" = "2.7" ]; then
-        # No Python 2.7 builds after conda=4.8.4 available.
-        conda install python=$PYTHON_VERSION conda\<=4.8.4 -y
-      else
-        conda install python=$PYTHON_VERSION -y
-      fi
+      conda install python=$PYTHON_VERSION -y
       mkdir -p $(Build.StagingDirectory)/miniconda/locks
       mkdir -p $(Build.StagingDirectory)/miniconda/bin
       chmod -w $(Build.StagingDirectory)/miniconda/locks
@@ -205,7 +195,7 @@ jobs:
 
   - publish: $(Build.ArtifactStagingDirectory)/pytest-replay
     artifact: 'macOS-$(CONDA_VERSION)-Py$(PYTHON_VERSION)'
-    condition: and(always(), ne(variables['PYTHON_VERSION'], '2.7'))
+    condition: always()
 
 
 - job: 'Windows'
@@ -237,8 +227,14 @@ jobs:
     inputs:
       versionSpec: '$(PYTHON_VERSION)'
 
-  - script: |
-      choco install vcpython27 -fdv -y --debug
+  # SHA256 checksum (070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf) is identical for
+  # https://web.archive.org/web/20210106040224/https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi
+  # and
+  # https://github.com/GeoNode/geonode-win-installer/raw/ffb76c7cbf1d6b4970c6c25f79c3c7682a3aa035/VCForPython27.msi
+  - powershell: |
+      $wc = New-Object net.webclient
+      $wc.Downloadfile("https://github.com/GeoNode/geonode-win-installer/raw/ffb76c7cbf1d6b4970c6c25f79c3c7682a3aa035/VCForPython27.msi", "VCForPython27.msi")
+      Start-Process "VCForPython27.msi" /qn -Wait
     displayName: Install vcpython27.msi
 
   - script: |
@@ -281,8 +277,7 @@ jobs:
       set PATH
       call %UserProfile%\Miniconda3\condabin\activate.bat base||exit 1
       set PATH
-      :: No Python 2.7 builds after conda=4.8.4 available.
-      if "%PYTHON_VERSION%" == "2.7" (call conda install python="%PYTHON_VERSION%" "conda<=4.8.4" -y||exit 1) else (call conda install python="%PYTHON_VERSION%" -y||exit 1)
+      call conda install python="%PYTHON_VERSION%" -y||exit 1
       if "%CONDA_VERSION%" == "canary" (call conda update -c conda-canary conda||exit 1) else (call conda update -q conda||exit 1)
       call conda config --set always_yes yes
       call conda config --set auto_update_conda no
@@ -292,12 +287,10 @@ jobs:
       python -c "import sys; print(sys.prefix)"
       call conda update -q --all||exit 1
       call conda install -q pip python-libarchive-c pytest git pytest-cov jinja2 m2-patch flake8 mock requests contextlib2 chardet glob2 perl pyflakes pycrypto posix m2-git anaconda-client numpy beautifulsoup4 pytest-xdist pytest-mock filelock pkginfo psutil pytz tqdm conda-package-handling pytest-azurepipelines||exit 1
-      if "%PYTHON_VERSION%" == "2.7" call conda install -c defaults -c conda-forge pytest-replay pytest-rerunfailures "coverage<5.0|>5.0" -y||exit 1
-      if "%PYTHON_VERSION%" NEQ "2.7" call conda install pytest-replay pytest-rerunfailures -y||exit 1
+      call conda install pytest-replay pytest-rerunfailures -y||exit 1
       echo safety_checks: disabled >> %UserProfile%\.condarc
       echo local_repodata_ttl: 1800 >> %UserProfile%\.condarc
-      if "%PYTHON_VERSION%" == "2.7" call conda install -q scandir pathlib2||exit 1
-      if "%PYTHON_VERSION%" NEQ "2.7" call conda install -q py-lief||exit 1
+      call conda install -q py-lief||exit 1
       python --version
       python -c "import struct; print(struct.calcsize('P') * 8)"
       pip install --no-deps .
@@ -311,8 +304,7 @@ jobs:
       call conda create -n blarg -yq --download-only python=3.8||exit 1
       call conda create -n blarg -yq --download-only python cmake||exit 1
       mkdir $(Build.ArtifactStagingDirectory)\\pytest-replay
-      set "PYTEST_REPLAY_OPTIONS=--replay-record-dir=$(Build.ArtifactStagingDirectory)\\pytest-replay"
-      if "%PYTHON_VERSION%" NEQ "2.7" set "PYTEST_REPLAY_OPTIONS=--replay-record-dir=$(Build.ArtifactStagingDirectory)\\pytest-replay --replay-base-name=Win-%CONDA_VERSION%-Py%PYTHON_VERSION%"
+      set "PYTEST_REPLAY_OPTIONS=--replay-record-dir=$(Build.ArtifactStagingDirectory)\\pytest-replay --replay-base-name=Win-%CONDA_VERSION%-Py%PYTHON_VERSION%"
       echo "##vso[task.setvariable variable=PYTEST_REPLAY_OPTIONS]%PYTEST_REPLAY_OPTIONS%"
     displayName: 'Configuration'
 

conda_build/skeletons/pypi.py

@@ -667,35 +667,58 @@ def convert_version(version):
     return pin_compatible
 
 
-MARKER_RE = re.compile(r"(?P<name>^[^=<>!~\s]+)"
+MARKER_RE = re.compile(r"(?P<name>^[^=<>!~\s;]+)"
                        r"\s*"
                        r"(?P<constraint>[=!><~]=?\s*[^\s;]+)?"
-                       r"(?:\s+;\s+)?(?P<env_mark_name>[^=<>!~\s;]+)?"
+                       r"(?:\s*;\s+)?(?P<env_mark_name>[^=<>!~\s;]+)?"
                        r"\s*"
                        r"(?P<env_mark_constraint>[=<>!\s]+[^=<>!~\s]+)?"
                        )
 
 
+def _get_env_marker_operator_and_value(constraint):
+    operator, value = constraint.split()
+    value = value.strip("'").strip('"').strip()
+    return operator, value
+
+
 def _translate_python_constraint(constraint):
-    parts = constraint.split()
     translation = constraint
-    if len(parts) == 2:
-        operator, value = parts
-        value = "".join(value.strip("'").strip('"').split(".")[:2])
+    if len(constraint.split()) == 2:
+        operator, value = _get_env_marker_operator_and_value(constraint)
+        value = "".join(value.split(".")[:2])
         translation = " ".join((operator, value))
-    return translation
+    return "py {}".format(translation)
+
+
+def _translate_platform_system_constraint(constraint):
+    operator, value = _get_env_marker_operator_and_value(constraint)
+    system = {
+        "Linux": "linux",
+        "Darwin": "osx",
+        "Windows": "win",
+    }.get(value, value.lower())
+    return "{}{}".format("not " if operator == "!=" else "", system)
+
+
+def _translate_sys_platform_constraint(constraint):
+    operator, value = _get_env_marker_operator_and_value(constraint)
+    # Only take the "letter" part to translate, e.g., "linux2"->"linux", "win32"->"win".
+    system = re.match('^[a-z]*', value, re.I)[0]
+    return "{}{}".format("not " if operator == "!=" else "", system)
 
 
 def env_mark_lookup(env_mark_name, env_mark_constraint):
     """returns translated variable name and corresponding function to run to normalize the
     version constraint to conda style"""
     # TODO: implement more of these from PEP 508 as necessary:
     #   https://www.python.org/dev/peps/pep-0508/
-    env_mark_table = {'python_version': {"repl": "py",
-                                         "constraint_trans_fn": _translate_python_constraint},
-                      }
-    marker = " ".join((env_mark_table[env_mark_name]["repl"],
-                    env_mark_table[env_mark_name]['constraint_trans_fn'](env_mark_constraint)))
+    env_mark_table = {
+        "python_version": _translate_python_constraint,
+        "platform_system": _translate_platform_system_constraint,
+        "sys_platform": _translate_sys_platform_constraint,
+    }
+    marker = env_mark_table[env_mark_name](env_mark_constraint)
     return '  # [ ' + marker + ' ]'
 
 
@@ -1201,7 +1224,7 @@ def run_setuppy(src_dir, temp_dir, python_version, extra_specs, config, setup_op
     # specs = ['python %s*' % python_version, 'pyyaml', 'setuptools', 'six', 'packaging', 'appdirs']
     subdir = config.host_subdir
     specs = ['python {}*'.format(python_version),
-             'pyyaml', 'setuptools'] + (['m2-patch', 'm2-gcc-libs'] if config.host_subdir.startswith('win')
+             'pip', 'pyyaml', 'setuptools'] + (['m2-patch', 'm2-gcc-libs'] if config.host_subdir.startswith('win')
                                                     else ['patch'])
     with open(os.path.join(src_dir, "setup.py")) as setup:
         text = setup.read()

news/gh-4232-Add-limited-support-for-platform_system-sys_platform-env-markers-in-PyPI-skeleton.rst

@@ -0,0 +1,25 @@
+Enhancements:
+-------------
+
+* Add limited support for platform_system/sys_platform env markers in PyPI skeleton
+
+Bug fixes:
+----------
+
+* <news item>
+
+Deprecations:
+-------------
+
+* <news item>
+
+Docs:
+-----
+
+* <news item>
+
+Other:
+------
+
+* <news item>
+

tests/test_published_examples.py

@@ -13,12 +13,11 @@
 
 @pytest.mark.sanity
 def test_skeleton_pypi(testing_workdir):
-    """published in docs at http://conda.pydata.org/docs/build_tutorials/pkgs.html"""
+    """published in docs at https://docs.conda.io/projects/conda-build/en/latest/user-guide/tutorials/build-pkgs-skeleton.html"""
     conda_path = os.path.join(sys.prefix, 'Scripts' if sys.platform == 'win32' else 'bin', 'conda')
-    # Note, if you use Click here instead it will fail.
-    cmd = conda_path + ' skeleton pypi Flask'
+    cmd = conda_path + ' skeleton pypi click'
     check_call_env(cmd.split())
-    cmd = conda_path + ' build flask'
+    cmd = conda_path + ' build click'
     check_call_env(cmd.split())