Make recipes beautiful again

appveyor.yml

@@ -6,16 +6,16 @@ environment:
     CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\tools\\appveyor\\run_with_env.cmd"
 
   matrix:
-    - PYTHON: "C:\\Miniconda36-x64"
-      PYTHON_VERSION: "3.6"
-      PYTHON_ARCH: "64"
+    - SYS_PYTHON: "C:\\Miniconda36-x64"
+      SYS_PYTHON_VERSION: "3.6"
+      SYS_PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Miniconda-x64"
-      PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "64"
+    - SYS_PYTHON: "C:\\Miniconda-x64"
+      SYS_PYTHON_VERSION: "2.7"
+      SYS_PYTHON_ARCH: "64"
 
 init:
-  - ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH% %HOME%
+  - ECHO %SYS_PYTHON% %SYS_PYTHON_VERSION% %SYS_PYTHON_ARCH% %HOME%
 
 
 install:
@@ -30,8 +30,8 @@ install:
         throw "There are newer queued builds for this pull request, failing early." }
   # these correspond to folder naming of miniconda installs on appveyor.  See
   # https://www.appveyor.com/docs/installed-software#python
-  - if "%PYTHON_VERSION%" == "3.6" set "BASE_PYTHON_VERSION=36"
-  - if "%PYTHON_ARCH%" == "64" set "ARCH_LABEL=-x64"
+  - if "%SYS_PYTHON_VERSION%" == "3.6" set "BASE_PYTHON_VERSION=36"
+  - if "%SYS_PYTHON_ARCH%" == "64" set "ARCH_LABEL=-x64"
   # These are already installed on appveyor.  Update them.
   - set "CONDA_ROOT=C:\Miniconda%BASE_PYTHON_VERSION%%ARCH_LABEL%"
   - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%"
@@ -44,7 +44,7 @@ install:
   - python -c "import sys; print(sys.prefix)"
   - conda update -q --all
   - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests contextlib2 chardet glob2 perl pyflakes pycrypto posix m2-git anaconda-client numpy conda-verify beautifulsoup4 pytest-xdist pytest-mock filelock pkginfo psutil
-  - if "%PYTHON_VERSION%" == "2.7" conda install -q scandir
+  - if "%SYS_PYTHON_VERSION%" == "2.7" conda install -q scandir
   # this is to ensure dependencies
   - python --version
   - python -c "import struct; print(struct.calcsize('P') * 8)"
@@ -71,7 +71,6 @@ test_script:
   - mkdir C:\cbtmp
   # unset other language env vars - we only want to test if conda-build sets them itself
   - set PERL=
-  - set PYTHON=
   - set LUA=
   - set R=
   # cache several files to avoid race condition errors
@@ -81,7 +80,7 @@ test_script:
   - conda create -n blarg -yq --download-only python=3.4 libpng=1.6.17
   - conda create -n blarg -yq --download-only python=3.5 libpng=1.6.17
   # no libpng here because we already have the right version from the 3.5 install
-  - conda create -n blarg -yq --download-only python=3.6
+  - conda create -n blarg -yq --download-only python
   - conda create -n blarg -yq --download-only cmake
   - py.test --color=yes -v --cov conda_build --cov-report xml tests --basetemp C:\cbtmp -n 0 -m "serial"
   # remove all folders to avoid permission errors.  Leave root files (may have coverage info there)
@@ -96,4 +95,4 @@ test_script:
 
 on_success:
   - conda install -c conda-forge codecov
-  - codecov --env PYTHON_VERSION --file C:\projects\conda-build\coverage.xml
+  - codecov --env SYS_PYTHON_VERSION --file C:\projects\conda-build\coverage.xml

ci/travis/run.sh

@@ -19,7 +19,7 @@ else
     conda create -n blarg1 -yq python=2.7
     conda create -n blarg2 -yq python=3.4
     conda create -n blarg3 -yq python=3.5
-    conda create -n blarg4 -yq python=3.6
+    conda create -n blarg4 -yq python numpy
     conda create -n blarg5 -yq libpng=1.6.17
     /opt/conda/bin/py.test -v -n 0 --basetemp /tmp/cb --cov conda_build --cov-report xml -m "serial" tests
     /opt/conda/bin/py.test -v -n 2 --basetemp /tmp/cb --cov conda_build --cov-append --cov-report xml -m "not serial" tests --forked

conda_build/build.py

@@ -1398,6 +1398,10 @@ def build(m, stats, post=None, need_source_download=True, need_reparse_in_env=Fa
                         env = environ.get_dict(m=m)
                     env["CONDA_BUILD_STATE"] = "BUILD"
 
+                    # hard-code this because we never want pip's build isolation
+                    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
+                    env["PIP_NO_BUILD_ISOLATION"] = False
+
                     work_file = join(m.config.work_dir, 'conda_build.sh')
                     with open(work_file, 'w') as bf:
                         for k, v in env.items():

conda_build/environ.py

@@ -242,18 +242,18 @@ def get_hg_build_info(repo):
     return d
 
 
-def get_dict(m, prefix=None, for_env=True, skip_build_id=False):
+def get_dict(m, prefix=None, for_env=True, skip_build_id=False, escape_backslash=False):
     if not prefix:
         prefix = m.config.host_prefix
 
     # conda-build specific vars
     d = conda_build_vars(prefix, m.config)
 
     # languages
-    d.update(python_vars(m, prefix))
-    d.update(perl_vars(m, prefix))
-    d.update(lua_vars(m, prefix))
-    d.update(r_vars(m, prefix))
+    d.update(python_vars(m, prefix, escape_backslash))
+    d.update(perl_vars(m, prefix, escape_backslash))
+    d.update(lua_vars(m, prefix, escape_backslash))
+    d.update(r_vars(m, prefix, escape_backslash))
 
     if m:
         d.update(meta_vars(m, skip_build_id=skip_build_id))
@@ -302,21 +302,33 @@ def conda_build_vars(prefix, config):
     }
 
 
-def python_vars(metadata, prefix):
+def python_vars(metadata, prefix, escape_backslash):
     py_ver = get_py_ver(metadata.config)
+    stdlib_dir = utils.get_stdlib_dir(prefix, py_ver)
+    sp_dir = utils.get_site_packages(prefix, py_ver)
+
+    if utils.on_win and escape_backslash:
+        stdlib_dir = stdlib_dir.replace('\\', '\\\\')
+        sp_dir = sp_dir.replace('\\', '\\\\')
+
     vars_ = {
             'CONDA_PY': ''.join(py_ver.split('.')[:2]),
             'PY3K': str(int(int(py_ver[0]) >= 3)),
             'PY_VER': py_ver,
-            'STDLIB_DIR': utils.get_stdlib_dir(prefix, py_ver),
-            'SP_DIR': utils.get_site_packages(prefix, py_ver),
+            'STDLIB_DIR': stdlib_dir,
+            'SP_DIR': sp_dir,
             }
     build_or_host = 'host' if metadata.is_cross else 'build'
     deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)]
     if 'python' in deps or metadata.name(fail_ok=True) == 'python':
+        python_bin = metadata.config.python_bin(prefix, metadata.config.host_subdir)
+
+        if utils.on_win and escape_backslash:
+            python_bin = python_bin.replace('\\', '\\\\')
+
         vars_.update({
             # host prefix is always fine, because it is the same as build when is_cross is False
-            'PYTHON': metadata.config.python_bin(prefix, metadata.config.host_subdir),
+            'PYTHON': python_bin,
         })
 
     np_ver = metadata.config.variant.get('numpy', get_default_variant(metadata.config)['numpy'])
@@ -326,38 +338,49 @@ def python_vars(metadata, prefix):
     return vars_
 
 
-def perl_vars(metadata, prefix):
+def perl_vars(metadata, prefix, escape_backslash):
     vars_ = {
             'PERL_VER': get_perl_ver(metadata.config),
             'CONDA_PERL': get_perl_ver(metadata.config),
              }
     build_or_host = 'host' if metadata.is_cross else 'build'
     deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)]
     if 'perl' in deps or metadata.name(fail_ok=True) == 'perl':
+        perl_bin = metadata.config.perl_bin(prefix, metadata.config.host_subdir)
+
+        if utils.on_win and escape_backslash:
+            perl_bin = perl_bin.replace('\\', '\\\\')
+
         vars_.update({
             # host prefix is always fine, because it is the same as build when is_cross is False
-            'PERL': metadata.config.perl_bin(prefix, metadata.config.host_subdir),
+            'PERL': perl_bin,
         })
     return vars_
 
 
-def lua_vars(metadata, prefix):
+def lua_vars(metadata, prefix, escape_backslash):
     vars_ = {
             'LUA_VER': get_lua_ver(metadata.config),
             'CONDA_LUA': get_lua_ver(metadata.config),
              }
     build_or_host = 'host' if metadata.is_cross else 'build'
     deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)]
     if 'lua' in deps:
+        lua_bin = metadata.config.lua_bin(prefix, metadata.config.host_subdir)
+        lua_include_dir = get_lua_include_dir(metadata.config)
+
+        if utils.on_win and escape_backslash:
+            lua_bin = lua_bin.replace('\\', '\\\\')
+            lua_include_dir = lua_include_dir.replace('\\', '\\\\')
+
         vars_.update({
-            'LUA': metadata.config.lua_bin(prefix,
-                                           metadata.config.host_subdir),
-            'LUA_INCLUDE_DIR': get_lua_include_dir(metadata.config),
+            'LUA': lua_bin,
+            'LUA_INCLUDE_DIR': lua_include_dir,
         })
     return vars_
 
 
-def r_vars(metadata, prefix):
+def r_vars(metadata, prefix, escape_backslash):
     vars_ = {
             'R_VER': get_r_ver(metadata.config),
             'CONDA_R': get_r_ver(metadata.config),
@@ -367,8 +390,13 @@ def r_vars(metadata, prefix):
     deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)]
     if 'r-base' in deps or 'mro-base' in deps or metadata.name(fail_ok=True) in (
             'r-base', 'mro-base'):
+        r_bin = metadata.config.r_bin(prefix, metadata.config.host_subdir)
+
+        if utils.on_win and escape_backslash:
+            r_bin = r_bin.replace('\\', '\\\\')
+
         vars_.update({
-            'R': metadata.config.r_bin(prefix, metadata.config.host_subdir),
+            'R': r_bin,
         })
     return vars_
 

conda_build/jinja_context.py

@@ -492,7 +492,8 @@ def context_processor(initial_metadata, recipe_dir, config, permit_undefined_jin
     initial_metadata: Augment the context with values from this MetaData object.
                       Used to bootstrap metadata contents via multiple parsing passes.
     """
-    ctx = get_environ(m=initial_metadata, for_env=False, skip_build_id=skip_build_id)
+    ctx = get_environ(m=initial_metadata, for_env=False, skip_build_id=skip_build_id,
+                      escape_backslash=True)
     environ = dict(os.environ)
     environ.update(get_environ(m=initial_metadata, skip_build_id=skip_build_id))
 

conda_build/metadata.py

@@ -38,8 +38,6 @@
 except:
     loader = yaml.Loader
 
-from yaml.resolver import Resolver
-
 on_win = (sys.platform == 'win32')
 
 # arches that don't follow exact names in the subdir need to be mapped here
@@ -220,6 +218,8 @@ def yamlize(data):
                 jinja2  # Avoid pyflakes failure: 'jinja2' imported but unused
             except ImportError:
                 raise exceptions.UnableToParseMissingJinja2(original=e)
+        print("Problematic recipe:", file=sys.stderr)
+        print(data, file=sys.stderr)
         raise exceptions.UnableToParse(original=e)
 
 
@@ -801,12 +801,14 @@ def trim_build_only_deps(metadata, requirements_used):
 def stringify_numbers():
     # ensure that numbers are not interpreted as ints or floats.  That trips up versions
     #     with trailing zeros.
-    implicit_resolver_backup = Resolver.yaml_implicit_resolvers.copy()
-    for ch in list(u'-+0123456789'):
-        del Resolver.yaml_implicit_resolvers[ch]
+    implicit_resolver_backup = loader.yaml_implicit_resolvers.copy()
+    for ch in list(u'0123456789'):
+        if ch in loader.yaml_implicit_resolvers:
+            del loader.yaml_implicit_resolvers[ch]
     yield
-    for ch in list(u'-+0123456789'):
-        Resolver.yaml_implicit_resolvers[ch] = implicit_resolver_backup[ch]
+    for ch in list(u'0123456789'):
+        if ch in implicit_resolver_backup:
+            loader.yaml_implicit_resolvers[ch] = implicit_resolver_backup[ch]
 
 
 class MetaData(object):
@@ -2093,13 +2095,13 @@ def _get_used_vars_meta_yaml(self, force_top_level=False, force_global=False):
         # make variant dict hashable so that memoization works
         variant_keys = tuple(sorted(self.config.variant.keys()))
 
-        reqs_text, recipe_text = self._get_used_vars_meta_yaml_helper(force_top_level=force_top_level,
-                                    force_global=force_global, apply_selectors=False)
+        reqs_text, recipe_text = self._get_used_vars_meta_yaml_helper(
+            force_top_level=force_top_level, force_global=force_global, apply_selectors=False)
         all_used_selectors = variants.find_used_variables_in_text(variant_keys, recipe_text,
                                                                     selectors=True)
 
-        reqs_text, recipe_text = self._get_used_vars_meta_yaml_helper(force_top_level=force_top_level,
-                                    force_global=force_global, apply_selectors=True)
+        reqs_text, recipe_text = self._get_used_vars_meta_yaml_helper(
+            force_top_level=force_top_level, force_global=force_global, apply_selectors=True)
         all_used_reqs = variants.find_used_variables_in_text(variant_keys, recipe_text,
                                                                     selectors=False)