Merge pull request #1102 from msarahan/skip_external_repos

allow skip-existing to examine any url in condarc

conda_build/build.py

@@ -334,12 +334,13 @@ def create_info_files(m, files, include_recipe=True):
                         join(config.info_dir, 'icon.png'))
 
 
-def get_build_index(clear_cache=True):
+def get_build_index(clear_cache=True, arg_channels=None):
     if clear_cache:
         # remove the cache such that a refetch is made,
         # this is necessary because we add the local build repo URL
         fetch_index.cache = {}
-    return get_index(channel_urls=[url_path(config.croot)] + list(channel_urls),
+    arg_channels = [] if not arg_channels else arg_channels
+    return get_index(channel_urls=[url_path(config.croot)] + list(channel_urls) + arg_channels,
                      prepend=not override_channels)
 
 
@@ -475,7 +476,8 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
                 vcs_source = m.uses_vcs_in_build()
                 if vcs_source and vcs_source not in specs:
                     vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
-                    has_vcs_available = os.path.isfile(external.find_executable(vcs_executable) or "")
+                    has_vcs_available = os.path.isfile(external.find_executable(vcs_executable) or
+                                                       "")
                     if not has_vcs_available:
                         if (vcs_source != "mercurial" or
                                 not any(spec.startswith('python') and "3." in spec

conda_build/index.py

@@ -76,7 +76,7 @@ def update_index(dir_path, verbose=False, force=False, check_md5=False, remove=T
         sys.exit("""\
 Error:
     Indexing a copy of the Anaconda conda package channel is neither
-    necessary nor supported.  If you which to add your own packages,
+    necessary nor supported.  If you wish to add your own packages,
     you can do so by adding them to a separate channel.
 """)
     for fn in files:

conda_build/main_build.py

@@ -149,9 +149,12 @@ def handle_binstar_upload(path, args):
     from conda_build.external import find_executable
 
     upload = False
+    if args.token or args.user:
+        args.yes = True
+        upload = True
     # this is the default, for no explicit argument.
     # remember that args.binstar_upload takes defaults from condarc
-    if args.binstar_upload is None:
+    elif args.binstar_upload is None:
         args.yes = False
         args.dry_run = False
     # rc file has uploading explicitly turned off
@@ -220,6 +223,8 @@ def execute(args, parser):
     from os import makedirs
     from os.path import abspath, isdir, isfile
 
+    import conda.config as cc
+
     import conda_build.build as build
     import conda_build.source as source
     from conda_build.config import config
@@ -252,7 +257,11 @@ def execute(args, parser):
             if not isdir(d):
                 makedirs(d)
             update_index(d)
-        index = build.get_build_index(clear_cache=True)
+        arg_channels = ['local']
+        if args.channel:
+            arg_channels.extend(args.channel)
+        index = build.get_build_index(clear_cache=True,
+                                      arg_channels=arg_channels)
 
     already_built = set()
     to_build_recursive = []
@@ -298,11 +307,14 @@ def execute(args, parser):
                     "configuration." % m.dist())
             continue
         if args.skip_existing:
-            # 'or m.pkg_fn() in index' is for conda <4.1 and could be removed in the future.
-            if ('local::' + m.pkg_fn() in index or
-                    m.pkg_fn() in index or
-                    m.pkg_fn() in already_built):
-                print(m.dist(), "is already built, skipping.")
+            urls = cc.get_rc_urls() + cc.get_local_urls() + ['local', ]
+            if args.channel:
+                urls.extend(args.channel)
+
+            # will be empty if none found, and evalute to False
+            package_exists = [url for url in urls if url + '::' + m.pkg_fn() in index]
+            if (package_exists or m.pkg_fn() in index or m.pkg_fn() in already_built):
+                print(m.dist(), "is already built in {0}, skipping.".format(package_exists))
                 continue
         if args.output:
             print(bldpkg_path(m))

tests/test_build_recipes.py

@@ -10,13 +10,28 @@
 from conda.compat import PY3, TemporaryDirectory
 from conda.config import subdir
 from conda.fetch import download
+
 from conda_build.source import _guess_patch_strip_level, apply_patch
+import conda_build.config as config
+
+if PY3:
+    import urllib.parse as urlparse
+    import urllib.request as urllib
+else:
+    import urlparse
+    import urllib
 
 thisdir = os.path.dirname(os.path.realpath(__file__))
 metadata_dir = os.path.join(thisdir, 'test-recipes', 'metadata')
 fail_dir = os.path.join(thisdir, 'test-recipes', 'fail')
 
 
+# Used for translating local paths into url (file://) paths
+#   http://stackoverflow.com/a/14298190/1170370
+def path2url(path):
+    return urlparse.urljoin('file:', urllib.pathname2url(path))
+
+
 def is_valid_dir(parent_dir, dirname):
     valid = os.path.isdir(os.path.join(parent_dir, dirname))
     valid &= not dirname.startswith("_")
@@ -297,23 +312,100 @@ def test_skip_existing():
                                                                                     "build_number"))
     process = subprocess.Popen(cmd.split(),
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output, _ = process.communicate()
+    output, error = process.communicate()
     output = output.decode('utf-8')
-    assert "is already built, skipping." in output
+    error = error.decode('utf-8')
+    assert "is already built" in output, error
+
+
+# def test_skip_existing_anaconda_org():
+#     """This test may give false errors, because multiple tests running in parallel (on different
+#     platforms) will all use the same central anaconda.org account.  Thus, this test is only reliable
+#     if it is being run by one person on one machine at a time."""
+#     # generated with conda_test_account user, command:
+#     #    anaconda auth --create --name CONDA_BUILD_UPLOAD_TEST --scopes 'api repos conda'
+#     token = "co-79de533f-926f-4e5e-a766-d393e33ae98f"
+#     cmd = 'conda build --token {} {}'.format(token, os.path.join(metadata_dir, "empty_sections"))
+#     subprocess.check_call(cmd.split())
+
+#     try:
+#         # ensure that we skip with the package in the anaconda.org channel
+#         cmd = ('conda build --no-anaconda-upload --override-channels '
+#                '-c conda_test_account --skip-existing {}'
+#                 .format(os.path.join(metadata_dir, "empty_sections")))
+#         process = subprocess.Popen(cmd.split(),
+#                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+#         output, error = process.communicate()
+#         output = output.decode('utf-8')
+#         error = error.decode('utf-8')
+#     except:
+#         raise
+#     finally:
+#         # clean up: remove the package
+#         cmd = 'anaconda --token {} remove --force conda_test_account/empty_sections'\
+#             .format(token)
+#         subprocess.check_call(cmd.split())
+
+#     assert "is already built" in output, error
+#     assert "conda_test_account" in output, error
+
+
+def test_skip_existing_url():
+    # make sure that it is built
+    cmd = 'conda build --no-anaconda-upload {}'.format(os.path.join(metadata_dir, "empty_sections"))
+    subprocess.check_call(cmd.split())
+
+    output_file = os.path.join(config.croot, subdir, "empty_sections-0.0-0.tar.bz2")
+
+    with TemporaryDirectory() as tmp:
+        platform = os.path.join(tmp, subdir)
+        os.makedirs(platform)
+        shutil.copy2(output_file, os.path.join(platform, os.path.basename(output_file)))
+
+        # create the index so conda can find the file
+        subprocess.check_call(["conda", "index"], cwd=platform)
+
+        channel_url = path2url(tmp)
+        cmd = ('conda build --override-channels --skip-existing '
+               '--no-anaconda-upload -c {} {}'.format(channel_url,
+                                                      os.path.join(metadata_dir, "empty_sections")))
+        process = subprocess.Popen(cmd.split(),
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        output, error = process.communicate()
+        output = output.decode('utf-8')
+        error = error.decode('utf-8')
+
+        assert "is already built" in output, (output + error)
+        assert channel_url in output, (output + error)
 
 
 def test_token_upload():
     # generated with conda_test_account user, command:
     #    anaconda auth --create --name CONDA_BUILD_UPLOAD_TEST --scopes 'api repos conda'
     token = "co-79de533f-926f-4e5e-a766-d393e33ae98f"
+
+    # clean up - we don't actually want this package to exist yet
+    cmd = 'anaconda --token {} remove --force conda_test_account/empty_sections'.format(token)
+    subprocess.check_call(cmd.split())
+
+    show_package = "anaconda show conda_test_account/empty_sections"
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(show_package.split())
     # the folder with the test recipe to upload
     cmd = 'conda build --token {} {}'.format(token, os.path.join(metadata_dir, "empty_sections"))
     subprocess.check_call(cmd.split())
+
+    # make sure that the package is available (should raise CalledProcessError if it doesn't)
+    subprocess.check_call(show_package.split())
+
     # clean up - we don't actually want this package to exist
-    cmd = 'anaconda --token {} remove --force conda_test_account/conda-build-test-empty_sections'\
-    .format(token)
+    cmd = 'anaconda --token {} remove --force conda_test_account/empty_sections'.format(token)
     subprocess.check_call(cmd.split())
 
+    # verify cleanup:
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(show_package.split())
+
 
 @pytest.mark.parametrize("service_name", ["binstar", "anaconda"])
 def test_no_anaconda_upload_condarc(service_name):
@@ -331,7 +423,6 @@ def test_no_anaconda_upload_condarc(service_name):
         output, error = process.communicate()
         output = output.decode('utf-8')
         error = error.decode('utf-8')
-        sys.stderr.write(output)
         assert "Automatic uploading is disabled" in output, error