Merge pull request #9273 from rapidsai/branch-21.08

Forward-merge `branch-21.08` into `branch-21.10`

ci/gpu/build.sh

@@ -101,8 +101,8 @@ function install_dask {
     # Install the main version of dask, distributed, and streamz
     gpuci_logger "Install the main version of dask, distributed, and streamz"
     set -x
-    pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps
-    pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps
+    pip install "git+https://github.com/dask/distributed.git@2021.07.1" --upgrade --no-deps
+    pip install "git+https://github.com/dask/dask.git@2021.07.1" --upgrade --no-deps
     # Need to uninstall streamz that is already in the env.
     pip uninstall -y streamz
     pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps

conda/environments/cudf_dev_cuda11.0.yml

@@ -58,7 +58,7 @@ dependencies:
   - transformers
   - pydata-sphinx-theme
   - pip:
-      - git+https://github.com/dask/dask.git@main
-      - git+https://github.com/dask/distributed.git@main
+      - git+https://github.com/dask/dask.git@2021.07.1
+      - git+https://github.com/dask/distributed.git@2021.07.1
       - git+https://github.com/python-streamz/streamz.git@master
       - pyorc

conda/environments/cudf_dev_cuda11.2.yml

@@ -58,7 +58,7 @@ dependencies:
   - transformers
   - pydata-sphinx-theme
   - pip:
-      - git+https://github.com/dask/dask.git@main
-      - git+https://github.com/dask/distributed.git@main
+      - git+https://github.com/dask/dask.git@2021.07.1
+      - git+https://github.com/dask/distributed.git@2021.07.1
       - git+https://github.com/python-streamz/streamz.git@master
       - pyorc

python/cudf/setup.py

@@ -6,13 +6,24 @@
 import subprocess
 import sys
 import sysconfig
+
+# Must import in this order:
+#   setuptools -> Cython.Distutils.build_ext -> setuptools.command.build_ext
+# Otherwise, setuptools.command.build_ext ends up inheriting from
+# Cython.Distutils.old_build_ext which we do not want
+import setuptools
+
+try:
+    from Cython.Distutils.build_ext import new_build_ext as _build_ext
+except ImportError:
+    from setuptools.command.build_ext import build_ext as _build_ext
+
 from distutils.spawn import find_executable
 from distutils.sysconfig import get_python_lib
 
 import numpy as np
 import pyarrow as pa
-from Cython.Build import cythonize
-from Cython.Distutils import build_ext
+import setuptools.command.build_ext
 from setuptools import find_packages, setup
 from setuptools.extension import Extension
 
@@ -105,22 +116,46 @@ def get_cuda_version_from_header(cuda_include_dir, delimeter=""):
     ),
 )
 
-try:
-    nthreads = int(os.environ.get("PARALLEL_LEVEL", "0") or "0")
-except Exception:
-    nthreads = 0
 
-cmdclass = versioneer.get_cmdclass()
+class build_ext_and_proto_no_debug(_build_ext):
+    def build_extensions(self):
+        def remove_flags(compiler, *flags):
+            for flag in flags:
+                try:
+                    compiler.compiler_so = list(
+                        filter((flag).__ne__, compiler.compiler_so)
+                    )
+                except Exception:
+                    pass
 
+        # Full optimization
+        self.compiler.compiler_so.append("-O3")
+        # Silence '-Wunknown-pragmas' warning
+        self.compiler.compiler_so.append("-Wno-unknown-pragmas")
+        # No debug symbols, full optimization, no '-Wstrict-prototypes' warning
+        remove_flags(
+            self.compiler, "-g", "-G", "-O1", "-O2", "-Wstrict-prototypes"
+        )
+        super().build_extensions()
 
-class build_ext_and_proto(build_ext):
-    def build_extensions(self):
-        try:
-            # Silence the '-Wstrict-prototypes' warning
-            self.compiler.compiler_so.remove("-Wstrict-prototypes")
-        except Exception:
-            pass
-        build_ext.build_extensions(self)
+    def finalize_options(self):
+        if self.distribution.ext_modules:
+            # Delay import this to allow for Cython-less installs
+            from Cython.Build.Dependencies import cythonize
+
+            nthreads = getattr(self, "parallel", None)  # -j option in Py3.5+
+            nthreads = int(nthreads) if nthreads else None
+            self.distribution.ext_modules = cythonize(
+                self.distribution.ext_modules,
+                nthreads=nthreads,
+                force=self.force,
+                gdb_debug=False,
+                compiler_directives=dict(
+                    profile=False, language_level=3, embedsignature=True
+                ),
+            )
+        # Skip calling super() and jump straight to setuptools
+        setuptools.command.build_ext.build_ext.finalize_options(self)
 
     def run(self):
         # Get protoc
@@ -158,11 +193,9 @@ def run(self):
                     src.write(new_src_content)
 
         # Run original Cython build_ext command
-        build_ext.run(self)
+        _build_ext.run(self)
 
 
-cmdclass["build_ext"] = build_ext_and_proto
-
 extensions = [
     Extension(
         "*",
@@ -196,6 +229,10 @@ def run(self):
     )
 ]
 
+cmdclass = versioneer.get_cmdclass()
+cmdclass["build_ext"] = build_ext_and_proto_no_debug
+
+
 setup(
     name="cudf",
     version=versioneer.get_version(),
@@ -214,13 +251,7 @@ def run(self):
     ],
     # Include the separately-compiled shared library
     setup_requires=["cython", "protobuf"],
-    ext_modules=cythonize(
-        extensions,
-        nthreads=nthreads,
-        compiler_directives=dict(
-            profile=False, language_level=3, embedsignature=True
-        ),
-    ),
+    ext_modules=extensions,
     packages=find_packages(include=["cudf", "cudf.*"]),
     package_data=dict.fromkeys(
         find_packages(include=["cudf._lib*"]), ["*.pxd"],