Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[REVIEW] Improve Cython Build with Custom build_ext #2638

Merged
merged 17 commits into from
Sep 10, 2020
Merged

[REVIEW] Improve Cython Build with Custom build_ext #2638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
fa8cb8c
Updating the Cython build with a custom build_ext to allow for late d…
mdemoret-nv Aug 3, 2020
cc39311
Adding CHANGELOG message
mdemoret-nv Aug 3, 2020
57eaa61
Fixing build system so it works in CI
mdemoret-nv Aug 4, 2020
a97579e
Reverting clean changes back to original
mdemoret-nv Aug 5, 2020
4f32eb9
Fixing style issues
mdemoret-nv Aug 5, 2020
d2a97d2
Adding debug code for when raft is not found and must be cloned
mdemoret-nv Aug 5, 2020
916bda8
Reverting the clean command and cleaning up the comments to add insight.
mdemoret-nv Aug 5, 2020
8b42951
Merge branch 'branch-0.15' into fea-improve-cython-build_ext
mdemoret-nv Aug 5, 2020
5988dfe
Merge branch 'branch-0.16' into fea-improve-cython-build_ext
mdemoret-nv Aug 26, 2020
b475bae
Updating build.sh since --inplace is set in setup.cfg now
mdemoret-nv Aug 26, 2020
44cd3cb
Moving the CHANGELOG PR from 0.15 to 0.16
mdemoret-nv Aug 26, 2020
6574dd4
Adding more documentation and comments based on PR feedback. Also add…
mdemoret-nv Sep 2, 2020
41655e4
Merge branch 'branch-0.16' into fea-improve-cython-build_ext
mdemoret-nv Sep 2, 2020
a62815d
Removing the compiler directives from each file since its set globall…
mdemoret-nv Sep 2, 2020
65dde3a
Merge remote-tracking branch 'upstream/branch-0.16' into fea-improve-…
mdemoret-nv Sep 9, 2020
81ffd4a
Fixing copyright and removing cython directives from .py and .pxd fil…
mdemoret-nv Sep 9, 2020
34fa7a7
Removing `# distutils:` from *.pxd files
mdemoret-nv Sep 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## New Features
- PR #2698: Distributed TF-IDF transformer
- PR #2638: Improve cython build with custom `build_ext`

## Improvements
- PR #2796: Remove tokens of length 1 by default for text vectorizers
Expand Down
5 changes: 2 additions & 3 deletions build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,10 +226,9 @@ fi
if completeBuild || hasArg cuml || hasArg pydocs; then
cd ${REPODIR}/python
if [[ ${INSTALL_TARGET} != "" ]]; then
python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace ${SINGLEGPU_PYTHON_FLAG}
python setup.py install --single-version-externally-managed --record=record.txt ${SINGLEGPU_PYTHON_FLAG}
python setup.py build_ext -j${PARALLEL_LEVEL:-1} ${SINGLEGPU_PYTHON_FLAG} install --single-version-externally-managed --record=record.txt
else
python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace --library-dir=${LIBCUML_BUILD_DIR} ${SINGLEGPU_PYTHON_FLAG}
python setup.py build_ext -j${PARALLEL_LEVEL:-1} --library-dir=${LIBCUML_BUILD_DIR} ${SINGLEGPU_PYTHON_FLAG}
fi

if hasArg pydocs; then
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/cluster/dbscan.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import cudf
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/cluster/kmeans.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import cudf
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/cluster/kmeans_mg.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import cudf
Expand Down
5 changes: 0 additions & 5 deletions python/cuml/cluster/kmeans_utils.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
from libcpp cimport bool

Expand Down
3 changes: 0 additions & 3 deletions python/cuml/common/base.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import cuml
import cuml.common.cuda
Expand Down
6 changes: 0 additions & 6 deletions python/cuml/common/cuda.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


# Populate this with more typedef's (eg: events) as and when needed
cdef extern from * nogil:
ctypedef void* _Stream "cudaStream_t"
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/common/cuda.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import functools
from libcpp.string cimport string
Expand Down
6 changes: 0 additions & 6 deletions python/cuml/common/handle.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


from libcpp.memory cimport shared_ptr
cimport cuml.common.cuda

Expand Down
3 changes: 0 additions & 3 deletions python/cuml/common/handle.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


import cuml
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/common/logger.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


from libcpp.string cimport string
Expand Down
4 changes: 0 additions & 4 deletions python/cuml/common/opg_data_utils_mg.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

# Util functions, will be moved to their own file as the other methods are
# refactored
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/common/pointer_utils.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from libc.stdint cimport uintptr_t

Expand Down
3 changes: 0 additions & 3 deletions python/cuml/dask/common/comms_utils.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from libc.stdlib cimport malloc, free
from cython.operator cimport dereference as deref
Expand Down
5 changes: 0 additions & 5 deletions python/cuml/dask/linear_model/elastic_net.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from cuml.dask.solvers import CD
from cuml.dask.common.base import BaseEstimator

Expand Down
5 changes: 0 additions & 5 deletions python/cuml/dask/linear_model/lasso.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from cuml.dask.solvers import CD
from cuml.dask.common.base import BaseEstimator

Expand Down
5 changes: 0 additions & 5 deletions python/cuml/dask/metrics/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import cupy as cp


Expand Down
3 changes: 0 additions & 3 deletions python/cuml/datasets/arima.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import cuml
import numpy as np
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/datasets/regression.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import cuml
import numpy as np
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/decomposition/base_mg.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


import ctypes
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/decomposition/pca.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import cudf
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/decomposition/pca_mg.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


import ctypes
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/decomposition/tsvd.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import cudf
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/decomposition/tsvd_mg.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import cudf
Expand Down
5 changes: 0 additions & 5 deletions python/cuml/decomposition/utils.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from libcpp cimport bool

ctypedef int underlying_type_t_solver
Expand Down
5 changes: 0 additions & 5 deletions python/cuml/ensemble/randomforest_shared.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import ctypes
import math
import numpy as np
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/ensemble/randomforest_shared.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from libcpp.vector cimport vector
from cython.operator cimport dereference as deref, preincrement as inc
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/ensemble/randomforestclassifier.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import numpy as np
import rmm
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/ensemble/randomforestregressor.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import numpy as np
import rmm
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/fil/fil.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

import copy
import cudf
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/internals/internals.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


from libc.stdint cimport uintptr_t
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/linear_model/base_mg.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3


import ctypes
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/linear_model/elastic_net.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from cuml.solvers import CD
from cuml.common.base import Base, RegressorMixin
Expand Down
3 changes: 0 additions & 3 deletions python/cuml/linear_model/lasso.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
# limitations under the License.
#

# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
# cython: language_level = 3

from cuml.solvers import CD
from cuml.common.base import Base, RegressorMixin
Expand Down
Loading