From af21f48ef860e334139b8136710f2849c0c9cef5 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 11 May 2021 10:26:31 -0500 Subject: [PATCH 1/2] :ENH Updates to get_raft and setuputils for changes for using rapids-cmake --- cpp/cmake/thirdparty/get_raft.cmake | 36 +++++++++++++------ python/setup.py | 5 +-- python/setuputils.py | 56 ++++++++++++++--------------- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 3e143b660c2..6bfa214276b 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -14,23 +14,37 @@ # limitations under the License. #============================================================================= -function(find_and_configure_raft VERSION) +function(find_and_configure_raft) - rapids_cpm_find(raft ${VERSION} - GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET cugraph-exports - INSTALL_EXPORT_SET cugraph-exports + set(oneValueArgs VERSION FORK PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(raft ${PKG_VERSION} + GLOBAL_TARGETS raft::raft + BUILD_EXPORT_SET cuml-exports + INSTALL_EXPORT_SET cuml-exports CPM_ARGS - # GIT_REPOSITORY https://github.com/rapidsai/raft.git - # GIT_TAG branch-${VERSION} - GIT_REPOSITORY https://github.com/robertmaynard/raft.git - GIT_TAG use_rapids_cmake + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" ) + message(VERBOSE "CUML: Using RAFT located in ${raft_SOURCE_DIR}") + endfunction() -set(CUGRAPH_MIN_VERSION_raft "${CUGRAPH_VERSION_MAJOR}.${CUGRAPH_VERSION_MINOR}") +set(CUML_MIN_VERSION_raft "${CUML_VERSION_MAJOR}.${CUML_VERSION_MINOR}") + +# Change pinned tag and fork here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# RPM_raft_SOURCE=/path/to/local/raft +find_and_configure_raft(VERSION ${CUML_MIN_VERSION_raft} + FORK dantegd + PINNED_TAG 020-fea-cpm + ) + -find_and_configure_raft(${CUGRAPH_MIN_VERSION_raft}) diff --git a/python/setup.py b/python/setup.py index 799cb805afa..d614eef24d6 100644 --- a/python/setup.py +++ b/python/setup.py @@ -65,8 +65,9 @@ raft_path = get_environment_option('RAFT_PATH') # FIXME: This could clone RAFT, even if it's not needed (eg. running --clean). -raft_include_dir = use_raft_package(raft_path, libcugraph_path, - git_info_file='../cpp/CMakeLists.txt') +# deprecated: This functionality will go away after +# https://github.com/rapidsai/raft/issues/83 +raft_include_dir = use_raft_package(raft_path, libcugraph_path) class CleanCommand(Command): diff --git a/python/setuputils.py b/python/setuputils.py index d93051d05ef..3a9492c6f97 100644 --- a/python/setuputils.py +++ b/python/setuputils.py @@ -71,7 +71,7 @@ def clean_folder(path): def use_raft_package(raft_path, cpp_build_path, - git_info_file='../cpp/cmake/Dependencies.cmake'): + git_info_file=None): """ Function to use the python code in RAFT in package.raft @@ -139,10 +139,13 @@ def use_raft_package(raft_path, cpp_build_path, def clone_repo_if_needed(name, cpp_build_path=None, git_info_file=None): if git_info_file is None: - git_info_file = _get_repo_path() + '/cpp/CMakeLists.txt' + git_info_file = \ + _get_repo_path() + '/cpp/cmake/thirdparty/get_{}.cmake'.format( + name + ) if cpp_build_path is None or cpp_build_path is False: - cpp_build_path = _get_repo_path() + '/cpp/build/' + cpp_build_path = _get_repo_path() + '/cpp/build/_deps/' repo_cloned = get_submodule_dependency(name, cpp_build_path=cpp_build_path, @@ -152,7 +155,7 @@ def clone_repo_if_needed(name, cpp_build_path=None, repo_path = ( _get_repo_path() + '/python/_external_repositories/' + name + '/') else: - repo_path = os.path.join(cpp_build_path, '_deps', name + '-src') + repo_path = os.path.join(cpp_build_path, name + '-src/') return repo_path, repo_cloned @@ -192,7 +195,7 @@ def get_submodule_dependency(repo, repo_info = get_repo_cmake_info(repos, git_info_file) - if os.path.exists(cpp_build_path): + if os.path.exists(os.path.join(cpp_build_path, repos[0] + '-src/')): print("-- Third party modules found succesfully in the libcugraph++ " "build folder.") @@ -200,11 +203,11 @@ def get_submodule_dependency(repo, else: - warnings.warn("-- Third party repositories have not been found so they" - "will be cloned. To avoid this set the environment " - "variable CUGRAPH_BUILD_PATH, containing the relative " - "path of the root of the repository to the folder " - "where libcugraph++ was built.") + print("-- Third party repositories have not been found so they" + "will be cloned. To avoid this set the environment " + "variable CUGRAPH_BUILD_PATH, containing the relative " + "path of the root of the repository to the folder " + "where libcugraph++ was built.") for repo in repos: clone_repo(repo, repo_info[repo][0], repo_info[repo][1]) @@ -262,8 +265,8 @@ def get_repo_cmake_info(names, file_path): `ExternalProject_Add(name` file_path : String Relative path of the location of the CMakeLists.txt (or the cmake - module which contains FetchContent_Declare or ExternalProject_Add - definitions) to extract the information. + module which contains ExternalProject_Add definitions) to extract + the information. Returns ------- @@ -272,31 +275,24 @@ def get_repo_cmake_info(names, file_path): where results[name][0] is the url of the repo and repo_info[repo][1] is the tag/commit hash to be cloned as specified by cmake. + """ - with open(file_path, encoding='utf-8') as f: + with open(file_path) as f: s = f.read() results = {} - cmake_ext_proj_decls = ["FetchContent_Declare", "ExternalProject_Add"] + print(file_path) for name in names: - res = None - for decl in cmake_ext_proj_decls: - res = re.search(f'{decl}\(\s*' # noqa: W605 - + '(' + re.escape(name) + ')' - + '\s+.*GIT_REPOSITORY\s+(\S+)\s+.+' # noqa: W605 - + '\s+.*GIT_TAG\s+(\S+)', # noqa: W605 - s) - if res: - break - if res is None: - raise RuntimeError('Could not find any of the following ' - f'statements: {cmake_ext_proj_decls}, for ' - f'module "{name}" in file "{file_path}" with ' - 'GIT_REPOSITORY and GIT_TAG settings') - - results[res.group(1)] = [res.group(2), res.group(3)] + repo = re.findall(r'\s.*GIT_REPOSITORY.*', s) + repo = repo[-1].split()[-1] + fork = re.findall(r'\s.*FORK.*', s) + fork = fork[-1].split()[-1] + repo = repo.replace("${PKG_FORK}", fork) + tag = re.findall(r'\s.*PINNED_TAG.*', s) + tag = tag[-1].split()[-1] + results[name] = [repo, tag] return results From 0b35e06f63a50a583db2d496ee65d25a05a0dde0 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 11 May 2021 10:31:14 -0500 Subject: [PATCH 2/2] FIX Remove print --- python/setuputils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/setuputils.py b/python/setuputils.py index 3a9492c6f97..ee738799a2a 100644 --- a/python/setuputils.py +++ b/python/setuputils.py @@ -282,8 +282,6 @@ def get_repo_cmake_info(names, file_path): results = {} - print(file_path) - for name in names: repo = re.findall(r'\s.*GIT_REPOSITORY.*', s) repo = repo[-1].split()[-1]