diff --git a/.github/workflows/scripts/check_clang_tidy.sh b/.github/workflows/scripts/check_clang_tidy.sh index d9db1c9a3433f..4155421d86716 100755 --- a/.github/workflows/scripts/check_clang_tidy.sh +++ b/.github/workflows/scripts/check_clang_tidy.sh @@ -5,8 +5,5 @@ CI_SETUP_CMAKE_ARGS=$1 cd taichi python3 -m pip install -r requirements_dev.txt -rm -rf build && mkdir build && cd build -cmake $CI_SETUP_CMAKE_ARGS .. - -cd .. -python3 ./scripts/run_clang_tidy.py $PWD/taichi -clang-tidy-binary clang-tidy-10 -checks=-*,performance-inefficient-string-concatenation,readability-identifier-naming -header-filter=$PWD/taichi -p $PWD/build -j2 +export CI_SETUP_CMAKE_ARGS +python3 ./scripts/run_clang_tidy.py $PWD/taichi -clang-tidy-binary clang-tidy-10 -checks=-*,performance-inefficient-string-concatenation,readability-identifier-naming -header-filter=$PWD/taichi -j2 diff --git a/.github/workflows/scripts/win_build.ps1 b/.github/workflows/scripts/win_build.ps1 index e58c179ee6952..86ad4243742e5 100644 --- a/.github/workflows/scripts/win_build.ps1 +++ b/.github/workflows/scripts/win_build.ps1 @@ -76,7 +76,6 @@ python -m venv venv . venv\Scripts\activate.ps1 python -m pip install wheel python -m pip install -r requirements_dev.txt -python -m pip install -r requirements_test.txt if (-not $?) { exit 1 } WriteInfo("Building Taichi") $env:TAICHI_CMAKE_ARGS += " -DCLANG_EXECUTABLE=$libsDir\\taichi_clang\\bin\\clang++.exe" diff --git a/.gitignore b/.gitignore index fd39d08f9acea..958272e9a7b4d 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,4 @@ _build !docs/**/*.json imgui.ini /venv/ +/_skbuild/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 17db545ac8194..7387e8b648b56 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,7 +49,6 @@ else () endif () set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build") -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build") find_program(CCACHE_PROGRAM ccache) if(CCACHE_PROGRAM) @@ -157,6 +156,7 @@ foreach(arch IN LISTS HOST_ARCH CUDA_ARCH) WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/taichi/runtime/llvm" ) add_dependencies(${CORE_LIBRARY_NAME} "generate_llvm_runtime_${arch}") + install(FILES "${PROJECT_SOURCE_DIR}/taichi/runtime/llvm/runtime_${arch}.bc" DESTINATION ${CMAKE_INSTALL_PREFIX}/python/taichi/_lib/runtime) endforeach() configure_file(taichi/common/version.h.in ${CMAKE_SOURCE_DIR}/taichi/common/version.h) diff --git a/MANIFEST.in b/MANIFEST.in index eff0f4f8f88c7..3c1e64aa11c27 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,3 @@ -include MANIFEST.in -include version.txt -include python/*.txt -include python/*.py -include *.cfg include python/taichi/*.md recursive-include python/taichi/assets * recursive-include python/taichi/examples *.py diff --git a/cmake/PythonNumpyPybind11.cmake b/cmake/PythonNumpyPybind11.cmake index 311630dba74a8..65a231e04f64b 100644 --- a/cmake/PythonNumpyPybind11.cmake +++ b/cmake/PythonNumpyPybind11.cmake @@ -1,93 +1,16 @@ # Python, numpy, and pybind11 +execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pybind11 --cmake + OUTPUT_VARIABLE pybind11_DIR OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import numpy;print(numpy.get_include())" + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE) -if (PYTHON_EXECUTABLE) - message("Using ${PYTHON_EXECUTABLE} as python executable.") -else () - if (WIN32) - message("Using 'python' as python interpreter.") - set(PYTHON_EXECUTABLE python) - else () - message("Using 'python3' as python interpreter.") - set(PYTHON_EXECUTABLE python3) - endif() -endif () +message("-- Python: Using ${PYTHON_EXECUTABLE} as the interpreter") +message(" version: ${PYTHON_VERSION_STRING}") +message(" include: ${PYTHON_INCLUDE_DIR}") +message(" library: ${PYTHON_LIBRARY}") +message(" numpy include: ${NUMPY_INCLUDE_DIR}") -if (WIN32) - execute_process(COMMAND where ${PYTHON_EXECUTABLE} - OUTPUT_VARIABLE PYTHON_EXECUTABLE_PATHS) - if (${PYTHON_EXECUTABLE_PATHS}) - string(FIND ${PYTHON_EXECUTABLE_PATHS} "\n" _LINE_BREAK_LOC) - string(SUBSTRING ${PYTHON_EXECUTABLE_PATHS} 0 ${_LINE_BREAK_LOC} PYTHON_EXECUTABLE_PATH) - else () - set(PYTHON_EXECUTABLE_PATH ${PYTHON_EXECUTABLE}) - endif () -else () - execute_process(COMMAND which ${PYTHON_EXECUTABLE} - OUTPUT_VARIABLE PYTHON_EXECUTABLE_PATH) -endif() -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import sys;\ - from distutils import sysconfig;\ - sys.stdout.write(sysconfig.get_python_version())" - OUTPUT_VARIABLE PYTHON_VERSION) -execute_process(COMMAND ${PYTHON_EXECUTABLE} --version) -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import sys;\ - from distutils import sysconfig;\ - sys.stdout.write(\ - (sysconfig.get_config_var('INCLUDEPY')\ - if sysconfig.get_config_var('INCLUDEDIR') is not None else None)\ - or sysconfig.get_python_inc())" - OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS) -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import sys;\ - from distutils import sysconfig;\ - sys.stdout.write((sysconfig.get_config_var('LIBDIR') or sysconfig.get_python_lib()).replace('\\\\','/'))" - OUTPUT_VARIABLE PYTHON_LIBRARY_DIR) +include_directories(${NUMPY_INCLUDE_DIR}) -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import sys;\ - sys.stdout.write(str(sys.version_info[1]))" - OUTPUT_VARIABLE PYTHON_MINOR_VERSION) - - -if (WIN32) - execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import sys;sys.stdout.write(sys.base_prefix.replace('\\\\', '/'))" - OUTPUT_VARIABLE PYTHON_BASE_PREFIX) - link_directories(${PYTHON_BASE_PREFIX}/libs) - set(PYTHON_LIBRARIES ${PYTHON_BASE_PREFIX}/libs/python3.lib) - set(PYTHON_LIBRARIES ${PYTHON_BASE_PREFIX}/libs/python3${PYTHON_MINOR_VERSION}.lib) -else() - find_library(PYTHON_LIBRARY NAMES python${PYTHON_VERSION} python${PYTHON_VERSION}m PATHS ${PYTHON_LIBRARY_DIR} - NO_DEFAULT_PATH NO_SYSTEM_ENVIRONMENT_PATH PATH_SUFFIXES x86_64-linux-gnu) - set(PYTHON_LIBRARIES ${PYTHON_LIBRARY}) -endif() - - -include_directories(${PYTHON_INCLUDE_DIRS}) -message(" version: ${PYTHON_VERSION}") -message(" include: ${PYTHON_INCLUDE_DIRS}") -message(" library: ${PYTHON_LIBRARIES}") - -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import numpy.distutils, sys;\ - sys.stdout.write(':'.join(numpy.distutils.misc_util.get_numpy_include_dirs()))" - OUTPUT_VARIABLE PYTHON_NUMPY_INCLUDE_DIR) - -message(" numpy include: ${PYTHON_NUMPY_INCLUDE_DIR}") -include_directories(${PYTHON_NUMPY_INCLUDE_DIR}) - -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import sys; import pybind11; sys.stdout.write(pybind11.get_include() + ';' + pybind11.get_include(True))" - OUTPUT_VARIABLE PYBIND11_INCLUDE_DIR - RESULT_VARIABLE PYBIND11_IMPORT_RET) -if (NOT PYBIND11_IMPORT_RET) - # returns zero if success - message(" pybind11 include: ${PYBIND11_INCLUDE_DIR}") -else () - message(FATAL_ERROR "Cannot import pybind11. Please install. ([sudo] pip3 install --user pybind11)") -endif () - -include_directories(${PYBIND11_INCLUDE_DIR}) +find_package(pybind11 CONFIG REQUIRED) diff --git a/cmake/TaichiCore.cmake b/cmake/TaichiCore.cmake index 9bc218d9cc9ea..687130d9e19d5 100644 --- a/cmake/TaichiCore.cmake +++ b/cmake/TaichiCore.cmake @@ -16,6 +16,7 @@ option(TI_EMSCRIPTENED "Build using emscripten" OFF) # projects. set(CMAKE_CXX_VISIBILITY_PRESET hidden) set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +set(INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/python/taichi/_lib) if(ANDROID) set(TI_WITH_VULKAN ON) @@ -384,6 +385,9 @@ if (TI_WITH_VULKAN) find_library(MOLTEN_VK libMoltenVK.dylib PATHS $HOMEBREW_CELLAR/molten-vk $VULKAN_SDK REQUIRED) configure_file(${MOLTEN_VK} ${CMAKE_BINARY_DIR}/libMoltenVK.dylib COPYONLY) message(STATUS "MoltenVK library ${MOLTEN_VK}") + if (EXISTS ${CMAKE_BINARY_DIR}/libMoltenVK.dylib) + install(FILES ${CMAKE_BINARY_DIR}/libMoltenVK.dylib DESTINATION ${INSTALL_LIB_DIR}/runtime) + endif() endif() endif () @@ -437,7 +441,7 @@ if(NOT TI_EMSCRIPTENED) # Cannot compile Python source code with Android, but TI_EXPORT_CORE should be set and # Android should only use the isolated library ignoring those source code. if (NOT ANDROID) - add_library(${CORE_WITH_PYBIND_LIBRARY_NAME} SHARED ${TAICHI_PYBIND_SOURCE}) + pybind11_add_module(${CORE_WITH_PYBIND_LIBRARY_NAME} ${TAICHI_PYBIND_SOURCE}) else() add_library(${CORE_WITH_PYBIND_LIBRARY_NAME} SHARED) endif () @@ -459,6 +463,10 @@ if(NOT TI_EMSCRIPTENED) set_target_properties(${CORE_WITH_PYBIND_LIBRARY_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/runtimes") endif () + + install(TARGETS ${CORE_WITH_PYBIND_LIBRARY_NAME} + RUNTIME DESTINATION ${INSTALL_LIB_DIR}/core + LIBRARY DESTINATION ${INSTALL_LIB_DIR}/core) endif() if(TI_EMSCRIPTENED) @@ -487,3 +495,8 @@ endif() target_link_libraries(${CORE_LIBRARY_NAME} imgui) endif() + +if (NOT APPLE) + install(FILES ${CMAKE_SOURCE_DIR}/external/cuda_libdevice/slim_libdevice.10.bc + DESTINATION ${INSTALL_LIB_DIR}/runtime) +endif() diff --git a/pyproject.toml b/pyproject.toml index c4a857f234460..f5d511fad16dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,9 @@ [build-system] -requires = ["setuptools", "wheel", "numpy", "pybind11", "cmake"] +requires = [ + "setuptools", "wheel", + "numpy", "pybind11", "cmake", + "scikit-build", "ninja; platform_system != 'Windows'", +] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] diff --git a/requirements_dev.txt b/requirements_dev.txt index e8849dc3cb1aa..2cc165f61719d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -14,3 +14,6 @@ twine wheel astunparse pre-commit +scikit-build +numpy +ninja; platform_system != 'Windows' diff --git a/scripts/run_clang_tidy.py b/scripts/run_clang_tidy.py index 229d91bfdf4d0..4e0abeb45fcc4 100644 --- a/scripts/run_clang_tidy.py +++ b/scripts/run_clang_tidy.py @@ -77,6 +77,18 @@ def make_absolute(f, directory): return os.path.normpath(os.path.join(directory, f)) +def cmake_configure(source_path='.'): + import shlex + + from skbuild.cmaker import CMaker + from skbuild.constants import CMAKE_BUILD_DIR + + cmaker = CMaker() + cmake_args = shlex.split(os.getenv('CI_SETUP_CMAKE_ARGS', '')) + cmaker.configure(cmake_args) + return CMAKE_BUILD_DIR() + + def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path, header_filter, extra_arg, extra_arg_before, quiet, config): @@ -265,8 +277,7 @@ def main(): if args.build_path is not None: build_path = args.build_path else: - # Find our database - build_path = find_compilation_database(db_path) + build_path = cmake_configure('.') try: invocation = [args.clang_tidy_binary, '-list-checks'] diff --git a/setup.py b/setup.py index 2110a34697234..a776e2d18d26f 100644 --- a/setup.py +++ b/setup.py @@ -8,17 +8,17 @@ import glob import multiprocessing import os -import platform import shutil import subprocess import sys from distutils.command.clean import clean from distutils.dir_util import remove_tree -from setuptools import Extension, find_packages, setup -from setuptools.command.build_ext import build_ext -from setuptools.command.build_py import build_py -from setuptools.command.egg_info import egg_info +from setuptools import find_packages +from skbuild import setup +from skbuild.command.egg_info import egg_info + +root_dir = os.path.dirname(os.path.abspath(__file__)) classifiers = [ 'Development Status :: 2 - Pre-Alpha', @@ -58,43 +58,11 @@ def get_version(): # Our python package root dir is python/ package_dir = 'python' -root_dir = os.path.abspath(os.path.dirname(__file__)) - - -def get_python_executable(): - return sys.executable.replace('\\', '/') - - -def get_os_name(): - name = platform.platform() - # in python 3.8, platform.platform() uses mac_ver() on macOS - # it will return 'macOS-XXXX' instead of 'Darwin-XXXX' - if name.lower().startswith('darwin') or name.lower().startswith('macos'): - return 'osx' - elif name.lower().startswith('windows'): - return 'win' - elif name.lower().startswith('linux'): - return 'linux' - elif 'bsd' in name.lower(): - return 'unix' - assert False, "Unknown platform name %s" % name - def remove_tmp(taichi_dir): shutil.rmtree(os.path.join(taichi_dir, 'assets'), ignore_errors=True) -def remove_files_with_extension(dir_name, extension): - for file in os.listdir(dir_name): - if file.endswith(extension): - os.remove(os.path.join(dir_name, file)) - - -class CMakeExtension(Extension): - def __init__(self, name): - Extension.__init__(self, name, sources=[]) - - class EggInfo(egg_info): def run(self): taichi_dir = os.path.join(package_dir, 'taichi') @@ -105,148 +73,15 @@ def run(self): egg_info.run(self) -# python setup.py build runs the following commands in order: -# python setup.py build_py -# python setup.py build_ext -class BuildPy(build_py): - def run(self): - build_py.run(self) - taichi_dir = os.path.join(package_dir, 'taichi') - remove_tmp(taichi_dir) - - -class CMakeBuild(build_ext): - def parse_cmake_args_from_env(self): - # Source: TAICHI_CMAKE_ARGS=... python setup.py ... - import shlex - cmake_args = os.getenv('TAICHI_CMAKE_ARGS', '') - return shlex.split(cmake_args.strip()) - - def run(self): - try: - subprocess.check_call(['cmake', '--version']) - except OSError: - raise RuntimeError( - "CMake must be installed to build the following extensions: " + - ", ".join(e.name for e in self.extensions)) - - # CMakeLists.txt is in the same directory as this setup.py file - cmake_list_dir = root_dir - self.build_temp = os.path.join(cmake_list_dir, 'build') - - build_directory = os.path.abspath(self.build_temp) - - cmake_args = self.parse_cmake_args_from_env() - - cmake_args += [ - f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={build_directory}', - f'-DPYTHON_EXECUTABLE={get_python_executable()}', - f'-DTI_VERSION_MAJOR={TI_VERSION_MAJOR}', - f'-DTI_VERSION_MINOR={TI_VERSION_MINOR}', - f'-DTI_VERSION_PATCH={TI_VERSION_PATCH}', - ] - - emscriptened = os.getenv('TI_EMSCRIPTENED', '0') in ('1', 'ON') - if emscriptened: - cmake_args += ['-DTI_EMSCRIPTENED=ON'] - - if shutil.which('ninja'): - cmake_args += ['-GNinja'] - - cfg = 'Release' - if (os.getenv('DEBUG', '0') in ('1', 'ON')): - cfg = 'Debug' - elif (os.getenv('RELWITHDEBINFO', '0') in ('1', 'ON')): - cfg = 'RelWithDebInfo' - elif (os.getenv('MINSIZEREL', '0') in ('1', 'ON')): - cfg = 'MinSizeRel' - - build_args = ['--config', cfg] - - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] - - # Assuming Makefiles - if get_os_name() != 'win': - num_threads = os.getenv('BUILD_NUM_THREADS', - multiprocessing.cpu_count()) - build_args += ['--', f'-j{num_threads}'] - - self.build_args = build_args - - env = os.environ.copy() - os.makedirs(self.build_temp, exist_ok=True) - - print('-' * 10, 'Running CMake prepare', '-' * 40) - print(' '.join(['cmake', cmake_list_dir] + cmake_args)) - subprocess.check_call(['cmake', cmake_list_dir] + cmake_args, - cwd=self.build_temp, - env=env) - - print('-' * 10, 'Building extensions', '-' * 40) - cmake_cmd = ['cmake', '--build', '.'] + self.build_args - subprocess.check_call(cmake_cmd, cwd=self.build_temp) - - self.prepare_package() - - def prepare_package(self): - # We need to make sure these additional files are ready for - # - develop mode: must exist in local python/taichi/lib/ folder - # - install mode: must exist in self.build_lib/taichi/lib - base_dir = package_dir if self.inplace else self.build_lib - taichi_lib_dir = os.path.join(base_dir, 'taichi', '_lib') - - runtime_dir = os.path.join(taichi_lib_dir, "runtime") - core_dir = os.path.join(taichi_lib_dir, "core") - os.makedirs(runtime_dir, exist_ok=True) - os.makedirs(core_dir, exist_ok=True) - - if (get_os_name() == 'linux' or get_os_name() == 'unix' - or get_os_name() == 'osx'): - remove_files_with_extension(core_dir, ".so") - else: - remove_files_with_extension(core_dir, ".pyd") - if get_os_name() == 'osx': - remove_files_with_extension(runtime_dir, ".dylib") - remove_files_with_extension(runtime_dir, ".bc") - - if get_os_name() == 'linux' or get_os_name() == 'unix': - self.copy_file(os.path.join(self.build_temp, 'libtaichi_core.so'), - os.path.join(core_dir, 'taichi_core.so')) - elif get_os_name() == 'osx': - self.copy_file( - os.path.join(self.build_temp, 'libtaichi_core.dylib'), - os.path.join(core_dir, 'taichi_core.so')) - moltenvk_path = os.path.join(self.build_temp, 'libMoltenVK.dylib') - if os.path.exists(moltenvk_path): - self.copy_file(moltenvk_path, - os.path.join(runtime_dir, 'libMoltenVK.dylib')) - else: - self.copy_file('runtimes/taichi_core.dll', - os.path.join(core_dir, 'taichi_core.pyd')) - - if get_os_name() != 'osx': - libdevice_path = 'external/cuda_libdevice/slim_libdevice.10.bc' - print("copying libdevice:", libdevice_path) - assert os.path.exists(libdevice_path) - self.copy_file(libdevice_path, - os.path.join(runtime_dir, 'slim_libdevice.10.bc')) - - llvm_runtime_dir = 'taichi/runtime/llvm' - for f in os.listdir(llvm_runtime_dir): - if f.startswith('runtime_') and f.endswith('.bc'): - print(f"Fetching runtime file {f} to {taichi_lib_dir} folder") - self.copy_file(os.path.join(llvm_runtime_dir, f), runtime_dir) - - class Clean(clean): def run(self): super().run() - self.build_temp = os.path.join(root_dir, 'build') + self.build_temp = os.path.join(root_dir, '_skbuild') if os.path.exists(self.build_temp): remove_tree(self.build_temp, dry_run=self.dry_run) generated_folders = ('bin', 'dist', 'python/taichi/assets', - 'python/taichi/_lib/runtime', - 'python/taichi.egg-info') + 'python/taichi/_lib/runtime', 'taichi.egg-info', + 'python/taichi.egg-info', 'build') for d in generated_folders: if os.path.exists(d): remove_tree(d, dry_run=self.dry_run) @@ -263,6 +98,45 @@ def run(self): os.remove(f) +def get_cmake_args(): + import shlex + + num_threads = os.getenv('BUILD_NUM_THREADS', multiprocessing.cpu_count()) + cmake_args = shlex.split(os.getenv('TAICHI_CMAKE_ARGS', '').strip()) + + if (os.getenv('DEBUG', '0') in ('1', 'ON')): + cfg = 'Debug' + elif (os.getenv('RELWITHDEBINFO', '0') in ('1', 'ON')): + cfg = 'RelWithDebInfo' + elif (os.getenv('MINSIZEREL', '0') in ('1', 'ON')): + cfg = 'MinSizeRel' + else: + cfg = None + if cfg: + sys.argv[2:2] = ['--build-type', cfg] + + cmake_args += [ + f'-DTI_VERSION_MAJOR={TI_VERSION_MAJOR}', + f'-DTI_VERSION_MINOR={TI_VERSION_MINOR}', + f'-DTI_VERSION_PATCH={TI_VERSION_PATCH}', + ] + emscriptened = os.getenv('TI_EMSCRIPTENED', '0') in ('1', 'ON') + if emscriptened: + cmake_args += ['-DTI_EMSCRIPTENED=ON'] + + if sys.platform != 'win32': + os.environ['SKBUILD_BUILD_OPTIONS'] = f'-j{num_threads}' + return cmake_args + + +def exclude_paths(manifest_files): + return [ + f for f in manifest_files + if f.endswith(('.so', 'pyd', + '.bc')) or os.path.basename(f) == 'libMoltenVK.dylib' + ] + + setup(name=project_name, packages=packages, package_dir={"": package_dir}, @@ -287,9 +161,10 @@ def run(self): ], }, classifiers=classifiers, - ext_modules=[CMakeExtension('taichi_core')], - cmdclass=dict(egg_info=EggInfo, - build_py=BuildPy, - build_ext=CMakeBuild, - clean=Clean), + cmake_args=get_cmake_args(), + cmake_process_manifest_hook=exclude_paths, + cmdclass={ + 'egg_info': EggInfo, + 'clean': Clean + }, has_ext_modules=lambda: True)