diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..e950239 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,62 @@ +Checks: > + *, + -llvmlibc*, + -fuchsia*, + -altera*, + -android*, + +CheckOptions: + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.ClassMemberCase + value: lower_case + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariablePrefix + value: k + - key: readability-identifier-naming.EnumCase + value: lower_case + - key: readability-identifier-naming.EnumPrefix + value: _ + - key: readability-identifier-naming.EnumConstantCase + value: UPPER_CASE + - key: readability-identifier-naming.EnumConstantPrefix + value: '' + - key: readability-identifier-naming.FunctionCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantCase + value: UPPER_CASE + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.StaticVariableCase + value: lower_case + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE + - key: readability-identifier-naming.MacroDefinitionIgnoredRegexp + value: "(^[A-Z]+(_[A-Z]+)*_$|^_.*)" + - key: readability-identifier-naming.MemberCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberSuffix + value: _ + - key: readability-identifier-naming.PublicMemberSuffix + value: "" + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.TypeAliasCase + value: camelBack + - key: readability-identifier-naming.TypedefCase + value: lower_case + - key: readability-identifier-naming.TypeDefSuffix + value: _t + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.IgnoreMainLikeFunctions + value: 1 + - key: readability-identifier-naming.MethodCase + value: camelBack + - key: readability-identifier-naming.StructCase + value: lower_case diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ceb42ac..52b6ef1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,16 +1,16 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.194.3/containers/ubuntu/.devcontainer/base.Dockerfile +# See here for image contents: https://github.com/devcontainers/images/tree/main/src/base-ubuntu -# [Choice] Ubuntu version: hirsute, bionic, focal -ARG VARIANT="focal" +ARG VARIANT="jammy" FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} # [Optional] Uncomment this section to install additional OS packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends -RUN sudo apt update \ - && sudo apt -y install python-is-python3 pip build-essential cmake ninja-build cppcheck \ - && sudo apt full-upgrade -y -RUN sudo apt update \ - && wget http://sdk-releases.upmem.com/2021.3.0/ubuntu_20.04/upmem_2021.3.0_amd64.deb \ - && sudo apt install -y ./upmem_2021.3.0_amd64.deb +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install build-essential ninja-build cppcheck nox pre-commit\ + && apt-get dist-upgrade -y +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem_2024.2.0_amd64.deb \ + && apt-get install -y ./upmem_2024.2.0_amd64.deb \ + && rm upmem_2024.2.0_amd64.deb diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b001699..4eb5da4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,53 +5,51 @@ "build": { "dockerfile": "Dockerfile", // Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic - "args": { "VARIANT": "focal" } + "args": { + "VARIANT": "jammy" + } }, // "runArgs": [ "--init", "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - - // Set *default* container specific settings.json values on container create. - "settings": { - "python.pythonPath": "/usr/bin/python", - "python.languageServer": "Pylance", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.formatting.provider": "black", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", - "python.linting.pylintArgs": [ - "--extension-pkg-whitelist=dpu_kmeans._core", - "--ignored-modules=dpu_kmeans._core" - ], + "customizations": { + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.pythonPath": "/usr/bin/python", + "python.languageServer": "Pylance", + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff" + }, + "ruff.lint.select": [ + "ALL" + ] + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "llvm-vs-code-extensions.vscode-clangd", + "twxs.cmake", + "donjayamanne.githistory", + "ms-python.python", + "ms-python.vscode-pylance", + "cschlosser.doxdocgen", + "charliermarsh.ruff", + "mhutchie.git-graph", + "tamasfe.even-better-toml" + ] + } }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "llvm-vs-code-extensions.vscode-clangd", - "twxs.cmake", - "donjayamanne.githistory", - "ms-python.python", - "ms-python.vscode-pylance", - "cschlosser.doxdocgen" - ], - "features": { - "git": "os-provided", - "python": "os-provided" + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/python:1": { + "installTools": false + } }, // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip install --upgrade pip nox pre-commit scikit-build pybind11[global] twine setuptools_scm hurry.filesize && pre-commit install && python setup.py clean && pip install --no-build-isolation -e .[test]", + "postCreateCommand": "pre-commit install && pip install -e .[test]", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", + "remoteUser": "vscode" } diff --git a/CMakeLists.txt b/CMakeLists.txt index 5629843..e3ffea3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,37 +27,24 @@ ExternalProject_Add( dpu_program SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_program INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_kmeans - # INSTALL_DIR ${SKBUILD_PLATLIB_DIR}/${SKBUILD_PROJECT_NAME} INSTALL_DIR - # ${SKBUILD_PROJECT_NAME} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= -DNR_TASKLETS=${NR_TASKLETS} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) -# BUILD_ALWAYS TRUE) USES_TERMINAL_BUILD TRUE) - -# ExternalProject_Get_Property(dpu_program INSTALL_DIR) -# set(DPU_PROGRAM_INSTALL_DIR ${INSTALL_DIR}) - -# install(DIRECTORY ${DPU_PROGRAM_INSTALL_DIR} DESTINATION .) + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + # temporary workaround until SDK distributes fixed toolchain file + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/src/dpu.cmake) # =================== BUILDING THE HOST BINARY ====================== find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) find_package(pybind11 CONFIG REQUIRED) -# set(PKG_CONFIG_EXECUTABLE "dpu-pkg-config") find_package(PkgConfig REQUIRED) -# pkg_search_module(DPU REQUIRED IMPORTED_TARGET dpu) +include(CheckIPOSupported) include(${UPMEM_HOME}/share/upmem/cmake/include/host/DpuHost.cmake) find_package(OpenMP REQUIRED) -set(CMAKE_C_STANDARD 99) -set(CMAKE_C_STANDARD_REQUIRED TRUE) - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Ofast") -include(CheckIPOSupported) -check_ipo_supported(RESULT ipo_supported OUTPUT error) - pybind11_add_module( _core MODULE src/main.cpp src/host_program/trivial_checksum_example_host.c src/host_program/lloyd_iter.c src/host_program/dimm_manager.c) @@ -66,11 +53,14 @@ target_link_libraries(_core PRIVATE ${DPU_HOST_LIBRARIES} OpenMP::OpenMP_CXX target_include_directories(_core PUBLIC ${DPU_HOST_INCLUDE_DIRECTORIES}) target_link_directories(_core PUBLIC ${DPU_HOST_LINK_DIRECTORIES}) +target_compile_features(_core PUBLIC c_std_99) + target_compile_definitions( _core PUBLIC VERSION_INFO=${SKBUILD_PROJECT_VERSION} PUBLIC NR_TASKLETS=${NR_TASKLETS}) +check_ipo_supported(RESULT ipo_supported OUTPUT error) if(ipo_supported) message(STATUS "IPO / LTO enabled") set_target_properties(_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE @@ -79,9 +69,10 @@ else() message(STATUS "IPO / LTO not supported: <${error}>") endif() +# install in-source rather than in the temporary build directory that way +# linting tools can find the compiled extension install(TARGETS _core LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_kmeans) -# install(TARGETS _core LIBRARY DESTINATION ${SKBUILD_PROJECT_NAME}) # copying the compilation database for language servers add_custom_target( diff --git a/docs/conf.py b/docs/conf.py index 55d9f31..64ceea5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,10 +50,6 @@ autosummary_generate = True -# myst_gfm_only = True - -# html_extra_path = ["../README.md"] - # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/pyproject.toml b/pyproject.toml index 90b751e..5df4afd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ Repository = "https://github.com/upmem/dpu_kmeans" [project.optional-dependencies] test = ["pytest", "pybind11-stubgen"] -benchmarks = ["pytest", "pandas", "pyarrow"] +benchmarks = ["pytest", "pandas", "pyarrow", "hurry.filesize"] doc = ["sphinx", "pydata-sphinx-theme", "myst-parser[linkify]"] [tool.setuptools_scm] diff --git a/src/dpu.cmake b/src/dpu.cmake new file mode 100644 index 0000000..0ac3d33 --- /dev/null +++ b/src/dpu.cmake @@ -0,0 +1,47 @@ +# +# Copyright (c) 2014-2024 - UPMEM +# + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR dpu) + +if(NOT DEFINED UPMEM_HOME) + if("$ENV{UPMEM_HOME}" STREQUAL "") + set(UPMEM_HOME "/usr") + else() + set(UPMEM_HOME $ENV{UPMEM_HOME}) + endif() +endif() + +set(CMAKE_SYSROOT ${UPMEM_HOME}/share/upmem/include) + +set(tools ${UPMEM_HOME}/bin) + +set(triple dpu-upmem-dpurte) + +set(CMAKE_C_COMPILER ${tools}/clang) +set(CMAKE_C_COMPILER_TARGET ${triple}) +set(CMAKE_CXX_COMPILER ${tools}/clang++) +set(CMAKE_CXX_COMPILER_TARGET ${triple}) +set(CMAKE_ASM_COMPILER ${tools}/clang) +set(CMAKE_ASM_COMPILER_TARGET ${triple}) + +# temporary workaround until SDK distributes llvm-ranlib +set(CMAKE_C_COMPILER_RANLIB echo) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_C_COMPILER_FORCED 1) + +set(CHIP_VERSION "v1A") # Default value +if(EXISTS "/sys/class/dpu_rank/dpu_rank0/dpu_chip_id") + file(READ "/sys/class/dpu_rank/dpu_rank0/dpu_chip_id" CHIP_ID_NUMBER) + if(CHIP_ID_NUMBER GREATER 8) + set(CHIP_VERSION "v1B") + endif() +endif() + +set(CMAKE_C_FLAGS_INIT "-mcpu=${CHIP_VERSION} -g") diff --git a/src/dpu_program/CMakeLists.txt b/src/dpu_program/CMakeLists.txt index 4a06618..534cec7 100644 --- a/src/dpu_program/CMakeLists.txt +++ b/src/dpu_program/CMakeLists.txt @@ -13,20 +13,25 @@ if(NOT DEFINED UPMEM_HOME) endif() endif() -include("${UPMEM_HOME}/share/upmem/cmake/dpu.cmake") -# include(CheckIPOSupported) check_ipo_supported(RESULT ipo_supported OUTPUT -# error) - -# set(CMAKE_THREAD_PREFER_PTHREAD FALSE) set(THREADS_PREFER_PTHREAD_FLAG FALSE) -set(CMAKE_THREAD_LIBS_INIT) +# include("${UPMEM_HOME}/share/upmem/cmake/dpu.cmake") +# CMAKE_FORCE_C_COMPILER(${tools}/dpu-upmem-dpurte-clang Clang) +message(INFO "ranlib : ${CMAKE_RANLIB}, ${CMAKE_C_COMPILER_RANLIB}") +include(CheckIPOSupported) +check_ipo_supported( + RESULT ipo_supported + OUTPUT error + LANGUAGES C) +if(NOT ipo_supported) + message(FATAL_ERROR "IPO / LTO not supported: <${error}>") +endif() +# message(FATAL_ERROR "Compiler ID: ${CMAKE_C_COMPILER_ID}") if(NOT CMAKE_BUILD_TYPE) message(FATAL_ERROR "CMAKE_BUILD_TYPE must be set") endif() set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og") -# set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -g0") -set(CMAKE_C_FLAGS_RELEASE "-O3 -g0") +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -g0") add_executable(trivial_checksum_example trivial_checksum_example.c) add_executable(kmeans_dpu_kernel kmeans_dpu_kernel.c) @@ -34,10 +39,13 @@ add_executable(kmeans_dpu_kernel kmeans_dpu_kernel.c) target_compile_definitions(kmeans_dpu_kernel PUBLIC NR_TASKLETS=${NR_TASKLETS}) target_link_options(kmeans_dpu_kernel PUBLIC -DNR_TASKLETS=${NR_TASKLETS}) -# if( ipo_supported ) message(STATUS "IPO / LTO enabled") -# set_target_properties(kmeans_dpu_kernel PROPERTIES -# INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) else() message(STATUS "IPO / LTO -# not supported: <${error}>") endif() +if(ipo_supported) + message(STATUS "IPO / LTO enabled") + set_target_properties(kmeans_dpu_kernel + PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) +else() + message(STATUS "IPO / LTO not supported: <${error}>") +endif() install(TARGETS trivial_checksum_example DESTINATION ${PROJECT_NAME}) install(TARGETS kmeans_dpu_kernel DESTINATION ${PROJECT_NAME}) diff --git a/src/dpu_program/kmeans_dpu_kernel.c b/src/dpu_program/kmeans_dpu_kernel.c index 6508134..672470b 100644 --- a/src/dpu_program/kmeans_dpu_kernel.c +++ b/src/dpu_program/kmeans_dpu_kernel.c @@ -254,7 +254,7 @@ void initialize(uint8_t tasklet_id) { */ #ifndef PERF_COUNTER void task_reduce(uint8_t tasklet_id, uint8_t icluster, int point_global_index, - uint16_t point_base_index, int_feature *w_features) { + uint16_t point_base_index, const int_feature *w_features) { #else void task_reduce(uint8_t tasklet_id, uint8_t icluster, int point_global_index, uint16_t point_base_index, int_feature *w_features,