fix devcontainer compile

upmem · Oct 24, 2024 · 1694245 · 1694245
1 parent e4cbb7e
commit 1694245
Show file tree

Hide file tree

Showing 9 changed files with 184 additions and 82 deletions.
diff --git a/.clang-tidy b/.clang-tidy
@@ -0,0 +1,62 @@
+Checks: >
+  *,
+  -llvmlibc*,
+  -fuchsia*,
+  -altera*,
+  -android*,
+
+CheckOptions:
+  - key: readability-identifier-naming.ClassCase
+    value: CamelCase
+  - key: readability-identifier-naming.ClassMemberCase
+    value: lower_case
+  - key: readability-identifier-naming.ConstexprVariableCase
+    value: CamelCase
+  - key: readability-identifier-naming.ConstexprVariablePrefix
+    value: k
+  - key: readability-identifier-naming.EnumCase
+    value: lower_case
+  - key: readability-identifier-naming.EnumPrefix
+    value: _
+  - key: readability-identifier-naming.EnumConstantCase
+    value: UPPER_CASE
+  - key: readability-identifier-naming.EnumConstantPrefix
+    value: ''
+  - key: readability-identifier-naming.FunctionCase
+    value: lower_case
+  - key: readability-identifier-naming.GlobalConstantCase
+    value: UPPER_CASE
+  - key: readability-identifier-naming.StaticConstantCase
+    value: CamelCase
+  - key: readability-identifier-naming.StaticConstantPrefix
+    value: k
+  - key: readability-identifier-naming.StaticVariableCase
+    value: lower_case
+  - key: readability-identifier-naming.MacroDefinitionCase
+    value: UPPER_CASE
+  - key: readability-identifier-naming.MacroDefinitionIgnoredRegexp
+    value: "(^[A-Z]+(_[A-Z]+)*_$|^_.*)"
+  - key: readability-identifier-naming.MemberCase
+    value: lower_case
+  - key: readability-identifier-naming.PrivateMemberSuffix
+    value: _
+  - key: readability-identifier-naming.PublicMemberSuffix
+    value: ""
+  - key: readability-identifier-naming.NamespaceCase
+    value: lower_case
+  - key: readability-identifier-naming.ParameterCase
+    value: lower_case
+  - key: readability-identifier-naming.TypeAliasCase
+    value: camelBack
+  - key: readability-identifier-naming.TypedefCase
+    value: lower_case
+  - key: readability-identifier-naming.TypeDefSuffix
+    value: _t
+  - key: readability-identifier-naming.VariableCase
+    value: lower_case
+  - key: readability-identifier-naming.IgnoreMainLikeFunctions
+    value: 1
+  - key: readability-identifier-naming.MethodCase
+    value: camelBack
+  - key: readability-identifier-naming.StructCase
+    value: lower_case
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,16 +1,16 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.194.3/containers/ubuntu/.devcontainer/base.Dockerfile
+# See here for image contents: https://github.com/devcontainers/images/tree/main/src/base-ubuntu
 
-# [Choice] Ubuntu version: hirsute, bionic, focal
-ARG VARIANT="focal"
+ARG VARIANT="jammy"
 FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT}
 
 # [Optional] Uncomment this section to install additional OS packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 #     && apt-get -y install --no-install-recommends <your-package-list-here>
 
-RUN sudo apt update \
-    && sudo apt -y install python-is-python3 pip build-essential cmake ninja-build cppcheck \
-    && sudo apt full-upgrade -y
-RUN sudo apt update \
-    && wget http://sdk-releases.upmem.com/2021.3.0/ubuntu_20.04/upmem_2021.3.0_amd64.deb \
-    && sudo apt install -y ./upmem_2021.3.0_amd64.deb
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install build-essential ninja-build cppcheck nox pre-commit\
+    && apt-get dist-upgrade -y
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem_2024.2.0_amd64.deb \
+    && apt-get install -y ./upmem_2024.2.0_amd64.deb \
+    && rm upmem_2024.2.0_amd64.deb
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -5,53 +5,51 @@
     "build": {
         "dockerfile": "Dockerfile",
         // Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic
-        "args": { "VARIANT": "focal" }
+        "args": {
+            "VARIANT": "jammy"
+        }
     },
     // "runArgs": [ "--init", "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
-
-    // Set *default* container specific settings.json values on container create.
-    "settings": {
-        "python.pythonPath": "/usr/bin/python",
-        "python.languageServer": "Pylance",
-        "python.linting.enabled": true,
-        "python.linting.pylintEnabled": true,
-        "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-        "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-        "python.formatting.provider": "black",
-        "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-        "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-        "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-        "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-        "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-        "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
-        "python.linting.pylintArgs": [
-            "--extension-pkg-whitelist=dpu_kmeans._core",
-            "--ignored-modules=dpu_kmeans._core"
-        ],
+    "customizations": {
+        "vscode": {
+            // Set *default* container specific settings.json values on container create.
+            "settings": {
+                "python.pythonPath": "/usr/bin/python",
+                "python.languageServer": "Pylance",
+                "[python]": {
+                    "editor.defaultFormatter": "charliermarsh.ruff"
+                },
+                "ruff.lint.select": [
+                    "ALL"
+                ]
+            },
+            // Add the IDs of extensions you want installed when the container is created.
+            "extensions": [
+                "llvm-vs-code-extensions.vscode-clangd",
+                "twxs.cmake",
+                "donjayamanne.githistory",
+                "ms-python.python",
+                "ms-python.vscode-pylance",
+                "cschlosser.doxdocgen",
+                "charliermarsh.ruff",
+                "mhutchie.git-graph",
+                "tamasfe.even-better-toml"
+            ]
+        }
     },
-
-    // Add the IDs of extensions you want installed when the container is created.
-    "extensions": [
-        "llvm-vs-code-extensions.vscode-clangd",
-        "twxs.cmake",
-        "donjayamanne.githistory",
-        "ms-python.python",
-        "ms-python.vscode-pylance",
-        "cschlosser.doxdocgen"
-    ],
-
     "features": {
-        "git": "os-provided",
-        "python": "os-provided"
+        "ghcr.io/devcontainers/features/git:1": {},
+        "ghcr.io/devcontainers/features/python:1": {
+            "installTools": false
+        }
     },
 
     // Use 'forwardPorts' to make a list of ports inside the container available locally.
     // "forwardPorts": [],
 
     // Use 'postCreateCommand' to run commands after the container is created.
-    "postCreateCommand": "pip install --upgrade pip nox pre-commit scikit-build pybind11[global] twine setuptools_scm hurry.filesize && pre-commit install && python setup.py clean && pip install --no-build-isolation -e .[test]",
+    "postCreateCommand": "pre-commit install && pip install -e .[test]",
 
     // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-    "remoteUser": "vscode",
+    "remoteUser": "vscode"
 }
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,37 +27,24 @@ ExternalProject_Add(
   dpu_program
   SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_program
   INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_kmeans
-  # INSTALL_DIR ${SKBUILD_PLATLIB_DIR}/${SKBUILD_PROJECT_NAME} INSTALL_DIR
-  # ${SKBUILD_PROJECT_NAME}
   CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
              -DNR_TASKLETS=${NR_TASKLETS}
-             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
-# BUILD_ALWAYS TRUE) USES_TERMINAL_BUILD TRUE)
-
-# ExternalProject_Get_Property(dpu_program INSTALL_DIR)
-# set(DPU_PROGRAM_INSTALL_DIR ${INSTALL_DIR})
-
-# install(DIRECTORY ${DPU_PROGRAM_INSTALL_DIR} DESTINATION .)
+             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+             # temporary workaround until SDK distributes fixed toolchain file
+             -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/src/dpu.cmake)
 
 # =================== BUILDING THE HOST BINARY ======================
 
 find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
 find_package(pybind11 CONFIG REQUIRED)
 
-# set(PKG_CONFIG_EXECUTABLE "dpu-pkg-config") find_package(PkgConfig REQUIRED)
-# pkg_search_module(DPU REQUIRED IMPORTED_TARGET dpu)
+include(CheckIPOSupported)
 include(${UPMEM_HOME}/share/upmem/cmake/include/host/DpuHost.cmake)
 
 find_package(OpenMP REQUIRED)
 
-set(CMAKE_C_STANDARD 99)
-set(CMAKE_C_STANDARD_REQUIRED TRUE)
-
 set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Ofast")
 
-include(CheckIPOSupported)
-check_ipo_supported(RESULT ipo_supported OUTPUT error)
-
 pybind11_add_module(
   _core MODULE src/main.cpp src/host_program/trivial_checksum_example_host.c
   src/host_program/lloyd_iter.c src/host_program/dimm_manager.c)
@@ -66,11 +53,14 @@ target_link_libraries(_core PRIVATE ${DPU_HOST_LIBRARIES} OpenMP::OpenMP_CXX
 target_include_directories(_core PUBLIC ${DPU_HOST_INCLUDE_DIRECTORIES})
 target_link_directories(_core PUBLIC ${DPU_HOST_LINK_DIRECTORIES})
 
+target_compile_features(_core PUBLIC c_std_99)
+
 target_compile_definitions(
   _core
   PUBLIC VERSION_INFO=${SKBUILD_PROJECT_VERSION}
   PUBLIC NR_TASKLETS=${NR_TASKLETS})
 
+check_ipo_supported(RESULT ipo_supported OUTPUT error)
 if(ipo_supported)
   message(STATUS "IPO / LTO enabled")
   set_target_properties(_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE
@@ -79,9 +69,10 @@ else()
   message(STATUS "IPO / LTO not supported: <${error}>")
 endif()
 
+# install in-source rather than in the temporary build directory that way
+# linting tools can find the compiled extension
 install(TARGETS _core
         LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_kmeans)
-# install(TARGETS _core LIBRARY DESTINATION ${SKBUILD_PROJECT_NAME})
 
 # copying the compilation database for language servers
 add_custom_target(

diff --git a/docs/conf.py b/docs/conf.py
@@ -50,10 +50,6 @@
 
 autosummary_generate = True
 
-# myst_gfm_only = True
-
-# html_extra_path = ["../README.md"]
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -48,7 +48,7 @@ Repository = "https://github.com/upmem/dpu_kmeans"
 
 [project.optional-dependencies]
 test = ["pytest", "pybind11-stubgen"]
-benchmarks = ["pytest", "pandas", "pyarrow"]
+benchmarks = ["pytest", "pandas", "pyarrow", "hurry.filesize"]
 doc = ["sphinx", "pydata-sphinx-theme", "myst-parser[linkify]"]
 
 [tool.setuptools_scm]

diff --git a/src/dpu.cmake b/src/dpu.cmake
@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2014-2024 - UPMEM
+#
+
+set(CMAKE_SYSTEM_NAME Generic)
+set(CMAKE_SYSTEM_PROCESSOR dpu)
+
+if(NOT DEFINED UPMEM_HOME)
+  if("$ENV{UPMEM_HOME}" STREQUAL "")
+    set(UPMEM_HOME "/usr")
+  else()
+    set(UPMEM_HOME $ENV{UPMEM_HOME})
+  endif()
+endif()
+
+set(CMAKE_SYSROOT ${UPMEM_HOME}/share/upmem/include)
+
+set(tools ${UPMEM_HOME}/bin)
+
+set(triple dpu-upmem-dpurte)
+
+set(CMAKE_C_COMPILER ${tools}/clang)
+set(CMAKE_C_COMPILER_TARGET ${triple})
+set(CMAKE_CXX_COMPILER ${tools}/clang++)
+set(CMAKE_CXX_COMPILER_TARGET ${triple})
+set(CMAKE_ASM_COMPILER ${tools}/clang)
+set(CMAKE_ASM_COMPILER_TARGET ${triple})
+
+# temporary workaround until SDK distributes llvm-ranlib
+set(CMAKE_C_COMPILER_RANLIB echo)
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+
+set(CMAKE_C_COMPILER_FORCED 1)
+
+set(CHIP_VERSION "v1A") # Default value
+if(EXISTS "/sys/class/dpu_rank/dpu_rank0/dpu_chip_id")
+  file(READ "/sys/class/dpu_rank/dpu_rank0/dpu_chip_id" CHIP_ID_NUMBER)
+  if(CHIP_ID_NUMBER GREATER 8)
+    set(CHIP_VERSION "v1B")
+  endif()
+endif()
+
+set(CMAKE_C_FLAGS_INIT "-mcpu=${CHIP_VERSION} -g")
diff --git a/src/dpu_program/CMakeLists.txt b/src/dpu_program/CMakeLists.txt
@@ -13,31 +13,39 @@ if(NOT DEFINED UPMEM_HOME)
   endif()
 endif()
 
-include("${UPMEM_HOME}/share/upmem/cmake/dpu.cmake")
-# include(CheckIPOSupported) check_ipo_supported(RESULT ipo_supported OUTPUT
-# error)
-
-# set(CMAKE_THREAD_PREFER_PTHREAD FALSE) set(THREADS_PREFER_PTHREAD_FLAG FALSE)
-set(CMAKE_THREAD_LIBS_INIT)
+# include("${UPMEM_HOME}/share/upmem/cmake/dpu.cmake")
+# CMAKE_FORCE_C_COMPILER(${tools}/dpu-upmem-dpurte-clang Clang)
+message(INFO "ranlib : ${CMAKE_RANLIB}, ${CMAKE_C_COMPILER_RANLIB}")
+include(CheckIPOSupported)
+check_ipo_supported(
+  RESULT ipo_supported
+  OUTPUT error
+  LANGUAGES C)
+if(NOT ipo_supported)
+  message(FATAL_ERROR "IPO / LTO not supported: <${error}>")
+endif()
+# message(FATAL_ERROR "Compiler ID: ${CMAKE_C_COMPILER_ID}")
 
 if(NOT CMAKE_BUILD_TYPE)
   message(FATAL_ERROR "CMAKE_BUILD_TYPE must be set")
 endif()
 
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og")
-# set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -g0")
-set(CMAKE_C_FLAGS_RELEASE "-O3 -g0")
+set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -g0")
 
 add_executable(trivial_checksum_example trivial_checksum_example.c)
 add_executable(kmeans_dpu_kernel kmeans_dpu_kernel.c)
 
 target_compile_definitions(kmeans_dpu_kernel PUBLIC NR_TASKLETS=${NR_TASKLETS})
 target_link_options(kmeans_dpu_kernel PUBLIC -DNR_TASKLETS=${NR_TASKLETS})
 
-# if( ipo_supported ) message(STATUS "IPO / LTO enabled")
-# set_target_properties(kmeans_dpu_kernel PROPERTIES
-# INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) else() message(STATUS "IPO / LTO
-# not supported: <${error}>") endif()
+if(ipo_supported)
+  message(STATUS "IPO / LTO enabled")
+  set_target_properties(kmeans_dpu_kernel
+                        PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
+else()
+  message(STATUS "IPO / LTO not supported: <${error}>")
+endif()
 
 install(TARGETS trivial_checksum_example DESTINATION ${PROJECT_NAME})
 install(TARGETS kmeans_dpu_kernel DESTINATION ${PROJECT_NAME})

diff --git a/src/dpu_program/kmeans_dpu_kernel.c b/src/dpu_program/kmeans_dpu_kernel.c
@@ -254,7 +254,7 @@ void initialize(uint8_t tasklet_id) {
  */
 #ifndef PERF_COUNTER
 void task_reduce(uint8_t tasklet_id, uint8_t icluster, int point_global_index,
-                 uint16_t point_base_index, int_feature *w_features) {
+                 uint16_t point_base_index, const int_feature *w_features) {
 #else
 void task_reduce(uint8_t tasklet_id, uint8_t icluster, int point_global_index,
                  uint16_t point_base_index, int_feature *w_features,