From 4cf742322604c7c1b21b0a018f19fbca8a338d9a Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:03:41 +0200 Subject: [PATCH 01/14] update pre-commit: - adopt ruff for formatting and linting over black and pylint - add clang tidy rules --- .clang-format | 1 + .clang-tidy | 62 +++++++++++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 27 +++++++----------- .pylintrc | 12 -------- 4 files changed, 73 insertions(+), 29 deletions(-) create mode 100644 .clang-format create mode 100644 .clang-tidy delete mode 100644 .pylintrc diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..f6cb8ad --- /dev/null +++ b/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..e950239 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,62 @@ +Checks: > + *, + -llvmlibc*, + -fuchsia*, + -altera*, + -android*, + +CheckOptions: + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.ClassMemberCase + value: lower_case + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariablePrefix + value: k + - key: readability-identifier-naming.EnumCase + value: lower_case + - key: readability-identifier-naming.EnumPrefix + value: _ + - key: readability-identifier-naming.EnumConstantCase + value: UPPER_CASE + - key: readability-identifier-naming.EnumConstantPrefix + value: '' + - key: readability-identifier-naming.FunctionCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantCase + value: UPPER_CASE + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.StaticVariableCase + value: lower_case + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE + - key: readability-identifier-naming.MacroDefinitionIgnoredRegexp + value: "(^[A-Z]+(_[A-Z]+)*_$|^_.*)" + - key: readability-identifier-naming.MemberCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberSuffix + value: _ + - key: readability-identifier-naming.PublicMemberSuffix + value: "" + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.TypeAliasCase + value: camelBack + - key: readability-identifier-naming.TypedefCase + value: lower_case + - key: readability-identifier-naming.TypeDefSuffix + value: _t + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.IgnoreMainLikeFunctions + value: 1 + - key: readability-identifier-naming.MethodCase + value: camelBack + - key: readability-identifier-naming.StructCase + value: lower_case diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0931f82..8077d9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,10 +12,12 @@ # # See https://github.com/pre-commit/pre-commit +exclude: '(_core.pyi|_kmeans_w_perf.py)' + repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -27,29 +29,20 @@ repos: - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace - - id: fix-encoding-pragma - -# isort -- repo: https://github.com/PyCQA/isort - rev: 5.10.1 - hooks: - - id: isort - name: isort (python) - args: ["--profile", "black", "--filter-files"] -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: 22.3.0 +# format with Ruff +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.2 hooks: - - id: black - exclude: ^(docs) + - id: ruff-format + exclude: ^(docs) -# C/C++ files code formatting +# C/C++ files code formatting and linting - repo: https://github.com/pocc/pre-commit-hooks rev: v1.3.5 hooks: - id: clang-format - args: ["--style=Google", "-i"] + args: ["-i"] - id: cppcheck # Changes tabs to spaces diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 52ec9fb..0000000 --- a/.pylintrc +++ /dev/null @@ -1,12 +0,0 @@ -[BASIC] - -# Good variable names which should always be accepted, separated by a comma. -good-names=X, _X, X_int, _X_int, X_mean, _allocated, _kernel, _data_id, _data_checksum, _requested_dpus, ctr, i, y, h, X_checksum, Xt - -[MESSAGES CONTROL] - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). -disable=attribute-defined-outside-init From 04300bd15487810748fbd6fc5ccfa87229b13903 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:04:02 +0200 Subject: [PATCH 02/14] map different email addresses --- .mailmap | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..c40d15a --- /dev/null +++ b/.mailmap @@ -0,0 +1,2 @@ +# All emails used by Sylvan Brocard +SylvanBrocard <81555313+SylvanBrocard@users.noreply.github.com> From 0d977d30c55f46c489c6b27c3e53c54223036443 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:05:05 +0200 Subject: [PATCH 03/14] remove leftover script --- debug_script.py | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 debug_script.py diff --git a/debug_script.py b/debug_script.py deleted file mode 100644 index efe2fca..0000000 --- a/debug_script.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -import numpy as np - -from dpu_kmeans import KMeans - -X = np.array([[1, 2], [1, 4], [1, 0], [10, 2], [10, 4], [10, 0]]) -kmeans = KMeans(n_clusters=2, random_state=0, n_dpu=2, verbose=False).fit(X) - -print(f"centroids: {kmeans.cluster_centers_}") From d13f4f15e3ee24b06ac2e98f8806c054f22488d2 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:05:35 +0200 Subject: [PATCH 04/14] format --- benchmarks/benchmark.py | 30 +++++++++++++++--------------- benchmarks/dimension/CPU.py | 1 - benchmarks/dimension/DPU.py | 1 - benchmarks/higgs/CPU+DPU.py | 7 ++++--- benchmarks/higgs/DPU_profiling.py | 7 ++++--- benchmarks/higgs/GPU.py | 17 ++++++++++++----- benchmarks/strong_scaling/GPU.py | 10 ++++++++-- benchmarks/weak_scaling/GPU.py | 10 ++++++++-- 8 files changed, 51 insertions(+), 32 deletions(-) diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py index b1dd306..95e0a8a 100644 --- a/benchmarks/benchmark.py +++ b/benchmarks/benchmark.py @@ -224,12 +224,12 @@ def log_cpu( ).all(axis=1) df.loc[cpu_index, ("results", "cpu", "times")] = cpu_time df.loc[cpu_index, ("results", "cpu", "train_times")] = CPU_kmeans.train_time_ - df.loc[ - cpu_index, ("results", "cpu", "preprocessing_times") - ] = CPU_kmeans.preprocessing_timer_ - df.loc[ - cpu_index, ("results", "cpu", "single_kmeans_times") - ] = CPU_kmeans.main_loop_timer_ + df.loc[cpu_index, ("results", "cpu", "preprocessing_times")] = ( + CPU_kmeans.preprocessing_timer_ + ) + df.loc[cpu_index, ("results", "cpu", "single_kmeans_times")] = ( + CPU_kmeans.main_loop_timer_ + ) df.loc[cpu_index, ("results", "cpu", "iterations")] = CPU_kmeans.n_iter_ df.loc[cpu_index, ("results", "cpu", "times_one_iter")] = ( CPU_kmeans.main_loop_timer_ / CPU_kmeans.n_iter_ @@ -264,18 +264,18 @@ def log_dpu( df.loc[dimm_index, ("results", "dpu", "times")] = dpu_time df.loc[dimm_index, ("results", "dpu", "train_times")] = DPU_kmeans.train_time_ df.loc[dimm_index, ("results", "dpu", "init_times")] = DPU_init_time - df.loc[ - dimm_index, ("results", "dpu", "preprocessing_times") - ] = DPU_kmeans.preprocessing_timer_ + df.loc[dimm_index, ("results", "dpu", "preprocessing_times")] = ( + DPU_kmeans.preprocessing_timer_ + ) df.loc[dimm_index, ("results", "dpu", "cpu_pim_times")] = DPU_kmeans.cpu_pim_time_ df.loc[dimm_index, ("results", "dpu", "pim_cpu_times")] = DPU_kmeans.pim_cpu_time_ df.loc[dimm_index, ("results", "dpu", "inertia_times")] = DPU_kmeans.inertia_timer_ - df.loc[ - dimm_index, ("results", "dpu", "reallocate_times") - ] = DPU_kmeans.reallocate_timer_ - df.loc[ - dimm_index, ("results", "dpu", "single_kmeans_times") - ] = DPU_kmeans.main_loop_timer_ + df.loc[dimm_index, ("results", "dpu", "reallocate_times")] = ( + DPU_kmeans.reallocate_timer_ + ) + df.loc[dimm_index, ("results", "dpu", "single_kmeans_times")] = ( + DPU_kmeans.main_loop_timer_ + ) df.loc[dimm_index, ("results", "dpu", "kernel_runtime")] = DPU_kmeans.dpu_run_time_ df.loc[dimm_index, ("results", "dpu", "inter_pim_core_times")] = ( DPU_kmeans.main_loop_timer_ - DPU_kmeans.dpu_run_time_ diff --git a/benchmarks/dimension/CPU.py b/benchmarks/dimension/CPU.py index cb06d31..cf643f1 100644 --- a/benchmarks/dimension/CPU.py +++ b/benchmarks/dimension/CPU.py @@ -37,7 +37,6 @@ n_clusters_iter = [] for n_clusters in tqdm(n_cluster_set, file=sys.stdout): - timer = 0 iter_counter = 0 inertia = np.inf diff --git a/benchmarks/dimension/DPU.py b/benchmarks/dimension/DPU.py index ef2d6aa..aff7a9c 100644 --- a/benchmarks/dimension/DPU.py +++ b/benchmarks/dimension/DPU.py @@ -45,7 +45,6 @@ n_clusters_iter = [] for n_clusters in tqdm(n_cluster_set, file=sys.stdout): - timer = 0 iter_counter = 0 diff --git a/benchmarks/higgs/CPU+DPU.py b/benchmarks/higgs/CPU+DPU.py index 460c993..9bfa906 100644 --- a/benchmarks/higgs/CPU+DPU.py +++ b/benchmarks/higgs/CPU+DPU.py @@ -59,9 +59,10 @@ higgs_file = "data/higgs.pq" df = pd.read_parquet(higgs_file) -data, tags = np.require( - df.iloc[:, 1:].to_numpy(dtype=np.float32), requirements=["C", "A", "O"] -), np.require(df.iloc[:, 0].to_numpy(dtype=int), requirements=["O"]) +data, tags = ( + np.require(df.iloc[:, 1:].to_numpy(dtype=np.float32), requirements=["C", "A", "O"]), + np.require(df.iloc[:, 0].to_numpy(dtype=int), requirements=["O"]), +) n_points, n_dim = data.shape diff --git a/benchmarks/higgs/DPU_profiling.py b/benchmarks/higgs/DPU_profiling.py index 886ca65..7ab9aab 100644 --- a/benchmarks/higgs/DPU_profiling.py +++ b/benchmarks/higgs/DPU_profiling.py @@ -32,9 +32,10 @@ higgs_file = "data/higgs.pq" df = pd.read_parquet(higgs_file) -data, tags = np.require( - df.iloc[:, 1:].to_numpy(dtype=np.float32), requirements=["C", "A", "O"] -), np.require(df.iloc[:, 0].to_numpy(dtype=int), requirements=["O"]) +data, tags = ( + np.require(df.iloc[:, 1:].to_numpy(dtype=np.float32), requirements=["C", "A", "O"]), + np.require(df.iloc[:, 0].to_numpy(dtype=int), requirements=["O"]), +) n_points, n_dim = data.shape diff --git a/benchmarks/higgs/GPU.py b/benchmarks/higgs/GPU.py index a138645..fc6f98f 100644 --- a/benchmarks/higgs/GPU.py +++ b/benchmarks/higgs/GPU.py @@ -51,9 +51,10 @@ def np2cudf(df): higgs_file = "data/higgs.pq" df = pd.read_parquet(higgs_file) -data, tags = np.require( - df.iloc[:, 1:].to_numpy(dtype=np.float32), requirements=["C", "A", "O"] -), np.require(df.iloc[:, 0].to_numpy(dtype=int), requirements=["O"]) +data, tags = ( + np.require(df.iloc[:, 1:].to_numpy(dtype=np.float32), requirements=["C", "A", "O"]), + np.require(df.iloc[:, 0].to_numpy(dtype=int), requirements=["O"]), +) n_points, n_dim = data.shape @@ -92,7 +93,10 @@ def np2cudf(df): toc = time.perf_counter() # read output -(GPU_centroids, GPU_iter_counter,) = ( +( + GPU_centroids, + GPU_iter_counter, +) = ( GPU_kmeans.cluster_centers_, GPU_kmeans.n_iter_, ) @@ -119,7 +123,10 @@ def np2cudf(df): toc = time.perf_counter() # read output -(CPU_centroids, CPU_iter_counter,) = ( +( + CPU_centroids, + CPU_iter_counter, +) = ( CPU_kmeans.cluster_centers_, CPU_kmeans.n_iter_, ) diff --git a/benchmarks/strong_scaling/GPU.py b/benchmarks/strong_scaling/GPU.py index 0dee32f..6611380 100644 --- a/benchmarks/strong_scaling/GPU.py +++ b/benchmarks/strong_scaling/GPU.py @@ -90,7 +90,10 @@ def np2cudf(df): toc = time.perf_counter() # read output -(GPU_centroids, GPU_iter_counter,) = ( +( + GPU_centroids, + GPU_iter_counter, +) = ( GPU_kmeans.cluster_centers_, GPU_kmeans.n_iter_, ) @@ -117,7 +120,10 @@ def np2cudf(df): toc = time.perf_counter() # read output -(CPU_centroids, CPU_iter_counter,) = ( +( + CPU_centroids, + CPU_iter_counter, +) = ( CPU_kmeans.cluster_centers_, CPU_kmeans.n_iter_, ) diff --git a/benchmarks/weak_scaling/GPU.py b/benchmarks/weak_scaling/GPU.py index df4459a..b80fff8 100644 --- a/benchmarks/weak_scaling/GPU.py +++ b/benchmarks/weak_scaling/GPU.py @@ -100,7 +100,10 @@ def np2cudf(df): toc = time.perf_counter() # read output - (GPU_centroids, GPU_iter_counter,) = ( + ( + GPU_centroids, + GPU_iter_counter, + ) = ( GPU_kmeans.cluster_centers_, GPU_kmeans.n_iter_, ) @@ -127,7 +130,10 @@ def np2cudf(df): toc = time.perf_counter() # read output - (CPU_centroids, CPU_iter_counter,) = ( + ( + CPU_centroids, + CPU_iter_counter, + ) = ( CPU_kmeans.cluster_centers_, CPU_kmeans.n_iter_, ) From f11299a593d63950dca79976ea1e9dc7691134bc Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:07:17 +0200 Subject: [PATCH 05/14] add git ignoreRevsFile --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..e933b5a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# ruff format + +d13f4f15e3ee24b06ac2e98f8806c054f22488d2 From 4672831d24d335458b7aa8e26188e157f93a4032 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:10:31 +0200 Subject: [PATCH 06/14] Modernize build system: - migrate from scikit-build to scikit-build-core - remove obsolescent files MANIFEST.in, setup.cfg an setup.py - move entire configuration to pyproject.toml - add project classifiers - enforce older versions of numpy and scikit-learn (this project relies on older sklearn implementation details) --- MANIFEST.in | 5 ---- pyproject.toml | 64 +++++++++++++++++++++++++++++++++++++++++++------- setup.cfg | 10 -------- setup.py | 46 ------------------------------------ 4 files changed, 55 insertions(+), 70 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index f04ecbc..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include pyproject.toml -include README.md -include LICENSE -include CMakeLists.txt -recursive-include src *.py *.c *.cpp *.h *.txt diff --git a/pyproject.toml b/pyproject.toml index 6d4d825..255b8c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,59 @@ [build-system] requires = [ - "setuptools>=42", - "wheel", - "pybind11>=2.7.1", - "cmake>=3.21", - "scikit-build>=0.12", - "ninja", - "pkgconfig", - "setuptools_scm>=6.2", + "pybind11", + "scikit-build-core>=0.10", ] +build-backend = "scikit_build_core.build" -build-backend = "setuptools.build_meta" +[project] +dynamic = ["version"] +name = "dpu_kmeans" +description = "A package for the k-means algorithm on DPU." +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX :: Linux", + "Topic :: Scientific/Engineering", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: Implementation :: CPython", +] +authors = [{ name = "Sylvan Brocard", email = "sbrocard@upmem.com" }] +requires-python = ">=3.7" +dependencies = [ + "numpy<2", + "scikit-learn<1.1", + "importlib_resources; python_version<'3.9'", + "xxhash", +] + +[project.urls] +Repository = "https://github.com/upmem/dpu_kmeans" + +[project.optional-dependencies] +test = ["pytest", "pybind11-stubgen"] +benchmarks = ["pytest", "pandas", "pyarrow", "hurry.filesize"] +doc = ["sphinx", "pydata-sphinx-theme", "myst-parser[linkify]"] + +[tool.setuptools_scm] + +[tool.scikit-build] +minimum-version = "build-system.requires" +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" +sdist.exclude = ["benchmarks", ".dvc", ".dvcignore", ".github", ".devcontainer"] +sdist.include = ["src/dpu_kmeans/dpu_program", "src/dpu_kmeans/_core.*.so"] + +[tool.scikit-build.cmake] +build-type = "Release" + +[tool.scikit-build.cmake.define] +NR_TASKLETS = "16" +UPMEM_HOME = {env = "UPMEM_HOME", default = "/usr"} diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c47e2ed..0000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[metadata] -name = dpu_kmeans -description = A package for the k-means algorithm on DPU. -long_description = file: README.md -long_description_content_type = text/markdown -license = MIT -license_files = LICENSE -url = https://github.com/upmem/dpu_kmeans -author = Sylvan Brocard -author_email = sylvan.brocard@gmail.com diff --git a/setup.py b/setup.py deleted file mode 100644 index 5b6d8c5..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys - -try: - from skbuild import setup -except ImportError: - print( - "Please update pip, you need pip 10 or greater,\n" - " or you need to install the PEP 518 requirements in pyproject.toml yourself", - file=sys.stderr, - ) - raise - -from setuptools import find_packages -from setuptools_scm import get_version - -VERSION = get_version(local_scheme="no-local-version") -VERSION = "".join([c for c in VERSION if c.isdigit() or c == "."]) - -# compilation of the host library -setup( - use_scm_version={ - "root": ".", - "relative_to": __file__, - }, - packages=find_packages(where="src"), - package_dir={"": "src"}, - cmake_install_dir="src/dpu_kmeans", - include_package_data=True, - install_requires=[ - "numpy", - "scikit-learn", - "importlib_resources;python_version<'3.9'", - "xxhash", - ], - extras_require={ - "test": ["pytest"], - "benchmarks": ["pytest", "pandas", "pyarrow"], - }, - zip_safe=False, - cmake_args=[ - "-DNR_TASKLETS=16", # number of parallel tasklets on each DPU - f"-DVERSION={VERSION}", - ], -) From 9ca1d2739e743f433c84405965bc6c6c5fe34398 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:14:29 +0200 Subject: [PATCH 07/14] CMake updates: - temporarily add custom DPU toolchain file until the SDK gets the same fix - modernize - handle LTO through CMake --- CMakeLists.txt | 75 ++++++++++++++++++---------------- README.md | 6 +++ src/dpu.cmake | 47 +++++++++++++++++++++ src/dpu_program/CMakeLists.txt | 26 ++++++++---- 4 files changed, 111 insertions(+), 43 deletions(-) create mode 100644 src/dpu.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e9fbbe8..e3ffea3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,22 @@ -cmake_minimum_required(VERSION 3.15...3.19) +cmake_minimum_required(VERSION 3.15...3.27) set(CMAKE_C_COMPILER "gcc") set(CMAKE_CXX_COMPILER "g++") -project(dpu_kmeans VERSION ${VERSION}) +project(${SKBUILD_PROJECT_NAME} VERSION ${SKBUILD_PROJECT_VERSION}) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # set(CMAKE_VERBOSE_MAKEFILE ON) if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) + message(FATAL_ERROR "CMAKE_BUILD_TYPE must be set") +endif() + +if(NOT DEFINED UPMEM_HOME) + if("$ENV{UPMEM_HOME}" STREQUAL "") + set(UPMEM_HOME "/usr") + else() + set(UPMEM_HOME $ENV{UPMEM_HOME}) + endif() endif() # =================== BUILDING THE DPU BINARY ====================== @@ -18,58 +26,53 @@ include(ExternalProject) ExternalProject_Add( dpu_program SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_program - INSTALL_DIR ./dpu_program + INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_kmeans CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= - -DNR_TASKLETS=${NR_TASKLETS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - BUILD_ALWAYS TRUE - USES_TERMINAL_BUILD TRUE) - -ExternalProject_Get_Property(dpu_program INSTALL_DIR) -set(DPU_PROGRAM_INSTALL_DIR ${INSTALL_DIR}) - -install(DIRECTORY ${DPU_PROGRAM_INSTALL_DIR} DESTINATION .) + -DNR_TASKLETS=${NR_TASKLETS} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + # temporary workaround until SDK distributes fixed toolchain file + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/src/dpu.cmake) # =================== BUILDING THE HOST BINARY ====================== -if(SKBUILD) - # Scikit-Build does not add your site-packages to the search path - # automatically, so we need to add it _or_ the pybind11 specific directory - # here. - execute_process( - COMMAND "${PYTHON_EXECUTABLE}" -c - "import pybind11; print(pybind11.get_cmake_dir())" - OUTPUT_VARIABLE _tmp_dir - OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ECHO STDOUT) - list(APPEND CMAKE_PREFIX_PATH "${_tmp_dir}") -endif() - -# Now we can find pybind11 +find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) find_package(pybind11 CONFIG REQUIRED) -set(PKG_CONFIG_EXECUTABLE "dpu-pkg-config") -find_package(PkgConfig REQUIRED) -pkg_search_module(DPU REQUIRED IMPORTED_TARGET dpu) +include(CheckIPOSupported) +include(${UPMEM_HOME}/share/upmem/cmake/include/host/DpuHost.cmake) find_package(OpenMP REQUIRED) +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Ofast") + pybind11_add_module( _core MODULE src/main.cpp src/host_program/trivial_checksum_example_host.c src/host_program/lloyd_iter.c src/host_program/dimm_manager.c) -target_link_libraries(_core PRIVATE PkgConfig::DPU OpenMP::OpenMP_CXX +target_link_libraries(_core PRIVATE ${DPU_HOST_LIBRARIES} OpenMP::OpenMP_CXX OpenMP::OpenMP_C) -target_include_directories(_core PUBLIC PkgConfig::DPU) +target_include_directories(_core PUBLIC ${DPU_HOST_INCLUDE_DIRECTORIES}) +target_link_directories(_core PUBLIC ${DPU_HOST_LINK_DIRECTORIES}) + +target_compile_features(_core PUBLIC c_std_99) target_compile_definitions( _core - PUBLIC VERSION_INFO=${PROJECT_VERSION} + PUBLIC VERSION_INFO=${SKBUILD_PROJECT_VERSION} PUBLIC NR_TASKLETS=${NR_TASKLETS}) -set(CMAKE_C_STANDARD 99) -set(CMAKE_C_STANDARD_REQUIRED TRUE) - -set(CMAKE_C_FLAGS_RELEASE "-Ofast") +check_ipo_supported(RESULT ipo_supported OUTPUT error) +if(ipo_supported) + message(STATUS "IPO / LTO enabled") + set_target_properties(_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE + TRUE) +else() + message(STATUS "IPO / LTO not supported: <${error}>") +endif() -install(TARGETS _core DESTINATION .) +# install in-source rather than in the temporary build directory that way +# linting tools can find the compiled extension +install(TARGETS _core + LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/src/dpu_kmeans) # copying the compilation database for language servers add_custom_target( diff --git a/README.md b/README.md index de0a03c..8468ec6 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ Installation - install the [UPMEM SDK](https://sdk.upmem.com/) - `pip install dpu-kmeans` +If you get an error about RANLIB, create a symlink to `llvm-ar`: + +```bash +sudo ln -s /usr/bin/llvm-ar /usr/bin/llvm-ranlib +``` + Usage ----- diff --git a/src/dpu.cmake b/src/dpu.cmake new file mode 100644 index 0000000..0ac3d33 --- /dev/null +++ b/src/dpu.cmake @@ -0,0 +1,47 @@ +# +# Copyright (c) 2014-2024 - UPMEM +# + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR dpu) + +if(NOT DEFINED UPMEM_HOME) + if("$ENV{UPMEM_HOME}" STREQUAL "") + set(UPMEM_HOME "/usr") + else() + set(UPMEM_HOME $ENV{UPMEM_HOME}) + endif() +endif() + +set(CMAKE_SYSROOT ${UPMEM_HOME}/share/upmem/include) + +set(tools ${UPMEM_HOME}/bin) + +set(triple dpu-upmem-dpurte) + +set(CMAKE_C_COMPILER ${tools}/clang) +set(CMAKE_C_COMPILER_TARGET ${triple}) +set(CMAKE_CXX_COMPILER ${tools}/clang++) +set(CMAKE_CXX_COMPILER_TARGET ${triple}) +set(CMAKE_ASM_COMPILER ${tools}/clang) +set(CMAKE_ASM_COMPILER_TARGET ${triple}) + +# temporary workaround until SDK distributes llvm-ranlib +set(CMAKE_C_COMPILER_RANLIB echo) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_C_COMPILER_FORCED 1) + +set(CHIP_VERSION "v1A") # Default value +if(EXISTS "/sys/class/dpu_rank/dpu_rank0/dpu_chip_id") + file(READ "/sys/class/dpu_rank/dpu_rank0/dpu_chip_id" CHIP_ID_NUMBER) + if(CHIP_ID_NUMBER GREATER 8) + set(CHIP_VERSION "v1B") + endif() +endif() + +set(CMAKE_C_FLAGS_INIT "-mcpu=${CHIP_VERSION} -g") diff --git a/src/dpu_program/CMakeLists.txt b/src/dpu_program/CMakeLists.txt index 7e1ea34..9d98c9a 100644 --- a/src/dpu_program/CMakeLists.txt +++ b/src/dpu_program/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...3.19) +cmake_minimum_required(VERSION 3.15...3.27) project(dpu_program) @@ -13,14 +13,14 @@ if(NOT DEFINED UPMEM_HOME) endif() endif() -include("${UPMEM_HOME}/share/upmem/cmake/dpu.cmake") +include(CheckIPOSupported) if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) + message(FATAL_ERROR "CMAKE_BUILD_TYPE must be set") endif() -set(CMAKE_C_FLAGS_DEBUG "-g -Og") -set(CMAKE_C_FLAGS_RELEASE "-g0 -O3 -flto=thin") +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og") +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -g0") add_executable(trivial_checksum_example trivial_checksum_example.c) add_executable(kmeans_dpu_kernel kmeans_dpu_kernel.c) @@ -28,8 +28,20 @@ add_executable(kmeans_dpu_kernel kmeans_dpu_kernel.c) target_compile_definitions(kmeans_dpu_kernel PUBLIC NR_TASKLETS=${NR_TASKLETS}) target_link_options(kmeans_dpu_kernel PUBLIC -DNR_TASKLETS=${NR_TASKLETS}) -install(TARGETS trivial_checksum_example DESTINATION .) -install(TARGETS kmeans_dpu_kernel DESTINATION .) +check_ipo_supported( + RESULT ipo_supported + OUTPUT error + LANGUAGES C) +if(ipo_supported) + message(STATUS "IPO / LTO enabled") + set_target_properties(kmeans_dpu_kernel + PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) +else() + message(FATAL_ERROR "IPO / LTO not supported: <${error}>") +endif() + +install(TARGETS trivial_checksum_example DESTINATION ${PROJECT_NAME}) +install(TARGETS kmeans_dpu_kernel DESTINATION ${PROJECT_NAME}) # copying the compilation database for language servers add_custom_target( From 816e4a8b23362cea614b23cc1d58d2ff1527949d Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:17:14 +0200 Subject: [PATCH 08/14] doc: - publish it to GH pages - include the README in it - adopt new sklearn theme - update usage example to current API --- .github/workflows/documentation.yml | 37 +++++++++++++++++++++++++++++ README.md | 25 +++++++++---------- docs/README.md | 3 +++ docs/conf.py | 18 +++++++++----- docs/dpu_kmeans.rst | 5 ++++ docs/index.rst | 11 ++++----- src/dpu_kmeans/_kmeans.py | 11 ++++----- 7 files changed, 80 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/documentation.yml create mode 100644 docs/README.md diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..f8b2320 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,37 @@ +name: documentation + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +permissions: + contents: write + +jobs: + docs: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Download SDK + run: wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem_2024.2.0_amd64.deb + - name: Install SDK + run: sudo apt install -y ./upmem_2024.2.0_amd64.deb + - name: Build and install + run: pip install --verbose .[doc] + - name: Sphinx build + run: | + sphinx-build docs _build + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + with: + publish_branch: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: _build/ + force_orphan: true diff --git a/README.md b/README.md index 8468ec6..1b7e777 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,22 @@ -dpu_kmeans -========== +dpu_kmeans README +================= -| CI | status | -|----------------------|--------| -| pip builds | [![Pip Actions Status][actions-pip-badge]][actions-pip-link] | +| CI | status | +| ---------- | ------------------------------------------------------------ | +| pip builds | [![Pip Actions Status][actions-pip-badge]][actions-pip-link] | +| docs | [![Doc Status][actions-docs-badge]][actions-docs-link] | A project built with [pybind11](https://github.com/pybind/pybind11) and [scikit-build](https://github.com/scikit-build/scikit-build), running the KMeans algorithm on in-memory processors with the UPMEM SDK. -[actions-badge]: https://github.com/upmem/dpu_kmeans/workflows/Tests/badge.svg -[actions-conda-link]: https://github.com/upmem/dpu_kmeans/actions?query=workflow%3AConda -[actions-conda-badge]: https://github.com/upmem/dpu_kmeans/workflows/Conda/badge.svg [actions-pip-link]: https://github.com/upmem/dpu_kmeans/actions?query=workflow%3APip [actions-pip-badge]: https://github.com/upmem/dpu_kmeans/workflows/Pip/badge.svg -[actions-wheels-link]: https://github.com/upmem/dpu_kmeans/actions?query=workflow%3AWheels -[actions-wheels-badge]: https://github.com/upmem/dpu_kmeans/workflows/Wheels/badge.svg +[actions-docs-link]: https://github.com/upmem/dpu_kmeans/actions?query=workflow%3Adocumentation +[actions-docs-badge]: https://github.com/upmem/dpu_kmeans/workflows/documentation/badge.svg + +Documentation +------------- + +- [API documentation](https://upmem.github.io/dpu_kmeans/) Installation ------------ @@ -115,5 +118,3 @@ dpu_kmeans.test_checksum() ``` Expected return: `0x007f8000` - -[`cibuildwheel`]: https://cibuildwheel.readthedocs.io diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..a00dc70 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +```{include} ../README.md +:relative-docs: docs/ +``` diff --git a/docs/conf.py b/docs/conf.py index 6005185..64ceea5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,6 +45,7 @@ "sphinx.ext.githubpages", "sphinx.ext.autosummary", "sphinx.ext.napoleon", + "myst_parser", ] autosummary_generate = True @@ -55,8 +56,7 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = [".rst", ".md"] # The master toctree document. master_doc = "index" @@ -66,7 +66,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -82,13 +82,16 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "alabaster" +html_theme = "pydata_sphinx_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # -# html_theme_options = {} +html_theme_options = { + "github_url": "https://github.com/upmem/dpu_kmeans", + "show_toc_level": 1, +} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -188,4 +191,7 @@ # -- Extension configuration ------------------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} +intersphinx_mapping = { + "python": ("https://docs.python.org/", None), + "scikit-learn": ("https://scikit-learn.org/stable/", None), +} diff --git a/docs/dpu_kmeans.rst b/docs/dpu_kmeans.rst index e7b76f3..82afbad 100644 --- a/docs/dpu_kmeans.rst +++ b/docs/dpu_kmeans.rst @@ -1 +1,6 @@ +dpu\_kmeans API +=============== + .. automodule:: dpu_kmeans + :members: + :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index ccd5c70..695660f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,19 +1,18 @@ -.. dpu_trees documentation master file, created by +.. dpu_kmeans documentation master file, created by sphinx-quickstart on Tue Oct 12 13:23:17 2021. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to dpu_trees's documentation! -===================================== +Welcome to dpu_kmeans's documentation! +====================================== .. toctree:: - :maxdepth: 2 + :maxdepth: 3 :caption: Contents: + README.md dpu_kmeans - - Indices and tables ================== diff --git a/src/dpu_kmeans/_kmeans.py b/src/dpu_kmeans/_kmeans.py index 7a2740e..929a2f8 100644 --- a/src/dpu_kmeans/_kmeans.py +++ b/src/dpu_kmeans/_kmeans.py @@ -344,12 +344,11 @@ class KMeans(KMeansCPU): -------- >>> import numpy as np - >>> from dpu_kmeans import DIMM_data, KMeans - >>> X = np.array([[1, 2], [1, 4], [1, 0],[10, 2], [10, 4], [10, 0]]) - >>> dimm_data = DIMM_data(X) + >>> from dpu_kmeans import KMeans + >>> X = np.array([[1, 2], [1, 4], [1, 0], [10, 2], [10, 4], [10, 0]]) >>> kmeans = KMeans(2) - >>> centroids, iterations, time = kmeans.fit(dimm_data) - >>> print(centroids) + >>> kmeans.fit(X) + >>> print(kmeans.cluster_centers_) [[ 0.9998627 2. ] [10.000137 2. ]] """ @@ -386,7 +385,7 @@ def fit(self, X, y=None, sample_weight=None): Returns ------- - self : object + self : KMeans Fitted estimator. """ tic = time.perf_counter() From 4eb516b9aad1411bed9fc3136c6794856c8a100b Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:17:42 +0200 Subject: [PATCH 09/14] modernize dev container --- .devcontainer/Dockerfile | 18 ++++----- .devcontainer/devcontainer.json | 72 ++++++++++++++++----------------- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ceb42ac..52b6ef1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,16 +1,16 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.194.3/containers/ubuntu/.devcontainer/base.Dockerfile +# See here for image contents: https://github.com/devcontainers/images/tree/main/src/base-ubuntu -# [Choice] Ubuntu version: hirsute, bionic, focal -ARG VARIANT="focal" +ARG VARIANT="jammy" FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} # [Optional] Uncomment this section to install additional OS packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends -RUN sudo apt update \ - && sudo apt -y install python-is-python3 pip build-essential cmake ninja-build cppcheck \ - && sudo apt full-upgrade -y -RUN sudo apt update \ - && wget http://sdk-releases.upmem.com/2021.3.0/ubuntu_20.04/upmem_2021.3.0_amd64.deb \ - && sudo apt install -y ./upmem_2021.3.0_amd64.deb +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install build-essential ninja-build cppcheck nox pre-commit\ + && apt-get dist-upgrade -y +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem_2024.2.0_amd64.deb \ + && apt-get install -y ./upmem_2024.2.0_amd64.deb \ + && rm upmem_2024.2.0_amd64.deb diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b001699..4eb5da4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,53 +5,51 @@ "build": { "dockerfile": "Dockerfile", // Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic - "args": { "VARIANT": "focal" } + "args": { + "VARIANT": "jammy" + } }, // "runArgs": [ "--init", "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - - // Set *default* container specific settings.json values on container create. - "settings": { - "python.pythonPath": "/usr/bin/python", - "python.languageServer": "Pylance", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.formatting.provider": "black", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", - "python.linting.pylintArgs": [ - "--extension-pkg-whitelist=dpu_kmeans._core", - "--ignored-modules=dpu_kmeans._core" - ], + "customizations": { + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.pythonPath": "/usr/bin/python", + "python.languageServer": "Pylance", + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff" + }, + "ruff.lint.select": [ + "ALL" + ] + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "llvm-vs-code-extensions.vscode-clangd", + "twxs.cmake", + "donjayamanne.githistory", + "ms-python.python", + "ms-python.vscode-pylance", + "cschlosser.doxdocgen", + "charliermarsh.ruff", + "mhutchie.git-graph", + "tamasfe.even-better-toml" + ] + } }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "llvm-vs-code-extensions.vscode-clangd", - "twxs.cmake", - "donjayamanne.githistory", - "ms-python.python", - "ms-python.vscode-pylance", - "cschlosser.doxdocgen" - ], - "features": { - "git": "os-provided", - "python": "os-provided" + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/python:1": { + "installTools": false + } }, // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip install --upgrade pip nox pre-commit scikit-build pybind11[global] twine setuptools_scm hurry.filesize && pre-commit install && python setup.py clean && pip install --no-build-isolation -e .[test]", + "postCreateCommand": "pre-commit install && pip install -e .[test]", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", + "remoteUser": "vscode" } From dcd2a3be5c045ce85577d29455fc4f8e8aee77f4 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:18:06 +0200 Subject: [PATCH 10/14] update pip CI --- .github/workflows/pip.yml | 28 ++++++++++++++++------------ .github/workflows/wheels.yml | 6 +++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 0425870..7994f0b 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -13,23 +13,25 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-latest] + platform: [ubuntu-22.04] python-version: ["3.7", "3.8", "3.9", "3.10"] runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Download SDK - run: wget http://sdk-releases.upmem.com/2021.2.0/ubuntu_20.04/upmem_2021.2.0_amd64.deb + run: wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem_2024.2.0_amd64.deb - name: Install SDK - run: sudo apt install -y ./upmem_2021.2.0_amd64.deb + run: sudo apt install -y ./upmem_2024.2.0_amd64.deb - name: Build and install run: pip install --verbose .[test] @@ -42,30 +44,32 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-latest] + platform: [ubuntu-22.04] python-version: ["3.7", "3.8", "3.9", "3.10"] runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Download SDK - run: wget http://sdk-releases.upmem.com/2021.2.0/ubuntu_20.04/upmem-2021.2.0-Linux-x86_64.tar.gz + run: wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem-2024.2.0-Linux-x86_64.tar.gz - name: Untar SDK - run: tar -zxvf upmem-2021.2.0-Linux-x86_64.tar.gz + run: tar -zxvf upmem-2024.2.0-Linux-x86_64.tar.gz - name: Build and install run: | - source upmem-2021.2.0-Linux-x86_64/upmem_env.sh + source upmem-2024.2.0-Linux-x86_64/upmem_env.sh pip install --verbose .[test] - name: Test run: | - source upmem-2021.2.0-Linux-x86_64/upmem_env.sh + source upmem-2024.2.0-Linux-x86_64/upmem_env.sh pytest diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b443c98..e0a3fbe 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -18,17 +18,17 @@ env: jobs: build_sdist: name: Build SDist - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 with: submodules: true - name: Download SDK - run: wget http://sdk-releases.upmem.com/2021.2.0/ubuntu_20.04/upmem_2021.2.0_amd64.deb + run: wget http://sdk-releases.upmem.com/2024.2.0/ubuntu_22.04/upmem_2024.2.0_amd64.deb - name: Install SDK - run: sudo apt install -y ./upmem_2021.2.0_amd64.deb + run: sudo apt install -y ./upmem_2024.2.0_amd64.deb - name: Build SDist run: pipx run build --sdist From 423a49d58899eb9871577fa946a4f481b2664750 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:18:24 +0200 Subject: [PATCH 11/14] autogenerate pybind11 stub and test stub generation --- src/dpu_kmeans/_core.pyi | 80 ++++++++++++++++++++++++++++++++++++++++ tests/test_stubgen.py | 32 ++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 src/dpu_kmeans/_core.pyi create mode 100644 tests/test_stubgen.py diff --git a/src/dpu_kmeans/_core.pyi b/src/dpu_kmeans/_core.pyi new file mode 100644 index 0000000..bb6804e --- /dev/null +++ b/src/dpu_kmeans/_core.pyi @@ -0,0 +1,80 @@ +""" + + DPU kmeans plugin + ----------------- + + .. currentmodule:: dpu_kmeans._core + + .. autosummary:: + :toctree: _generate + + add + subtract + checksum + Container + +""" +from __future__ import annotations +import numpy +__all__ = ['Container', 'FEATURE_TYPE', 'add', 'checksum', 'subtract'] +class Container: + """ + + Container object to interface with the DPUs + + """ + @staticmethod + def _pybind11_conduit_v1_(*args, **kwargs): + ... + def __init__(self) -> None: + ... + def allocate(self) -> None: + ... + def allocate_host_memory(self) -> None: + ... + def compute_inertia(self, arg0: numpy.ndarray[numpy.int16]) -> int: + ... + def deallocate_host_memory(self) -> None: + ... + def free_data(self) -> None: + ... + def free_dpus(self) -> None: + ... + def get_cpu_pim_time(self) -> float: + ... + def get_dpu_run_time(self) -> float: + ... + def get_nr_dpus(self) -> int: + ... + def get_pim_cpu_time(self) -> float: + ... + def lloyd_iter(self, arg0: numpy.ndarray[numpy.int16], arg1: numpy.ndarray[numpy.int64], arg2: numpy.ndarray[numpy.int32]) -> None: + ... + def load_array_data(self, arg0: numpy.ndarray[numpy.int16], arg1: int, arg2: int, arg3: int) -> None: + ... + def load_kernel(self, arg0: str) -> None: + ... + def load_n_clusters(self, arg0: int) -> None: + ... + def reset_timer(self) -> None: + ... + def set_nr_dpus(self, arg0: int) -> None: + ... +def add(arg0: int, arg1: int) -> int: + """ + Add two numbers + + Some other explanation about the add function. + """ +def checksum(arg0: str) -> int: + """ + Checksum test on dpus + """ +def subtract(arg0: int, arg1: int) -> int: + """ + Subtract two numbers + + Some other explanation about the subtract function. + """ +FEATURE_TYPE: int = 16 +__version__: str = '0.1.9' diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py new file mode 100644 index 0000000..02a190f --- /dev/null +++ b/tests/test_stubgen.py @@ -0,0 +1,32 @@ +# Author: Sylvan Brocard +# License: MIT +"""Module for testing the _core module stub generation.""" + +import subprocess +from pathlib import Path + + +def test_stub(tmp_path): + """Test _core module stub generation and verify its correctness.""" + # generate stub + result = subprocess.run( + ["pybind11-stubgen", "dpu_kmeans._core", "--output", tmp_path], + check=True, + ) + assert result.returncode == 0, f"Failed to generate stub: {result.stderr}" + + # compare the generated stub with the distributed one + with Path("src/dpu_kmeans/_core.pyi").open( + "r", + encoding="utf-8", + ) as distributed_stub: + expected = distributed_stub.read() + with Path(f"{tmp_path}/dpu_kmeans/_core.pyi").open( + "r", + encoding="utf-8", + ) as generated_stub: + generated = generated_stub.read() + assert expected == generated, ( + "Generated stub is different from the distributed one, " + "re-run `pybind11-stubgen dpu_kmeans._core --output src`" + ) From a08a9cc991cfb6365d513840ecb00debd7e792c3 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:19:19 +0200 Subject: [PATCH 12/14] `from dpu_kmeans import *` now only import KMeans --- src/dpu_kmeans/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dpu_kmeans/__init__.py b/src/dpu_kmeans/__init__.py index 399336f..6d44897 100644 --- a/src/dpu_kmeans/__init__.py +++ b/src/dpu_kmeans/__init__.py @@ -4,9 +4,12 @@ # License: MIT from ._checksum import test_checksum -from ._core import __doc__, __version__, add, subtract +from ._core import __version__, add, subtract from ._kmeans import KMeans def add2(x: int, y: int) -> int: return add(x, y) + + +__all__ = ["KMeans"] From f2bca2364c78626116aafa7eb8c4504303513088 Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:19:48 +0200 Subject: [PATCH 13/14] address new cpplint diagnostic --- src/dpu_program/kmeans_dpu_kernel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dpu_program/kmeans_dpu_kernel.c b/src/dpu_program/kmeans_dpu_kernel.c index 6508134..672470b 100644 --- a/src/dpu_program/kmeans_dpu_kernel.c +++ b/src/dpu_program/kmeans_dpu_kernel.c @@ -254,7 +254,7 @@ void initialize(uint8_t tasklet_id) { */ #ifndef PERF_COUNTER void task_reduce(uint8_t tasklet_id, uint8_t icluster, int point_global_index, - uint16_t point_base_index, int_feature *w_features) { + uint16_t point_base_index, const int_feature *w_features) { #else void task_reduce(uint8_t tasklet_id, uint8_t icluster, int point_global_index, uint16_t point_base_index, int_feature *w_features, From 059fb4da371f4a6bec9c305140ad6e9a794e6d6e Mon Sep 17 00:00:00 2001 From: SylvanBrocard Date: Thu, 24 Oct 2024 18:20:41 +0200 Subject: [PATCH 14/14] ruff format --- src/dpu_kmeans/perf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dpu_kmeans/perf.py b/src/dpu_kmeans/perf.py index 1f893d8..4bce29f 100644 --- a/src/dpu_kmeans/perf.py +++ b/src/dpu_kmeans/perf.py @@ -12,7 +12,14 @@ def reset_timers(): - global host_reduction_timer, host_quantize_timer, host_unquantize_timer, host_average_timer, host_c_timer, alloc_dealloc_timer, host_reduction_timer + global \ + host_reduction_timer, \ + host_quantize_timer, \ + host_unquantize_timer, \ + host_average_timer, \ + host_c_timer, \ + alloc_dealloc_timer, \ + host_reduction_timer host_quantize_timer = 0 host_unquantize_timer = 0 host_average_timer = 0