Skip to content

Commit

Permalink
Merge pull request #34 from ssciwr/restructure
Browse files Browse the repository at this point in the history
Restructure project layout to allow mixing of pure Python and compiled module
  • Loading branch information
lkeegan authored Jan 12, 2024
2 parents 1be961d + 05b9a91 commit 7e4813a
Show file tree
Hide file tree
Showing 31 changed files with 199 additions and 159 deletions.
24 changes: 7 additions & 17 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,13 @@ jobs:
with:
submodules: "recursive"

- name: Make build directory
run: cmake -E make_directory ${{runner.workspace}}/build

- name: Configure cmake
shell: bash
working-directory: ${{runner.workspace}}/build
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Debug -DBUILD_DOCS=OFF -DBUILD_TESTING=ON -DBUILD_PYTHON=OFF

- name: Build
shell: bash
working-directory: ${{runner.workspace}}/build
run: cmake --build .

- name: Run tests
shell: bash
working-directory: ${{runner.workspace}}/build
run: ctest
- name: Build and run c++ tests
run: |
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_DOCS=OFF -DBUILD_PYTHON=OFF -DBUILD_TESTING=ON ..
cmake --build .
ctest
python:
name: "${{ matrix.os }} :: Python ${{ matrix.python-version }}"
Expand Down
4 changes: 2 additions & 2 deletions .readthedocs.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
version: 2

build:
os: "ubuntu-20.04"
os: "ubuntu-22.04"
tools:
python: "3.9"
python: "3.12"

sphinx:
builder: html
Expand Down
28 changes: 14 additions & 14 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ cmake_minimum_required(VERSION 3.15..3.26)
# Set a name and a version number for your project:
project(
pybind11-numpy-example
VERSION 0.0.7
VERSION 0.0.9
LANGUAGES CXX)

# Initialize some default paths
include(GNUInstallDirs)

# Define the minimum C++ standard that is required
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

set(CMAKE_POSITION_INDEPENDENT_CODE ON)
Expand All @@ -19,15 +19,15 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
option(BUILD_PYTHON "Enable building of Python bindings" ON)
option(BUILD_DOCS "Enable building of documentation" ON)

# Build the library
add_subdirectory(src)
# Build the c++ library
add_subdirectory(lib)

# Build the tests
# Build the c++ tests
include(CTest)
if(BUILD_TESTING)
add_subdirectory(ext/Catch2)
include(./ext/Catch2/extras/Catch.cmake)
add_subdirectory(tests)
add_subdirectory(tests/cpp)
endif()

# Build the documentation
Expand All @@ -37,26 +37,26 @@ endif()

# Build the python bindings
if(BUILD_PYTHON)
add_subdirectory(python)
add_subdirectory(src)
endif()

# Add an alias target for use if this project is included as a subproject in
# another project
add_library(pybind11-numpy-example::pybind11-numpy-example ALIAS
pybind11-numpy-example)
add_library(pybind11_numpy_example::pybind11_numpy_example ALIAS
pybind11_numpy_example)

# Install targets and configuration
install(
TARGETS pybind11-numpy-example
EXPORT pybind11-numpy-example-config
TARGETS pybind11_numpy_example
EXPORT pybind11_numpy_example_config
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})

install(
EXPORT pybind11-numpy-example-config
NAMESPACE pybind11-numpy-example::
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pybind11-numpy-example)
EXPORT pybind11_numpy_example_config
NAMESPACE pybind11_numpy_example::
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pybind11_numpy_example)

install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ a Numpy array is much faster and uses a lot less memory:

# How

The pybind11 code is in [python/pybind11-numpy-example_python.cpp](https://github.com/ssciwr/pybind11-numpy-example/blob/main/python/pybind11-numpy-example_python.cpp).
The pybind11 code is in [src/pybind11_numpy_example_python.cpp](https://github.com/ssciwr/pybind11-numpy-example/blob/main/src/pybind11_numpy_example_python.cpp).

The python project is defined in [pyproject.toml](https://github.com/ssciwr/pybind11-numpy-example/blob/main/pyproject.toml)
The python package is defined in [pyproject.toml](https://github.com/ssciwr/pybind11-numpy-example/blob/main/pyproject.toml)
and uses [scikit-build-core](https://github.com/scikit-build/scikit-build-core).

Each tagged commit triggers a [GitHub action job](https://github.com/ssciwr/pybind11-numpy-example/actions/workflows/pypi.yml)
Expand Down
2 changes: 1 addition & 1 deletion doc/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ How
===

The pybind11 code is in
`python/pybind11-numpy-example\_python.cpp <https://github.com/ssciwr/pybind11-numpy-example/blob/main/python/pybind11-numpy-example_python.cpp>`__.
`src/pybind11\_numpy\_example\_python.cpp <https://github.com/ssciwr/pybind11-numpy-example/blob/main/src/pybind11_numpy_example_python.cpp>`__.

The python project is defined in `pyproject.toml <https://github.com/ssciwr/pybind11-numpy-example/blob/main/pyproject.toml>`__
and uses `scikit-build-core <https://github.com/scikit-build/scikit-build-core>`__.
Expand Down
2 changes: 1 addition & 1 deletion ext/Catch2
Submodule Catch2 updated 171 files
5 changes: 5 additions & 0 deletions lib/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
add_library(pybind11_numpy_example pybind11_numpy_example.cpp)
target_include_directories(
pybind11_numpy_example
PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include/>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
1 change: 1 addition & 0 deletions lib/pybind11_numpy_example.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
#include "pybind11_numpy_example/pybind11_numpy_example.hpp"
5 changes: 4 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ description = "An example of using numpy with pybind11"
readme = "README.md"
license = {text = "MIT"}
authors=[{name="Liam Keegan", email="[email protected]"}]
maintainers=[{name="Liam Keegan", email="[email protected]"}]
requires-python = ">=3.7"
keywords = ["pybind11", "cibuildwheel", "c++", "pypi", "numpy", "simple", "example"]
classifiers=[
"Programming Language :: C++",
"Programming Language :: Python :: 3 :: Only",
Expand All @@ -29,6 +31,7 @@ classifiers=[

[project.urls]
Github = "https://github.com/ssciwr/pybind11-numpy-example"
Documentation = "https://pybind11-numpy-example.readthedocs.io"

[project.optional-dependencies]
test = ["pytest", "numpy"]
Expand All @@ -41,5 +44,5 @@ BUILD_DOCS = "OFF"

[tool.cibuildwheel]
test-extras = "test"
test-command = "python -m pytest {project}/python/tests -v"
test-command = "python -m pytest {project}/tests/python -v"
test-skip = "pp* *-musllinux* *-manylinux_i686"
7 changes: 0 additions & 7 deletions python/CMakeLists.txt

This file was deleted.

10 changes: 6 additions & 4 deletions scripts/benchmarks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ echo "#n time (seconds)" > time_list.txt
cp time_list.txt time_array.txt
cp time_list.txt time_array_nocopy.txt

for n in 1000 10000 100000 1000000 10000000 50000000 100000000 200000000 300000000 400000000
for n in 1000 10000 100000 1000000 10000000 50000000 100000000 200000000 300000000 400000000 600000000 800000000 1000000000 1200000000
do
echo $n
m_list=$(./memory.py $n 0)
m_array=$(./memory.py $n 1)
m_array_nocopy=$(./memory.py $n 2)
Expand All @@ -24,9 +25,9 @@ do
echo "${n} ${t_array}" >> time_array.txt
echo "${n} ${t_array_nocopy}" >> time_array_nocopy.txt
done

for n in 1000000000 2000000000 3000000000 4000000000
for n in 2000000000 3000000000 4000000000 6000000000 8000000000 10000000000 12000000000
do
echo $n
m_array=$(./memory.py $n 1)
echo "${n} ${m_array}" >> mem_array.txt
m_array_nocopy=$(./memory.py $n 2)
Expand All @@ -38,8 +39,9 @@ do
echo "${n} ${t_array_nocopy}" >> time_array_nocopy.txt
done

for n in 5000000000 6000000000 7000000000 8000000000
for n in 14000000000 16000000000 18000000000 20000000000 24000000000
do
echo $n
m_array_nocopy=$(./memory.py $n 2)
echo "${n} ${m_array_nocopy}" >> mem_array_nocopy.txt

Expand Down
35 changes: 21 additions & 14 deletions scripts/mem_array.txt
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
#n memory (kb)
1000 18896
10000 18864
100000 19560
1000000 23260
10000000 58380
50000000 214536
100000000 410024
200000000 800452
300000000 1191012
400000000 1581412
1000000000 3925392
2000000000 7831684
3000000000 11737892
4000000000 15644044
1000 20648
10000 20664
100000 20840
1000000 24520
10000000 59672
50000000 215920
100000000 411412
200000000 801848
300000000 1192548
400000000 1583112
600000000 2363992
800000000 3145808
1000000000 3926680
1200000000 4708296
2000000000 7832800
3000000000 11739192
4000000000 15645488
6000000000 23458288
8000000000 31270592
10000000000 39083108
12000000000 46895616
44 changes: 26 additions & 18 deletions scripts/mem_array_nocopy.txt
Original file line number Diff line number Diff line change
@@ -1,19 +1,27 @@
#n memory (kb)
1000 18808
10000 18864
100000 18936
1000000 20912
10000000 38284
50000000 116396
100000000 214164
200000000 409508
300000000 604768
400000000 800084
1000000000 1971984
2000000000 3925016
3000000000 5878236
4000000000 7831396
5000000000 9784604
6000000000 11737484
7000000000 13690668
8000000000 15643960
1000 20656
10000 20668
100000 20844
1000000 22596
10000000 39936
50000000 118080
100000000 215964
200000000 411264
300000000 606336
400000000 801716
600000000 1192512
800000000 1583148
1000000000 1973368
1200000000 2364404
2000000000 3926908
3000000000 5880000
4000000000 7832964
6000000000 11739264
8000000000 15645624
10000000000 19551908
12000000000 23457968
14000000000 27364404
16000000000 31270648
18000000000 35176512
20000000000 39083136
24000000000 46895548
24 changes: 14 additions & 10 deletions scripts/mem_list.txt
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
#n memory (kb)
1000 0
10000 444
100000 4164
1000000 41356
10000000 414140
50000000 2069920
100000000 4139380
200000000 8278588
300000000 12418828
400000000 16557252
1000 192
10000 384
100000 4224
1000000 41088
10000000 410304
50000000 2050752
100000000 4101504
200000000 8203200
300000000 12304704
400000000 16405824
600000000 24609216
800000000 32811840
1000000000 41014464
1200000000 49218432
Binary file modified scripts/memory.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 4 additions & 4 deletions scripts/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# simple script to print approx memory usage

import pybind11numpyexample
import pybind11_numpy_example
import resource
import sys

Expand All @@ -11,11 +11,11 @@
max_mem_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

if data_type == 0:
a = pybind11numpyexample.vector_as_list(n)
a = pybind11_numpy_example.vector_as_list(n)
elif data_type == 1:
a = pybind11numpyexample.vector_as_array(n)
a = pybind11_numpy_example.vector_as_array(n)
elif data_type == 2:
a = pybind11numpyexample.vector_as_array_nocopy(n)
a = pybind11_numpy_example.vector_as_array_nocopy(n)

max_mem_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

Expand Down
Binary file modified scripts/time.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 4 additions & 4 deletions scripts/time.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# simple script to print approx time taken

import pybind11numpyexample
import pybind11_numpy_example
import timeit
import sys

Expand All @@ -13,11 +13,11 @@

def doit():
if data_type == 0:
return pybind11numpyexample.vector_as_list(n)
return pybind11_numpy_example.vector_as_list(n)
elif data_type == 1:
return pybind11numpyexample.vector_as_array(n)
return pybind11_numpy_example.vector_as_array(n)
elif data_type == 2:
return pybind11numpyexample.vector_as_array_nocopy(n)
return pybind11_numpy_example.vector_as_array_nocopy(n)


print(timeit.timeit(doit, number=iters) / iters)
Loading

0 comments on commit 7e4813a

Please sign in to comment.