From dbb28e73dce8a0069a1f7be62d30f899ffd7e155 Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Sat, 9 Nov 2024 20:21:56 +0000 Subject: [PATCH 01/12] feat: Add VSCode devcontainer (#148) - Installs python, git, gdb for dev environment - Python venv created in /tools/venv (in container, not on local machine) with build requirements pre-installed. - vcpkg cloned and setup in /tools/vcpkg. Note the FIXME on VCPKG_FORCE_SYSTEM_BINARIES added to get working on MacOS running a Linux container, as it may need to be made conditional on platform. - PYTHONPATH and PATH setup so integrated terminal works. - First draft of developer guide README.md. - devcontainer.json has cmake and python settings as in Dockerfile --- .devcontainer/Dockerfile | 47 +++++++++++++++++++++++++++++++++ .devcontainer/README.md | 31 ++++++++++++++++++++++ .devcontainer/devcontainer.json | 39 +++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/README.md create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..8a0e1705 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:22.04 + +# Install python +RUN apt-get update +RUN apt-get install -y \ + python3 \ + python3-venv \ + python3-pip + +# Dev tools require an additional update +RUN apt-get update +RUN apt-get install -y \ + git \ + gdb \ + curl \ + zip \ + unzip \ + tar \ + autoconf \ + libtool \ + libtool-bin \ + pkg-config + +WORKDIR /tools + +# Create new venv at container root and install libs +RUN python3 -m venv /tools/venv +ENV PATH /tools/venv/bin:$PATH +RUN pip install --upgrade pip setuptools wheel +RUN pip install \ + cmake \ + ninja \ + pybind11 \ + numpy \ + pytest \ + pytest-cov + +# FIXME make conditional on arm, s390x, ppc64le and riscv +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 + +# Install vcpkg in checkout location +RUN git clone https://github.com/Microsoft/vcpkg.git /tools/vcpkg +RUN /tools/vcpkg/bootstrap-vcpkg.sh +ENV PATH /tools/vcpkg:$PATH + +# This is required to be able to run pytest in the VSCode terminal +ENV PYTHONPATH=. \ No newline at end of file diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..019f933c --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,31 @@ +# Build and debug in VSCode devcontainer + +The `.devcontainer/` folder provides a consistent build and debug environment that can be run in VSCode locally or in GitHub Codespaces remotely. + +## Opening devcontainer locally + +When you open the cloned RoughPy folder in VSCode, accept the popup to open in a Dev Container or run the editor command `Dev Containers: Reopen in Dev Container`. + +VSCode will restart inside a container that will have mapped your checkout folder to `/workspaces/`, so be mindful that editing files is affecting this folder on your machine. + +The devcontainer `Dockerfile` sets up a root `/tools` folder that contains a `venv` with build dependencies pre-installed, and a clone of `vcpkg`; both of which are added to `$PATH`. + +## Building, testing and debugging + +Run the editor command `CMake: Select Configure Preset` and select `Debug Develop` to compile with debug symbols. Then run `CMake: build`. + +The devcontainer automatically installs the pytest and CTest extension, so after building you can launch any of the tests from the VSCode test explorer panel. Alternatively, you can run directly `pytest` directly from the integrated terminal. + +VSCode loads debug scenarios from `.vscode/launch.json`. However, developers frequently need to edit this file with changes that should not be committed, so here they are stored in `example_launch.json` which must be manually copied over with: + +```sh + cp .vscode/example_launch.json .vscode/launch.json +``` + +Select a debug scenario in the Run and Debug panel, add a breakpoint on the code to stop on, and launch with F5. + +The three example scenarios are: + +- `debugpy pytest` - Debug a specific `pytest` unit test in a python debugger. +- `gdb pytest` - Start a python test and debug any underlying C++. +- `gdb gtest` - Debug a specific GTest unit test in a C++ debugger. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..3a42f3c5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,39 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/cpp +{ + "name": "RoughPy", + + "build": { + "dockerfile": "Dockerfile" + }, + + "customizations": { + "vscode":{ + "extensions": [ + "ms-python.debugpy", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-vscode.cpptools-extension-pack" + ], + "settings": { + "python.defaultInterpreterPath": "/tools/venv/bin/python", + "python.languageServer": "Pylance", + "python.testing.pytestEnabled": true, + "cmake.cmakePath": "/tools/venv/bin/cmake", + "cmake.configureArgs": [ + "-DCMAKE_MAKE_PROGRAM=/tools/venv/bin/ninja", + "-DCMAKE_TOOLCHAIN_FILE=/tools/vcpkg/scripts/buildsystems/vcpkg.cmake", + "-DROUGHPY_BUILD_TESTS=ON", + "-DROUGHPY_BUILD_PYLIB_INPLACE=ON" + ] + } + } + }, + + // Enable debugging in container + "capAdd": [ + "SYS_PTRACE" + ] +} From ae5d8419224ba8fc249bef7d95fbadb8e9d66d2e Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Sat, 9 Nov 2024 20:31:26 +0000 Subject: [PATCH 02/12] fix: Custom vscode triplet for MacOS devcontainer (#148) - When running a Linux container on MacOS, the mpg123 library fails with REAL_IS_FIXED, which I think means that this distro/arch combination fails to infer it automatically. As a workaround I explicitly set HAVE_FPU. If this is OK, I can remove the FIXME comment, but it's there for now for discussion in PR. --- tools/triplets/arm64-linux.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tools/triplets/arm64-linux.cmake diff --git a/tools/triplets/arm64-linux.cmake b/tools/triplets/arm64-linux.cmake new file mode 100644 index 00000000..768033f9 --- /dev/null +++ b/tools/triplets/arm64-linux.cmake @@ -0,0 +1,10 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +# FIXME for discussion in PR. I had to add this to get containers working in MacOS, as it seems the macro is not set correctly by default. +if (PORT MATCHES "mpg123") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DHAVE_FPU=1") +endif() From aff1e7f3b052237612e1b67f7de409b7d14579ff Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Sat, 9 Nov 2024 20:39:32 +0000 Subject: [PATCH 03/12] feat: example_launch.json for debug scenarios (#148) - Contains various python and C++ debug scenarios. - Copy to .vscode/launch.json to use in development. - See .devcontainer/README.md for full information. --- .devcontainer/README.md | 4 +- .devcontainer/example_launch.json | 73 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 .devcontainer/example_launch.json diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 019f933c..ff247eaa 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -16,10 +16,10 @@ Run the editor command `CMake: Select Configure Preset` and select `Debug Develo The devcontainer automatically installs the pytest and CTest extension, so after building you can launch any of the tests from the VSCode test explorer panel. Alternatively, you can run directly `pytest` directly from the integrated terminal. -VSCode loads debug scenarios from `.vscode/launch.json`. However, developers frequently need to edit this file with changes that should not be committed, so here they are stored in `example_launch.json` which must be manually copied over with: +VSCode loads debug scenarios from `.vscode/launch.json`. However, developers frequently need to edit this file with changes that should not be committed, so instead it is stored in `.devcontainer/example_launch.json`. Manually copy this over with: ```sh - cp .vscode/example_launch.json .vscode/launch.json + cp .devcontainer/example_launch.json .vscode/launch.json ``` Select a debug scenario in the Run and Debug panel, add a breakpoint on the code to stop on, and launch with F5. diff --git a/.devcontainer/example_launch.json b/.devcontainer/example_launch.json new file mode 100644 index 00000000..71d6548f --- /dev/null +++ b/.devcontainer/example_launch.json @@ -0,0 +1,73 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "debugpy pytest", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "console": "integratedTerminal", + "args": [ + "-k", "test_exp_log_roundtrip_poly_coeffs" + ], + "env": {}, + "preLaunchTask": "CMake: build" + }, + { + "name": "gdb pytest", + "type": "cppdbg", + "request": "launch", + "program": "${config:python.defaultInterpreterPath}", + "args": [ + "-m", "pytest", + "-k", "test_exp_log_roundtrip_poly_coeffs" + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ], + "preLaunchTask": "CMake: build" + }, + { + "name": "gdb gtest", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/dev-debug/intervals/src/test_intervals", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ], + "preLaunchTask": "CMake: build" + } + ] +} \ No newline at end of file From 45c40b5700446491bd396cfb1d8d072c97941b6c Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 14:47:00 +0000 Subject: [PATCH 04/12] build: remove redundant devcontainer flags (#148) - These flags are automatically set via `CMakePresets.json`. --- .devcontainer/devcontainer.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3a42f3c5..55e47d9a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,9 +24,7 @@ "cmake.cmakePath": "/tools/venv/bin/cmake", "cmake.configureArgs": [ "-DCMAKE_MAKE_PROGRAM=/tools/venv/bin/ninja", - "-DCMAKE_TOOLCHAIN_FILE=/tools/vcpkg/scripts/buildsystems/vcpkg.cmake", - "-DROUGHPY_BUILD_TESTS=ON", - "-DROUGHPY_BUILD_PYLIB_INPLACE=ON" + "-DCMAKE_TOOLCHAIN_FILE=/tools/vcpkg/scripts/buildsystems/vcpkg.cmake" ] } } From 2f99a9d4b9b270d16be49cb7598362985b70b82d Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 14:52:51 +0000 Subject: [PATCH 05/12] docs: update devcontainer README (#148) - Added notes on the need to wait while CMake is configuring. This needs to finish for tests to be available. --- .devcontainer/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index ff247eaa..d3412270 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -4,12 +4,15 @@ The `.devcontainer/` folder provides a consistent build and debug environment th ## Opening devcontainer locally -When you open the cloned RoughPy folder in VSCode, accept the popup to open in a Dev Container or run the editor command `Dev Containers: Reopen in Dev Container`. +When you open the cloned RoughPy folder in VSCode, it will detect the `.devcontainer/` folder. Accept the popup to open in a Dev Container or run the editor command `Dev Containers: Reopen in Dev Container`. If the container image needs to be rebuilt, it will take a few extra minutes. -VSCode will restart inside a container that will have mapped your checkout folder to `/workspaces/`, so be mindful that editing files is affecting this folder on your machine. +VSCode will restart inside a container that mounts your checkout folder to `/workspaces/`. Be mindful that editing files is affecting this folder on your machine. The devcontainer `Dockerfile` sets up a root `/tools` folder that contains a `venv` with build dependencies pre-installed, and a clone of `vcpkg`; both of which are added to `$PATH`. +On the first run, the project will need to be configured in CMake and additional build dependencies will be fetched by vcpkg. You must wait for this to finish before you can build and run any tests. Progress can be tracked under 'CMake/Build' in the VSCode 'Output' panel. + + ## Building, testing and debugging Run the editor command `CMake: Select Configure Preset` and select `Debug Develop` to compile with debug symbols. Then run `CMake: build`. From 9c925b5e7a2acf1b467b0e72a1161e413af8e390 Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 14:53:22 +0000 Subject: [PATCH 06/12] docs: devcontainer CHANGELOG (#148) --- CHANGELOG | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index ba382149..8164d748 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +Unreleased + - Added VSCode `.devcontainer` settings for containerised developer environment for debugging tests. See `.devcontainer/README.md` for instructions (@alexallmont #148) + Version 0.1.2 - Documentation is now in sync with the main branch (@philipparubin PR #85) - Fixed a fault when constructing with larger than expected array (Issue #70, @inakleinbottle PR #72) From 927656865af033fe6a67248badd184df6b6bae4a Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 16:31:36 +0000 Subject: [PATCH 07/12] build: conditional vcpkg tools for aarch64 (#148) --- .devcontainer/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8a0e1705..a6eb3110 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -35,8 +35,11 @@ RUN pip install \ pytest \ pytest-cov -# FIXME make conditional on arm, s390x, ppc64le and riscv -ENV VCPKG_FORCE_SYSTEM_BINARIES=1 +# Use system tools for vcpkg when building aarch64 (Linux on MacOS) +RUN if [ "$(uname -m)" = "aarch64" ]; then \ + echo "Setting VCPKG_FORCE_SYSTEM_BINARIES for MacOS"; \ + export VCPKG_FORCE_SYSTEM_BINARIES=1; \ + fi # Install vcpkg in checkout location RUN git clone https://github.com/Microsoft/vcpkg.git /tools/vcpkg From 7bddeced2b2f9b582dc458b3b953bb2351641f1d Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 16:32:47 +0000 Subject: [PATCH 08/12] build: devcontainer uses ubuntu 24.04 (#148) --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a6eb3110..549513c5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 # Install python RUN apt-get update From 0d797c157b9884de064fc45b5420eee031727344 Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 16:55:25 +0000 Subject: [PATCH 09/12] fix: set devcontainer env default (#148) - It seems that without setting ENV, that vcpkg does not always pick up this variable and setup can fail. Committing here to test on a fresh clone. --- .devcontainer/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 549513c5..3cbc183e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -36,6 +36,7 @@ RUN pip install \ pytest-cov # Use system tools for vcpkg when building aarch64 (Linux on MacOS) +ENV VCPKG_FORCE_SYSTEM_BINARIES=0 RUN if [ "$(uname -m)" = "aarch64" ]; then \ echo "Setting VCPKG_FORCE_SYSTEM_BINARIES for MacOS"; \ export VCPKG_FORCE_SYSTEM_BINARIES=1; \ From 0cc2845bf88ad262109aa04a32409247fdcf1a02 Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 16:59:42 +0000 Subject: [PATCH 10/12] build: set devcontainer VCPKG_ROOT instead of amending to PATH (#148) --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3cbc183e..ef5e21c5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -45,7 +45,7 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then \ # Install vcpkg in checkout location RUN git clone https://github.com/Microsoft/vcpkg.git /tools/vcpkg RUN /tools/vcpkg/bootstrap-vcpkg.sh -ENV PATH /tools/vcpkg:$PATH +ENV VCPKG_ROOT=/tools/vcpkg # This is required to be able to run pytest in the VSCode terminal ENV PYTHONPATH=. \ No newline at end of file From 82b9dd101aad22260479ac3a77be2caba7640836 Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 17:29:43 +0000 Subject: [PATCH 11/12] docs: devcontainer launch.json setup notes (#148) --- .devcontainer/README.md | 1 + .gitignore | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index d3412270..c0080ba7 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -22,6 +22,7 @@ The devcontainer automatically installs the pytest and CTest extension, so after VSCode loads debug scenarios from `.vscode/launch.json`. However, developers frequently need to edit this file with changes that should not be committed, so instead it is stored in `.devcontainer/example_launch.json`. Manually copy this over with: ```sh + mkdir -p .vscode cp .devcontainer/example_launch.json .vscode/launch.json ``` diff --git a/.gitignore b/.gitignore index e8726459..491072c4 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,9 @@ dkms.conf # *.iml # *.ipr +# VSCode local settings +.vscode/ + # CMake cmake-build-*/ build/ From ff80fc471a2510775d8639cbc9fd136ffe6364c4 Mon Sep 17 00:00:00 2001 From: Alex Allmont Date: Wed, 20 Nov 2024 17:43:19 +0000 Subject: [PATCH 12/12] docs: tidy comment for HAVE_FPU (#148) - Replace FIXME with clearer comment. So far this setting is only required when building containerized on MacOS, presumably because mgp123 wants specific instructions for cross-compilation or embedded work. Here it is fine to set this, as the Linux-on-MacOS target platform will definitely have an FPU. --- tools/triplets/arm64-linux.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/triplets/arm64-linux.cmake b/tools/triplets/arm64-linux.cmake index 768033f9..31d25a8a 100644 --- a/tools/triplets/arm64-linux.cmake +++ b/tools/triplets/arm64-linux.cmake @@ -4,7 +4,8 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Linux) -# FIXME for discussion in PR. I had to add this to get containers working in MacOS, as it seems the macro is not set correctly by default. +# HAVE_FPU is on for all native platforms, but not by default when building in a +# Linux container on MacOS (e.g. in devcontainer), so it is set explicitly here. if (PORT MATCHES "mpg123") set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DHAVE_FPU=1") endif()