diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 08938701..2366573c 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -24,16 +24,16 @@ concurrency: jobs: duckdb-latest-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.0 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: duckdb_version: main extension_name: spatial - ci_tools_version: v1.1.3 + ci_tools_version: main duckdb-latest-deploy: name: Deploy extension binaries needs: duckdb-latest-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.0 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: duckdb_version: main diff --git a/.gitmodules b/.gitmodules index 7114de7a..45589912 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "duckdb"] path = duckdb url = https://github.com/duckdb/duckdb.git +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = https://github.com/duckdb/extension-ci-tools.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 95792382..7067bedc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ if(NOT EXISTS ${CMAKE_BINARY_DIR}/deps) -DVCPKG_INSTALLED_DIR='${CMAKE_BINARY_DIR}/vcpkg_installed' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' -S ${CMAKE_CURRENT_SOURCE_DIR}/deps -B ${CMAKE_BINARY_DIR}/deps RESULT_VARIABLE DEPENDENCIES_GENERATE_RESULT) if(NOT DEPENDENCIES_GENERATE_RESULT EQUAL 0) diff --git a/Makefile b/Makefile index 830e5a45..a8ea67db 100644 --- a/Makefile +++ b/Makefile @@ -1,180 +1,8 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -all: release +# Configuration of extension +EXT_NAME=excel +EXT_CONFIG=${PROJ_DIR}extension_config.cmake -DUCKDB_SRCDIR ?= "./duckdb/" - -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -PROJ_DIR := $(dir $(MKFILE_PATH)) - -TEST_PATH="/test/unittest" -DUCKDB_PATH="/duckdb" - -# For non-MinGW windows the path is slightly different -ifeq ($(OS),Windows_NT) -ifneq ($(CXX),g++) - TEST_PATH="/test/Release/unittest.exe" - DUCKDB_PATH="/Release/duckdb.exe" -endif -endif - -#### OSX config -OSX_BUILD_FLAG= -ifneq (${OSX_BUILD_ARCH}, "") - OSX_BUILD_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH} -endif - -#### VCPKG config -VCPKG_TOOLCHAIN_PATH?= -ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' -endif -ifneq ("${VCPKG_TARGET_TRIPLET}", "") - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' -endif -ifeq (${USE_MERGED_VCPKG_MANIFEST}, 1) - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}build/extension_configuration' -else - TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -endif - -#### Enable Ninja as generator -ifeq ($(GEN),ninja) - GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1 -endif - -EXT_NAME=spatial - -#### Configuration for this extension -EXTENSION_NAME=SPATIAL -EXTENSION_FLAGS=\ --DENABLE_SANITIZER=OFF \ --DDUCKDB_EXTENSION_NAMES="spatial" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)spatial/include" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_LINKED_LIBS="../../deps/local/lib/*.a" - -#### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed) -EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="parquet;json" - -BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' -ifeq (${BUILD_SHELL}, 0) - BUILD_FLAGS += -DBUILD_SHELL=0 -else - ifeq ($(OS),Windows_NT) - BUILD_FLAGS += -DBUILD_SHELL=0 - endif -endif - -ifeq (${BUILD_BENCHMARK}, 1) - BUILD_FLAGS += -DBUILD_BENCHMARKS=1 -endif - -CLIENT_FLAGS:= - -#### Main build -# For regular CLI build, we link the spatial extension directly into the DuckDB executable -CLIENT_FLAGS=-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 - -debug: - mkdir -p build/debug && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \ - cmake --build build/debug --config Debug - -release: - mkdir -p build/release && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ - cmake --build build/release --config Release - -reldebug: - mkdir -p build/reldebug && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=RelWithDebInfo -S ./duckdb/ -B build/reldebug && \ - cmake --build build/reldebug --config RelWithDebInfo - -##### Client build -JS_BUILD_FLAGS=-DBUILD_NODE=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 -PY_BUILD_FLAGS=-DBUILD_PYTHON=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 - -debug_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS) -debug_js: debug -debug_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS) -debug_python: debug -release_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS) -release_js: release -release_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS) -release_python: release - -# Main tests -test: test_release - -test_release: release - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" - -test_debug: debug - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" - -#### Client tests -DEBUG_EXT_PATH='$(PROJ_DIR)build/debug/extension/spatial/spatial.duckdb_extension' -RELEASE_EXT_PATH='$(PROJ_DIR)build/release/extension/spatial/spatial.duckdb_extension' - -test_js: test_debug_js - -test_debug_js: debug_js - cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js" - -test_release_js: release_js - cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js" - -test_python: test_debug_python - -test_debug_python: debug_python - cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) python3 -m pytest - -test_release_python: release_python - cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) python3 -m pytest - -#### Misc -format: - find spatial/src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i - find spatial/include -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i - cmake-format -i spatial/src/CMakeLists.txt - -update: - git submodule update --remote --merge - -pull: - git submodule init - git submodule update --recursive --remote - -clean: - rm -rf build - rm -rf testext -#cd duckdb && make clean -#cd duckdb && make clean-python - -# WASM config -VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(VCPKG_EMSDK_FLAGS) -WASM_CXX_MVP_FLAGS= -WASM_CXX_EH_FLAGS=$(WASM_CXX_MVP_FLAGS) -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -WASM_CXX_THREADS_FLAGS=$(WASM_COMPILE_TIME_EH_FLAGS) -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread -WASM_LINK_TIME_FLAGS=-O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_${EXT_NAME}_version,_${EXT_NAME}_init" - -# WASM targets -wasm_mvp: - mkdir -p build/wasm_mvp - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="$(WASM_CXX_MVP_FLAGS)" -S $(DUCKDB_SRCDIR) -DDUCKDB_EXPLICIT_PLATFORM=wasm_mvp -DDUCKDB_CUSTOM_PLATFORM=wasm_mvp - emmake make -j8 -Cbuild/wasm_mvp - -wasm_eh: - mkdir -p build/wasm_eh - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="$(WASM_CXX_EH_FLAGS)" -S $(DUCKDB_SRCDIR) -DDUCKDB_EXPLICIT_PLATFORM=wasm_eh -DDUCKDB_CUSTOM_PLATFORM=wasm_eh - emmake make -j8 -Cbuild/wasm_eh - -wasm_threads: - mkdir -p ./build/wasm_threads - emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="$(WASM_CXX_THREADS_FLAGS)" -S $(DUCKDB_SRCDIR) -DDUCKDB_EXPLICIT_PLATFORM=wasm_threads -DDUCKDB_CUSTOM_PLATFORM=wasm_threads - emmake make -j8 -Cbuild/wasm_threads +# Include the Makefile from extension-ci-tools +include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index f26869ff..e9fc6822 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -11,6 +11,10 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() +if(${VCPKG_TARGET_TRIPLET} STREQUAL "wasm32-emscripten") + set(EMSCRIPTEN ON) +endif() + if (NOT EMSCRIPTEN) if (OSX_BUILD_ARCH) message(STATUS "building for OSX architecture: ${OSX_BUILD_ARCH}") @@ -36,37 +40,15 @@ if (EMSCRIPTEN) set(OGR_ENABLE_DRIVER_OPENFILEGDB OFF) endif() -# ZLIB -ExternalProject_Add( - ZLIB - URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/zlib1213.zip - CONFIGURE_HANDLED_BY_BUILD TRUE - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX:PATH=${LOCAL_INSTALL_DIR} - -DCMAKE_PREFIX_PATH=${LOCAL_INSTALL_DIR} - -DCMAKE_MODULE_PATH=${LOCAL_INSTALL_DIR}/lib/cmake - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED} - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DBUILD_SHARED_LIBS=OFF - ${CMAKE_FLAGS_POINTER_SIZE} - # vcpkg options - -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}' - -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' - -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' - -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' -) - # Use the system sqlite binary on Emscripten, iOS and Android if (EMSCRIPTEN OR IOS OR ANDROID) find_program(EXE_SQLITE3 sqlite3) endif() -set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} ZLIB) +set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES}) # SQLite3 ExternalProject_Add( SQLITE3 - DEPENDS ZLIB URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/sqlite3 CONFIGURE_HANDLED_BY_BUILD TRUE CMAKE_ARGS @@ -80,6 +62,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' ) set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} SQLITE3) @@ -87,7 +70,6 @@ if(SPATIAL_USE_NETWORK) # CURL ExternalProject_Add( CURL - DEPENDS ZLIB URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/curl-7.87.0.zip CONFIGURE_HANDLED_BY_BUILD TRUE CMAKE_ARGS @@ -102,6 +84,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' -DBUILD_SHARED_LIBS=OFF -DHTTP_ONLY=ON # Disable some defaults for now @@ -118,7 +101,7 @@ endif() # PROJ ExternalProject_Add( PROJ - DEPENDS SQLITE3 ZLIB + DEPENDS SQLITE3 URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/proj-9.1.1.zip CONFIGURE_HANDLED_BY_BUILD TRUE PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/proj${PATCH_NAME_ENDING}.patch" @@ -136,6 +119,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' # PROJ options -DBUILD_SHARED_LIBS=OFF -DBUILD_APPS=OFF @@ -164,6 +148,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' # EXPAT options -DEXPAT_SHARED_LIBS=OFF -DEXPAT_BUILD_TESTS=OFF @@ -192,6 +177,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' # GEOS options -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF @@ -219,6 +205,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' # GeographicLib options -DBUILD_SHARED_LIBS=OFF -DGEOGRAPHICLIB_DATA=${LOCAL_INSTALL_DIR}/share/GeographicLib @@ -247,6 +234,7 @@ ExternalProject_Add( -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}' -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}' -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}' # GDAL Options -DGDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE=ON # this is needed for GDAL to build with -fPIC -DBUILD_TESTING=OFF diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 00000000..916d4ef4 --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit 916d4ef4371068ca98a007378b52582c3e46b4e5 diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 00000000..a47a64cd --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,10 @@ +# This file is included by DuckDB's build system. It specifies which extension to load + +# Extension from this repo +duckdb_extension_load(spatial + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/spatial/include + LOAD_TESTS + DONT_LINK + LINKED_LIBS "../../deps/local/lib/*.a" +) diff --git a/vcpkg.json b/vcpkg.json index 0fa0cf26..e438d290 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,5 +1,6 @@ { "dependencies": [ - "openssl" + "openssl", + "zlib" ] }