diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml new file mode 100644 index 0000000..74e831e --- /dev/null +++ b/.github/workflows/distribution.yml @@ -0,0 +1,41 @@ +# +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Stable Extension Distribution Pipeline +on: + pull_request: + branches: + - main + paths-ignore: + - '**/README.md' + - 'doc/**' + push: + branches: + - main + paths-ignore: + - '**/README.md' + - 'doc/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-stable-build: + name: Build extension binaries + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.0 + with: + duckdb_version: v1.1.0 + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" + extension_name: substrait + + duckdb-stable-deploy: + name: Deploy extension binaries + needs: duckdb-stable-build + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.0 + secrets: inherit + with: + duckdb_version: v1.1.0 + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" + extension_name: substrait diff --git a/.github/workflows/main_distribution.yml b/.github/workflows/main_distribution.yml new file mode 100644 index 0000000..b974a08 --- /dev/null +++ b/.github/workflows/main_distribution.yml @@ -0,0 +1,27 @@ +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Main Extension Distribution Pipeline +on: + pull_request: + paths-ignore: + - '**/README.md' + - 'doc/**' + push: + paths-ignore: + - '**/README.md' + - 'doc/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-main-build: + name: Build extension binaries + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main + with: + duckdb_version: main + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools" + extension_name: substrait + diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 274a102..5597e7d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,16 +18,12 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: true - uses: actions/setup-python@v2 with: python-version: '3.11' - - name: Update DuckDB submodule - run: | - git config --global --add safe.directory '*' - make pull - - name: Build DuckDB (Python) run: | cd duckdb/tools/pythonpkg @@ -45,4 +41,4 @@ jobs: - name: Test Python run: | - (cd test/python && python -m pytest) \ No newline at end of file + (cd test/python && python -m pytest) diff --git a/.github/workflows/sql.yml b/.github/workflows/sql.yml deleted file mode 100644 index a849445..0000000 --- a/.github/workflows/sql.yml +++ /dev/null @@ -1,31 +0,0 @@ -on: [push, pull_request,repository_dispatch] - -defaults: - run: - shell: bash - -jobs: - sql: - name: SQL - runs-on: macos-latest - env: - GEN: ninja - - steps: - - name: Install Ninja - run: brew install ninja - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Update DuckDB submodule - run: | - git config --global --add safe.directory '*' - make pull - - - name: Build - run: make release - - - name: Test - run: make test_release \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 81fceeb..82a23a2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "substrait"] path = substrait url = https://github.com/substrait-io/substrait +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = https://github.com/duckdb/extension-ci-tools.git diff --git a/Makefile b/Makefile index c211163..36723f9 100644 --- a/Makefile +++ b/Makefile @@ -1,47 +1,25 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) all: release -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -PROJ_DIR := $(dir $(MKFILE_PATH)) - -EXTRA_CMAKE_VARIABLES := - -# These flags will make DuckDB build the extension -EXTRA_CMAKE_VARIABLES += -DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="tpch;json" ${OSX_ARCH_FLAG} -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_NAMES="substrait" -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_SHOULD_LINK=1 -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_LOAD_TESTS=1 -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_PATH=$(PROJ_DIR) -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_TEST_PATH=$(PROJ_DIR)test -EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_INCLUDE_PATH="$(PROJ_DIR)src/include" -export - -DUCKDB_DIRECTORY= -ifndef DUCKDB_DIR - DUCKDB_DIRECTORY=./duckdb -else - DUCKDB_DIRECTORY=${DUCKDB_DIR} +# Configuration of extension +EXT_NAME=substrait +EXT_CONFIG=${PROJ_DIR}extension_config.cmake + +CORE_EXTENSIONS='tpch;json' + +# Set this flag during building to enable the benchmark runner +ifeq (${BUILD_BENCHMARK}, 1) + TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DBUILD_BENCHMARKS=1 endif +# Include the Makefile from extension-ci-tools +include extension-ci-tools/makefiles/duckdb_extension.Makefile + pull: git submodule init git submodule update --recursive --remote -clean: - rm -rf ${DUCKDB_DIRECTORY}/build - rm -rf testext - cd ${DUCKDB_DIRECTORY} && make clean - -# Main builds -debug: -# Have to actually cd here because the makefile assumes it's called from within duckdb - cd ${DUCKDB_DIRECTORY} && $(MAKE) -C . debug - -release: -# Have to actually cd here because the makefile assumes it's called from within duckdb - cd ${DUCKDB_DIRECTORY} && $(MAKE) -C . release - # Client builds %_js: export BUILD_NODE=1 debug_js: debug @@ -58,14 +36,8 @@ release_r: release debug_python: debug release_python: release -# Main tests -test: test_release - -test_release: release - ${DUCKDB_DIRECTORY}/build/release/test/unittest "$(PROJ_DIR)test/*" - test_debug: debug - ${DUCKDB_DIRECTORY}/build/debug/test/unittest "$(PROJ_DIR)test/*" + build/debug/test/unittest "$(PROJ_DIR)test/*" # Client tests test_python: test_debug_python @@ -88,4 +60,4 @@ format: rm .clang-format update: - git submodule update --remote --merge \ No newline at end of file + git submodule update --remote --merge diff --git a/README.md b/README.md index bbe9c86..3f86a18 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ make To run, run the bundled `duckdb` shell: ``` - ./duckdb/build/release/duckdb + ./build/release/duckdb ``` Then, load the Substrait - DuckDB extension like so: diff --git a/duckdb b/duckdb index a6e32b1..fa5c2fe 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit a6e32b115826ba543e32a733cb92f68fd0549186 +Subproject commit fa5c2fe15f3da5f32397b009196c0895fce60820 diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 0000000..69ec500 --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit 69ec500f2330913633481af5602728a3cd4b7468 diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 0000000..7665aeb --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,8 @@ +# This file is included by DuckDB's build system. It specifies which extension to load + +# Extension from this repo +duckdb_extension_load(substrait + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/include + LOAD_TESTS +) diff --git a/test/python/conftest.py b/test/python/conftest.py index d83ce39..2a19f0c 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -23,7 +23,7 @@ def add_extension(extension_name, conn: Union[str, DuckDBPyConnection] = '') -> 'allow_unsigned_extensions' : 'true' } conn = duckdb.connect(conn or '', config=config) - file_path = f"'{dir}/../../duckdb/build/{build_type}/extension/{extension_name}/{extension_name}.duckdb_extension'" + file_path = f"'{dir}/../../build/{build_type}/extension/{extension_name}/{extension_name}.duckdb_extension'" conn.execute(f"LOAD {file_path}") return conn diff --git a/test/python/test_adbc.py b/test/python/test_adbc.py index 1d577ae..e80198b 100644 --- a/test/python/test_adbc.py +++ b/test/python/test_adbc.py @@ -19,7 +19,7 @@ def find_substrait(): # Paths to search for extensions - build = normpath(join(dirname(__file__), "../../duckdb/build/")) + build = normpath(join(dirname(__file__), "../../build/")) extension = "extension/*/*.duckdb_extension" extension_search_patterns = [ diff --git a/test/r/test_substrait.R b/test/r/test_substrait.R index 8a6610f..0caa43a 100644 --- a/test/r/test_substrait.R +++ b/test/r/test_substrait.R @@ -7,7 +7,7 @@ load_extension <- function() { file_directory <- getwd() con <- dbConnect(duckdb::duckdb(config=list("allow_unsigned_extensions"="true"))) - dbExecute(con, sprintf("LOAD '%s/../../duckdb/build/%s/extension/substrait/substrait.duckdb_extension';", file_directory, build_type)) + dbExecute(con, sprintf("LOAD '%s/../../build/%s/extension/substrait/substrait.duckdb_extension';", file_directory, build_type)) return (con) } diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..9757fe1 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,4 @@ +{ + "dependencies": [ + ] +}