Move to extension-ci-tools, enable distribution of nightly versions

.github/workflows/distribution.yml

@@ -0,0 +1,41 @@
+#
+# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension
+#
+name: Stable Extension Distribution Pipeline
+on:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '**/README.md'
+      - 'doc/**'
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/README.md'
+      - 'doc/**'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  duckdb-stable-build:
+    name: Build extension binaries
+    uses: duckdb/extension-ci-tools/.github/workflows/[email protected]
+    with:
+      duckdb_version: v1.1.0
+      exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools"
+      extension_name: substrait
+
+  duckdb-stable-deploy:
+    name: Deploy extension binaries
+    needs: duckdb-stable-build
+    uses: duckdb/extension-ci-tools/.github/workflows/[email protected]
+    secrets: inherit
+    with:
+      duckdb_version: v1.1.0
+      exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools"
+      extension_name: substrait

.github/workflows/main_distribution.yml

@@ -0,0 +1,27 @@
+# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension
+#
+name: Main Extension Distribution Pipeline
+on:
+  pull_request:
+    paths-ignore:
+      - '**/README.md'
+      - 'doc/**'
+  push:
+    paths-ignore:
+      - '**/README.md'
+      - 'doc/**'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  duckdb-main-build:
+    name: Build extension binaries
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
+    with:
+      duckdb_version: main
+      exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64;windows_amd64_rtools"
+      extension_name: substrait
+

.github/workflows/python.yml

@@ -18,16 +18,12 @@ jobs:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
+        submodules: true
 
     - uses: actions/setup-python@v2
       with:
         python-version: '3.11'
 
-    - name: Update DuckDB submodule
-      run: |
-        git config --global --add safe.directory '*'
-        make pull
-
     - name: Build DuckDB (Python)
       run: |
         cd duckdb/tools/pythonpkg
@@ -45,4 +41,4 @@ jobs:
 
     - name: Test Python
       run: |
-        (cd test/python && python -m pytest)
+        (cd test/python && python -m pytest)

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "substrait"]
 	path = substrait
 	url = https://github.com/substrait-io/substrait
+[submodule "extension-ci-tools"]
+	path = extension-ci-tools
+	url = https://github.com/duckdb/extension-ci-tools.git

Makefile

@@ -1,47 +1,25 @@
-.PHONY: all clean format debug release duckdb_debug duckdb_release pull update
+PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 all: release
 
-MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
-PROJ_DIR := $(dir $(MKFILE_PATH))
-
-EXTRA_CMAKE_VARIABLES :=
-
-# These flags will make DuckDB build the extension
-EXTRA_CMAKE_VARIABLES += -DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="tpch;json" ${OSX_ARCH_FLAG}
-EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_NAMES="substrait"
-EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_SHOULD_LINK=1
-EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_LOAD_TESTS=1
-EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_PATH=$(PROJ_DIR)
-EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_TEST_PATH=$(PROJ_DIR)test
-EXTRA_CMAKE_VARIABLES += -DDUCKDB_EXTENSION_SUBSTRAIT_INCLUDE_PATH="$(PROJ_DIR)src/include"
-export
-
-DUCKDB_DIRECTORY=
-ifndef DUCKDB_DIR
-	DUCKDB_DIRECTORY=./duckdb
-else
-	DUCKDB_DIRECTORY=${DUCKDB_DIR}
+# Configuration of extension
+EXT_NAME=substrait
+EXT_CONFIG=${PROJ_DIR}extension_config.cmake
+
+CORE_EXTENSIONS='tpch;json'
+
+# Set this flag during building to enable the benchmark runner
+ifeq (${BUILD_BENCHMARK}, 1)
+	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DBUILD_BENCHMARKS=1
 endif
 
+# Include the Makefile from extension-ci-tools
+include extension-ci-tools/makefiles/duckdb_extension.Makefile
+
 pull:
 	git submodule init
 	git submodule update --recursive --remote
 
-clean:
-	rm -rf ${DUCKDB_DIRECTORY}/build
-	rm -rf testext
-	cd ${DUCKDB_DIRECTORY} && make clean
-
-# Main builds
-debug:
-# Have to actually cd here because the makefile assumes it's called from within duckdb
-	cd ${DUCKDB_DIRECTORY} && $(MAKE) -C . debug
-
-release:
-# Have to actually cd here because the makefile assumes it's called from within duckdb
-	cd ${DUCKDB_DIRECTORY} && $(MAKE) -C . release
-
 # Client builds
 %_js: export BUILD_NODE=1
 debug_js: debug
@@ -58,14 +36,8 @@ release_r: release
 debug_python: debug
 release_python: release
 
-# Main tests
-test: test_release
-
-test_release: release
-	${DUCKDB_DIRECTORY}/build/release/test/unittest "$(PROJ_DIR)test/*"
-
 test_debug: debug
-	${DUCKDB_DIRECTORY}/build/debug/test/unittest "$(PROJ_DIR)test/*"
+	build/debug/test/unittest "$(PROJ_DIR)test/*"
 
 # Client tests
 test_python: test_debug_python
@@ -88,4 +60,4 @@ format:
 	rm .clang-format
 
 update:
-	git submodule update --remote --merge
+	git submodule update --remote --merge

README.md

@@ -11,7 +11,7 @@ make
 
 To run, run the bundled `duckdb` shell:
 ```
- ./duckdb/build/release/duckdb 
+ ./build/release/duckdb 
 ```
 
 Then, load the Substrait - DuckDB extension like so:

extension_config.cmake

@@ -0,0 +1,8 @@
+# This file is included by DuckDB's build system. It specifies which extension to load
+
+# Extension from this repo
+duckdb_extension_load(substrait
+    SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
+    INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/include
+    LOAD_TESTS
+)

test/python/conftest.py

@@ -23,7 +23,7 @@ def add_extension(extension_name, conn: Union[str, DuckDBPyConnection] = '') ->
             'allow_unsigned_extensions' : 'true'
         }
         conn = duckdb.connect(conn or '', config=config)
-    file_path = f"'{dir}/../../duckdb/build/{build_type}/extension/{extension_name}/{extension_name}.duckdb_extension'"
+    file_path = f"'{dir}/../../build/{build_type}/extension/{extension_name}/{extension_name}.duckdb_extension'"
     conn.execute(f"LOAD {file_path}")
     return conn
 

test/python/test_adbc.py

@@ -19,7 +19,7 @@
 
 def find_substrait():
     # Paths to search for extensions
-    build = normpath(join(dirname(__file__), "../../duckdb/build/"))
+    build = normpath(join(dirname(__file__), "../../build/"))
     extension = "extension/*/*.duckdb_extension"
 
     extension_search_patterns = [

test/r/test_substrait.R

@@ -7,7 +7,7 @@ load_extension <- function() {
 
   file_directory <- getwd()
   con <- dbConnect(duckdb::duckdb(config=list("allow_unsigned_extensions"="true")))
-  dbExecute(con, sprintf("LOAD '%s/../../duckdb/build/%s/extension/substrait/substrait.duckdb_extension';", file_directory, build_type))
+  dbExecute(con, sprintf("LOAD '%s/../../build/%s/extension/substrait/substrait.duckdb_extension';", file_directory, build_type))
   return (con)
 }
 

vcpkg.json

@@ -0,0 +1,4 @@
+{
+    "dependencies": [
+    ]
+}