diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml deleted file mode 100644 index f5206125..00000000 --- a/.github/workflows/Linux.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Linux -on: - push: - paths-ignore: - - '**/README.md' - - 'doc/**' - pull_request: - paths-ignore: - - '**/README.md' - - 'doc/**' - repository_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - linux: - name: Linux Release - runs-on: ubuntu-latest - container: ${{ matrix.container }} - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] - vcpkg_version: [ '2023.04.15' ] - include: - - arch: 'linux_amd64_gcc4' - container: 'quay.io/pypa/manylinux2014_x86_64' - vcpkg_triplet: 'x64-linux' - - arch: 'linux_amd64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'x64-linux' - - arch: 'linux_arm64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'arm64-linux' - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - TREAT_WARNINGS_AS_ERRORS: 1 - - steps: - - name: Install required ubuntu packages - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq lsb-release ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - - - name: Install Git 2.18.5 - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - # Setup ccache (not on _gcc4) - - name: ccache - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ github.job }}-${{ matrix.arch }} - - - name: Setup ManyLinux2014 - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} - run: | - ./duckdb/scripts/setup_manylinux2014.sh general ccache openssl - - - name: Setup Ubuntu - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - uses: ./duckdb/.github/actions/ubuntu_18_setup - with: - aarch64_cross_compile: ${{ matrix.arch == 'linux_arm64' && 1 }} - openssl: 0 - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: 9edb1b8e590cc086563301d735cae4b6e732d2d2 - - # Build extension - - name: Build extension - env: - GEN: ninja - STATIC_OPENSSL: 1 - STATIC_LIBCPP: 1 - CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }} - CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }} - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - make release - - - name: Build extension - if: ${{ matrix.arch != 'linux_arm64'}} - run: | - make test_release - -# - uses: actions/upload-artifact@v2 -# with: -# name: ${{matrix.arch}}-extensions-${{ matrix.duckdb_version == '' && 'latest' || matrix.duckdb_version }} -# path: | -# build/release/extension/spatial/spatial.duckdb_extension -# -# - name: Deploy -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} -# BUCKET_NAME: ${{ secrets.S3_BUCKET }} -# run: | -# git config --global --add safe.directory '*' -# cd duckdb -# git fetch --tags -# export DUCKDB_VERSION=`git tag --points-at HEAD` -# export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} -# cd .. -# if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then -# echo 'No key set, skipping' -# elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then -# python3 -m pip install pip awscli -# ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true -# elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then -# python3 -m pip install pip awscli -# ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false -# fi \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml deleted file mode 100644 index fdbfc60f..00000000 --- a/.github/workflows/MacOS.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: MacOS -on: - push: - paths-ignore: - - '**/README.md' - - 'doc/**' - pull_request: - paths-ignore: - - '**/README.md' - - 'doc/**' - repository_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - macos: - name: MacOS Release (${{ matrix.osx_build_arch }}) - runs-on: macos-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] - vcpkg_triplet: [ 'x64-osx', 'arm64-osx' ] - include: - - vcpkg_triplet: 'x64-osx' - osx_build_arch: 'x86_64' - duckdb_arch: 'osx_amd64' - - vcpkg_triplet: 'arm64-osx' - osx_build_arch: 'arm64' - duckdb_arch: 'osx_arm64' - - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }} - GEN: Ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Setup Ninja - run: brew install ninja - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11 - with: - vcpkgGitCommitId: 9edb1b8e590cc086563301d735cae4b6e732d2d2 - - - name: Setup ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ runner.os }}-${{ matrix.osx_build_arch }}-${{ github.job }} - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - # Build extension - - name: Build extension - shell: bash - run: | - make release - - # Test extension (only on x86_64) - - name: Test Extension - if: ${{ matrix.osx_build_arch == 'x86_64'}} - shell: bash - run: | - make test - - # Upload artefact -# - uses: actions/upload-artifact@v2 -# with: -# name: osx-${{ matrix.osx_build_arch }}-extension-${{ matrix.duckdb_version == '' && 'latest' || matrix.duckdb_version }} -# path: | -# build/release/extension/spatial/spatial.duckdb_extension -# -# - name: Deploy -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} -# BUCKET_NAME: ${{ secrets.S3_BUCKET }} -# run: | -# git config --global --add safe.directory '*' -# cd duckdb -# git fetch --tags -# export DUCKDB_VERSION=`git tag --points-at HEAD` -# export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} -# cd .. -# if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then -# echo 'No key set, skipping' -# elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then -# python3 -m pip install pip awscli -# ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME true -# elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then -# python3 -m pip install pip awscli -# ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME false -# fi \ No newline at end of file diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 1899a90f..016a83b2 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -20,7 +20,7 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@3bc9b9f0c24d09e86cc407bb1d106dd4ae1b77b3 with: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml deleted file mode 100644 index 3702863e..00000000 --- a/.github/workflows/Windows.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Windows -on: - push: - paths-ignore: - - '**/README.md' - - 'doc/**' - pull_request: - paths-ignore: - - '**/README.md' - - 'doc/**' - repository_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - windows: - name: Release - runs-on: windows-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] - vcpkg_triplet: ['x64-windows'] - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: Ninja - VCPKG_ROOT: ${{ github.workspace }}\vcpkg - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11 - with: - vcpkgGitCommitId: 9edb1b8e590cc086563301d735cae4b6e732d2d2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Setup CCache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ runner.os }}-${{ github.job }} - - - name: Checkout DuckDB to version - # Add commits/tags to build against other DuckDB versions - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Build extension - run: | - make release - build/release/test/Release/unittest.exe - env: - # Disable building shell. Since it includes sqlite3 which we statically link to as well it causes a conflict - # and the build fails. Fixing this is a bit more involved so for now we just disable building the shell on windows. - BUILD_SHELL: 0 - -# - uses: actions/upload-artifact@v2 -# with: -# name: windows-amd64-extensions-${{ matrix.duckdb_version == '' && 'latest' || matrix.duckdb_version }} -# path: | -# build/release/extension/spatial/spatial.duckdb_extension -# -# - name: Deploy -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} -# AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} -# BUCKET_NAME: ${{ secrets.S3_BUCKET }} -# run: | -# cd duckdb -# git fetch --tags -# export DUCKDB_VERSION=`git tag --points-at HEAD` -# export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} -# cd .. -# if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then -# echo 'No key set, skipping' -# elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then -# python -m pip install awscli -# ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true -# elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then -# python -m pip install awscli -# ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false -# fi \ No newline at end of file diff --git a/spatial/include/spatial/core/io/shapefile.hpp b/spatial/include/spatial/core/io/shapefile.hpp index 6931b529..e359d485 100644 --- a/spatial/include/spatial/core/io/shapefile.hpp +++ b/spatial/include/spatial/core/io/shapefile.hpp @@ -14,7 +14,6 @@ struct SHPHandleDeleter { }; using SHPHandlePtr = unique_ptr; - struct DBFHandleDeleter { void operator()(DBFInfo *info) { if (info) { @@ -35,11 +34,9 @@ struct SHPObjectDeleter { using SHPObjectPtr = unique_ptr; - DBFHandlePtr OpenDBFFile(FileSystem &fs, const string &filename); SHPHandlePtr OpenSHPFile(FileSystem &fs, const string &filename); - enum class AttributeEncoding { UTF8, LATIN1, @@ -47,15 +44,20 @@ enum class AttributeEncoding { }; struct EncodingUtil { - static inline uint8_t GetUTF8ByteLength (data_t first_char) { - if (first_char < 0x80) return 1; - if (!(first_char & 0x20)) return 2; - if (!(first_char & 0x10)) return 3; - if (!(first_char & 0x08)) return 4; - if (!(first_char & 0x04)) return 5; + static inline uint8_t GetUTF8ByteLength(data_t first_char) { + if (first_char < 0x80) + return 1; + if (!(first_char & 0x20)) + return 2; + if (!(first_char & 0x10)) + return 3; + if (!(first_char & 0x08)) + return 4; + if (!(first_char & 0x04)) + return 5; return 6; } - static inline data_t UTF8ToLatin1Char (const_data_ptr_t ptr) { + static inline data_t UTF8ToLatin1Char(const_data_ptr_t ptr) { auto len = GetUTF8ByteLength(*ptr); if (len == 1) { return *ptr; @@ -70,7 +72,7 @@ struct EncodingUtil { // Convert UTF-8 to ISO-8859-1 // out must be at least the size of in - static void UTF8ToLatin1Buffer (const_data_ptr_t in, data_ptr_t out) { + static void UTF8ToLatin1Buffer(const_data_ptr_t in, data_ptr_t out) { while (*in) { *out++ = UTF8ToLatin1Char(in); } @@ -96,7 +98,6 @@ struct EncodingUtil { } }; +} // namespace core -} - -} \ No newline at end of file +} // namespace spatial \ No newline at end of file diff --git a/spatial/include/spatial/gdal/file_handler.hpp b/spatial/include/spatial/gdal/file_handler.hpp index 4f2dbd0f..15c6c465 100644 --- a/spatial/include/spatial/gdal/file_handler.hpp +++ b/spatial/include/spatial/gdal/file_handler.hpp @@ -11,13 +11,14 @@ class DuckDBFileSystemHandler; class GDALClientContextState : public ClientContextState { string client_prefix; - DuckDBFileSystemHandler* fs_handler; + DuckDBFileSystemHandler *fs_handler; + public: - explicit GDALClientContextState(ClientContext& context); + explicit GDALClientContextState(ClientContext &context); ~GDALClientContextState() override; void QueryEnd() override; - const string& GetPrefix() const; - static GDALClientContextState& GetOrCreate(ClientContext& context); + const string &GetPrefix() const; + static GDALClientContextState &GetOrCreate(ClientContext &context); }; } // namespace gdal diff --git a/spatial/include/spatial/gdal/functions.hpp b/spatial/include/spatial/gdal/functions.hpp index 740556e6..9bf2945c 100644 --- a/spatial/include/spatial/gdal/functions.hpp +++ b/spatial/include/spatial/gdal/functions.hpp @@ -57,7 +57,7 @@ struct GdalCopyFunction { }; struct GdalMetadataFunction { - static void Register(DatabaseInstance &db); + static void Register(DatabaseInstance &db); }; } // namespace gdal diff --git a/spatial/src/spatial/core/functions/scalar/st_asgeojson.cpp b/spatial/src/spatial/core/functions/scalar/st_asgeojson.cpp index b80a21bd..e5723663 100644 --- a/spatial/src/spatial/core/functions/scalar/st_asgeojson.cpp +++ b/spatial/src/spatial/core/functions/scalar/st_asgeojson.cpp @@ -219,23 +219,21 @@ static Point PointFromGeoJSON(yyjson_val *coord_array, GeometryFactory &factory, // empty point return factory.CreateEmptyPoint(); } - if(len < 2) { - throw InvalidInputException("GeoJSON input coordinates field is not an array of at least length 2: %s", - raw.GetString()); - } - auto x_val = yyjson_arr_get_first(coord_array); - if (!yyjson_is_num(x_val)) { - throw InvalidInputException("GeoJSON input coordinates field is not an array of numbers: %s", - raw.GetString()); - } - auto y_val = len == 2 ? yyjson_arr_get_last(coord_array) : yyjson_arr_get(coord_array, 1); - if (!yyjson_is_num(y_val)) { - throw InvalidInputException("GeoJSON input coordinates field is not an array of numbers: %s", - raw.GetString()); - } - auto x = yyjson_get_num(x_val); - auto y = yyjson_get_num(y_val); - return factory.CreatePoint(x, y); + if (len < 2) { + throw InvalidInputException("GeoJSON input coordinates field is not an array of at least length 2: %s", + raw.GetString()); + } + auto x_val = yyjson_arr_get_first(coord_array); + if (!yyjson_is_num(x_val)) { + throw InvalidInputException("GeoJSON input coordinates field is not an array of numbers: %s", raw.GetString()); + } + auto y_val = len == 2 ? yyjson_arr_get_last(coord_array) : yyjson_arr_get(coord_array, 1); + if (!yyjson_is_num(y_val)) { + throw InvalidInputException("GeoJSON input coordinates field is not an array of numbers: %s", raw.GetString()); + } + auto x = yyjson_get_num(x_val); + auto y = yyjson_get_num(y_val); + return factory.CreatePoint(x, y); } static VertexVector VerticesFromGeoJSON(yyjson_val *coord_array, GeometryFactory &factory, const string_t &raw) { @@ -253,8 +251,8 @@ static VertexVector VerticesFromGeoJSON(yyjson_val *coord_array, GeometryFactory } auto coord_len = yyjson_arr_size(coord_val); if (coord_len < 2) { - throw InvalidInputException("GeoJSON input coordinates field is not an array of arrays of length >= 2: %s", - raw.GetString()); + throw InvalidInputException( + "GeoJSON input coordinates field is not an array of arrays of length >= 2: %s", raw.GetString()); } auto x_val = yyjson_arr_get_first(coord_val); if (!yyjson_is_num(x_val)) { @@ -316,8 +314,8 @@ static MultiPoint MultiPointFromGeoJSON(yyjson_val *coord_array, GeometryFactory raw.GetString()); } if (yyjson_arr_size(point_val) < 2) { - throw InvalidInputException("GeoJSON input coordinates field is not an array of arrays of length >= 2: %s", - raw.GetString()); + throw InvalidInputException( + "GeoJSON input coordinates field is not an array of arrays of length >= 2: %s", raw.GetString()); } multi_point[idx] = PointFromGeoJSON(point_val, factory, raw); } diff --git a/spatial/src/spatial/core/functions/scalar/st_collect.cpp b/spatial/src/spatial/core/functions/scalar/st_collect.cpp index 6fc64301..8d48f674 100644 --- a/spatial/src/spatial/core/functions/scalar/st_collect.cpp +++ b/spatial/src/spatial/core/functions/scalar/st_collect.cpp @@ -31,7 +31,7 @@ static void CollectFunction(DataChunk &args, ExpressionState &state, Vector &res auto geometry_blob = ((string_t *)format.data)[mapped_idx]; auto geometry = lstate.factory.Deserialize(geometry_blob); // Dont add empty geometries - if(!geometry.IsEmpty()) { + if (!geometry.IsEmpty()) { geometries.push_back(geometry); } } diff --git a/spatial/src/spatial/core/functions/scalar/st_distance.cpp b/spatial/src/spatial/core/functions/scalar/st_distance.cpp index 5ee8c6cf..894487fe 100644 --- a/spatial/src/spatial/core/functions/scalar/st_distance.cpp +++ b/spatial/src/spatial/core/functions/scalar/st_distance.cpp @@ -65,7 +65,7 @@ static void PointToPointDistanceFunction(DataChunk &args, ExpressionState &state out_data[i] = std::sqrt(std::pow(left_x[i] - right_x[i], 2) + std::pow(left_y[i] - right_y[i], 2)); } - if(count == 1) { + if (count == 1) { result.SetVectorType(VectorType::CONSTANT_VECTOR); } } diff --git a/spatial/src/spatial/core/functions/scalar/st_quadkey.cpp b/spatial/src/spatial/core/functions/scalar/st_quadkey.cpp index b0770498..9b0fcb5b 100644 --- a/spatial/src/spatial/core/functions/scalar/st_quadkey.cpp +++ b/spatial/src/spatial/core/functions/scalar/st_quadkey.cpp @@ -14,14 +14,15 @@ namespace spatial { namespace core { -static void GetQuadKey(double lon, double lat, int32_t level, char* buffer) { +static void GetQuadKey(double lon, double lat, int32_t level, char *buffer) { lat = std::max(-85.05112878, std::min(85.05112878, lat)); lon = std::max(-180.0, std::min(180.0, lon)); double lat_rad = lat * M_PI / 180.0; auto x = static_cast((lon + 180.0) / 360.0 * (1 << level)); - auto y = static_cast((1.0 - std::log(std::tan(lat_rad) + 1.0 / std::cos(lat_rad)) / M_PI) / 2.0 * (1 << level)); + auto y = + static_cast((1.0 - std::log(std::tan(lat_rad) + 1.0 / std::cos(lat_rad)) / M_PI) / 2.0 * (1 << level)); for (int i = level; i > 0; --i) { char digit = '0'; @@ -45,14 +46,15 @@ static void CoordinateQuadKeyFunction(DataChunk &args, ExpressionState &state, V auto &level = args.data[2]; auto count = args.size(); - TernaryExecutor::Execute(lon_in, lat_in, level, result, count, [&](double lon, double lat, int32_t level) { - if(level < 1 || level > 23) { - throw InvalidInputException("ST_QuadKey: Level must be between 1 and 23"); - } - char buffer[64]; - GetQuadKey(lon, lat, level, buffer); - return StringVector::AddString(result, buffer, level); - }); + TernaryExecutor::Execute( + lon_in, lat_in, level, result, count, [&](double lon, double lat, int32_t level) { + if (level < 1 || level > 23) { + throw InvalidInputException("ST_QuadKey: Level must be between 1 and 23"); + } + char buffer[64]; + GetQuadKey(lon, lat, level, buffer); + return StringVector::AddString(result, buffer, level); + }); } //------------------------------------------------------------------------------ @@ -65,27 +67,28 @@ static void GeometryQuadKeyFunction(DataChunk &args, ExpressionState &state, Vec auto &level = args.data[1]; auto count = args.size(); - BinaryExecutor::Execute(geom, level, result, count, [&](string_t input, int32_t level) { - auto header = GeometryHeader::Get(input); - if(header.type != GeometryType::POINT) { - throw InvalidInputException("ST_QuadKey: Only POINT geometries are supported"); - } - auto point = ctx.factory.Deserialize(input); - if(point.IsEmpty()) { - throw InvalidInputException("ST_QuadKey: Empty geometries are not supported"); - } - auto vertex = point.GetPoint().GetVertex(); - auto x = vertex.x; - auto y = vertex.y; - - if(level < 1 || level > 23) { - throw InvalidInputException("ST_QuadKey: Level must be between 1 and 23"); - } - - char buffer[64]; - GetQuadKey(x, y, level, buffer); - return StringVector::AddString(result, buffer, level); - }); + BinaryExecutor::Execute( + geom, level, result, count, [&](string_t input, int32_t level) { + auto header = GeometryHeader::Get(input); + if (header.type != GeometryType::POINT) { + throw InvalidInputException("ST_QuadKey: Only POINT geometries are supported"); + } + auto point = ctx.factory.Deserialize(input); + if (point.IsEmpty()) { + throw InvalidInputException("ST_QuadKey: Empty geometries are not supported"); + } + auto vertex = point.GetPoint().GetVertex(); + auto x = vertex.x; + auto y = vertex.y; + + if (level < 1 || level > 23) { + throw InvalidInputException("ST_QuadKey: Level must be between 1 and 23"); + } + + char buffer[64]; + GetQuadKey(x, y, level, buffer); + return StringVector::AddString(result, buffer, level); + }); } //------------------------------------------------------------------------------ @@ -95,10 +98,11 @@ void CoreScalarFunctions::RegisterStQuadKey(DatabaseInstance &db) { ScalarFunctionSet set("ST_QuadKey"); - set.AddFunction(ScalarFunction({LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::INTEGER}, LogicalType::VARCHAR, - CoordinateQuadKeyFunction)); + set.AddFunction(ScalarFunction({LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::INTEGER}, + LogicalType::VARCHAR, CoordinateQuadKeyFunction)); set.AddFunction(ScalarFunction({GeoTypes::GEOMETRY(), LogicalType::INTEGER}, LogicalType::VARCHAR, - GeometryQuadKeyFunction, nullptr, nullptr, nullptr, GeometryFunctionLocalState::Init)); + GeometryQuadKeyFunction, nullptr, nullptr, nullptr, + GeometryFunctionLocalState::Init)); ExtensionUtil::RegisterFunction(db, set); } diff --git a/spatial/src/spatial/core/io/shapefile/read_shapefile.cpp b/spatial/src/spatial/core/io/shapefile/read_shapefile.cpp index 0971df09..fae9829c 100644 --- a/spatial/src/spatial/core/io/shapefile/read_shapefile.cpp +++ b/spatial/src/spatial/core/io/shapefile/read_shapefile.cpp @@ -37,20 +37,13 @@ struct ShapefileBindData : TableFunctionData { vector attribute_types; explicit ShapefileBindData(string file_name_p) - : file_name(std::move(file_name_p)), - shape_count(0), - shape_type(0), - min_bound{0, 0, 0, 0}, - max_bound{0, 0, 0, 0}, - attribute_encoding(AttributeEncoding::LATIN1) - { } + : file_name(std::move(file_name_p)), shape_count(0), + shape_type(0), min_bound {0, 0, 0, 0}, max_bound {0, 0, 0, 0}, attribute_encoding(AttributeEncoding::LATIN1) { + } }; - -static unique_ptr Bind(ClientContext &context, - TableFunctionBindInput &input, - vector &return_types, - vector &names) { +static unique_ptr Bind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { auto file_name = StringValue::Get(input.inputs[0]); auto result = make_uniq(file_name); @@ -62,9 +55,9 @@ static unique_ptr Bind(ClientContext &context, SHPGetInfo(shp_handle.get(), &result->shape_count, &result->shape_type, result->min_bound, result->max_bound); // Ensure we have a supported shape type - auto valid_types = { SHPT_NULL, SHPT_POINT, SHPT_ARC, SHPT_POLYGON, SHPT_MULTIPOINT }; + auto valid_types = {SHPT_NULL, SHPT_POINT, SHPT_ARC, SHPT_POLYGON, SHPT_MULTIPOINT}; bool is_valid_type = false; - for(auto type : valid_types) { + for (auto type : valid_types) { if (result->shape_type == type) { is_valid_type = true; break; @@ -92,23 +85,23 @@ static unique_ptr Bind(ClientContext &context, } } - for(auto &kv : input.named_parameters) { - if(kv.first == "encoding") { + for (auto &kv : input.named_parameters) { + if (kv.first == "encoding") { auto encoding = StringUtil::Lower(StringValue::Get(kv.second)); - if(encoding == "utf-8") { + if (encoding == "utf-8") { result->attribute_encoding = AttributeEncoding::UTF8; - } else if(encoding == "iso-8859-1") { + } else if (encoding == "iso-8859-1") { result->attribute_encoding = AttributeEncoding::LATIN1; - } else if(encoding == "blob") { + } else if (encoding == "blob") { // Otherwise, parse as blob result->attribute_encoding = AttributeEncoding::BLOB; } else { - vector candidates = { "utf-8", "iso-8859-1", "blob" }; + vector candidates = {"utf-8", "iso-8859-1", "blob"}; auto msg = StringUtil::CandidatesErrorMessage(candidates, encoding, "encoding"); throw InvalidInputException("Invalid encoding %s", encoding.c_str()); } } - if(kv.first == "spatial_filter_box") { + if (kv.first == "spatial_filter_box") { auto filter_box = StructValue::GetChildren(kv.second); } } @@ -140,7 +133,7 @@ static unique_ptr Bind(ClientContext &context, type = LogicalType::INTEGER; break; case FTDouble: - if(field_precision == 0 && field_width < 19) { + if (field_precision == 0 && field_width < 19) { type = LogicalType::BIGINT; } else { type = LogicalType::DOUBLE; @@ -167,10 +160,10 @@ static unique_ptr Bind(ClientContext &context, names.push_back("geom"); // Deduplicate field names if necessary - for(size_t i = 0; i < names.size(); i++) { + for (size_t i = 0; i < names.size(); i++) { idx_t count = 1; - for(size_t j = i + 1; j < names.size(); j++) { - if(names[i] == names[j]) { + for (size_t j = i + 1; j < names.size(); j++) { + if (names[i] == names[j]) { names[j] += "_" + std::to_string(count++); } } @@ -221,7 +214,7 @@ struct ConvertPoint { struct ConvertLineString { static Geometry Convert(SHPObjectPtr &shape, GeometryFactory &factory) { - if(shape->nParts == 1) { + if (shape->nParts == 1) { // Single LineString auto line_string = factory.CreateLineString(shape->nVertices); for (int i = 0; i < shape->nVertices; i++) { @@ -252,18 +245,18 @@ struct ConvertPolygon { // Each polygon is identified by a part with clockwise winding order // we calculate the winding order by checking the sign of the area vector polygon_part_starts; - for(int i = 0; i < shape->nParts; i++) { + for (int i = 0; i < shape->nParts; i++) { auto start = shape->panPartStart[i]; auto end = i == shape->nParts - 1 ? shape->nVertices : shape->panPartStart[i + 1]; double area = 0; - for(int j = start; j < end - 1; j++) { + for (int j = start; j < end - 1; j++) { area += (shape->padfX[j] * shape->padfY[j + 1]) - (shape->padfX[j + 1] * shape->padfY[j]); } - if(area < 0) { + if (area < 0) { polygon_part_starts.push_back(i); } } - if(polygon_part_starts.size() < 2) { + if (polygon_part_starts.size() < 2) { // Single polygon, every part is an interior ring // Even if the polygon is counter-clockwise (which should not happen for shapefiles). // we still fall back and convert it to a single polygon. @@ -279,15 +272,13 @@ struct ConvertPolygon { start = end; } return polygon; - } - else { + } else { // MultiPolygon auto multi_polygon = factory.CreateMultiPolygon(polygon_part_starts.size()); - for(size_t polygon_idx = 0; polygon_idx < polygon_part_starts.size(); polygon_idx++) { + for (size_t polygon_idx = 0; polygon_idx < polygon_part_starts.size(); polygon_idx++) { auto part_start = polygon_part_starts[polygon_idx]; - auto part_end = polygon_idx == polygon_part_starts.size() - 1 - ? shape->nParts - : polygon_part_starts[polygon_idx + 1]; + auto part_end = polygon_idx == polygon_part_starts.size() - 1 ? shape->nParts + : polygon_part_starts[polygon_idx + 1]; auto polygon = factory.CreatePolygon(part_end - part_start); for (auto ring_idx = part_start; ring_idx < part_end; ring_idx++) { @@ -316,11 +307,12 @@ struct ConvertMultiPoint { } }; -template -static void ConvertGeomLoop(Vector &result, int record_start, idx_t count, SHPHandle &shp_handle, GeometryFactory &factory) { - for(idx_t result_idx = 0; result_idx < count; result_idx++) { +template +static void ConvertGeomLoop(Vector &result, int record_start, idx_t count, SHPHandle &shp_handle, + GeometryFactory &factory) { + for (idx_t result_idx = 0; result_idx < count; result_idx++) { auto shape = SHPObjectPtr(SHPReadObject(shp_handle, record_start++)); - if(shape->nSHPType == SHPT_NULL) { + if (shape->nSHPType == SHPT_NULL) { FlatVector::SetNull(result, result_idx, true); } else { FlatVector::GetData(result)[result_idx] = factory.Serialize(result, OP::Convert(shape, factory)); @@ -328,8 +320,9 @@ static void ConvertGeomLoop(Vector &result, int record_start, idx_t count, SHPHa } } -static void ConvertGeometryVector(Vector &result, int record_start, idx_t count, SHPHandle shp_handle, GeometryFactory &factory, int geom_type) { - switch(geom_type) { +static void ConvertGeometryVector(Vector &result, int record_start, idx_t count, SHPHandle shp_handle, + GeometryFactory &factory, int geom_type) { + switch (geom_type) { case SHPT_NULL: FlatVector::Validity(result).SetAllInvalid(count); break; @@ -356,7 +349,7 @@ static void ConvertGeometryVector(Vector &result, int record_start, idx_t count, struct ConvertBlobAttribute { using TYPE = string_t; - static string_t Convert(Vector& result, DBFHandle dbf_handle, int record_idx, int field_idx) { + static string_t Convert(Vector &result, DBFHandle dbf_handle, int record_idx, int field_idx) { auto value = DBFReadStringAttribute(dbf_handle, record_idx, field_idx); return StringVector::AddString(result, const_char_ptr_cast(value)); } @@ -364,28 +357,28 @@ struct ConvertBlobAttribute { struct ConvertIntegerAttribute { using TYPE = int32_t; - static int32_t Convert(Vector&, DBFHandle dbf_handle, int record_idx, int field_idx) { + static int32_t Convert(Vector &, DBFHandle dbf_handle, int record_idx, int field_idx) { return DBFReadIntegerAttribute(dbf_handle, record_idx, field_idx); } }; struct ConvertBigIntAttribute { using TYPE = int64_t; - static int64_t Convert(Vector&, DBFHandle dbf_handle, int record_idx, int field_idx) { + static int64_t Convert(Vector &, DBFHandle dbf_handle, int record_idx, int field_idx) { return static_cast(DBFReadDoubleAttribute(dbf_handle, record_idx, field_idx)); } }; struct ConvertDoubleAttribute { using TYPE = double; - static double Convert(Vector&, DBFHandle dbf_handle, int record_idx, int field_idx) { + static double Convert(Vector &, DBFHandle dbf_handle, int record_idx, int field_idx) { return DBFReadDoubleAttribute(dbf_handle, record_idx, field_idx); } }; struct ConvertDateAttribute { using TYPE = date_t; - static date_t Convert(Vector&, DBFHandle dbf_handle, int record_idx, int field_idx) { + static date_t Convert(Vector &, DBFHandle dbf_handle, int record_idx, int field_idx) { // XBase stores dates as 8-char strings (without separators) // but DuckDB expects a date string with separators. auto value = DBFReadStringAttribute(dbf_handle, record_idx, field_idx); @@ -402,28 +395,30 @@ struct ConvertDateAttribute { struct ConvertBooleanAttribute { using TYPE = bool; - static bool Convert(Vector& result, DBFHandle dbf_handle, int record_idx, int field_idx) { + static bool Convert(Vector &result, DBFHandle dbf_handle, int record_idx, int field_idx) { return *DBFReadLogicalAttribute(dbf_handle, record_idx, field_idx) == 'T'; } }; -template +template static void ConvertAttributeLoop(Vector &result, int record_start, idx_t count, DBFHandle dbf_handle, int field_idx) { int record_idx = record_start; - for(idx_t row_idx = 0; row_idx < count; row_idx++) { - if(DBFIsAttributeNULL(dbf_handle, record_idx, field_idx)) { + for (idx_t row_idx = 0; row_idx < count; row_idx++) { + if (DBFIsAttributeNULL(dbf_handle, record_idx, field_idx)) { FlatVector::SetNull(result, row_idx, true); } else { - FlatVector::GetData(result)[row_idx] = OP::Convert(result, dbf_handle, record_idx, field_idx); + FlatVector::GetData(result)[row_idx] = + OP::Convert(result, dbf_handle, record_idx, field_idx); } record_idx++; } } -static void ConvertStringAttributeLoop(Vector &result, int record_start, idx_t count, DBFHandle dbf_handle, int field_idx, AttributeEncoding attribute_encoding) { +static void ConvertStringAttributeLoop(Vector &result, int record_start, idx_t count, DBFHandle dbf_handle, + int field_idx, AttributeEncoding attribute_encoding) { int record_idx = record_start; vector conversion_buffer; - for(idx_t row_idx = 0; row_idx < count; row_idx++) { + for (idx_t row_idx = 0; row_idx < count; row_idx++) { if (DBFIsAttributeNULL(dbf_handle, record_idx, field_idx)) { FlatVector::SetNull(result, row_idx, true); } else { @@ -431,13 +426,15 @@ static void ConvertStringAttributeLoop(Vector &result, int record_start, idx_t c string_t result_str; if (attribute_encoding == AttributeEncoding::LATIN1) { conversion_buffer.reserve(strlen(string_bytes) * 2 + 1); // worst case (all non-ascii chars) - auto out_len = EncodingUtil::LatinToUTF8Buffer(const_data_ptr_cast(string_bytes), conversion_buffer.data()); + auto out_len = + EncodingUtil::LatinToUTF8Buffer(const_data_ptr_cast(string_bytes), conversion_buffer.data()); result_str = StringVector::AddString(result, const_char_ptr_cast(conversion_buffer.data()), out_len); } else { result_str = StringVector::AddString(result, const_char_ptr_cast(string_bytes)); } if (!Utf8Proc::IsValid(result_str.GetDataUnsafe(), result_str.GetSize())) { - throw InvalidInputException("Could not decode VARCHAR field as valid UTF-8, try passing encoding='blob' to skip decoding of string attributes"); + throw InvalidInputException("Could not decode VARCHAR field as valid UTF-8, try passing " + "encoding='blob' to skip decoding of string attributes"); } FlatVector::GetData(result)[row_idx] = result_str; } @@ -447,7 +444,7 @@ static void ConvertStringAttributeLoop(Vector &result, int record_start, idx_t c static void ConvertAttributeVector(Vector &result, int record_start, idx_t count, DBFHandle dbf_handle, int field_idx, AttributeEncoding attribute_encoding) { - switch(result.GetType().id()) { + switch (result.GetType().id()) { case LogicalTypeId::BLOB: ConvertAttributeLoop(result, record_start, count, dbf_handle, field_idx); break; @@ -488,18 +485,20 @@ static void Execute(ClientContext &context, TableFunctionInput &input, DataChunk // Calculate how many record we can fit in the output auto output_size = std::min(STANDARD_VECTOR_SIZE, bind_data.shape_count - gstate.shape_idx); int record_start = gstate.shape_idx; - for(auto col_idx = 0; col_idx < output.ColumnCount(); col_idx++) { + for (auto col_idx = 0; col_idx < output.ColumnCount(); col_idx++) { // Projected column indices auto projected_col_idx = gstate.column_ids[col_idx]; auto &col_vec = output.data[col_idx]; - if(col_vec.GetType() == GeoTypes::GEOMETRY()) { - ConvertGeometryVector(col_vec, record_start, output_size, gstate.shp_handle.get(), gstate.factory, bind_data.shape_type); + if (col_vec.GetType() == GeoTypes::GEOMETRY()) { + ConvertGeometryVector(col_vec, record_start, output_size, gstate.shp_handle.get(), gstate.factory, + bind_data.shape_type); } else { // The geometry is always last, so we can use the projected column index directly auto field_idx = projected_col_idx; - ConvertAttributeVector(col_vec, record_start, output_size, gstate.dbf_handle.get(), (int)field_idx, bind_data.attribute_encoding); + ConvertAttributeVector(col_vec, record_start, output_size, gstate.dbf_handle.get(), (int)field_idx, + bind_data.attribute_encoding); } } // Update the shape index @@ -513,8 +512,7 @@ static void Execute(ClientContext &context, TableFunctionInput &input, DataChunk // Progress, Cardinality and Replacement Scans //------------------------------------------------------------------------------ -static double GetProgress(ClientContext &context, - const FunctionData *bind_data_p, +static double GetProgress(ClientContext &context, const FunctionData *bind_data_p, const GlobalTableFunctionState *global_state) { auto &gstate = global_state->Cast(); @@ -534,7 +532,8 @@ static unique_ptr GetCardinality(ClientContext &context, const F return result; } -static unique_ptr GetReplacementScan(ClientContext &context, const string &table_name, ReplacementScanData *data) { +static unique_ptr GetReplacementScan(ClientContext &context, const string &table_name, + ReplacementScanData *data) { // Check if the table name ends with .shp if (!StringUtil::EndsWith(StringUtil::Lower(table_name), ".shp")) { return nullptr; @@ -564,8 +563,6 @@ void CoreTableFunctions::RegisterShapefileTableFunction(DatabaseInstance &db) { config.replacement_scans.emplace_back(GetReplacementScan); } +} // namespace core -} - -} - +} // namespace spatial diff --git a/spatial/src/spatial/core/io/shapefile/read_shapefile_meta.cpp b/spatial/src/spatial/core/io/shapefile/read_shapefile_meta.cpp index 327a2854..b3728059 100644 --- a/spatial/src/spatial/core/io/shapefile/read_shapefile_meta.cpp +++ b/spatial/src/spatial/core/io/shapefile/read_shapefile_meta.cpp @@ -25,24 +25,24 @@ struct ShapeFileMetaBindData : public TableFunctionData { struct ShapeTypeEntry { int shp_type; - const char* shp_name; + const char *shp_name; }; static ShapeTypeEntry shape_type_map[] = { - { SHPT_NULL, "NULL" }, - { SHPT_POINT, "POINT" }, - { SHPT_ARC, "LINESTRING" }, - { SHPT_POLYGON, "POLYGON" }, - { SHPT_MULTIPOINT, "MULTIPOINT" }, - { SHPT_POINTZ, "POINTZ" }, - { SHPT_ARCZ, "LINESTRINGZ" }, - { SHPT_POLYGONZ, "POLYGONZ" }, - { SHPT_MULTIPOINTZ, "MULTIPOINTZ" }, - { SHPT_POINTM, "POINTM" }, - { SHPT_ARCM, "LINESTRINGM" }, - { SHPT_POLYGONM, "POLYGONM" }, - { SHPT_MULTIPOINTM, "MULTIPOINTM" }, - { SHPT_MULTIPATCH, "MULTIPATCH" }, + {SHPT_NULL, "NULL"}, + {SHPT_POINT, "POINT"}, + {SHPT_ARC, "LINESTRING"}, + {SHPT_POLYGON, "POLYGON"}, + {SHPT_MULTIPOINT, "MULTIPOINT"}, + {SHPT_POINTZ, "POINTZ"}, + {SHPT_ARCZ, "LINESTRINGZ"}, + {SHPT_POLYGONZ, "POLYGONZ"}, + {SHPT_MULTIPOINTZ, "MULTIPOINTZ"}, + {SHPT_POINTM, "POINTM"}, + {SHPT_ARCM, "LINESTRINGM"}, + {SHPT_POLYGONM, "POLYGONM"}, + {SHPT_MULTIPOINTM, "MULTIPOINTM"}, + {SHPT_MULTIPATCH, "MULTIPATCH"}, }; static unique_ptr ShapeFileMetaBind(ClientContext &context, TableFunctionBindInput &input, @@ -50,7 +50,7 @@ static unique_ptr ShapeFileMetaBind(ClientContext &context, TableF auto result = make_uniq(); auto files = MultiFileReader::GetFileList(context, input.inputs[0], "ShapeFiles", FileGlobOptions::ALLOW_EMPTY); for (auto &file : files) { - if(StringUtil::EndsWith(StringUtil::Lower(file), ".shp")) { + if (StringUtil::EndsWith(StringUtil::Lower(file), ".shp")) { result->files.push_back(file); } } @@ -83,7 +83,8 @@ struct ShapeFileMetaGlobalState : public GlobalTableFunctionState { vector files; }; -static unique_ptr ShapeFileMetaInitGlobal(ClientContext &context, TableFunctionInitInput &input) { +static unique_ptr ShapeFileMetaInitGlobal(ClientContext &context, + TableFunctionInitInput &input) { auto &bind_data = input.bind_data->Cast(); auto result = make_uniq(); @@ -126,8 +127,9 @@ static void ShapeFileMetaExecute(ClientContext &context, TableFunctionInput &inp SHPGetInfo(shp_handle.get(), &record_count, &shape_type, min_bound, max_bound); file_name_data[out_idx] = StringVector::AddString(file_name_vector, file_name); shape_type_data[out_idx] = 0; - for(auto shape_type_idx = 0; shape_type_idx < sizeof(shape_type_map) / sizeof(ShapeTypeEntry); shape_type_idx++) { - if(shape_type_map[shape_type_idx].shp_type == shape_type) { + for (auto shape_type_idx = 0; shape_type_idx < sizeof(shape_type_map) / sizeof(ShapeTypeEntry); + shape_type_idx++) { + if (shape_type_map[shape_type_idx].shp_type == shape_type) { shape_type_data[out_idx] = shape_type_idx; break; } @@ -161,13 +163,13 @@ static unique_ptr ShapeFileMetaCardinality(ClientContext &contex void CoreTableFunctions::RegisterShapefileMetaTableFunction(DatabaseInstance &db) { - TableFunction meta_func("shapefile_meta", {LogicalType::VARCHAR}, ShapeFileMetaExecute, ShapeFileMetaBind, ShapeFileMetaInitGlobal); + TableFunction meta_func("shapefile_meta", {LogicalType::VARCHAR}, ShapeFileMetaExecute, ShapeFileMetaBind, + ShapeFileMetaInitGlobal); meta_func.table_scan_progress = ShapeFileMetaProgress; meta_func.cardinality = ShapeFileMetaCardinality; ExtensionUtil::RegisterFunction(db, MultiFileReader::CreateFunctionSet(meta_func)); - } -} +} // namespace core -} \ No newline at end of file +} // namespace spatial \ No newline at end of file diff --git a/spatial/src/spatial/core/io/shapefile/shapefile_common.cpp b/spatial/src/spatial/core/io/shapefile/shapefile_common.cpp index a8c50582..00db00bc 100644 --- a/spatial/src/spatial/core/io/shapefile/shapefile_common.cpp +++ b/spatial/src/spatial/core/io/shapefile/shapefile_common.cpp @@ -15,7 +15,7 @@ namespace core { //------------------------------------------------------------------------------ // Shapefile filesystem abstractions //------------------------------------------------------------------------------ -static SAFile DuckDBShapefileOpen(void* userData, const char *filename, const char *access_mode) { +static SAFile DuckDBShapefileOpen(void *userData, const char *filename, const char *access_mode) { try { auto &fs = *reinterpret_cast(userData); auto file_handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ); @@ -23,25 +23,25 @@ static SAFile DuckDBShapefileOpen(void* userData, const char *filename, const ch return nullptr; } return reinterpret_cast(file_handle.release()); - } catch(...) { + } catch (...) { return nullptr; } } static SAOffset DuckDBShapefileRead(void *p, SAOffset size, SAOffset nmemb, SAFile file) { - auto handle = reinterpret_cast(file); + auto handle = reinterpret_cast(file); auto read_bytes = handle->Read(p, size * nmemb); return read_bytes / size; } static SAOffset DuckDBShapefileWrite(const void *p, SAOffset size, SAOffset nmemb, SAFile file) { - auto handle = reinterpret_cast(file); - auto written_bytes = handle->Write(const_cast(p), size * nmemb); + auto handle = reinterpret_cast(file); + auto written_bytes = handle->Write(const_cast(p), size * nmemb); return written_bytes / size; } static SAOffset DuckDBShapefileSeek(SAFile file, SAOffset offset, int whence) { - auto file_handle = reinterpret_cast(file); + auto file_handle = reinterpret_cast(file); switch (whence) { case SEEK_SET: file_handle->Seek(offset); @@ -59,7 +59,7 @@ static SAOffset DuckDBShapefileSeek(SAFile file, SAOffset offset, int whence) { } static SAOffset DuckDBShapefileTell(SAFile file) { - auto handle = reinterpret_cast(file); + auto handle = reinterpret_cast(file); return handle->SeekPosition(); } @@ -68,18 +68,18 @@ static int DuckDBShapefileFlush(SAFile file) { auto handle = reinterpret_cast(file); handle->Sync(); return 0; - } catch(...) { + } catch (...) { return -1; } } static int DuckDBShapefileClose(SAFile file) { try { - auto handle = reinterpret_cast(file); + auto handle = reinterpret_cast(file); handle->Close(); delete handle; return 0; - } catch(...) { + } catch (...) { return -1; } } @@ -95,7 +95,7 @@ static int DuckDBShapefileRemove(void *userData, const char *filename) { fs.RemoveFile(filename); } return 0; - } catch(...) { + } catch (...) { return -1; } } @@ -132,7 +132,6 @@ static SAHooks GetDuckDBHooks(FileSystem &fs) { return hooks; } - DBFHandlePtr OpenDBFFile(FileSystem &fs, const string &filename) { auto hooks = GetDuckDBHooks(fs); auto handle = DBFOpenLL(filename.c_str(), "rb", &hooks); @@ -153,7 +152,6 @@ SHPHandlePtr OpenSHPFile(FileSystem &fs, const string &filename) { return SHPHandlePtr(handle); } +} // namespace core -} - -} \ No newline at end of file +} // namespace spatial \ No newline at end of file diff --git a/spatial/src/spatial/gdal/file_handler.cpp b/spatial/src/spatial/gdal/file_handler.cpp index 511ca391..abed205c 100644 --- a/spatial/src/spatial/gdal/file_handler.cpp +++ b/spatial/src/spatial/gdal/file_handler.cpp @@ -20,10 +20,10 @@ namespace gdal { class DuckDBFileHandle : public VSIVirtualHandle { private: unique_ptr file_handle; + public: - explicit DuckDBFileHandle(unique_ptr file_handle_p) - : file_handle(std::move(file_handle_p)) - { } + explicit DuckDBFileHandle(unique_ptr file_handle_p) : file_handle(std::move(file_handle_p)) { + } vsi_l_offset Tell() override { return static_cast(file_handle->SeekPosition()); @@ -47,14 +47,17 @@ class DuckDBFileHandle : public VSIVirtualHandle { size_t Read(void *pBuffer, size_t nSize, size_t nCount) override { auto remaining_bytes = nSize * nCount; - while(remaining_bytes > 0) { - auto read_bytes = file_handle->Read(pBuffer, remaining_bytes); - if(read_bytes == 0) { - break; + try { + while (remaining_bytes > 0) { + auto read_bytes = file_handle->Read(pBuffer, remaining_bytes); + if (read_bytes == 0) { + break; + } + remaining_bytes -= read_bytes; + // Note we performed a cast back to void* + pBuffer = static_cast(pBuffer) + read_bytes; } - remaining_bytes -= read_bytes; - // Note we performed a cast back to void* - pBuffer = static_cast(pBuffer) + read_bytes; + } catch (...) { } return nCount - (remaining_bytes / nSize); } @@ -63,9 +66,12 @@ class DuckDBFileHandle : public VSIVirtualHandle { return file_handle->SeekPosition() == file_handle->GetFileSize() ? TRUE : FALSE; } - size_t Write(const void *pBuffer, size_t nSize, size_t nCount) override { - auto written_bytes = file_handle->Write(const_cast(pBuffer), nSize * nCount); + size_t written_bytes = 0; + try { + written_bytes = file_handle->Write(const_cast(pBuffer), nSize * nCount); + } catch (...) { + } // Return the number of items written return static_cast(written_bytes / nSize); } @@ -88,25 +94,24 @@ class DuckDBFileHandle : public VSIVirtualHandle { // VSIRangeStatus GetRangeStatus(vsi_l_offset nOffset, vsi_l_offset nLength) override; }; - //-------------------------------------------------------------------------- // GDAL DuckDB File system wrapper //-------------------------------------------------------------------------- class DuckDBFileSystemHandler : public VSIFilesystemHandler { private: string client_prefix; - ClientContext& context; + ClientContext &context; + public: - DuckDBFileSystemHandler(string client_prefix, ClientContext& context) - : client_prefix(std::move(client_prefix)), context(context) { }; + DuckDBFileSystemHandler(string client_prefix, ClientContext &context) + : client_prefix(std::move(client_prefix)), context(context) {}; - const char* StripPrefix(const char *pszFilename) - { + const char *StripPrefix(const char *pszFilename) { return pszFilename + client_prefix.size(); } - VSIVirtualHandle *Open(const char *prefixed_file_name, const char *access, - bool bSetError, CSLConstList /* papszOptions */) override { + VSIVirtualHandle *Open(const char *prefixed_file_name, const char *access, bool bSetError, + CSLConstList /* papszOptions */) override { auto file_name = StripPrefix(prefixed_file_name); auto &fs = FileSystem::GetFileSystem(context); @@ -146,10 +151,20 @@ class DuckDBFileSystemHandler : public VSIFilesystemHandler { try { string path(file_name); + // Check if the file is a directory + +#ifdef _WIN32 + if (fs.DirectoryExists(path) && (flags & FileFlags::FILE_FLAGS_READ)) { + // We can't open a directory for reading on windows without special flags + // so just open nul instead, gdal will reject it when it tries to read + auto file = fs.OpenFile("nul", flags); + return new DuckDBFileHandle(std::move(file)); + } +#endif auto file = fs.OpenFile(file_name, flags); return new DuckDBFileHandle(std::move(file)); } catch (std::exception &ex) { - if(bSetError) { + if (bSetError) { VSIError(VSIE_FileError, "Failed to open file %s: %s", file_name, ex.what()); } return nullptr; @@ -162,6 +177,17 @@ class DuckDBFileSystemHandler : public VSIFilesystemHandler { memset(pstatbuf, 0, sizeof(VSIStatBufL)); + if (!(fs.FileExists(file_name) || fs.DirectoryExists(file_name))) { + return -1; + } + +#ifdef _WIN32 + if(fs.DirectoryExists(file_name)) { + pstatbuf->st_mode = S_IFDIR; + return 0; + } +#endif + unique_ptr file; try { file = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ); @@ -169,7 +195,7 @@ class DuckDBFileSystemHandler : public VSIFilesystemHandler { return -1; } - if(!file) { + if (!file) { return -1; } @@ -190,10 +216,9 @@ class DuckDBFileSystemHandler : public VSIFilesystemHandler { break; default: // HTTPFS returns invalid type for everything basically. - if(FileSystem::IsRemoteFile(file_name)) { + if (FileSystem::IsRemoteFile(file_name)) { pstatbuf->st_mode = S_IFREG; - } - else { + } else { return -1; } } @@ -269,7 +294,6 @@ class DuckDBFileSystemHandler : public VSIFilesystemHandler { } }; - //-------------------------------------------------------------------------- // GDALClientContextState //-------------------------------------------------------------------------- @@ -278,7 +302,7 @@ class DuckDBFileSystemHandler : public VSIFilesystemHandler { // use their own attached file systems. This is necessary because GDAL is // not otherwise aware of the connection context. // -GDALClientContextState::GDALClientContextState(ClientContext& context) { +GDALClientContextState::GDALClientContextState(ClientContext &context) { // Create a new random prefix for this client client_prefix = StringUtil::Format("/vsiduckdb-%s/", UUID::ToString(UUID::GenerateRandomUUID())); @@ -298,19 +322,19 @@ GDALClientContextState::~GDALClientContextState() { delete fs_handler; } -void GDALClientContextState::QueryEnd(){ +void GDALClientContextState::QueryEnd() { }; -const string& GDALClientContextState::GetPrefix() const { +const string &GDALClientContextState::GetPrefix() const { return client_prefix; } -GDALClientContextState& GDALClientContextState::GetOrCreate(ClientContext& context) { - if(!context.registered_state["gdal"]) { +GDALClientContextState &GDALClientContextState::GetOrCreate(ClientContext &context) { + if (!context.registered_state["gdal"]) { context.registered_state["gdal"] = make_uniq(context); } - return *dynamic_cast(context.registered_state["gdal"].get()); + return *dynamic_cast(context.registered_state["gdal"].get()); } } // namespace gdal diff --git a/spatial/src/spatial/gdal/functions/st_read.cpp b/spatial/src/spatial/gdal/functions/st_read.cpp index eecc020e..c133c0b2 100644 --- a/spatial/src/spatial/gdal/functions/st_read.cpp +++ b/spatial/src/spatial/gdal/functions/st_read.cpp @@ -52,12 +52,12 @@ static string FilterToGdal(const TableFilter &filter, const string &column_name) switch (filter.filter_type) { case TableFilterType::CONSTANT_COMPARISON: { - auto &constant_filter = (const ConstantFilter &)filter; - return column_name + ExpressionTypeToOperator(constant_filter.comparison_type) + - constant_filter.constant.ToSQLString(); + auto &constant_filter = filter.Cast(); + return KeywordHelper::WriteOptionallyQuoted(column_name) + + ExpressionTypeToOperator(constant_filter.comparison_type) + constant_filter.constant.ToSQLString(); } case TableFilterType::CONJUNCTION_AND: { - auto &and_filter = (const ConjunctionAndFilter &)filter; + auto &and_filter = filter.Cast(); vector filters; for (const auto &child_filter : and_filter.child_filters) { filters.push_back(FilterToGdal(*child_filter, column_name)); @@ -65,7 +65,7 @@ static string FilterToGdal(const TableFilter &filter, const string &column_name) return StringUtil::Join(filters, " AND "); } case TableFilterType::CONJUNCTION_OR: { - auto &or_filter = (const ConjunctionOrFilter &)filter; + auto &or_filter = filter.Cast(); vector filters; for (const auto &child_filter : or_filter.child_filters) { filters.push_back(FilterToGdal(*child_filter, column_name)); @@ -73,10 +73,10 @@ static string FilterToGdal(const TableFilter &filter, const string &column_name) return StringUtil::Join(filters, " OR "); } case TableFilterType::IS_NOT_NULL: { - return column_name + " IS NOT NULL"; + return KeywordHelper::WriteOptionallyQuoted(column_name) + " IS NOT NULL"; } case TableFilterType::IS_NULL: { - return column_name + " IS NULL"; + return KeywordHelper::WriteOptionallyQuoted(column_name) + " IS NULL"; } default: throw NotImplementedException("FilterToGdal: filter type not implemented"); @@ -326,7 +326,8 @@ unique_ptr GdalTableFunction::Bind(ClientContext &context, TableFu auto column_name = string(attribute.name); auto duckdb_type = arrow_type->GetDuckType(); - if (duckdb_type.id() == LogicalTypeId::BLOB && attribute.metadata != nullptr && strncmp(attribute.metadata, ogc_flag, sizeof(ogc_flag)) == 0) { + if (duckdb_type.id() == LogicalTypeId::BLOB && attribute.metadata != nullptr && + strncmp(attribute.metadata, ogc_flag, sizeof(ogc_flag)) == 0) { // This is a WKB geometry blob result->arrow_table.AddColumn(col_idx, std::move(arrow_type)); @@ -334,7 +335,7 @@ unique_ptr GdalTableFunction::Bind(ClientContext &context, TableFu return_types.emplace_back(core::GeoTypes::WKB_BLOB()); } else { return_types.emplace_back(core::GeoTypes::GEOMETRY()); - if(column_name == "wkb_geometry") { + if (column_name == "wkb_geometry") { column_name = "geom"; } } @@ -591,8 +592,7 @@ unique_ptr GdalTableFunction::ReplacementScan(ClientContext &, const s auto lower_name = StringUtil::Lower(table_name); // Check if the table name ends with some common geospatial file extensions - if (StringUtil::EndsWith(lower_name, ".gpkg") || - StringUtil::EndsWith(lower_name, ".fgb")) { + if (StringUtil::EndsWith(lower_name, ".gpkg") || StringUtil::EndsWith(lower_name, ".fgb")) { auto table_function = make_uniq(); vector> children; diff --git a/spatial/src/spatial/gdal/functions/st_read_meta.cpp b/spatial/src/spatial/gdal/functions/st_read_meta.cpp index cc7e72a8..0c171841 100644 --- a/spatial/src/spatial/gdal/functions/st_read_meta.cpp +++ b/spatial/src/spatial/gdal/functions/st_read_meta.cpp @@ -28,13 +28,13 @@ static LogicalType GEOMETRY_FIELD_TYPE = LogicalType::STRUCT({ {"type", LogicalType::VARCHAR}, {"nullable", LogicalType::BOOLEAN}, {"crs", LogicalType::STRUCT({ - {"name", LogicalType::VARCHAR}, - {"auth_name", LogicalType::VARCHAR}, - {"auth_code", LogicalType::VARCHAR}, - {"wkt", LogicalType::VARCHAR}, - {"proj4", LogicalType::VARCHAR}, - {"projjson", LogicalType::VARCHAR}, - })}, + {"name", LogicalType::VARCHAR}, + {"auth_name", LogicalType::VARCHAR}, + {"auth_code", LogicalType::VARCHAR}, + {"wkt", LogicalType::VARCHAR}, + {"proj4", LogicalType::VARCHAR}, + {"projjson", LogicalType::VARCHAR}, + })}, }); static LogicalType STANDARD_FIELD_TYPE = LogicalType::STRUCT({ @@ -60,7 +60,8 @@ static unique_ptr Bind(ClientContext &context, TableFunctionBindIn auto file_name = input.inputs[0].GetValue(); auto result = make_uniq(); - result->file_names = MultiFileReader::GetFileList(context, input.inputs[0], "gdal metadata", FileGlobOptions::ALLOW_EMPTY); + result->file_names = + MultiFileReader::GetFileList(context, input.inputs[0], "gdal metadata", FileGlobOptions::ALLOW_EMPTY); names.push_back("file_name"); return_types.push_back(LogicalType::VARCHAR); @@ -108,17 +109,17 @@ static unique_ptr Init(ClientContext &context, TableFu static Value GetLayerData(GDALDatasetUniquePtr &dataset) { vector layer_values; - for(const auto &layer : dataset->GetLayers()) { + for (const auto &layer : dataset->GetLayers()) { child_list_t layer_value_fields; layer_value_fields.emplace_back("name", Value(layer->GetName())); layer_value_fields.emplace_back("feature_count", Value(static_cast(layer->GetFeatureCount()))); vector geometry_fields; - for(const auto &field : layer->GetLayerDefn()->GetGeomFields()) { + for (const auto &field : layer->GetLayerDefn()->GetGeomFields()) { child_list_t geometry_field_value_fields; auto field_name = field->GetNameRef(); - if(std::strlen(field_name) == 0) { + if (std::strlen(field_name) == 0) { field_name = "geom"; } geometry_field_value_fields.emplace_back("name", Value(field_name)); @@ -126,23 +127,23 @@ static Value GetLayerData(GDALDatasetUniquePtr &dataset) { geometry_field_value_fields.emplace_back("nullable", Value(static_cast(field->IsNullable()))); auto crs = field->GetSpatialRef(); - if(crs != nullptr) { + if (crs != nullptr) { child_list_t crs_value_fields; crs_value_fields.emplace_back("name", Value(crs->GetName())); crs_value_fields.emplace_back("auth_name", Value(crs->GetAuthorityName(nullptr))); crs_value_fields.emplace_back("auth_code", Value(crs->GetAuthorityCode(nullptr))); - char* wkt_ptr = nullptr; + char *wkt_ptr = nullptr; crs->exportToWkt(&wkt_ptr); crs_value_fields.emplace_back("wkt", wkt_ptr ? Value(wkt_ptr) : Value()); CPLFree(wkt_ptr); - char* proj4_ptr = nullptr; + char *proj4_ptr = nullptr; crs->exportToProj4(&proj4_ptr); crs_value_fields.emplace_back("proj4", proj4_ptr ? Value(proj4_ptr) : Value()); CPLFree(proj4_ptr); - char* projjson_ptr = nullptr; + char *projjson_ptr = nullptr; crs->exportToPROJJSON(&projjson_ptr, nullptr); crs_value_fields.emplace_back("projjson", projjson_ptr ? Value(projjson_ptr) : Value()); CPLFree(projjson_ptr); @@ -152,10 +153,11 @@ static Value GetLayerData(GDALDatasetUniquePtr &dataset) { geometry_fields.push_back(Value::STRUCT(geometry_field_value_fields)); } - layer_value_fields.emplace_back("geometry_fields", Value::LIST(GEOMETRY_FIELD_TYPE, std::move(geometry_fields))); + layer_value_fields.emplace_back("geometry_fields", + Value::LIST(GEOMETRY_FIELD_TYPE, std::move(geometry_fields))); vector standard_fields; - for(const auto &field : layer->GetLayerDefn()->GetFields()) { + for (const auto &field : layer->GetLayerDefn()->GetFields()) { child_list_t standard_field_value_fields; standard_field_value_fields.emplace_back("name", Value(field->GetNameRef())); standard_field_value_fields.emplace_back("type", Value(OGR_GetFieldTypeName(field->GetType()))); @@ -180,7 +182,7 @@ static void Scan(ClientContext &context, TableFunctionInput &input, DataChunk &o auto out_size = MinValue(STANDARD_VECTOR_SIZE, bind_data.file_names.size() - state.current_file_idx); - for(idx_t out_idx = 0; out_idx < out_size; out_idx++, state.current_file_idx++) { + for (idx_t out_idx = 0; out_idx < out_size; out_idx++, state.current_file_idx++) { auto file_name = bind_data.file_names[state.current_file_idx]; auto prefixed_file_name = GDALClientContextState::GetOrCreate(context).GetPrefix() + file_name; diff --git a/spatial/src/spatial/gdal/functions/st_write.cpp b/spatial/src/spatial/gdal/functions/st_write.cpp index 8ab228d7..59ac1e23 100644 --- a/spatial/src/spatial/gdal/functions/st_write.cpp +++ b/spatial/src/spatial/gdal/functions/st_write.cpp @@ -9,6 +9,7 @@ #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "spatial/core/types.hpp" #include "spatial/core/geometry/geometry_factory.hpp" +#include "spatial/core/geometry/geometry_type.hpp" #include "spatial/gdal/functions.hpp" #include "spatial/gdal/file_handler.hpp" @@ -28,6 +29,7 @@ struct BindData : public TableFunctionData { vector dataset_creation_options; vector layer_creation_options; string target_srs; + OGRwkbGeometryType geometry_type = wkbUnknown; BindData(string file_path, vector field_sql_types, vector field_names) : file_path(std::move(file_path)), field_sql_types(std::move(field_sql_types)), @@ -93,8 +95,35 @@ static unique_ptr Bind(ClientContext &context, const CopyInfo &inf } bind_data->dataset_creation_options.push_back(s.GetValue()); } + } else if (StringUtil::Upper(option.first) == "GEOMETRY_TYPE") { + auto &set = option.second.front(); + if (set.type().id() == LogicalTypeId::VARCHAR) { + auto type = set.GetValue(); + if (StringUtil::CIEquals(type, "POINT")) { + bind_data->geometry_type = wkbPoint; + } else if (StringUtil::CIEquals(type, "LINESTRING")) { + bind_data->geometry_type = wkbLineString; + } else if (StringUtil::CIEquals(type, "POLYGON")) { + bind_data->geometry_type = wkbPolygon; + } else if (StringUtil::CIEquals(type, "MULTIPOINT")) { + bind_data->geometry_type = wkbMultiPoint; + } else if (StringUtil::CIEquals(type, "MULTILINESTRING")) { + bind_data->geometry_type = wkbMultiLineString; + } else if (StringUtil::CIEquals(type, "MULTIPOLYGON")) { + bind_data->geometry_type = wkbMultiPolygon; + } else if (StringUtil::CIEquals(type, "GEOMETRYCOLLECTION")) { + bind_data->geometry_type = wkbGeometryCollection; + } else { + throw BinderException( + "Unknown geometry type '%s', expected one of 'POINT', 'LINESTRING', 'POLYGON', 'MULTIPOINT', " + "'MULTILINESTRING', 'MULTIPOLYGON', 'GEOMETRYCOLLECTION'", + type); + } + } else { + throw BinderException("Geometry type must be a string"); + } } else if (StringUtil::Upper(option.first) == "SRS") { - auto set = option.second.front(); + auto &set = option.second.front(); if (set.type().id() == LogicalTypeId::VARCHAR) { bind_data->target_srs = set.GetValue(); } else { @@ -116,6 +145,11 @@ static unique_ptr Bind(ClientContext &context, const CopyInfo &inf bind_data->layer_name = fs.ExtractBaseName(bind_data->file_path); } + // Driver specific checks + if (bind_data->driver_name == "OpenFileGDB" && bind_data->geometry_type == wkbUnknown) { + throw BinderException("OpenFileGDB requires 'GEOMETRY_TYPE' parameter to be set when writing!"); + } + return std::move(bind_data); } @@ -175,6 +209,10 @@ static unique_ptr OGRFieldTypeFromLogicalType(const string &name, case LogicalTypeId::TIME: return make_uniq(name.c_str(), OFTTime); case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_NS: + case LogicalTypeId::TIMESTAMP_MS: + case LogicalTypeId::TIMESTAMP_SEC: + case LogicalTypeId::TIMESTAMP_TZ: return make_uniq(name.c_str(), OFTDateTime); case LogicalTypeId::LIST: { auto child_type = ListType::GetChildType(type); @@ -217,7 +255,7 @@ static unique_ptr OGRFieldTypeFromLogicalType(const string &name, static unique_ptr InitGlobal(ClientContext &context, FunctionData &bind_data, const string &file_path) { - auto &gdal_data = (BindData &)bind_data; + auto &gdal_data = bind_data.Cast(); GDALDriver *driver = GetGDALDriverManager()->GetDriverByName(gdal_data.driver_name.c_str()); if (!driver) { throw IOException("Could not open driver"); @@ -252,7 +290,7 @@ static unique_ptr InitGlobal(ClientContext &context, Functio // so we have to pass nullptr if we want the default behavior. OGRSpatialReference *srs_ptr = gdal_data.target_srs.empty() ? nullptr : &srs; - auto layer = dataset->CreateLayer(gdal_data.layer_name.c_str(), srs_ptr, wkbUnknown, lco); + auto layer = dataset->CreateLayer(gdal_data.layer_name.c_str(), srs_ptr, gdal_data.geometry_type, lco); if (!layer) { throw IOException("Could not create layer"); } @@ -366,8 +404,76 @@ static void SetOgrFieldFromValue(OGRFeature *feature, int field_idx, const Logic auto day = Date::ExtractDay(date); feature->SetField(field_idx, year, month, day, 0, 0, 0, 0); } break; + case LogicalTypeId::TIME: { + auto time = value.GetValueUnsafe(); + auto hour = static_cast(time.micros / Interval::MICROS_PER_HOUR); + auto minute = static_cast((time.micros % Interval::MICROS_PER_HOUR) / Interval::MICROS_PER_MINUTE); + auto second = static_cast(static_cast(time.micros % Interval::MICROS_PER_MINUTE) / + static_cast(Interval::MICROS_PER_SEC)); + feature->SetField(field_idx, 0, 0, 0, hour, minute, second, 0); + } break; + case LogicalTypeId::TIMESTAMP: { + auto timestamp = value.GetValueUnsafe(); + auto date = Timestamp::GetDate(timestamp); + auto time = Timestamp::GetTime(timestamp); + auto year = Date::ExtractYear(date); + auto month = Date::ExtractMonth(date); + auto day = Date::ExtractDay(date); + auto hour = static_cast((time.micros % Interval::MICROS_PER_DAY) / Interval::MICROS_PER_HOUR); + auto minute = static_cast((time.micros % Interval::MICROS_PER_HOUR) / Interval::MICROS_PER_MINUTE); + auto second = static_cast(static_cast(time.micros % Interval::MICROS_PER_MINUTE) / + static_cast(Interval::MICROS_PER_SEC)); + feature->SetField(field_idx, year, month, day, hour, minute, second, 0); + } break; + case LogicalTypeId::TIMESTAMP_NS: { + auto timestamp = value.GetValueUnsafe(); + timestamp = Timestamp::FromEpochNanoSeconds(timestamp.value); + auto date = Timestamp::GetDate(timestamp); + auto time = Timestamp::GetTime(timestamp); + auto year = Date::ExtractYear(date); + auto month = Date::ExtractMonth(date); + auto day = Date::ExtractDay(date); + auto hour = static_cast((time.micros % Interval::MICROS_PER_DAY) / Interval::MICROS_PER_HOUR); + auto minute = static_cast((time.micros % Interval::MICROS_PER_HOUR) / Interval::MICROS_PER_MINUTE); + auto second = static_cast(static_cast(time.micros % Interval::MICROS_PER_MINUTE) / + static_cast(Interval::MICROS_PER_SEC)); + feature->SetField(field_idx, year, month, day, hour, minute, second, 0); + } break; + case LogicalTypeId::TIMESTAMP_MS: { + auto timestamp = value.GetValueUnsafe(); + timestamp = Timestamp::FromEpochMs(timestamp.value); + auto date = Timestamp::GetDate(timestamp); + auto time = Timestamp::GetTime(timestamp); + auto year = Date::ExtractYear(date); + auto month = Date::ExtractMonth(date); + auto day = Date::ExtractDay(date); + auto hour = static_cast((time.micros % Interval::MICROS_PER_DAY) / Interval::MICROS_PER_HOUR); + auto minute = static_cast((time.micros % Interval::MICROS_PER_HOUR) / Interval::MICROS_PER_MINUTE); + auto second = static_cast(static_cast(time.micros % Interval::MICROS_PER_MINUTE) / + static_cast(Interval::MICROS_PER_SEC)); + feature->SetField(field_idx, year, month, day, hour, minute, second, 0); + } break; + case LogicalTypeId::TIMESTAMP_SEC: { + auto timestamp = value.GetValueUnsafe(); + timestamp = Timestamp::FromEpochSeconds(timestamp.value); + auto date = Timestamp::GetDate(timestamp); + auto time = Timestamp::GetTime(timestamp); + auto year = Date::ExtractYear(date); + auto month = Date::ExtractMonth(date); + auto day = Date::ExtractDay(date); + auto hour = static_cast((time.micros % Interval::MICROS_PER_DAY) / Interval::MICROS_PER_HOUR); + auto minute = static_cast((time.micros % Interval::MICROS_PER_HOUR) / Interval::MICROS_PER_MINUTE); + auto second = static_cast(static_cast(time.micros % Interval::MICROS_PER_MINUTE) / + static_cast(Interval::MICROS_PER_SEC)); + feature->SetField(field_idx, year, month, day, hour, minute, second, 0); + } break; + case LogicalTypeId::TIMESTAMP_TZ: { + // Not sure what to with the timezone, just let GDAL parse it? + auto timestamp = value.GetValueUnsafe(); + auto time_str = Timestamp::ToString(timestamp); + feature->SetField(field_idx, time_str.c_str()); + } break; default: - // TODO: Add time types // TODO: Handle list types throw NotImplementedException("Unsupported field type"); } @@ -375,9 +481,9 @@ static void SetOgrFieldFromValue(OGRFeature *feature, int field_idx, const Logic static void Sink(ExecutionContext &context, FunctionData &bdata, GlobalFunctionData &gstate, LocalFunctionData &lstate, DataChunk &input) { - auto &bind_data = (BindData &)bdata; - auto &global_state = (GlobalState &)gstate; - auto &local_state = (LocalState &)lstate; + auto &bind_data = bdata.Cast(); + auto &global_state = gstate.Cast(); + auto &local_state = lstate.Cast(); local_state.factory.allocator.Reset(); lock_guard d_lock(global_state.lock); @@ -398,6 +504,15 @@ static void Sink(ExecutionContext &context, FunctionData &bdata, GlobalFunctionD if (IsGeometryType(type)) { // TODO: check how many geometry fields there are and use the correct one. auto geom = OGRGeometryFromValue(type, value, local_state.factory); + if (bind_data.geometry_type != wkbUnknown && geom->getGeometryType() != bind_data.geometry_type) { + auto got_name = + StringUtil::Replace(StringUtil::Upper(OGRGeometryTypeToName(geom->getGeometryType())), " ", ""); + auto expected_name = + StringUtil::Replace(StringUtil::Upper(OGRGeometryTypeToName(bind_data.geometry_type)), " ", ""); + throw InvalidInputException("Expected all geometries to be of type '%s', but got one of type '%s'", + expected_name, got_name); + } + if (feature->SetGeometry(geom.get()) != OGRERR_NONE) { throw IOException("Could not set geometry"); } @@ -417,7 +532,7 @@ static void Sink(ExecutionContext &context, FunctionData &bdata, GlobalFunctionD //===--------------------------------------------------------------------===// static void Combine(ExecutionContext &context, FunctionData &bind_data, GlobalFunctionData &gstate, - LocalFunctionData &lstate) { + LocalFunctionData &lstate) { } //===--------------------------------------------------------------------===// diff --git a/spatial/src/spatial/gdal/module.cpp b/spatial/src/spatial/gdal/module.cpp index dea5d476..e0cee7ed 100644 --- a/spatial/src/spatial/gdal/module.cpp +++ b/spatial/src/spatial/gdal/module.cpp @@ -31,7 +31,7 @@ void GdalModule::Register(DatabaseInstance &db) { // try to strip it off to make the errors more readable auto msg = string(raw_msg); auto path_pos = msg.find("/vsiduckdb-"); - if(path_pos != string::npos) { + if (path_pos != string::npos) { // We found a path, strip it off msg.erase(path_pos, 48); } @@ -64,7 +64,7 @@ void GdalModule::Register(DatabaseInstance &db) { GdalTableFunction::Register(db); GdalDriversTableFunction::Register(db); GdalCopyFunction::Register(db); - GdalMetadataFunction::Register(db); + GdalMetadataFunction::Register(db); } } // namespace gdal diff --git a/spatial/src/spatial/geographiclib/functions/st_area_spheroid.cpp b/spatial/src/spatial/geographiclib/functions/st_area_spheroid.cpp index b692b3c6..a69a7c07 100644 --- a/spatial/src/spatial/geographiclib/functions/st_area_spheroid.cpp +++ b/spatial/src/spatial/geographiclib/functions/st_area_spheroid.cpp @@ -91,7 +91,7 @@ static double PolygonArea(const core::Polygon &poly, GeographicLib::PolygonArea } double ring_area; double _perimeter; - // We use the absolute value here so that the actual winding order of the polygon rings dont matter. + // We use the absolute value here so that the actual winding order of the polygon rings dont matter. comp.Compute(false, true, _perimeter, ring_area); if (ring_idx == 0) { // Add outer ring diff --git a/spatial/src/spatial/geos/functions/scalar/st_makevalid.cpp b/spatial/src/spatial/geos/functions/scalar/st_makevalid.cpp index 44719baf..670feaf8 100644 --- a/spatial/src/spatial/geos/functions/scalar/st_makevalid.cpp +++ b/spatial/src/spatial/geos/functions/scalar/st_makevalid.cpp @@ -17,21 +17,19 @@ using namespace spatial::core; static void MakeValidFunction(DataChunk &args, ExpressionState &state, Vector &result) { auto &lstate = GEOSFunctionLocalState::ResetAndGet(state); auto &ctx = lstate.ctx.GetCtx(); - UnaryExecutor::Execute( - args.data[0], result, args.size(), [&](string_t input) { - auto geom = lstate.ctx.Deserialize(input); - auto valid = make_uniq_geos(ctx, GEOSMakeValid_r(ctx, geom.get())); - return lstate.ctx.Serialize(result, valid); - }); + UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t input) { + auto geom = lstate.ctx.Deserialize(input); + auto valid = make_uniq_geos(ctx, GEOSMakeValid_r(ctx, geom.get())); + return lstate.ctx.Serialize(result, valid); + }); } void GEOSScalarFunctions::RegisterStMakeValid(DatabaseInstance &db) { ScalarFunctionSet set("ST_MakeValid"); - set.AddFunction(ScalarFunction({GeoTypes::GEOMETRY()}, GeoTypes::GEOMETRY(), - MakeValidFunction, nullptr, nullptr, nullptr, - GEOSFunctionLocalState::Init)); + set.AddFunction(ScalarFunction({GeoTypes::GEOMETRY()}, GeoTypes::GEOMETRY(), MakeValidFunction, nullptr, nullptr, + nullptr, GEOSFunctionLocalState::Init)); ExtensionUtil::RegisterFunction(db, set); } diff --git a/spatial/src/spatial/proj/module.cpp b/spatial/src/spatial/proj/module.cpp index 18601f85..ff985e91 100644 --- a/spatial/src/spatial/proj/module.cpp +++ b/spatial/src/spatial/proj/module.cpp @@ -35,11 +35,11 @@ PJ_CONTEXT *ProjModule::GetThreadProjContext() { throw InternalException("Could not set proj.db path"); } - // Dont log errors to stderr - proj_log_level(ctx, PJ_LOG_NONE); + // Dont log errors to stderr + proj_log_level(ctx, PJ_LOG_NONE); - // Dont allow network - proj_context_set_enable_network(ctx, false); + // Dont allow network + proj_context_set_enable_network(ctx, false); return ctx; } diff --git a/test/sql/gdal/st_read_gdb.test b/test/sql/gdal/st_read_gdb.test new file mode 100644 index 00000000..9a2e3a69 --- /dev/null +++ b/test/sql/gdal/st_read_gdb.test @@ -0,0 +1,19 @@ +require spatial + +statement error +COPY (SELECT ST_Point(1,2) as geom, 10 as i) TO '__TEST_DIR__/test.gdb' WITH (FORMAT 'GDAL', DRIVER 'OpenFileGDB'); +---- +OpenFileGDB requires 'GEOMETRY_TYPE' parameter to be set when writing + +statement error +COPY (SELECT ST_Point(1,2) as geom, 10 as i) TO '__TEST_DIR__/test_fail.gdb' WITH (FORMAT 'GDAL', DRIVER 'OpenFileGDB', GEOMETRY_TYPE 'LINESTRING'); +---- +Expected all geometries to be of type 'LINESTRING', but got one of type 'POINT' + +statement ok +COPY (SELECT ST_Point(1,2) as geom, 10 as i) TO '__TEST_DIR__/test.gdb' WITH (FORMAT 'GDAL', DRIVER 'OpenFileGDB', GEOMETRY_TYPE 'POINT'); + +query II +FROM st_read('__TEST_DIR__/test.gdb'); +---- +10 POINT (1 2) \ No newline at end of file