Add omniscidb-cuda

.azure-pipelines/azure-pipelines-linux.yml

@@ -7,7 +7,7 @@ jobs:
     matrix:
       linux:
         CONFIG: azure-linux-64-comp7
-        IMAGE_NAME: condaforge/linux-anvil-comp7
+        IMAGE_NAME: condaforge/linux-anvil-cuda:10.1
         CF_MAX_PY_VER: 37
         AZURE: True
   timeoutInMinutes: 360

recipes/omniscidb-cuda/recipe/No-boost-cmake-4.8.patch

@@ -0,0 +1,13 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 27d2eb8f..05e1aa03 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -75,6 +75,8 @@ else()
+   add_definitions("-DBOOST_LOG_DYN_LINK")
+ endif()
+
++set(Boost_NO_BOOST_CMAKE 1)
++
+ option(ENABLE_JAVA_REMOTE_DEBUG "Enable Java Remote Debug" OFF )
+ if(ENABLE_JAVA_REMOTE_DEBUG)
+   add_definitions("-DENABLE_JAVA_REMOTE_DEBUG")

recipes/omniscidb-cuda/recipe/Skip-installing-Boost-libs.patch

@@ -0,0 +1,15 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 7bbdd6c1..65dcd35d 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -781,10 +781,6 @@ install(FILES mapd.thrift DESTINATION ".")
+ install(FILES common.thrift DESTINATION ".")
+ install(FILES QueryEngine/serialized_result_set.thrift DESTINATION "QueryEngine/")
+
+-if(NOT PREFER_STATIC_LIBS)
+-  install(FILES ${Boost_LIBRARIES} DESTINATION ThirdParty/lib)
+-endif()
+-
+ if("${MAPD_EDITION_LOWER}" STREQUAL "ee")
+   set(EULA_FILE "${CMAKE_SOURCE_DIR}/EULA-EE.txt")
+ else()

recipes/omniscidb-cuda/recipe/build.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Make sure -fPIC is not in CXXFLAGS (that some conda packages may
+# add), otherwise omniscidb server will crash when executing generated
+# machine code:
+export CXXFLAGS="`echo $CXXFLAGS | sed 's/-fPIC//'`"
+
+# Fixes https://github.com/Quansight/pearu-sandbox/issues/7
+#       https://github.com/omnisci/omniscidb/issues/374
+export CXXFLAGS="$CXXFLAGS -Dsecure_getenv=getenv"
+
+# Fixes `error: expected ')' before 'PRIxPTR'`
+export CXXFLAGS="$CXXFLAGS -D__STDC_FORMAT_MACROS"
+
+# Remove --as-needed to resolve undefined reference to `__vdso_clock_gettime@GLIBC_PRIVATE'
+export LDFLAGS="`echo $LDFLAGS | sed 's/-Wl,--as-needed//'`"
+
+export EXTRA_CMAKE_OPTIONS="$EXTRA_CMAKE_OPTIONS -DCMAKE_C_COMPILER=${CC} -DCMAKE_CXX_COMPILER=${CXX}"
+export EXTRA_CMAKE_OPTIONS="$EXTRA_CMAKE_OPTIONS -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_HOME"
+
+# Required for building on a CUDA enabled system without libcuda.so.1
+export EXTRA_CMAKE_OPTIONS="$EXTRA_CMAKE_OPTIONS -DCMAKE_LIBRARY_PATH=$CUDA_HOME/lib64/stubs"
+
+mkdir -p build
+cd build
+
+# TODO: when building on a system with no GPUs, using
+# -DENABLE_TESTS=off will save build time
+
+export INSTALL_BASE=opt/omnisci
+cmake -Wno-dev \
+    -DCMAKE_PREFIX_PATH=$PREFIX \
+    -DCMAKE_INSTALL_PREFIX=$PREFIX/$INSTALL_BASE \
+    -DCMAKE_BUILD_TYPE=release \
+    -DMAPD_DOCS_DOWNLOAD=off \
+    -DENABLE_AWS_S3=off \
+    -DENABLE_CUDA=on \
+    -DENABLE_FOLLY=off \
+    -DENABLE_JAVA_REMOTE_DEBUG=off \
+    -DENABLE_PROFILER=off \
+    -DENABLE_TESTS=on \
+    -DPREFER_STATIC_LIBS=off \
+    $EXTRA_CMAKE_OPTIONS \
+    ..
+
+make -j $CPU_COUNT
+
+# skip tests when libcuda.so is not available
+if [ "`ldd bin/initdb | grep "not found" | tr -d '[:space:]'`" == "libcuda.so.1=>notfound" ]; then
+    echo "SKIP RUNNING SANITY TESTS: libcuda.so.1 not found"
+else
+    # Omnisci UDF support uses CLangTool for parsing Load-time UDF C++
+    # code to AST. If the C++ code uses C++ std headers, we need to
+    # specify the locations of include directories:
+    . ${RECIPE_DIR}/get_cxx_include_path.sh
+    export CPLUS_INCLUDE_PATH=$(get_cxx_include_path)
+
+    # Running sanity tests requires CUDA enabled system with GPU(s).
+    mkdir tmp
+    $PREFIX/bin/initdb tmp
+    make sanity_tests
+    rm -rf tmp
+fi
+
+make install
+
+mkdir -p $PREFIX/bin
+cd $PREFIX/bin
+ln -s ../$INSTALL_BASE/bin/initdb omnisci_initdb
+ln -s ../$INSTALL_BASE/startomnisci startomnisci
+ln -s ../$INSTALL_BASE/insert_sample_data omnisci_insert_sample_data
+for FN in "omnisci_server" "omnisci_web_server" "omnisql"
+do
+    ln -s ../$INSTALL_BASE/bin/$FN $FN
+done
+cd -

recipes/omniscidb-cuda/recipe/conda_build_config.yaml

@@ -0,0 +1,11 @@
+cuda_compiler:
+- nvcc
+cuda_compiler_version:
+- 10.1
+docker_image:
+- condaforge/linux-anvil-cuda:10.1
+zip_keys:
+  - - cuda_compiler_version
+    - docker_image
+cgo_compiler:
+  - go-cgo

recipes/omniscidb-cuda/recipe/get_cxx_include_path.sh

@@ -0,0 +1,34 @@
+#
+# Usage:
+#
+#   source this file and call ${get_cxx_include_path} to get a full
+#   include path of ${CXX}
+#
+# Author: Pearu Peterson
+# Created: November 2019
+#
+function get_cxx_include_path() {
+    local cplus_include_path=""
+    local start_include_search=false
+    local sep=""
+    (${CXX} -E -x c++ - -v < /dev/null) 2>&1 |
+        while read -r LINE
+        do
+            case "$LINE" in
+                '#include <...> search starts here:' | '#include "..." search starts here:')
+                    start_include_search=true
+                    ;;
+                'End of search list.')
+                    echo "$cplus_include_path"
+                    break
+                    ;;
+                *)
+                    if [ "$start_include_search" = "true" ];
+                    then
+                        cplus_include_path="$cplus_include_path${sep}$(realpath ${LINE})"
+                        sep=":"
+                    fi
+                    ;;
+            esac
+        done
+}

recipes/omniscidb-cuda/recipe/meta.yaml

@@ -0,0 +1,106 @@
+{% set name = "omniscidb-cuda" %}
+{% set version = "5.0.0" %}
+{% set sha256 = "3115261258059f8ad61efc802cd6c13e8c0d62a07621ebcfad55bb6f8ceaa0ed" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://github.com/omnisci/omniscidb/archive/v{{ version }}.tar.gz
+  sha256: {{ sha256 }}
+  patches:
+    - Skip-installing-Boost-libs.patch
+    # Remove when
+    #   https://github.com/omnisci/omniscidb-internal/pull/3984
+    # has been merged and available in release:
+    - nvcc-boost-include-dirs.patch
+
+build:
+  number: 0
+  # OmniSciDB server is linux-only program
+  skip: True  # [not (linux64 and cuda_compiler_version == "10.1")]
+  missing_dso_whitelist:
+  - "*/libcuda.*"
+
+requirements:
+  build:
+    - {{ compiler('cxx') }}
+    - {{ compiler("cgo") }}
+    - {{ compiler("cuda") }}
+    # clang++ is used for generating the bytecodes of extension functions
+    - clangxx
+    - cmake
+    - make
+    - maven
+  host:
+    - llvmdev
+    - clangdev
+    - llvm
+    - boost-cpp
+    - double-conversion
+    - snappy
+    - gflags
+    - glog
+    - libarchive
+    - libkml
+    - libpng
+    - blosc
+    - libgdal >=2.3
+    # omniscidb 5.0 is not arrow-cpp 0.14+ ready
+    - arrow-cpp  >=0.12.1,<0.14
+    - thrift-cpp >=0.11.0
+    - ncurses
+    - flex
+    - bisonpp
+    - openssl
+    - librdkafka
+    - openldap
+  run:
+    - boost-cpp
+    - arrow-cpp >=0.12.1,<0.14
+    - snappy
+    - double-conversion
+    - libgdal >=2.3
+    - glog
+    - gflags
+    - openjdk 8.*
+    - ncurses
+    - xz
+    - bzip2
+    - zlib
+    - librdkafka
+    - openldap
+    # omnscidb Load-time UDF support calls clang++
+    - gxx_{{ target_platform }}
+    - clangxx
+
+test:
+  commands:
+    - omnisql -v
+    # the following programs require libcuda.so.1, so here we test
+    # that these are in PATH:
+    - which omnisci_server
+    - which omnisci_web_server
+    - which omnisci_initdb
+
+about:
+  home: https://www.omnisci.com/
+  license: Apache-2.0
+  license_family: APACHE
+  license_file: LICENSE.md
+  summary: 'The OmniSci database'
+
+  description: |
+    OmniSciDB is an in-memory, column store, SQL relational database 
+    that was designed from the ground up to run on GPUs.
+
+    This Omnisci database can be run on CUDA enabled GPUs.
+  doc_url: https://www.omnisci.com/docs/latest/
+  dev_url: https://github.com/omnisci/omniscidb
+
+extra:
+  recipe-maintainers:
+    - xmnlab
+    - pearu
+    - andrewseidl

recipes/omniscidb-cuda/recipe/nvcc-boost-include-dirs.patch

@@ -0,0 +1,16 @@
+diff --git a/QueryEngine/CMakeLists.txt b/QueryEngine/CMakeLists.txt
+index 99414188..44999d4e 100644
+--- a/QueryEngine/CMakeLists.txt
++++ b/QueryEngine/CMakeLists.txt
+@@ -1,6 +1,11 @@
+ set(EXECUTABLE_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR})
+
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -Wall -Wno-attributes")
++
++if(ENABLE_CUDA)
++  set(MAPD_HOST_COMPILER_FLAG "${MAPD_HOST_COMPILER_FLAG} -I${Boost_INCLUDE_DIRS}")
++endif()
++
+ set_source_files_properties(RuntimeFunctionsCodegenWithIncludes.cpp PROPERTIES COMPILE_FLAGS -O0)
+ set(query_engine_source_files
+     Allocators/CudaAllocator.cpp