[BUILD] Upgrade to LLVM6 (#1010)

* [BUILD] Upgrade to LLVM6

Includes:
  - Upgrade of LLVM to version 6.0.1 provided by dai-thirdparty-deps
  project
  - LVMlite upgrade
  - LLVM config path

The upgrade enables PPC64le tests and avoids segfaults reported by:
  - numba/numba#2451 
  - numba/numba#2848

Close #980

Makefile

@@ -220,7 +220,7 @@ ubuntu_docker_publish: Dockerfile-ubuntu.$(PLATFORM).tag
 	docker push $(CONTAINER_NAME_TAG)
 
 ARCH_NAME ?= $(shell uname -m)
-DOCKER_IMAGE_TAG ?= 0.6.0-PR-1147.4
+DOCKER_IMAGE_TAG ?= 0.6.0-PR-1010.4
 CENTOS_DOCKER_IMAGE_NAME ?= docker.h2o.ai/opsh2oai/datatable-build-$(ARCH_NAME)_centos7:$(DOCKER_IMAGE_TAG)
 UBUNTU_DOCKER_IMAGE_NAME ?= docker.h2o.ai/opsh2oai/datatable-build-$(ARCH_NAME)_ubuntu:$(DOCKER_IMAGE_TAG)
 

ci/Dockerfile-centos7.in

@@ -13,18 +13,19 @@ RUN \
         bzip2 && \
     wget -q https://repo.continuum.io/miniconda/Miniconda3-4.3.27-Linux-ARCH_SUBST.sh && \
     bash Miniconda3-4.3.27-Linux-ARCH_SUBST.sh -b -p /opt/h2oai/dai/python && \
-    wget -q https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/dai-thirdparty-deps-llvm/1.0-master-21/ARCH_SUBST-centos7/llvm.tar.bz2 && \
+    wget -q https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/dai-thirdparty-deps/1.0-master-2/ARCH_SUBST-centos7/llvm.tar.bz2 && \
     tar xvf llvm.tar.bz2 && \
     cp -r llvm/* /opt/h2oai/dai/ && \
     rm -rf llvm*
 
-ENV LLVM4=/opt/h2oai/dai
+ENV LLVM6=/opt/h2oai/dai
+ENV LLVM_CONFIG=$LLVM6/bin/llvm-config
 ENV PATH=/opt/h2oai/dai/python/bin:$PATH
 ENV PATH=/usr/local/bin:$PATH
-ENV PATH=$LLVM4/bin:$PATH
+ENV PATH=$LLVM6/bin:$PATH
 ENV PANDAS_VERSION=0.20.0
 
-RUN pip install llvmlite==0.20.0 wheel virtualenv && \
+RUN pip install llvmlite==0.23.0 wheel virtualenv && \
     \
     conda create -n datatable-py35-with-pandas python=3.5 && \
     . activate datatable-py35-with-pandas && \

ci/Dockerfile-ubuntu.in

@@ -8,6 +8,7 @@ MAINTAINER H2O.ai <[email protected]>
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+ENV LLVM_VERSION=6.0.1
 
 RUN \
   apt-get -q -y update && \
@@ -68,15 +69,15 @@ RUN \
 
 # Install LLVM for pydatatable
 RUN \
-  wget http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
-  tar xf clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
-  rm clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
-  cp /clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04/lib/libomp.so /usr/lib
+  wget http://releases.llvm.org/${LLVM_VERSION}/clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
+  tar xf clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
+  rm clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
+  cp /clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04/lib/libomp.so /usr/lib
 
 ENV \
-  LLVM4=/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04 \
-  LLVM_CONFIG=/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/llvm-config \
-  CLANG=/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang
+  LLVM6=/clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04 \
+  LLVM_CONFIG=/clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04/bin/llvm-config \
+  CLANG=/clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-16.04/bin/clang
 
 # A separated layer to collect core dumps
 RUN mkdir -p /tmp/cores && chmod a+rwx /tmp/cores

ci/Jenkinsfile2

@@ -34,12 +34,12 @@ RELEASE_BRANCH_PREFIX = 'rel-'
 CREDS_ID = 'h2o-ops-personal-auth-token'
 GITCONFIG_CRED_ID = 'master-gitconfig'
 RSA_CRED_ID = 'master-id-rsa'
-DOCKER_IMAGE_TAG = '0.6.0-PR-1147.4'
+DOCKER_IMAGE_TAG = '0.6.0-PR-1010.4'
 X86_64_CENTOS_DOCKER_IMAGE = "docker.h2o.ai/opsh2oai/datatable-build-x86_64_centos7:${DOCKER_IMAGE_TAG}"
 EXPECTED_SHAS = [
         files : [
-                'ci/Dockerfile-ubuntu.in' : '6446a6d1dd63c14db76d273c00f861817a4464ba',
-                'ci/Dockerfile-centos7.in': 'c6bf8d1ac424f81f3742b96218c052ef3f1c834c',
+                'ci/Dockerfile-ubuntu.in' : 'd5b19e748c079d65964e6c1e458f525854aee0f3',
+                'ci/Dockerfile-centos7.in': 'd9ef4a7f981719e05fff49487c6176b7ba445817',
         ]
 ]
 
@@ -79,7 +79,7 @@ properties([
     parameters([
         booleanParam(name: 'FORCE_BUILD_PPC64LE', defaultValue: false, description: '[BUILD] Trigger build of PPC64le artifacts'),
         booleanParam(name: 'DISABLE_ALL_TESTS', defaultValue: false, description: '[BUILD] Disable all tests'),
-        booleanParam(name: 'DISABLE_PPC64LE_TESTS', defaultValue: true, description: '[BUILD] Disable PPC64LE tests'),
+        booleanParam(name: 'DISABLE_PPC64LE_TESTS', defaultValue: false, description: '[BUILD] Disable PPC64LE tests'),
         booleanParam(name: 'DISABLE_COVERAGE', defaultValue: false, description: '[BUILD] Disable coverage'),
         booleanParam(name: 'FORCE_ALL_TESTS_IN_PR', defaultValue: false, description: '[BUILD] Trigger all tests even for PR')
     ]),