Rework Docker

- Add ubuntu-18.04
- add pkg-config to ubuntu(s)
- Rework Docker Makefile
- Add test job to verify archive
  - Add cc.Dockerfile
  - Add java.Dockerfile
  - Add dotnet.Dockerfile
- Build ortools in docker_<lang>
- Build archive and fz_archive in two separate containers
- Add dependencies to makefiles when building docker image
- Suppress previous archive if any
- Improve docker build by using export/archive when building language images
- Reduce build context using export/$* instead of .
  note: when running `docker build ... foo`, all files in `foo` are recursively
  send to the docker daemon before building. Since we have tons of Go of images
  it could take time to "upload".

tools/docker/centos-7.Dockerfile

@@ -1,66 +1,70 @@
 FROM centos:7
 
-ENV SRC_GIT_BRANCH master
-
-RUN yum -y update
-
-RUN yum -y install yum-utils
-
-RUN rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
-
-RUN yum-config-manager --add-repo http://download.mono-project.com/repo/centos7/
-
-RUN yum -y \
-    install \
-    wget \
-    git \
-    autoconf \
-    libtool \
-    zlib-devel \
-    gawk \
-    gcc-c++ \
-    curl \
-    subversion \
-    make \
-    mono-devel \
-    redhat-lsb-core \
-    python-devel \
-    java-1.8.0-openjdk \
-    java-1.8.0-openjdk-devel \
-    python-setuptools \
-    python-six \
-    python-wheel \
-    pcre-devel which
+#############
+##  SETUP  ##
+#############
+RUN yum -y update \
+&& yum -y install yum-utils \
+&& yum -y install \
+ wget git pkg-config make autoconf libtool zlib-devel gawk gcc-c++ curl subversion \
+ redhat-lsb-core pcre-devel which \
+ python-devel python-setuptools python-six python-wheel \
+ java-1.8.0-openjdk  java-1.8.0-openjdk-devel \
+&& yum clean all \
+&& rm -rf /var/cache/yum
+
+# Install dotnet
+RUN rpm -Uvh "https://packages.microsoft.com/config/rhel/7/packages-microsoft-prod.rpm" \
+&& yum -y update \
+&& yum -y install dotnet-sdk-2.1 \
+&& yum clean all \
+&& rm -rf /var/cache/yum
+
+# Install Mono
+#RUN rpm --import "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF" \
+#&& su -c 'curl https://download.mono-project.com/repo/centos7-stable.repo | tee /etc/yum.repos.d/mono-centos7-stable.repo'
+#&& yum -y update \
+#&& yum -y install mono-devel
+
+# Install CMake
+RUN wget "https://cmake.org/files/v3.8/cmake-3.8.2-Linux-x86_64.sh" \
+&& chmod 775 cmake-3.8.2-Linux-x86_64.sh \
+&& yes | ./cmake-3.8.2-Linux-x86_64.sh --prefix=/usr --exclude-subdir
+
+# Install Swig
+RUN wget "https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz" \
+&& tar xvf swig-3.0.12.tar.gz \
+&& rm swig-3.0.12.tar.gz \
+&& cd swig-3.0.12 \
+&& ./configure --prefix=/usr \
+&& make -j 4 \
+&& make install \
+&& cd .. \
+&& rm -rf swig-3.0.12
 
 ENV TZ=America/Los_Angeles
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
+################
+##  OR-TOOLS  ##
+################
+ARG SRC_GIT_BRANCH
+ENV SRC_GIT_BRANCH ${SRC_GIT_BRANCH:-master}
+ARG SRC_GIT_SHA1
+ENV SRC_GIT_SHA1 ${SRC_GIT_SHA1:-unknown}
+
+# Download sources
+# use SRC_GIT_SHA1 to modify the command
+# i.e. avoid docker reusing the cache when new commit is pushed
 WORKDIR /root
+RUN git clone -b "${SRC_GIT_BRANCH}" --single-branch https://github.com/google/or-tools \
+&& echo "sha1: $(cd or-tools && git rev-parse --verify HEAD)" \
+&& echo "expected sha1: ${SRC_GIT_SHA1}"
 
-RUN wget "https://cmake.org/files/v3.8/cmake-3.8.2-Linux-x86_64.sh"
-
-RUN chmod 775 cmake-3.8.2-Linux-x86_64.sh
-
-RUN yes | ./cmake-3.8.2-Linux-x86_64.sh --prefix=/usr --exclude-subdir
-
-RUN wget "https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz"
-
-RUN tar xvf swig-3.0.12.tar.gz
-
-WORKDIR /root/swig-3.0.12
-
-RUN ./configure --prefix=/usr
-
-RUN make -j 4
-
-RUN make install
-
-WORKDIR /root
-
-RUN git clone -b "$SRC_GIT_BRANCH" --single-branch https://github.com/google/or-tools
-
+# Prebuild
 WORKDIR /root/or-tools
-
-RUN make third_party
-
-RUN easy_install wheel
+RUN make detect && make third_party
+RUN make detect_cc && make cc
+RUN make detect_python && make python
+RUN make detect_java && make java
+RUN make detect_dotnet && make dotnet

tools/docker/debian-9.Dockerfile

@@ -1,18 +1,62 @@
 FROM debian:9
 
-ENV SRC_GIT_BRANCH master
-
-RUN apt-get update
-
-RUN apt-get -y install git wget autoconf libtool zlib1g-dev gawk g++ curl cmake subversion make mono-complete swig lsb-release python-dev default-jdk twine python-setuptools python-six python3-setuptools python3-dev python-wheel python3-wheel
+#############
+##  SETUP  ##
+#############
+RUN apt-get update -qq \
+&& apt-get install -qq \
+ git pkg-config wget make cmake autoconf libtool zlib1g-dev gawk g++ curl subversion \
+ swig lsb-release \
+ python-dev python-wheel python-setuptools python-six \
+ python3-dev python3-wheel python3-setuptools \
+ default-jdk \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Dotnet Install
+RUN apt-get update -qq \
+&& apt-get install -qq gpg apt-transport-https \
+&& wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg \
+&& mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ \
+&& wget -q https://packages.microsoft.com/config/debian/9/prod.list \
+&& mv prod.list /etc/apt/sources.list.d/microsoft-prod.list \
+&& apt-get update -qq \
+&& apt-get install -qq dotnet-sdk-2.1 \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Mono Install
+#RUN apt-get install -qq apt-transport-https dirmngr \
+#&& apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \
+#&& echo "deb https://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list \
+#&& apt-get update -qq \
+#&& apt-get install -qq mono-complete \
+#&& apt-get clean \
+#&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ENV TZ=America/Los_Angeles
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
+################
+##  OR-TOOLS  ##
+################
+ARG SRC_GIT_BRANCH
+ENV SRC_GIT_BRANCH ${SRC_GIT_BRANCH:-master}
+ARG SRC_GIT_SHA1
+ENV SRC_GIT_SHA1 ${SRC_GIT_SHA1:-unknown}
+
+# Download sources
+# use SRC_GIT_SHA1 to modify the command
+# i.e. avoid docker reusing the cache when new commit is pushed
 WORKDIR /root
+RUN git clone -b "${SRC_GIT_BRANCH}" --single-branch https://github.com/google/or-tools \
+&& echo "sha1: $(cd or-tools && git rev-parse --verify HEAD)" \
+&& echo "expected sha1: ${SRC_GIT_SHA1}"
 
-RUN git clone -b "$SRC_GIT_BRANCH" --single-branch https://github.com/google/or-tools
-
+# Prebuild
 WORKDIR /root/or-tools
-
-RUN make third_party
+RUN make detect && make third_party
+RUN make detect_cc && make cc
+RUN make detect_python && make python
+RUN make detect_java && make java
+RUN make detect_dotnet && make dotnet

tools/docker/manylinux1.Dockerfile

@@ -1,10 +1,12 @@
 FROM quay.io/pypa/manylinux1_x86_64:latest
 
-ENV SRC_ROOT /root/src
+ARG SRC_GIT_BRANCH
+ENV SRC_GIT_BRANCH ${SRC_GIT_BRANCH:-master}
+
 ENV BUILD_ROOT /root/build
 ENV EXPORT_ROOT /export
 ENV SRC_GIT_URL https://github.com/google/or-tools
-ENV SRC_GIT_BRANCH master
+ENV SRC_ROOT /root/src
 # The build of Python 2.6.x bindings is known to be broken.
 ENV SKIP_PLATFORMS "cp26-cp26m cp26-cp26mu"
 

tools/docker/test/centos-7/cc.Dockerfile

@@ -0,0 +1,14 @@
+FROM centos:7
+LABEL maintainer="[email protected]"
+
+RUN yum -y update \
+&& yum -y groupinstall 'Development Tools' \
+&& yum -y install which zlib-devel \
+&& yum clean all \
+&& rm -rf /var/cache/yum
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_centos-7_v*.tar.gz .

tools/docker/test/centos-7/dotnet.Dockerfile

@@ -0,0 +1,19 @@
+FROM centos:7
+LABEL maintainer="[email protected]"
+
+RUN yum -y update \
+&& yum -y groupinstall 'Development Tools' \
+&& yum -y install which zlib-devel \
+&& yum clean all \
+&& rm -rf /var/cache/yum
+
+# Install dotnet
+RUN rpm -Uvh "https://packages.microsoft.com/config/rhel/7/packages-microsoft-prod.rpm" \
+&& yum -y update \
+&& yum -y install dotnet-sdk-2.1
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_centos-7_v*.tar.gz .

tools/docker/test/centos-7/java.Dockerfile

@@ -0,0 +1,14 @@
+FROM centos:7
+LABEL maintainer="[email protected]"
+
+RUN yum -y update \
+&& yum -y groupinstall 'Development Tools' \
+&& yum -y install which zlib-devel \
+&& yum clean all \
+&& rm -rf /var/cache/yum
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_centos-7_v*.tar.gz .

tools/docker/test/debian-9/cc.Dockerfile

@@ -0,0 +1,13 @@
+FROM debian:9
+LABEL maintainer="[email protected]"
+
+RUN apt-get update \
+&& apt-get install -y -q build-essential zlib1g-dev \
+&& apt clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_debian-9_v*.tar.gz .

tools/docker/test/debian-9/dotnet.Dockerfile

@@ -0,0 +1,25 @@
+FROM debian:9
+LABEL maintainer="[email protected]"
+
+RUN apt-get update \
+&& apt-get install -y -q build-essential zlib1g-dev \
+&& apt clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install dotnet
+RUN apt-get update -qq \
+&& apt-get install -qq apt-transport-https \
+&& wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg \
+&& mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ \
+&& wget -q https://packages.microsoft.com/config/debian/9/prod.list \
+&& mv prod.list /etc/apt/sources.list.d/microsoft-prod.list \
+&& apt-get update -qq \
+&& apt-get install -qq dotnet-sdk-2.1 \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_debian-9_v*.tar.gz .

tools/docker/test/debian-9/java.Dockerfile

@@ -0,0 +1,13 @@
+FROM debian:9
+LABEL maintainer="[email protected]"
+
+RUN apt-get update \
+&& apt-get install -y -q build-essential zlib1g-dev \
+&& apt clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_debian-9_v*.tar.gz .

tools/docker/test/ubuntu-14.04/cc.Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:14.04
+
+RUN apt-get update \
+&& apt-get install -y -q build-essential zlib1g-dev \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_ubuntu-14.04_v*.tar.gz .

tools/docker/test/ubuntu-14.04/dotnet.Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:14.04
+
+RUN apt update \
+&& apt install -y -q build-essential zlib1g-dev \
+&& apt clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Dotnet install
+RUN apt-get update \
+&& wget -q https://packages.microsoft.com/config/ubuntu/14.04/packages-microsoft-prod.deb \
+&& dpkg -i packages-microsoft-prod.deb \
+&& apt-get install -qq apt-transport-https \
+&& apt-get update \
+&& apt-get install -qq dotnet-sdk-2.1 \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_ubuntu-14.04_v*.tar.gz .

tools/docker/test/ubuntu-14.04/java.Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:14.04
+
+RUN apt update \
+&& apt install -y -q build-essential zlib1g-dev \
+&& apt clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_ubuntu-14.04_v*.tar.gz .

tools/docker/test/ubuntu-16.04/cc.Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:16.04
+
+RUN apt update \
+&& apt install -y -q build-essential zlib1g-dev \
+&& apt clean \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+#ENV TZ=America/Los_Angeles
+#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+WORKDIR /root
+ADD or-tools_ubuntu-16.04_v*.tar.gz .