Py3, dev and gpu-py3 images. Refactoring.

iterative · Apr 18, 2020 · eb37f94 · eb37f94
1 parent 22b3796
commit eb37f94
Show file tree

Hide file tree

Showing 6 changed files with 141 additions and 45 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: ci
+name: "Publush CML dockers"
 
 on: [push, pull_request]
 
@@ -18,7 +18,7 @@ jobs:
     - name: "tests"
       run: npm run test
 
-    - name: Publish to dockerhub
+    - name: Publish dvc-cml docker image
       # only publish if push to master (dvcorg/dvc-cml:latest) 
       # or create a tag in the repo (dvcorg/dvc-cml:tag)
       if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master')
@@ -32,7 +32,31 @@ jobs:
         cache: true
         tag_names: true
 
-    - name: Publish image with GPU support to dockerhub
+    - name: Publish dvc-cml-py3 docker image
+      if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master')
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: dvcorg/dvc-cml-py3
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
+        dockerfile: ./docker/Dockerfile-py3
+        context: ./
+        cache: true
+        tag_names: true
+
+    - name: Publish dvc-cml-dev docker image
+      if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master')
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: dvcorg/dvc-cml-dev
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
+        dockerfile: ./docker/Dockerfile-dev
+        context: ./
+        cache: true
+        tag_names: true
+
+    - name: Publish dvc-cml-gpu docker image
       if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master')
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
@@ -43,3 +67,16 @@ jobs:
         context: ./
         cache: true
         tag_names: true
+
+    - name: Publish dvc-cml-gpu-py3 docker image
+      if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master')
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: dvcorg/dvc-cml-gpu-py3
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
+        dockerfile: ./docker/Dockerfile-gpu-py3
+        context: ./
+        cache: true
+        tag_names: true
+
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,31 +1,39 @@
 FROM ubuntu:18.04
 
-LABEL maintainer="Iterative Inc"
+LABEL maintainer="dvc.org"
 
-RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
-    build-essential \
-    apt-utils \
-    apt-transport-https \
-    ca-certificates \
-    software-properties-common \
-    pkg-config \
-    curl \
-    wget \
-    unzip \
-    gpg-agent && \
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --fix-missing \
+        build-essential \
+        apt-utils \
+        apt-transport-https \
+        ca-certificates \
+        software-properties-common \
+        pkg-config \
+        curl \
+        wget \
+        unzip \
+        gpg-agent \
+        locales && \
     add-apt-repository universe -y && \
     add-apt-repository ppa:git-core/ppa -y && \
     apt-get install -y git && \
     curl -sL https://deb.nodesource.com/setup_12.x | bash && \
     apt-get install -y nodejs  && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    locale-gen en_US.UTF-8 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
 
 # Install update pip and nodejs, install dvc and dvc-cml
 ADD "./" "/dvc-cml"
 RUN wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list && \
-    apt update && apt install dvc && \
+    apt update && \
+    apt install dvc && \
     npm install -g /dvc-cml && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY ./docker/entrypoint.sh ./
-CMD ["entrypoint.sh"]
+CMD ["entrypoint.sh"]
diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev
@@ -0,0 +1,16 @@
+FROM dvcorg/dvc-cml-py3:latest
+
+LABEL maintainer="dvc.org"
+
+RUN apt-get remove -y dvc && \
+    apt-get install -y --no-install-recommends --fix-missing \
+        git \
+        python3-setuptools \
+        python3-dev && \
+    git clone https://github.com/iterative/dvc/ && \
+    cd dvc && \
+    git checkout 128fa7f && \
+    pip install -U -e .[all,tests] && \
+    cd .. && \
+    rm -rf dvc
+
diff --git a/docker/Dockerfile-gpu b/docker/Dockerfile-gpu
@@ -4,25 +4,31 @@ ARG UBUNTU_VERSION=18.04
 
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
 
-LABEL maintainer="Iterative Inc"
+LABEL maintainer="dvc.org"
 
-RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
-    build-essential \
-    apt-utils \
-    apt-transport-https \
-    ca-certificates \
-    software-properties-common \
-    pkg-config \
-    curl \
-    wget \
-    unzip \
-    gpg-agent && \
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --fix-missing \
+        build-essential \
+        apt-utils \
+        apt-transport-https \
+        ca-certificates \
+        software-properties-common \
+        pkg-config \
+        curl \
+        wget \
+        unzip \
+        gpg-agent \
+        locales && \
     add-apt-repository universe -y && \
     add-apt-repository ppa:git-core/ppa -y && \
     apt-get install -y git && \
     curl -sL https://deb.nodesource.com/setup_12.x | bash && \
     apt-get install -y nodejs  && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    locale-gen en_US.UTF-8 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
 
 # Install update pip and nodejs, install dvc and dvc-cml
 ADD "./" "/dvc-cml"
@@ -34,38 +40,43 @@ RUN wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list && \
 # CUDNN
 ARG CUDA
 ARG CUDNN=7.6.4.38-1
-ARG CUDNN_MAJOR_VERSION=7
+ARG CUDNN_MAJOR=7
 ARG LIBNVINFER=6.0.1-1
-ARG LIBNVINFER_MAJOR_VERSION=6
+ARG LIBNVINFER_MAJOR=6
 ARG CUBLAS=10.2.1.243-1
-ARG CUBLAS_MAJOR_VERSION=10
+ARG CUBLAS_MAJOR=10
 
 SHELL ["/bin/bash", "-c"]
 
 RUN apt update && apt-get install -y --no-install-recommends \
     cuda-command-line-tools-${CUDA/./-} \
-    libcublas${CUBLAS_MAJOR_VERSION}=${CUBLAS} \ 
+    libcublas${CUBLAS_MAJOR}=${CUBLAS} \ 
     cuda-nvrtc-${CUDA/./-} \
     cuda-cufft-${CUDA/./-} \
     cuda-curand-${CUDA/./-} \
     cuda-cusolver-${CUDA/./-} \
     cuda-cusparse-${CUDA/./-} \
-    libcudnn${CUDNN_MAJOR_VERSION}=${CUDNN}+cuda${CUDA} \
+    libcudnn${CUDNN_MAJOR}=${CUDNN}+cuda${CUDA} \
     libfreetype6-dev \
     libhdf5-serial-dev \
     libzmq3-dev && \ 
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
 
 # TENSORFLOW
-RUN apt update && apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
-    libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
-    && apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
-    && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
-    && ldconfig
+RUN apt update && \
+    apt-get install -y --no-install-recommends \
+        libnvinfer${LIBNVINFER_MAJOR}=${LIBNVINFER}+cuda${CUDA} \
+        libnvinfer-plugin${LIBNVINFER_MAJOR}=${LIBNVINFER}+cuda${CUDA} && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    ln -s /usr/local/cuda/lib64/stubs/libcuda.so \
+            /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+    echo /usr/local/cuda/lib64/stubs > /etc/ld.so.conf.d/z-cuda-stubs.conf && \
+    ldconfig
 # TENSORFLOW ENDS
 
 COPY ./docker/entrypoint.sh ./
-CMD ["entrypoint.sh"]
+CMD ["entrypoint.sh"]
diff --git a/docker/Dockerfile-gpu-py3 b/docker/Dockerfile-gpu-py3
@@ -0,0 +1,12 @@
+FROM dvcorg/dvc-cml-gpu:latest
+
+LABEL maintainer="dvc.org"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --fix-missing \
+            python3 \
+            python3-pip && \
+    pip3 install --upgrade pip && \
+    update-alternatives --install \
+            /usr/bin/python python $(which python3) 10
+
diff --git a/docker/Dockerfile-py3 b/docker/Dockerfile-py3
@@ -0,0 +1,12 @@
+FROM dvcorg/dvc-cml:latest
+
+LABEL maintainer="dvc.org"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends --fix-missing \
+            python3 \
+            python3-pip && \
+    pip3 install --upgrade pip && \
+    update-alternatives --install \
+            /usr/bin/python python $(which python3) 10
+