[BEAM-11608] Development environment set up automation (#14584)

* [BEAM-11608] Add a local setup shell script for Linux and Mac * [BEAM-11608] Added GitHub workflow for local-env-setup.sh Co-authored-by: Isidro Martinez <[email protected]> * [BEAM-11608] Added GitHub workflow for local-env-setup.sh (cleaned) * Changed GOPATH behavior. Added extra information on comments. * Fixed GOPATH check * [BEAM-11608] Add link to install docs * [BEAM-11608] Moving dependencies list to external file for outside consumption * [BEAM-11608] backtracking change to build.gradle gcpProject variable * [BEAM-11608] Adding license comment to pkglist file * [BEAM-11608] Adding changes suggested in code review * [BEAM-11608] Installing virtualenv using pip3 * [BEAM-11608] Adding virtualenv dependency to Dockerfile Co-authored-by: Eduardo Barrera <[email protected]> Co-authored-by: ebarreraWL <[email protected]> Co-authored-by: Isidro Martinez <[email protected]> Co-authored-by: Fernando Morales <[email protected]>
apache · May 29, 2021 · d8c5da8 · d8c5da8
1 parent dd15d59
commit d8c5da8
Show file tree

Hide file tree

Showing 4 changed files with 228 additions and 27 deletions.
diff --git a/.github/workflows/local_env_tests.yml b/.github/workflows/local_env_tests.yml
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# To learn more about GitHub Actions in Apache Beam check the CI.md
+
+name: Local environment tests
+
+on:
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+    paths: ['dev-support/**', 'buildSrc/**', '**/build.gradle', 'sdks/python/setup.py', 'sdks/python/tox.ini']
+
+jobs:
+  run_local_env_install_ubuntu:
+    timeout-minutes: 25
+    name: "Ubuntu run local environment shell script"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Installing local env dependencies"
+        run: "sudo ./local-env-setup.sh"
+        id: local_env_install_ubuntu
+      - name: "Gradle check"
+        run: "./gradlew checkSetup"
+        id: local_env_install_gradle_check_ubuntu
+  run_local_env_install_mac:
+    timeout-minutes: 50
+    name: "Mac run local environment shell script"
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Installing local env dependencies"
+        run: "./local-env-setup.sh"
+        id: local_env_install_mac
+      - name: "Gradle check"
+        run: "./gradlew checkSetup"
+        id: local_env_install_gradle_check_mac
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
@@ -42,32 +42,11 @@ ENV DEBCONF_TERSE true
 ###
 RUN apt -q update \
    && apt install -y software-properties-common apt-utils apt-transport-https ca-certificates \
-   && add-apt-repository -y ppa:deadsnakes/ppa \
-   && apt-get -q install -y --no-install-recommends \
-        bash-completion \
-        build-essential \
-        bzip2 \
-        wget \
-        curl \
-        docker.io \
-        g++ \
-        gcc \
-        git \
-        gnupg-agent \
-        rsync \
-        sudo \
-        vim \
-        locales \
-        wget \
-        time \
-        openjdk-8-jdk \
-        python3-setuptools \
-        python3-pip \
-        python3.6 \
-        python3.7 \
-        python3.8 \
-        virtualenv \
-        tox
+   && add-apt-repository -y ppa:deadsnakes/ppa
+
+RUN mkdir /package
+COPY pkglist /package/pkglist
+RUN apt-get -q install -y --no-install-recommends $(grep -v '^#' /package/pkglist | cat)
 
 ###
 # Set the locale ( see https://stackoverflow.com/a/28406007/114196 )
@@ -87,7 +66,7 @@ RUN alias python=python3.6
 ###
 # Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work
 ###
-RUN pip3 install grpcio-tools mypy-protobuf
+RUN pip3 install grpcio-tools mypy-protobuf virtualenv
 
 ###
 # Install useful tools

diff --git a/dev-support/docker/pkglist b/dev-support/docker/pkglist
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bash-completion
+build-essential
+bzip2
+wget
+curl
+docker.io
+g++
+gcc
+git
+gnupg-agent
+rsync
+sudo
+vim
+locales
+wget
+time
+openjdk-8-jdk
+python3-setuptools
+python3-pip
+python3.6
+python3.7
+python3.8
+tox
+docker.io
diff --git a/local-env-setup.sh b/local-env-setup.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+darwin_install_pip3_packages() {
+    echo "Installing setuptools grpcio-tools virtualenv"
+    pip3 install setuptools grpcio-tools virtualenv
+    echo "Installing mypy-protobuf"
+    pip3 install --user mypy-protobuf
+}
+
+install_go_packages(){
+        echo "Installing goavro"
+        go get github.com/linkedin/goavro
+        # As we are using bash, we are assuming .bashrc exists.
+        grep -qxF "export GOPATH=${PWD}/sdks/go/examples/.gogradle/project_gopath" ~/.bashrc
+        gopathExists=$?
+        if [ $gopathExists -ne 0 ]; then
+            export GOPATH=${PWD}/sdks/go/examples/.gogradle/project_gopath && echo "GOPATH was set for this session to '$GOPATH'. Make sure to add this path to your ~/.bashrc file after the execution of this script."
+        fi
+}
+
+kernelname=$(uname -s)
+
+# Running on Linux
+if [ "$kernelname" = "Linux" ]; then
+    # Assuming Debian based Linux and the prerequisites in https://beam.apache.org/contribute/ are met:
+    apt-get update
+
+    echo "Installing dependencies listed in pkglist file"
+    apt-get install -y $(grep -v '^#' dev-support/docker/pkglist | cat) # pulling dependencies from pkglist file
+
+    type -P python3 > /dev/null 2>&1
+    python3Exists=$?
+    type -P pip3 > /dev/null 2>&1
+    pip3Exists=$?
+    if [ $python3Exists -eq 0  -a $pip3Exists -eq 0 ]; then
+        echo "Installing grpcio-tools mypy-protobuf virtualenv"
+        pip3 install grpcio-tools mypy-protobuf virtualenv
+    else
+        echo "Python3 and pip3 are required but failed to install. Install them manually and rerun the script."
+        exit
+    fi
+
+    type -P go > /dev/null 2>&1
+    goExists=$?
+    if [ $goExists -eq 0 ]; then
+        install_go_packages
+    else
+        echo "Go is required. Install it manually from https://golang.org/doc/install and rerun the script."
+        exit
+    fi
+
+# Running on Mac
+elif [ "$kernelname" = "Darwin" ]; then
+    # Check for Homebrew, install if we don't have it
+    type -P brew > /dev/null 2>&1
+    brewExists=$?
+    if [ $brewExists -ne 0 ]; then
+        echo "Installing homebrew"
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+    fi
+
+    # Update homebrew recipes
+    echo "Updating brew"
+    brew update
+
+    # Assuming we are using brew
+    if brew ls --versions openjdk@8 > /dev/null; then
+        echo "openjdk@8 already installed. Skipping"
+    else
+        echo "Installing openjdk@8"
+        brew install openjdk@8
+    fi
+
+    type -P python3 > /dev/null 2>&1
+    python3Exists=$?
+    type -P pip3 > /dev/null 2>&1
+    pip3Exists=$?
+    if [ $python3Exists -eq 0  -a $pip3Exists -eq 0 ]; then
+        darwin_install_pip3_packages
+    else
+        echo "Python3 and pip3 are required but failed to install. Install them manually and rerun the script."
+        exit
+    fi
+
+    type -P tox > /dev/null 2>&1
+    toxExists=$?
+    if [ $toxExists -eq 0 ]; then
+        echo "tox already installed. Skipping"
+    else
+        echo "Installing tox"
+        brew install tox
+    fi
+
+    type -P docker > /dev/null 2>&1
+    dockerExists=$?
+    if [ $dockerExists -eq 0 ]; then
+        echo "docker already installed. Skipping"
+    else
+        echo "Installing docker"
+        brew install --cask docker
+    fi
+
+    type -P go > /dev/null 2>&1
+    goExists=$?
+    if [ $goExists -eq 0 ]; then
+        install_go_packages
+    else
+        echo "Go is required. Install it manually from https://golang.org/doc/install and rerun the script."
+        exit
+    fi
+
+else echo "Unrecognized Kernel Name: $kernelname"
+fi