diff --git a/.github/workflows/local_env_tests.yml b/.github/workflows/local_env_tests.yml new file mode 100644 index 000000000000..2bf6af1c6a88 --- /dev/null +++ b/.github/workflows/local_env_tests.yml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Local environment tests + +on: + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['dev-support/**', 'buildSrc/**', '**/build.gradle', 'sdks/python/setup.py', 'sdks/python/tox.ini'] + +jobs: + run_local_env_install_ubuntu: + timeout-minutes: 25 + name: "Ubuntu run local environment shell script" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: "Installing local env dependencies" + run: "sudo ./local-env-setup.sh" + id: local_env_install_ubuntu + - name: "Gradle check" + run: "./gradlew checkSetup" + id: local_env_install_gradle_check_ubuntu + run_local_env_install_mac: + timeout-minutes: 50 + name: "Mac run local environment shell script" + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: "Installing local env dependencies" + run: "./local-env-setup.sh" + id: local_env_install_mac + - name: "Gradle check" + run: "./gradlew checkSetup" + id: local_env_install_gradle_check_mac diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 92e5a8cdf845..ff9bb2ab52ea 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -42,32 +42,11 @@ ENV DEBCONF_TERSE true ### RUN apt -q update \ && apt install -y software-properties-common apt-utils apt-transport-https ca-certificates \ - && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get -q install -y --no-install-recommends \ - bash-completion \ - build-essential \ - bzip2 \ - wget \ - curl \ - docker.io \ - g++ \ - gcc \ - git \ - gnupg-agent \ - rsync \ - sudo \ - vim \ - locales \ - wget \ - time \ - openjdk-8-jdk \ - python3-setuptools \ - python3-pip \ - python3.6 \ - python3.7 \ - python3.8 \ - virtualenv \ - tox + && add-apt-repository -y ppa:deadsnakes/ppa + +RUN mkdir /package +COPY pkglist /package/pkglist +RUN apt-get -q install -y --no-install-recommends $(grep -v '^#' /package/pkglist | cat) ### # Set the locale ( see https://stackoverflow.com/a/28406007/114196 ) @@ -87,7 +66,7 @@ RUN alias python=python3.6 ### # Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work ### -RUN pip3 install grpcio-tools mypy-protobuf +RUN pip3 install grpcio-tools mypy-protobuf virtualenv ### # Install useful tools diff --git a/dev-support/docker/pkglist b/dev-support/docker/pkglist new file mode 100644 index 000000000000..00fc31684725 --- /dev/null +++ b/dev-support/docker/pkglist @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash-completion +build-essential +bzip2 +wget +curl +docker.io +g++ +gcc +git +gnupg-agent +rsync +sudo +vim +locales +wget +time +openjdk-8-jdk +python3-setuptools +python3-pip +python3.6 +python3.7 +python3.8 +tox +docker.io diff --git a/local-env-setup.sh b/local-env-setup.sh new file mode 100755 index 000000000000..c52d15b4e741 --- /dev/null +++ b/local-env-setup.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +darwin_install_pip3_packages() { + echo "Installing setuptools grpcio-tools virtualenv" + pip3 install setuptools grpcio-tools virtualenv + echo "Installing mypy-protobuf" + pip3 install --user mypy-protobuf +} + +install_go_packages(){ + echo "Installing goavro" + go get github.com/linkedin/goavro + # As we are using bash, we are assuming .bashrc exists. + grep -qxF "export GOPATH=${PWD}/sdks/go/examples/.gogradle/project_gopath" ~/.bashrc + gopathExists=$? + if [ $gopathExists -ne 0 ]; then + export GOPATH=${PWD}/sdks/go/examples/.gogradle/project_gopath && echo "GOPATH was set for this session to '$GOPATH'. Make sure to add this path to your ~/.bashrc file after the execution of this script." + fi +} + +kernelname=$(uname -s) + +# Running on Linux +if [ "$kernelname" = "Linux" ]; then + # Assuming Debian based Linux and the prerequisites in https://beam.apache.org/contribute/ are met: + apt-get update + + echo "Installing dependencies listed in pkglist file" + apt-get install -y $(grep -v '^#' dev-support/docker/pkglist | cat) # pulling dependencies from pkglist file + + type -P python3 > /dev/null 2>&1 + python3Exists=$? + type -P pip3 > /dev/null 2>&1 + pip3Exists=$? + if [ $python3Exists -eq 0 -a $pip3Exists -eq 0 ]; then + echo "Installing grpcio-tools mypy-protobuf virtualenv" + pip3 install grpcio-tools mypy-protobuf virtualenv + else + echo "Python3 and pip3 are required but failed to install. Install them manually and rerun the script." + exit + fi + + type -P go > /dev/null 2>&1 + goExists=$? + if [ $goExists -eq 0 ]; then + install_go_packages + else + echo "Go is required. Install it manually from https://golang.org/doc/install and rerun the script." + exit + fi + +# Running on Mac +elif [ "$kernelname" = "Darwin" ]; then + # Check for Homebrew, install if we don't have it + type -P brew > /dev/null 2>&1 + brewExists=$? + if [ $brewExists -ne 0 ]; then + echo "Installing homebrew" + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + fi + + # Update homebrew recipes + echo "Updating brew" + brew update + + # Assuming we are using brew + if brew ls --versions openjdk@8 > /dev/null; then + echo "openjdk@8 already installed. Skipping" + else + echo "Installing openjdk@8" + brew install openjdk@8 + fi + + type -P python3 > /dev/null 2>&1 + python3Exists=$? + type -P pip3 > /dev/null 2>&1 + pip3Exists=$? + if [ $python3Exists -eq 0 -a $pip3Exists -eq 0 ]; then + darwin_install_pip3_packages + else + echo "Python3 and pip3 are required but failed to install. Install them manually and rerun the script." + exit + fi + + type -P tox > /dev/null 2>&1 + toxExists=$? + if [ $toxExists -eq 0 ]; then + echo "tox already installed. Skipping" + else + echo "Installing tox" + brew install tox + fi + + type -P docker > /dev/null 2>&1 + dockerExists=$? + if [ $dockerExists -eq 0 ]; then + echo "docker already installed. Skipping" + else + echo "Installing docker" + brew install --cask docker + fi + + type -P go > /dev/null 2>&1 + goExists=$? + if [ $goExists -eq 0 ]; then + install_go_packages + else + echo "Go is required. Install it manually from https://golang.org/doc/install and rerun the script." + exit + fi + +else echo "Unrecognized Kernel Name: $kernelname" +fi