diff --git a/.gitignore b/.gitignore index 894a44c..f82a459 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,5 @@ venv.bak/ # mypy .mypy_cache/ + +.idea/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..04ad75a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +sudo: required + +services: + - docker + +before_install: + - docker build -t heroku-load-balancer . -f Dockerfile + - docker run -d -p 7979:7979 -v $PWD:/heroku-load-balancer -e PORT=7979 -e HEROKU_API_KEY=$HEROKU_API_KEY -e PIPELINE_IDENTIFIER='f64cf79b-79ba-4c45-8039-57c9af5d4508' --name heroku-load-balancer heroku-load-balancer + +script: + - docker exec -it heroku-load-balancer bash -c "radon cc src -nb --total-average" + - docker exec -it heroku-load-balancer bash -c "cat requirements.txt requirements-dev.txt | safety check --stdin" + - docker exec -it heroku-load-balancer bash -c "bash <(curl -s https://linters.io/sort-requirements) requirements.txt requirements-dev.txt" + - docker exec -it heroku-load-balancer bash -c "bash <(curl -s https://linters.io/isort-diff) src" + - docker exec -it heroku-load-balancer bash -c "flake8 src" + +env: + global: + secure: I8Rr7rQOfSFkq+VjYFThFfbBE2YIs4d5y3f9kJEAbZ51p61QWu8h4Opu1/fXGPFA5hCOS2sd8hywj+wVAaavd4GUuJurYbUd4rGVhkpQDCSYlUVr2eGHXGC+JgpyZ14LKsPOAFIfdwumU4ZrmAgTmKuduhiXo/erQk2g086ivBjqUvjG/yRH3ZehlMVY1MU4QIOZa1JrWgG/XmXXIxaFbQpwIeNQw3Q5i10PcG+X+6Yoeg+IrJ4mKIExKzrwrBS3I/JEWh37TAB2AkQN8Ez2u8AktM8uAyKALxL4mThhr9sCsIjfrNHJOYENDKvzLM1Y0XDURSclrngsvp3ihOTo23JSXemBPzbxfP2jvhFV0nDePVq88fVIrhxFNw+Kd1Gvew6hFa1PftNq825eDfLv+oWD8o3J9SU8innqxF6TH5d/UfIkI+Z3ovciedL4Jy7/bGTIqKk5Zc04GylCfEGxm1YFotNpE1LYunMcmE6pJQvs35DWryhJw3ryGOI3CWQLXuSq8pbR3czJpPeZlisJ1pU7vh79x8lZmWsxsza5jYMvPAle6JgInnm8oYkvwNc4fi5u9Y4Tb9Q4QSIc3nX9o62mH1fu6F/N5lefe6xoRkTjz2xkOkwEHtiAFKrXimyI6E/wscOk5NgenMqp0cm4Sk5C3vlLnvCc79/zycCeIpw= diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2553ce3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:16.04 + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +RUN apt-get update && apt-get install -y \ + software-properties-common \ + nginx \ + curl + +RUN add-apt-repository ppa:deadsnakes/ppa && apt-get update && apt-get install -y \ + python3.6 \ + python3.6-dev \ + python3-pip \ + python3-setuptools + +RUN ln -sfn /usr/bin/python3.6 /usr/bin/python3 && ln -sfn /usr/bin/python3 /usr/bin/python + +WORKDIR /heroku-load-balancer +COPY . /heroku-load-balancer + +ENV PYTHONPATH="$PYTHONPATH:/heroku-load-balancer/src" + +RUN pip3 install -r /heroku-load-balancer/requirements.txt -r /heroku-load-balancer/requirements-dev.txt + +CMD /bin/bash -c "python3 src/entrypoint.py create-load-balancer --nginx-port=$PORT --heroku-api-key=$HEROKU_API_KEY --pipeline-identifier=$PIPELINE_IDENTIFIER" && mv nginx.conf /etc/nginx/nginx.conf && nginx -g 'daemon off;' diff --git a/README.md b/README.md new file mode 100644 index 0000000..8dc3368 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +Automatic (with no manual configuring) load balancer for your Heroku pipeline production applications. + +[![Release](https://img.shields.io/github/release/dmytrostriletskyi/heroku-load-balancer.svg)](https://github.com/dmytrostriletskyi/heroku-load-balancer/releases) +[![Build Status](https://travis-ci.com/dmytrostriletskyi/heroku-load-balancer.svg?branch=develop)](https://travis-ci.com/dmytrostriletskyi/heroku-load-balancer) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + + * [Getting started](#getting-started) + * [What is a load balancer](#what-is-a-load-balancer) + * [Motivation](#motivation) + * [How to use](#how-to-use) + * [How it works](#how-it-works) + * [Development](#development) + +## Getting started + +### What is a load balancer + +A load balancer is a device that distributes network or application traffic across a cluster of servers. A load balancer +sits between the client and the server farm accepting incoming network and application traffic and distributing the +traffic across multiple backend servers. By balancing application requests across multiple servers, a load balancer +reduces individual server load and prevents any one application server from becoming a single point of failure, +thus improving overall application availability and responsiveness. + +![Illustation on how load balancer works](https://habrastorage.org/webt/iy/-s/vx/iy-svxvpqwnquwvciv7qm3pfm1u.png) + +### Motivation + +[Heroku](https://heroku.com) does not have `load balancer` paid feature to balancing your applications. It is the drawback +comparing to [Digital Ocean](https://www.digitalocean.com/products/load-balancer) and [Amazon Web Services](https://aws.amazon.com/elasticloadbalancing/) +which do have it. + +### How to use + +1. Press the button named `Deploy to Heroku` below. + +[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/dmytrostriletskyi/heroku-load-balancer/tree/create-docs) + +2. Enter the name for the application which will host the load balancer. Choose the region and add to the pipeline if needed. + + + +3. Visit the [Heroku account setting page](https://dashboard.heroku.com/account), find the `API Key` section, reveal the key and +paste it to the `HEROKU_API_KEY` field. + + + +3. Open the preferable pipeline and copy its identifier from the URL. On the screenshoot it is `f64cf79b-79ba-4c45-8039-57c9af5d4508` mentioned by red arrow at the top. + + + +4. Return to the deploying page, paste the identifier to the `PIPELINE_IDENTIFER` field. + + + +5. Press the button named `Deploy app`. The process of deploying will start immediately as illustrated below. + + + +6. When build is finished, you can manage your application (rename, etc.) and view it (open URL in the browser). + + + +7. To check if load balancer works properly, just open logs of each production back-end servers +(`heroku logs --tail -a application-name` in the terminal), and send the request to the load balancer application. +As the result, the load balancer will proxy your request to the each back-end server in round-robin method (one by one in order). + + + +### How it works + +1. You specify pipeline's identifier (`PIPELINE_IDENTIFER`) to create load balancer for its applications in `production` stage. +2. Through the [Heroku API](https://devcenter.heroku.com/categories/platform-api) using your `HEROKU_API_KEY`, URLs of applications are fetched. +3. Then [configuration file for load balancing](http://nginx.org/en/docs/http/load_balancing.html) based on fetched URLS is created. +4. And served by the [Nginx](https://nginx.org/en) in round-robin method (one by one in order). + +## Development + +Clone the project with the following command: + +```bash +$ git clone https://github.com/dmytrostriletskyi/heroku-load-balancer.git +$ cd heroku-load-balancer +``` + +To build the project, use the following command: + +```bash +$ docker build -t heroku-load-balancer . -f Dockerfile +``` + +To run the project, use the following command. It will start the server and occupate current terminal session: + +```bash +$ docker run -p 7979:7979 -v $PWD:/heroku-load-balancer \ + -e PORT=7979 \ + -e HEROKU_API_KEY='8af7dbb9-e6b8-45bd-8c0a-87787b5ae881' \ + -e PIPELINE_IDENTIFIER='f64cf79b-79ba-4c45-8039-57c9af5d4508' \ + --name heroku-load-balancer heroku-load-balancer +``` + +If you need to enter the bash of the container, use the following command: + +```bash +$ docker exec -it heroku-load-balancer bash +``` + +Clean all containers with the following command: + +```bash +$ docker rm $(docker ps -a -q) -f +``` + +Clean all images with the following command: + +```bash +$ docker rmi $(docker images -q) -f +``` diff --git a/app.json b/app.json new file mode 100644 index 0000000..65d5255 --- /dev/null +++ b/app.json @@ -0,0 +1,18 @@ +{ + "name": "heroku-load-balancer", + "description": "Automatic (with no manual configuring) load balancer for your Heroku pipeline production applications.", + "repository": "https://github.com/dmytrostriletskyi/heroku-load-balancer", + "env": { + "HEROKU_API_KEY": { + "description": "Heroku API key from the personal account.", + "required": true + }, + "PIPELINE_IDENTIFIER": { + "description": "Pipeline identifier for creating load balancer.", + "required": true + } + }, + "logo": "https://habrastorage.org/webt/w3/fp/ep/w3fpepqfjjtumhnyul4ymts-qm8.png", + "keywords": ["nginx", "docker", "proxy", "balancing", "balancer"], + "stack": "container" +} diff --git a/heroku.yml b/heroku.yml new file mode 100644 index 0000000..8eec25b --- /dev/null +++ b/heroku.yml @@ -0,0 +1,3 @@ +build: + docker: + web: Dockerfile diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..8cfd09e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,11 @@ +add-trailing-comma==1.0.0 +flake8-commas==2.0.0 +flake8-comprehensions==2.1.0 +flake8-docstrings==1.3.0 +flake8-per-file-ignores==0.8.1 +flake8-print==3.1.0 +flake8==3.7.7 +isort==4.3.20 +pep8-naming==0.8.2 +radon==3.0.3 +safety==1.8.5 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ffa408c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +click==7.0 +requests==2.22.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..fedce7c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[isort] +line_length=120 +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=True +combine_as_imports=True + +[flake8] +max-line-length=120 +ignore=D200, D413, D107, D100 +per-file-ignores= + */__init__.py: D104, F401, D100, + */test_*: D205, + src/constants.py: E501 + +[coverage:run] +omit = + */.virtualenvs/*, + */virtualenv/*, + + */__init__.py, + tests/* diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/constants.py b/src/constants.py new file mode 100644 index 0000000..351ea7b --- /dev/null +++ b/src/constants.py @@ -0,0 +1,29 @@ +NGINX_LOAD_BALANCER_CONFIG_TEMPLATE = """events < + worker_connections 4096; +> + +http < + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + proxy_ssl_server_name on; + + # ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; + + upstream main < +{upstream_server_localhosts} + > + + server < + listen {port}; + + location / < + proxy_pass http://main; + > + > + +{upstream_server_configs} +> +""" diff --git a/src/entrypoint.py b/src/entrypoint.py new file mode 100644 index 0000000..c7db432 --- /dev/null +++ b/src/entrypoint.py @@ -0,0 +1,61 @@ +""" +Provide implementation of the command line interface. +""" +import click + +from src.heroku import ( + GetHerokuPipelineProductionApplicationsUrls, + HerokuApi, +) +from src.nginx import CreationNginxLoadBalancerConfigFile + + +@click.group() +def cli(): + """ + Command line interface root function. + """ + pass + + +@click.command() +@click.option( + '--nginx-port', + type=int, + required=True, + help='The port to the Nginx on.', +) +@click.option( + '--heroku-api-key', + type=str, + required=True, + help='The account\'s Heroku API key.', +) +@click.option( + '--pipeline-identifier', + type=str, + required=True, + help='Pipeline identifier to fetch applications for balancing.', +) +def create_load_balancer(nginx_port, heroku_api_key, pipeline_identifier): + """ + Create Nginx load balancer config file with pipeline's production applications URLs. + """ + heroku_api = HerokuApi( + key=heroku_api_key, + ) + + get_heroku_pipeline_production_applications_urls = GetHerokuPipelineProductionApplicationsUrls( + heroku_api=heroku_api, + ) + + pipeline_production_applications_urls = get_heroku_pipeline_production_applications_urls.by_pipeline_identifier( + identifier=pipeline_identifier, + ) + + CreationNginxLoadBalancerConfigFile(port=nginx_port).with_urls(urls=pipeline_production_applications_urls) + + +if __name__ == '__main__': + cli.add_command(create_load_balancer) + cli() diff --git a/src/heroku.py b/src/heroku.py new file mode 100644 index 0000000..355f430 --- /dev/null +++ b/src/heroku.py @@ -0,0 +1,73 @@ +""" +Provide implements of the Heroku domain. +""" +import requests + + +class HerokuApi: + """ + Implements Heroku API communicator. + """ + + def __init__(self, key: str): + """ + Constructor. + """ + self.headers = { + 'Accept': 'application/vnd.heroku+json; version=3', + 'Authorization': f'Bearer {key}', + } + + def fetch_pipeline_applications(self, identifier: str): + """ + Fetch a pipeline applications by its identifier. + """ + fetch_pipeline_applications_url = f'https://api.heroku.com/pipelines/{identifier}/pipeline-couplings' + + response = requests.get(fetch_pipeline_applications_url, headers=self.headers) + response_json = response.json() + + return response_json + + def fetch_application(self, identifier: str): + """ + Fetch an application by its identifier. + """ + fetch_pipeline_applications_url = f'https://api.heroku.com/apps/{identifier}' + + response = requests.get(fetch_pipeline_applications_url, headers=self.headers) + response_json = response.json() + + return response_json + + +class GetHerokuPipelineProductionApplicationsUrls: + """ + Implement getting Heroku's pipeline production applications' URLs transaction. + """ + + def __init__(self, heroku_api: HerokuApi): + """ + Constructor. + """ + self.heroku_api = heroku_api + + def by_pipeline_identifier(self, identifier): + """ + Get Heroku's pipeline production applications' URLs by pipeline identifier. + """ + production_applications_identifiers = [] + + for application in self.heroku_api.fetch_pipeline_applications(identifier=identifier): + if application.get('stage') == 'production': + application_identifier = application.get('app').get('id') + production_applications_identifiers.append(application_identifier) + + production_applications_urls = [] + + for application_identifier in production_applications_identifiers: + application = self.heroku_api.fetch_application(identifier=application_identifier) + application_url = application.get('web_url') + production_applications_urls.append(application_url) + + return production_applications_urls diff --git a/src/nginx.py b/src/nginx.py new file mode 100644 index 0000000..f212355 --- /dev/null +++ b/src/nginx.py @@ -0,0 +1,53 @@ +""" +Provide implements of the Nginx domain. +""" +from src.constants import NGINX_LOAD_BALANCER_CONFIG_TEMPLATE + + +class CreationNginxLoadBalancerConfigFile: + """ + Implements creation of Nginx load balancer config file transaction. + """ + + def __init__(self, port): + """ + Constructor. + """ + self.port = port + + @staticmethod + def get_host_from_url(url): + """ + Remove protocol and last slash from the URL. + """ + return url.replace('https://', '').replace('/', '') + + def with_urls(self, urls): + """ + Creation of Nginx load balancer config file with specified URLs. + """ + upstream_server_localhosts_text = '' + upstream_server_configs_text = '' + + for index, url in enumerate(urls): + url_without_last_slash = url[:-1] + host_from_url = self.get_host_from_url(url=url) + + upstream_server_configs_text += \ + f'\tserver <\n\t\tlisten 800{index};\n\n\t\tlocation / ' + \ + f'<\n\t\t\tproxy_set_header HOST {host_from_url};\n\t\t\t' + \ + f'proxy_pass {url_without_last_slash};\n\t\t>\n\t>\n\n' + + upstream_server_localhosts_text += f'\t\tserver 127.0.0.1:800{index};\n' + + nginx_load_balancer_config_file_ready_to_use = NGINX_LOAD_BALANCER_CONFIG_TEMPLATE.format( + port=self.port, + upstream_server_localhosts=upstream_server_localhosts_text, + upstream_server_configs=upstream_server_configs_text, + ) + + nginx_load_balancer_config_file_ready_to_use = \ + nginx_load_balancer_config_file_ready_to_use.replace('<', '{').replace('>', '}') + + with open('nginx.conf', 'w') as nginx_config: + nginx_config.write(nginx_load_balancer_config_file_ready_to_use)