diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 00000000..5c3a02e6 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Tolu Aina + +Contributors +------------ + +None yet. Why not be the first? diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..c9465c33 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,9 @@ +.. _changelog: + +Changelog +========= + +1.0.0 (2019-09-19) +------------------ + +* initial release diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..dd1a51b7 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,128 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/toluaina/essync/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +pgsync could always use more documentation, whether as part of the +official pgsync docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/toluaina/pgsync/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `pgsync` for local development. + +1. Fork the `pgsync` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:toluaina/pgsync.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv pgsync + $ cd pgsync/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the + tests, including testing other Python versions with tox:: + + $ flake8 pgsync tests + $ python setup.py test or py.test + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.6 and 3.7, and for PyPy. Check + https://github.com/toluaina/pgsync/pulls + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + +$ py.test tests.test_pgsync + + +Deploying +--------- + +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in HISTORY.rst). +Then run:: + +$ bumpversion patch # possible: major / minor / patch +$ git push +$ git push --tags + +Github CI will then deploy to PyPI if tests pass. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..84cabfde --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +FROM python:3.7-alpine + +ENV PYTHONUNBUFFERED 1 + +ENV PYTHONDONTWRITEBYTECODE 1 + +RUN apk add --no-cache \ + build-base \ + git \ + libffi-dev \ + openssh-client \ + openssl-dev \ + python-dev + +# Required for building psycopg2-binary: https://github.com/psycopg/psycopg2/issues/684 +RUN apk update && apk add postgresql-dev gcc python3-dev musl-dev + +# Install requirements +COPY requirements.txt /requirements.txt +RUN pip install --upgrade pip \ + && pip install --upgrade setuptools \ + && pip install --upgrade -r /requirements.txt \ + && rm -r /root/.cache + +ARG WORKDIR=/code + +RUN mkdir $WORKDIR + +ADD ./.env.sample $WORKDIR/.env +ADD ./codecov.yml $WORKDIR/codecov.yml +ADD ./bin/ $WORKDIR/bin +ADD ./pgsync/ $WORKDIR/pgsync +ADD ./tests/ $WORKDIR/tests +ADD ./examples/ $WORKDIR/examples +ADD ./supervisor/ $WORKDIR/supervisor +ADD ./requirements.in $WORKDIR/requirements.in +ADD ./requirements.txt $WORKDIR/requirements.txt + +WORKDIR $WORKDIR + +ENV PYTHONPATH=$WORKDIR/pgsync +ENV PATH=$PATH:$WORKDIR/bin +ENV SCHEMA=$WORKDIR/examples/airbnb/schema.json +ENV LOG_LEVEL=debug + +COPY supervisor/supervisord.conf /etc/supervisor/supervisord.conf +COPY supervisor/pgsync.conf /etc/supervisor/conf.d/ +ENTRYPOINT ["/bin/sh", "supervisor/supervisord_entrypoint.sh"] +CMD ["-c", "/etc/supervisor/supervisord.conf"] diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 00000000..ee8de944 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2019-01-22) +------------------ + +* First release on PyPI. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..022b2de4 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.md + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..6b47f819 --- /dev/null +++ b/Makefile @@ -0,0 +1,88 @@ +.PHONY: clean clean-test clean-pyc clean-build docs help +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +try: + from urllib import pathname2url +except: + from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + rm -fr .pytest_cache + +lint: ## check style with flake8 + flake8 pgsync tests + +test: ## run tests quickly with the default Python + py.test + +test-all: ## run tests on every Python version with tox + tox + +coverage: ## check code coverage quickly with the default Python + coverage run --source pgsync -m pytest + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + +docs: ## generate Sphinx HTML documentation, including API docs + rm -f docs/pgsync.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ pgsync + $(MAKE) -C docs clean + $(MAKE) -C docs html + $(BROWSER) docs/_build/html/index.html + +servedocs: docs ## compile the docs watching for changes + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: dist ## package and upload a release + twine upload dist/* + +dist: clean ## builds source and wheel package + python setup.py sdist + python setup.py bdist_wheel + ls -l dist + +install: clean ## install the package to the active Python's site-packages + python setup.py install diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..311352c3 --- /dev/null +++ b/NOTICE @@ -0,0 +1,5 @@ +PGSync + +Copyright (c) 2019, Tolu Aina. + +This product includes software developed at PGSync (https://www.pgsync.com). \ No newline at end of file diff --git a/README.md b/README.md index 347eddf0..f29cb47f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ +![alt text](docs/logo.png "PGSync logo") + # PGSync ## PostgreSQL to Elasticsearch sync -PGSync is a middleware for syncing data from [Postgres](https://www.postgresql.org) to [Elasticsearch](https://www.elastic.co/products/elastic-stack). +[PGSync](http://pgsync.com) is a middleware for syncing data from [Postgres](https://www.postgresql.org) to [Elasticsearch](https://www.elastic.co/products/elastic-stack). It allows you to keep [Postgres](https://www.postgresql.org) as your source of truth data source and expose structured denormalized documents in [Elasticsearch](https://www.elastic.co/products/elastic-stack). @@ -11,10 +13,10 @@ PGSync's advanced query builder then generates optimized SQL queries on the fly based on your schema. PGsync's advisory model allows you to quickly move and transform large volumes of data quickly whilst maintaining relational integrity. -Simply describe your document structure or schema in JSON and PGSync will +Simply describe your document structure or schema in JSON and [PGSync](http://pgsync.com) will continuously capture changes in your data and load it into [Elasticsearch](https://www.elastic.co/products/elastic-stack) without writing any code. -PGSync transforms your relational data into a structured document format. +[PGSync](http://pgsync.com) transforms your relational data into a structured document format. It allows you to take advantage of the expressive power and scalability of [Elasticsearch](https://www.elastic.co/products/elastic-stack) directly from [Postgres](https://www.postgresql.org). @@ -63,7 +65,7 @@ the search capabilities of [Elasticsearch](https://www.elastic.co/products/elast #### How it works -PGSync is written in Python (supporting version 3.4 onwards) and the stack is composed of: [Redis](https://redis.io), [Elasticsearch](https://www.elastic.co/products/elastic-stack), [Postgres](https://www.postgresql.org), and [SQlAlchemy](https://www.sqlalchemy.org). +PGSync is written in Python (supporting version 3.6 onwards) and the stack is composed of: [Redis](https://redis.io), [Elasticsearch](https://www.elastic.co/products/elastic-stack), [Postgres](https://www.postgresql.org), and [SQlAlchemy](https://www.sqlalchemy.org). PGSync leverages the [logical decoding](https://www.postgresql.org/docs/current/logicaldecoding.html) feature of [Postgres](https://www.postgresql.org) (introduced in PostgreSQL 9.4) to capture a continuous stream of change events. This feature needs to be enabled in your [Postgres](https://www.postgresql.org) configuration file by setting in the postgresql.conf file: @@ -94,32 +96,32 @@ There are several ways of installing and trying PGSync To startup all services with docker. Run: ``` -docker-compose up +$ docker-compose up ``` In another shell, run ``` -docker-compose up exec -it pgsync +$ docker-compose up exec -it pgsync ``` Create a sample database ``` -psql -d mydb < samples/schema.sql +$ psql -d mydb < samples/schema.sql ``` Load some data into the sample database ``` -psql -f samples/data.sql +$ psql -f samples/data.sql ``` Run PGSync ``` -./bin/pgsync +$ ./bin/pgsync ``` Show the content in Elasticsearch ``` -curl -X GET http://localhost:9200/[index_name] +$ curl -X GET http://localhost:9200/[index_name] ``` ##### Manual configuration @@ -130,15 +132,15 @@ curl -X GET http://localhost:9200/[index_name] - alternatively, ```ALTER SYSTEM SET wal_level = logical``` - Installation - - ```git clone https://github.com/toluaina/essync.git``` - - ```cd pgsync``` - - ```virtualenv -p python3 venv``` - - ```source venv/bin/activate``` - - ```pip install -r requirements.txt``` + - ```$ git clone https://github.com/toluaina/essync.git``` + - ```$ cd pgsync``` + - ```$ virtualenv -p python3 venv``` + - ```$ source venv/bin/activate``` + - ```$ pip install -r requirements.txt``` - create a schema.json for you document representation - - ```cp env_sample .env``` + - ```$ cp env_sample .env``` - edit the .env above - - ```source .env``` + - ```$ source .env``` - run the program with **_```pgsync```_** @@ -150,19 +152,19 @@ Key features of PGSync are: - Negligible impact on database performance. - Transactionally consistent output in Elasticsearch. This means: writes appear only when they are committed to the database, insert, update and delete (TG_OP's) operations appear in the same order as they were committed (as opposed to eventual consistency). - Fault-tolerant: does not lose data, even if processes crash or a network interruption occurs, etc. The process can be recovered from the last checkpoint. -- Returns the data directly as Postgres JSON from the database for speed +- Returns the data directly as Postgres JSON from the database for speed. - Transforms the data on the fly e.g rename labels before indexing. - Supports composite primary and foreign keys. - Supports an arbitrary depth of nested entities i.e Tables having long chain of relationship dependencies. - Supports Postgres JSON data fields. This means: we can extract JSON fields in a database table as a separate field in the resulting document. -- Customize the document structure. +- Fully customizable document structure. #### Requirements - [Python](https://www.python.org) 3.7 - [Postgres](https://www.postgresql.org) 9.4+ - [Redis](https://redis.io) 3.1.0 -- [Elasticsearch](https://www.https://www.elastic.co/products/elastic-stack) 6.3.1 +- [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1 - [SQlAlchemy](https://www.sqlalchemy.org) 1.3.4 #### Example @@ -196,6 +198,7 @@ Consider this example of a Book library catalogue. | 4 | 9781471331435 | 4 | With PGSync, we can simply define this [JSON](https://jsonapi.org) schema where the **_book_** table is the pivot. +A **_pivot_** table indicates the root of your document. ```json { @@ -281,54 +284,30 @@ e.g PGSync addresses the following challenges: - What if we update the author's name in the database? - What if we wanted to add another author for a book? -- What if there are lots of documents already with the same author -we wanted to change? +- What if we have lots of documents already with the same author we wanted to change? - What if we delete or update an author? - What if we truncate an entire table? #### Benefits -- PGsync aims to be simple to use out of the box compared to other solutions. +- PGsync aims to be a simple to use out of the box solution. - PGsync handles data deletions unlike Logstash. - PGSync requires little development effort. You simply define a config describing your data. -- PGsync generates advanced queries matching your schema directly. With Logstash you need to write this yourself. +- PGsync generates advanced queries matching your schema directly. With Logstash, you need to write this yourself. - PGSync allows you to easily rebuild your indexes in case of a schema change. - You can expose only the data you require in Elasticsearch. +#### Contributing + +Contributions are very welcome! Check out the [Contribution](CONTRIBUTING.rst) Guidelines for instructions. #### Credits -- This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. -.. _Cookiecutter: https://github.com/audreyr/cookiecutter -.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage +- This package was created with [Cookiecutter](https://github.com/audreyr/cookiecutter) - Elasticsearch is a trademark of Elasticsearch BV, registered in the U.S. and in other countries. #### License -Copyright (c) 2019, Tolu Aina -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of PGSync nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This code is released under the Apache 2.0 License. Please see [LICENSE](LICENSE) and [NOTICE](NOTICE) for more details. + +Copyright (c) 2019, Tolu Aina. diff --git a/bin/bootstrap b/bin/bootstrap new file mode 100755 index 00000000..ecba92d9 --- /dev/null +++ b/bin/bootstrap @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +"""PGSync bootstrap.""" +import json +import logging +import os + +import click + +import settings +from base import Base + +logger = logging.getLogger(__name__) + + +def create_es_mapping(): + # add implementation hook here... + pass + + +@click.command() +@click.option('--teardown', is_flag=True, help='Bootstrap teardown') +def main(teardown): + """Application onetime Bootstrap.""" + if not os.path.exists(settings.SCHEMA): + raise IOError(f'Schema config "{settings.SCHEMA}" not found') + + for database in set([ + schema.get('index') for schema in json.load( + open(settings.SCHEMA) + ) + ]): + if teardown: + Base(database).teardown() + else: + Base(database).setup() + create_es_mapping() + logger.info(f'Bootstrap: {database}') + + +if __name__ == '__main__': + main() diff --git a/bin/pgsync b/bin/pgsync new file mode 100755 index 00000000..7340284b --- /dev/null +++ b/bin/pgsync @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +"""PGSync runtime.""" +import sync + +if __name__ == '__main__': + sync.main() diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..882852f6 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,5 @@ +coverage: + status: + project: + default: + threshold: 90% \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..0ff2f6ab --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3.3' + +services: + postgres: + build: + context: ./docker + ports: + - "65432:5432" + environment: + - POSTGRES_USER=pgsync + - POSTGRES_DB=airbnb + - POSTGRES_PASSWORD=PLEASE_REPLACE_ME + redis: + image: redis + command: redis-server --requirepass PLEASE_REPLACE_ME + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0 + ports: + - "9201:9200" + - "9301:9300" + environment: + - xpack.security.enabled=false + - network.host=127.0.0.1 + - http.host=0.0.0.0 + pgsync: + build: + context: . + dockerfile: Dockerfile + labels: + org.label-schema.name: "Postgres-Elasticsearch-Sync" + org.label-schema.description: "Postgres to Elasticsearch ETL" + com.label-schema.service-type: "pgsync-daemon" + depends_on: + - postgres + - redis + - elasticsearch diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..ac539808 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:11-alpine +COPY conf.sql /docker-entrypoint-initdb.d/ +RUN chmod a+r /docker-entrypoint-initdb.d/conf.sql diff --git a/docker/conf.sql b/docker/conf.sql new file mode 100755 index 00000000..202e2a92 --- /dev/null +++ b/docker/conf.sql @@ -0,0 +1 @@ +ALTER SYSTEM SET wal_level = logical; \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..312bd721 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = pgsync +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 00000000..e122f914 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 00000000..565b0521 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1 @@ +.. include:: ../CHANGELOG.rst diff --git a/docs/complex.rst b/docs/complex.rst new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/docs/complex.rst @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 00000000..e57d1ee8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# pgsync documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 9 13:47:02 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +import pgsync + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'pgsync' +copyright = u"2019, Tolu Aina" +author = u"Tolu Aina" + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = pgsync.__version__ +# The full version, including alpha/beta/rc tags. +release = pgsync.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output --------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'pgsyncdoc' + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'pgsync.tex', + u'pgsync Documentation', + u'Tolu Aina', 'manual'), +] + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pgsync', + u'pgsync Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'pgsync', + u'pgsync Documentation', + author, + 'pgsync', + 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 00000000..e582053e --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/features.rst b/docs/features.rst new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/docs/features.rst @@ -0,0 +1,2 @@ + + diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 00000000..25064996 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..e2bfd3b4 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,21 @@ +Welcome to pgsync's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + installation + usage + complex + modules + contributing + authors + history + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 00000000..fb3c5050 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,51 @@ +.. highlight:: shell + +============ +Installation +============ + + +Stable release +-------------- + +To install pgsync, run this command in your terminal: + +.. code-block:: console + + $ pip install pgsync + +This is the preferred method to install pgsync, as it will always install the most recent stable release. + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + + +From sources +------------ + +The sources for pgsync can be downloaded from the `Github repo`_. + +You can either clone the public repository: + +.. code-block:: console + + $ git clone git://github.com/toluaina/essync + +Or download the `tarball`_: + +.. code-block:: console + + $ curl -OL https://github.com/toluaina/essync/tarball/master + +Once you have a copy of the source, you can install it with: + +.. code-block:: console + + $ python setup.py install + + +.. _Github repo: https://github.com/toluaina/essync +.. _tarball: https://github.com/toluaina/essync/tarball/master diff --git a/docs/logo.png b/docs/logo.png new file mode 100644 index 00000000..73e60cb8 Binary files /dev/null and b/docs/logo.png differ diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..abab839a --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=pgsync + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 00000000..72a33558 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 00000000..56c281ac --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,7 @@ +===== +Usage +===== + +To use pgsync in a project:: + + import pgsync diff --git a/examples/airbnb/bootstrap.sh b/examples/airbnb/bootstrap.sh new file mode 100755 index 00000000..0a0f7368 --- /dev/null +++ b/examples/airbnb/bootstrap.sh @@ -0,0 +1,17 @@ +#! /bin/sh +set -u +# create database prior to running this bootstrap +# ensure bin dir is in the PATH +source .path +export SCHEMA='./examples/airbnb/schema.json' +read -p "Are you sure you want to delete the 'airbnb' elasticserch index? [y/N] " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + exit 1 +fi +curl -X DELETE http://localhost:9200/airbnb +python examples/airbnb/schema.py +python examples/airbnb/data.py +bootstrap +pgsync \ No newline at end of file diff --git a/examples/airbnb/data.py b/examples/airbnb/data.py new file mode 100644 index 00000000..20208e40 --- /dev/null +++ b/examples/airbnb/data.py @@ -0,0 +1,212 @@ +import json +from datetime import datetime, timedelta + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +import settings +from base import pg_engine, subtransactions +from helper import teardown +from schema import Bookings, Cities, Countries, Hosts, Places, Reviews, Users + +Base = declarative_base() + + +def main(): + + teardown(drop_db=False) + schema = json.load(open(settings.SCHEMA)) + engine = pg_engine(database=schema[0].get('index')) + Session = sessionmaker(bind=engine, autoflush=True) + session = Session() + + # Bootstrap + users = [ + Users(email='stephanie.miller@aol.com'), + Users(email='nancy.gaines@ibm.com'), + Users(email='andrea.cabrera@gmail.com'), + Users(email='brandon86@yahoo.com'), + Users(email='traci.williams@amazon.com'), + Users(email='john.brown@apple.com'), + ] + + hosts = [ + Hosts(email='kermit@muppetlabs.com'), + Hosts(email='bert@sesamestreet.com'), + Hosts(email='big.bird@sesamestreet.com'), + Hosts(email='cookie.monster@sesamestreet.com'), + Hosts(email='mr.snuffleupagus@sesamestreet.com'), + Hosts(email='grover@sesamestreet.com'), + ] + + cities = [ + Cities( + name='Manila', + country=Countries( + name='Philippines', + country_code='PH' + ) + ), + Cities( + name='Lisbon', + country=Countries( + name='Portugal', + country_code='PT' + ), + ), + Cities( + name='Havana', + country=Countries( + name='Cuba', + country_code='PT' + ), + ), + Cities( + name='Copenagen', + country=Countries( + name='Denmark', + country_code='DK' + ), + ), + Cities( + name='London', + country=Countries( + name='United Kingdom', + country_code='UK' + ), + ), + Cities( + name='Casablanca', + country=Countries( + name='Morocco', + country_code='MA' + ), + ), + ] + + places = [ + Places( + host=hosts[0], + city=cities[0], + address='Quezon Boulevard' + ), + Places( + host=hosts[1], + city=cities[1], + address='Castelo de São Jorge' + ), + Places( + host=hosts[2], + city=cities[2], + address='Old Havana' + ), + Places( + host=hosts[3], + city=cities[3], + address='Tivoli Gardens' + ), + Places( + host=hosts[4], + city=cities[4], + address='Buckingham Palace' + ), + Places( + host=hosts[5], + city=cities[5], + address='Medina' + ), + ] + + reviews = [ + Reviews( + booking=Bookings( + user=users[0], + place=places[0], + start_date=datetime.now() + timedelta(days=1), + end_date=datetime.now() + timedelta(days=4), + price_per_night=100, + num_nights=4 + ), + rating=1, + review_body='Neque porro quisquam est qui dolorem' + ), + Reviews( + booking=Bookings( + user=users[1], + place=places[1], + start_date=datetime.now() + timedelta(days=2), + end_date=datetime.now() + timedelta(days=4), + price_per_night=150, + num_nights=3 + ), + rating=2, + review_body='Sed eget finibus massa, vel efficitur mauris' + ), + Reviews( + booking=Bookings( + user=users[2], + place=places[2], + start_date=datetime.now() + timedelta(days=15), + end_date=datetime.now() + timedelta(days=19), + price_per_night=120, + num_nights=4 + ), + rating=3, + review_body='Suspendisse cursus ex et turpis dignissim dignissim' + ), + Reviews( + booking=Bookings( + user=users[3], + place=places[3], + start_date=datetime.now() + timedelta(days=2), + end_date=datetime.now() + timedelta(days=7), + price_per_night=300, + num_nights=5 + ), + rating=4, + review_body='Suspendisse ultricies arcu lectus' + ), + Reviews( + booking=Bookings( + user=users[4], + place=places[4], + start_date=datetime.now() + timedelta(days=1), + end_date=datetime.now() + timedelta(days=10), + price_per_night=800, + num_nights=3 + ), + rating=5, + review_body='Putent sententiae scribentur ne vis' + ), + Reviews( + booking=Bookings( + user=users[5], + place=places[5], + start_date=datetime.now() + timedelta(days=2), + end_date=datetime.now() + timedelta(days=8), + price_per_night=80, + num_nights=10 + ), + rating=3, + review_body='Debet invenire sed ne' + ), + ] + + with subtransactions(session): + session.add_all(users) + + with subtransactions(session): + session.add_all(hosts) + + with subtransactions(session): + session.add_all(cities) + + with subtransactions(session): + session.add_all(places) + + with subtransactions(session): + session.add_all(reviews) + + +if __name__ == '__main__': + main() diff --git a/examples/airbnb/schema.json b/examples/airbnb/schema.json new file mode 100644 index 00000000..3e8f2b0a --- /dev/null +++ b/examples/airbnb/schema.json @@ -0,0 +1,71 @@ +[ + { + "index": "airbnb", + "doc_type": "airbnb", + "nodes": [ + { + "table": "users", + "columns": [ + + ], + "children": [ + { + "table": "bookings", + "columns": [ + ], + "relationship": { + "variant": "object", + "type": "one_to_one" + }, + "children":[ + { + "table": "places", + "relationship": { + "variant": "object", + "type": "one_to_one" + }, + "children": [ + { + "table": "cities", + "relationship": { + "variant": "object", + "type": "one_to_one" + }, + "children": [ + { + "table": "countries", + "relationship": { + "variant": "object", + "type": "one_to_one" + } + } + ] + }, + { + "table": "hosts", + "columns": [ + "email" + ], + "relationship": { + "variant": "scalar", + "type": "one_to_one" + } + } + ] + }, + { + "table": "reviews", + "columns": [ + ], + "relationship": { + "variant": "object", + "type": "one_to_one" + } + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/examples/airbnb/schema.py b/examples/airbnb/schema.py new file mode 100644 index 00000000..80e2753f --- /dev/null +++ b/examples/airbnb/schema.py @@ -0,0 +1,124 @@ +import json +from datetime import datetime + +import sqlalchemy as sa +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import UniqueConstraint + +import settings +from base import create_database, pg_engine +from helper import teardown + +Base = declarative_base() + + +class Users(Base): + __tablename__ = 'users' + __table_args__ = ( + UniqueConstraint('email', ), + ) + id = sa.Column(sa.Integer, primary_key=True) + email = sa.Column(sa.String, unique=True, nullable=False) + + +class Hosts(Base): + __tablename__ = 'hosts' + __table_args__ = ( + UniqueConstraint('email', ), + ) + id = sa.Column(sa.Integer, primary_key=True) + email = sa.Column(sa.String, unique=True, nullable=False) + + +class Countries(Base): + __tablename__ = 'countries' + __table_args__ = ( + UniqueConstraint('name', 'country_code'), + ) + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String, nullable=False) + country_code = sa.Column(sa.String, nullable=False) + + +class Cities(Base): + __tablename__ = 'cities' + __table_args__ = ( + UniqueConstraint('name', 'country_id'), + ) + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String, nullable=False) + country_id = sa.Column( + sa.Integer, sa.ForeignKey(Countries.id) + ) + country = sa.orm.relationship( + Countries, + backref=sa.orm.backref('countries') + ) + + +class Places(Base): + __tablename__ = 'places' + __table_args__ = ( + UniqueConstraint('host_id', 'address', 'city_id'), + ) + id = sa.Column(sa.Integer, primary_key=True) + host_id = sa.Column(sa.Integer, sa.ForeignKey(Hosts.id)) + address = sa.Column(sa.String, nullable=False) + city_id = sa.Column(sa.Integer, sa.ForeignKey(Cities.id)) + host = sa.orm.relationship( + Hosts, + backref=sa.orm.backref('hosts') + ) + city = sa.orm.relationship( + Cities, + backref=sa.orm.backref('cities') + ) + + +class Bookings(Base): + __tablename__ = 'bookings' + __table_args__ = ( + UniqueConstraint('user_id', 'place_id', 'start_date'), + ) + id = sa.Column(sa.Integer, primary_key=True) + user_id = sa.Column(sa.Integer, sa.ForeignKey(Users.id)) + place_id = sa.Column(sa.Integer, sa.ForeignKey(Places.id)) + start_date = sa.Column(sa.DateTime, default=datetime.now()) + end_date = sa.Column(sa.DateTime, default=datetime.now()) + price_per_night = sa.Column(sa.Float, default=0) + num_nights = sa.Column(sa.Integer, nullable=False, default=1) + user = sa.orm.relationship(Users) + place = sa.orm.relationship(Places) + + +class Reviews(Base): + __tablename__ = 'reviews' + __table_args__ = ( + UniqueConstraint('booking_id',), + ) + id = sa.Column(sa.Integer, primary_key=True) + booking_id = sa.Column(sa.Integer, sa.ForeignKey(Bookings.id)) + rating = sa.Column(sa.SmallInteger, nullable=True) + review_body = sa.Column(sa.Text, nullable=True) + booking = sa.orm.relationship( + Bookings, + backref=sa.orm.backref('bookings') + ) + + +def setup(): + for schema in json.load(open(settings.SCHEMA)): + create_database(schema.get('index')) + # create schema + engine = pg_engine(database=schema.get('index')) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + +def main(): + teardown() + setup() + + +if __name__ == '__main__': + main() diff --git a/pgsync/__init__.py b/pgsync/__init__.py new file mode 100644 index 00000000..08f1744b --- /dev/null +++ b/pgsync/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +"""Top-level package for PGSync.""" + +__author__ = """Tolu Aina""" +__email__ = 'toluaina@hotmail.com' +__version__ = '1.0.0' diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..7adb81aa --- /dev/null +++ b/requirements.in @@ -0,0 +1,35 @@ +pip +pip-tools +bumpversion +wheel +watchdog +tox +coverage +twine +sqlalchemy +# clip to psycopg2 2.7.7. 2.8+ has compatibility issues +psycopg2-binary==2.7.7 +elasticsearch-dsl>=6.0.0,<8.0.0 +sqlparse +redis +environs +newrelic +python-logstash +faker +pytest +pytest-cov +pytest-runner +pytest-mock +pytest-sugar +pytest-runner +mock +futures +flake8 +flake8_docstrings +flake8-debugger +flake8-print +flake8-todo +flake8-isort +pep8-naming +pydocstyle +supervisor diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..11c23afb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,88 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements.txt requirements.in +# +argh==0.26.2 # via watchdog +atomicwrites==1.3.0 # via pytest +attrs==19.1.0 # via packaging, pytest +bleach==3.1.0 # via readme-renderer +bumpversion==0.5.3 +certifi==2019.9.11 # via requests +chardet==3.0.4 # via requests +click==7.0 # via pip-tools +coverage==4.5.4 +docutils==0.15.2 # via readme-renderer +elasticsearch-dsl==7.0.0 +elasticsearch==7.0.4 # via elasticsearch-dsl +entrypoints==0.3 # via flake8 +environs==5.2.1 +faker==2.0.1 +filelock==3.0.12 # via tox +flake8-debugger==3.1.0 +flake8-docstrings==1.4.0 +flake8-isort==2.7.0 +flake8-polyfill==1.0.2 # via pep8-naming +flake8-print==3.1.0 +flake8-todo==0.7 +flake8==3.7.8 +futures==3.1.1 +idna==2.8 # via requests +importlib-metadata==0.22 # via pluggy, pytest, tox +isort==4.3.21 # via flake8-isort +marshmallow==3.0.4 # via environs +mccabe==0.6.1 # via flake8 +meld3==2.0.0 # via supervisor +mock==3.0.5 +more-itertools==7.2.0 # via pytest, zipp +newrelic==5.0.2.126 +packaging==19.1 # via pytest, pytest-sugar, tox +pathtools==0.1.2 # via watchdog +pep8-naming==0.8.2 +pip-tools==4.1.0 +pkginfo==1.5.0.1 # via twine +pluggy==0.13.0 # via pytest, tox +psycopg2-binary==2.7.7 +py==1.8.0 # via pytest, tox +pycodestyle==2.5.0 # via flake8, flake8-debugger, flake8-print, flake8-todo +pydocstyle==4.0.1 +pyflakes==2.1.1 # via flake8 +pygments==2.4.2 # via readme-renderer +pyparsing==2.4.2 # via packaging +pytest-cov==2.7.1 +pytest-mock==1.10.4 +pytest-runner==5.1 +pytest-sugar==0.9.2 +pytest==5.1.2 +python-dateutil==2.8.0 # via elasticsearch-dsl, faker +python-dotenv==0.10.3 # via environs +python-logstash==0.4.6 +pyyaml==5.1.2 # via watchdog +readme-renderer==24.0 # via twine +redis==3.3.8 +requests-toolbelt==0.9.1 # via twine +requests==2.22.0 # via requests-toolbelt, twine +six==1.12.0 # via bleach, elasticsearch-dsl, faker, flake8-print, mock, packaging, pip-tools, python-dateutil, readme-renderer, tox +snowballstemmer==1.9.1 # via pydocstyle +sqlalchemy==1.3.8 +sqlparse==0.3.0 +supervisor==4.0.4 +termcolor==1.1.0 # via pytest-sugar +testfixtures==6.10.0 # via flake8-isort +text-unidecode==1.2 # via faker +toml==0.10.0 # via tox +tox==3.14.0 +tqdm==4.35.0 # via twine +twine==1.14.0 +urllib3==1.25.3 # via elasticsearch, requests +virtualenv==16.7.5 # via tox +watchdog==0.9.0 +wcwidth==0.1.7 # via pytest +webencodings==0.5.1 # via bleach +wheel==0.33.6 +zipp==0.6.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip==19.2.3 +# setuptools==41.2.0 # via twine diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..bc75cf55 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,43 @@ +[bumpversion] +current_version = 1.0.0 +commit = True +tag = True + +[bumpversion:file:setup.py] +search = version='{current_version}' +replace = version='{new_version}' + +[bumpversion:file:pgsync/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bdist_wheel] +universal = 1 + +[flake8] +exclude = docs +max-line-length = 79 +max-complexity = 12 +doctests = True +statistics = True +benchmark = True + +[isort] +line_length=79 +multi_line_output=3 +include_trailing_commas=True +not_skip=__init__.py +order_by_type=False + +[aliases] +# Define setup.py command aliases here +test = pytest + +[tool:pytest] +collect_ignore = ['setup.py'] +# Always run with long traceback and local variables on failure +addopts = --showlocals --no-print-logs -p no:warnings +# ignore all hidden files and folders +norecursedirs = .* tmp* ci src +python_files = tests.py test_*.py *_tests.py +python_functions = test_* diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..318e1ea0 --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""The setup script.""" + +from setuptools import setup, find_packages + +with open('README.rst') as readme_file: + readme = readme_file.read() + +with open('HISTORY.rst') as history_file: + history = history_file.read() + +with open('requirements.txt') as requirements_file: + requirements = requirements_file.read() + +setup_requirements = ['pytest-runner', ] + +test_requirements = ['pytest', ] + +setup( + author="Tolu Aina", + author_email='toluaina@hotmail.com', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + description="Postgres to Elasticsearch sync", + install_requires=requirements, + long_description=readme + '\n\n' + history, + include_package_data=True, + keywords='pgsync', + name='pgsync', + packages=find_packages(include=['pgsync']), + setup_requires=setup_requirements, + test_suite='tests', + tests_require=test_requirements, + url='https://github.com/toluaina/essync', + version='1.0.0', + zip_safe=False, +) diff --git a/supervisor/pgsync.conf b/supervisor/pgsync.conf new file mode 100644 index 00000000..32ab94a2 --- /dev/null +++ b/supervisor/pgsync.conf @@ -0,0 +1,47 @@ +[program:schema] +directory=/code +command=python examples/airbnb/schema.py +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +autostart=true +autorestart=false +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:data] +directory=/code +command=python examples/airbnb/data.py +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +autostart=true +autorestart=false +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:bootstrap] +directory=/code +command=bootstrap +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +autostart=true +autorestart=false +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:pgsync] +directory=/code +command=pgsync +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 \ No newline at end of file diff --git a/supervisor/supervisord.conf b/supervisor/supervisord.conf new file mode 100644 index 00000000..adf2d5c4 --- /dev/null +++ b/supervisor/supervisord.conf @@ -0,0 +1,21 @@ +[unix_http_server] +file=/tmp/supervisor.sock ; path to your socket file + +[supervisord] +logfile=/tmp/supervisord.log ; supervisord log file +logfile_maxbytes=50MB ; maximum size of logfile before rotation +logfile_backups=10 ; number of backed up logfiles +loglevel=%(ENV_LOG_LEVEL)s ; info, debug, warn, trace +pidfile=/tmp/supervisord.pid ; pidfile location +nodaemon=true ; run supervisord as a daemon +minfds=1024 ; number of startup file descriptors +minprocs=200 ; number of process descriptors + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/supervisor/supervisord_entrypoint.sh b/supervisor/supervisord_entrypoint.sh new file mode 100644 index 00000000..48a64d28 --- /dev/null +++ b/supervisor/supervisord_entrypoint.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +set -e + +if [ $# -eq 0 ] || [ "${1#-}" != "$1" ]; then + set -- supervisord "$@" +fi + +exec "$@" + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..fbe89d09 --- /dev/null +++ b/tox.ini @@ -0,0 +1,21 @@ +[tox] +envlist = py36, py367, flake8 + +[testenv:flake8] +basepython = python +deps = flake8 +commands = flake8 pgsync + +[testenv] +setenv = + PYTHONPATH = {toxinidir} +deps = + -r{toxinidir}/requirements_dev.txt +; If you want to make tox run the tests with the same versions, create a +; requirements.txt with the pinned versions and uncomment the following line: +; -r{toxinidir}/requirements.txt +commands = + pip install -U pip + py.test --basetemp={envtmpdir} + +