Docker for production

.gitignore

@@ -5,6 +5,7 @@
 *.sqlite3
 htmlcov/
 npm-debug.log
+db/
 
 # cache, compiled & downloaded files
 *.pyc

.travis.yml

@@ -5,10 +5,6 @@ language: python
 python:
   - 3.5
 
-cache:
-  directories:
-    - sysconfcpus
-
 addons:
     postgresql: "9.6"
 
@@ -19,32 +15,17 @@ env:
   - DATABASE_URL="postgres://postgres@localhost/jarbas"
 
 before_install:
-  - |
-    if [ ! -d sysconfcpus/bin ];
-    then
-      git clone https://github.com/obmarg/libsysconfcpus.git;
-      cd libsysconfcpus;
-      ./configure --prefix=$TRAVIS_BUILD_DIR/sysconfcpus;
-      make && make install;
-      cd ..;
-    fi
-  - nvm install 6
-  - nvm use 6
-  - npm install -g yarn
   - python -m pip install -U pip
 
 install:
   - psql -U postgres -c 'create database "jarbas";'
-  - yarn install
   - python -m pip install -r requirements-dev.txt coveralls
 
 before_script:
-  - $TRAVIS_BUILD_DIR/sysconfcpus/bin/sysconfcpus -n 2 yarn assets
   - python manage.py migrate
   - python manage.py collectstatic --no-input
 
 script:
-  - $TRAVIS_BUILD_DIR/sysconfcpus/bin/sysconfcpus -n 2 yarn test
   - coverage run manage.py test -v 3
 
 after_success:

Dockerfile

@@ -1,12 +1,30 @@
-FROM python:3.5
-COPY requirements.txt /requirements.txt
-COPY requirements-dev.txt /requirements-dev.txt
-RUN python -m pip install -U pip
-RUN python -m pip install -r requirements-dev.txt
-RUN apt-get update && apt-get install -y postgresql postgresql-contrib
+FROM python:3.5.4-alpine
+ENV PYTHONUNBUFFERED 1
+
+# set timezone
+RUN apk update && apk add --no-cache tzdata && \
+    cp /usr/share/zoneinfo/America/Sao_Paulo /etc/localtime && \
+    echo "America/Sao_Paulo" >  /etc/timezone
+
+# pyscopg2 dependencies
+RUN apk update && apk add --no-cache \
+    gcc \
+    musl-dev \
+    postgresql-dev \
+    python3-dev
+
+# install git so we can pip install from github repos
+RUN apk update && apk add --no-cache git
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+COPY ./requirements-dev.txt /code/requirements-dev.txt
+RUN python -m pip install -U pip && \
+    python -m pip install -r requirements-dev.txt
+
 COPY manage.py /code/manage.py
 COPY jarbas /code/jarbas
-WORKDIR /code
-RUN echo "America/Sao_Paulo" > /etc/timezone && dpkg-reconfigure -f noninteractive tzdata
-VOLUME /code/staticfiles
-CMD ["gunicorn", "jarbas.wsgi:application", "--reload", "--bind", "0.0.0.0:8001", "--workers", "4"]
+RUN python manage.py collectstatic --no-input
+
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]

Dockerfile-elm

@@ -1,14 +1,20 @@
-FROM node:6.9.1
+FROM node:8.4.0
+
+# npm5 issue with updating itself: https://github.com/npm/npm/issues/18278
+# RUN npm install -g npm@latest
 
-RUN npm install yarn
-RUN ln -s /node_modules/yarn/bin/yarn.js /usr/bin/yarn
 WORKDIR /code
 
-COPY yarn.lock yarn.lock
 COPY package.json package.json
+COPY package-lock.json package-lock.json
 COPY elm-package.json elm-package.json
 COPY gulpfile.js gulpfile.js
-RUN yarn install --pure-lockfile
 
-COPY jarbas jarbas
-CMD ["yarn", "assets"]
+RUN npm install
+
+# npm5 issue w/ postinstall: https://github.com/npm/npm/issues/17316
+RUN ./node_modules/.bin/elm-package install --yes
+
+COPY ./jarbas /code/jarbas
+
+CMD ["npm", "run", "watch"]

Dockerfile-nginx

@@ -1,3 +1,5 @@
-FROM nginx:1.11.5
+FROM nginx:1.13.3-alpine
+
 COPY contrib/nginx.docker /etc/nginx/conf.d/default.conf
-RUN mkdir -p /opt/jarbas
+
+RUN mkdir -p /code/staticfiles

README.md

@@ -19,9 +19,9 @@ Jarbas is in charge of making data from [CEAP](https://github.com/datasciencebr/
     1. [Company](#company)
     1. [Tapioca Jarbas](#tapioca-jarbas)
 1. [Installing](#installing)
+    1. [Settings](#settings)
     1. [Using Docker](#using-docker)
     1. [Local install](#local-install)
-    1. [Settings](#settings)
 
 ## JSON API endpoints
 
@@ -132,63 +132,110 @@ There is also a [tapioca-wrapper](https://github.com/vintasoftware/tapioca-wrapp
 
 ## Installing
 
+### Settings
+
+Copy `contrib/.env.sample` as `.env` in the project's root folder and adjust your settings. These are the main variables:
+
+##### Django settings
+
+* `DEBUG` (_bool_) enable or disable [Django debug mode](https://docs.djangoproject.com/en/1.10/ref/settings/#debug)
+* `SECRET_KEY` (_str_) [Django's secret key](https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-SECRET_KEY)
+* `ALLOWED_HOSTS` (_str_) [Django's allowed hosts](https://docs.djangoproject.com/en/1.10/ref/settings/#allowed-hosts)
+* `USE_X_FORWARDED_HOST` (_bool_) [Whether to use the `X-Forwarded-Host` header](https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-USE_X_FORWARDED_HOST)
+* `CACHE_BACKEND` (_str_) [Cache backend](https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-CACHES-BACKEND) (e.g. `django.core.cache.backends.memcached.MemcachedCache`)
+* `CACHE_LOCATION` (_str_) [Cache location](https://docs.djangoproject.com/en/1.10/ref/settings/#location) (e.g. `localhost:11211`)
+* `SECURE_PROXY_SSL_HEADER` _(str)_ [Django secure proxy SSL header](https://docs.djangoproject.com/en/1.10/ref/settings/#secure-proxy-ssl-header) (e.g. `HTTP_X_FORWARDED_PROTO,https` transforms in tuple `('HTTP_X_FORWARDED_PROTO', 'https')`)
+
+##### Database
+
+* `DATABASE_URL` (_string_) [Database URL](https://github.com/kennethreitz/dj-database-url#url-schema), must be [PostgreSQL](https://www.postgresql.org) since Jarbas uses [JSONField](https://docs.djangoproject.com/en/1.10/ref/contrib/postgres/fields/#jsonfield).
+
+##### Amazon S3 settings
+
+* `AMAZON_S3_BUCKET` (_str_) Name of the Amazon S3 bucket to look for datasets (e.g. `serenata-de-amor-data`)
+* `AMAZON_S3_REGION` (_str_) Region of the Amazon S3 (e.g. `s3-sa-east-1`)
+* `AMAZON_S3_CEAPTRANSLATION_DATE` (_str_) File name prefix for dataset guide (e.g. `2016-08-08` for `2016-08-08-ceap-datasets.md`)
+
+##### Google settings
+
+* `GOOGLE_ANALYTICS` (_str_) Google Analytics tracking code (e.g. `UA-123456-7`)
+* `GOOGLE_STREET_VIEW_API_KEY` (_str_) Google Street View Image API key
+
+##### Twitter settings
+
+* `TWITTER_CONSUMER_KEY` (_str_) Twitter API key
+* `TWITTER_CONSUMER_SECRET` (_str_) Twitter API secret
+* `TWITTER_ACCESS_TOKEN` (_str_) Twitter access token
+* `TWITTER_ACCESS_SECRET` (_str_) Twitter access token secret
+
+To get this credentials follow [`python-twitter`
+instructions](https://python-twitter.readthedocs.io/en/latest/getting_started.html#getting-your-application-tokens).
 
 ### Using Docker
 
-With [Docker](https://docs.docker.com/engine/installation/) and [Docker Compose](https://docs.docker.com/compose/install/)) installed just run:
+There are two combinations in terms of With [Docker](https://docs.docker.com/engine/installation/) and [Docker Compose](https://docs.docker.com/compose/install/)
+ environments.
 
-```console
-$ make run.devel
-```
+* **Develoment**: simply running `docker-compose …` will trigger `docker-compose.yml` and `docker-compose.override.yml` with optimun configuration for developing such as:
+  * automatic serving static files through Django
+  * restarting the Django on Python files changes
+  * rebuilding JS from Elm files on save
+  * skipping server cache
+* **Production**: passing a specific configurarion as `docker-compose -f docker-compose.yml -f docker-compose.prod.yml …` will launch a more robust environment with production in mind, among others:
+  * `nginx` in front of Django
+  * server-side cache with memcached
+  * manually generate JS after edits on Elm files
+  * manually run `collectstatic` command is static changes
+  * manually restarting server on change
+
+That said instructions here keep it simple and runs with the development set up. To swicth always add `-f docker-compose.yml -f docker-compose.prod.yml` after `docker-compose`.
 
-or
+#### Build and start services
 
 ```console
 $ docker-compose up -d
-$ docker-compose run --rm jarbas python manage.py migrate
-$ docker-compose run --rm jarbas python manage.py ceapdatasets
-$ docker-compose run --rm jarbas python manage.py tweets
-$ docker-compose run --rm jarbas python manage.py collectstatic --no-input
 ```
 
-You can access it at [`localhost:8000`](http://localhost:8000/). However your database starts empty, but you can use sample data to development using this command:
+#### Create and seed the database with sample data
 
-```console
-$ make seed.sample
+Creating the database and applying migrations:
+
+```
+$ docker-compose run --rm django python manage.py migrate
 ```
 
-or
+Seeding it with sample data:
 
 ```console
-$ docker-compose run --rm jarbas python manage.py reimbursements contrib/sample-data/reimbursements_sample.xz
-$ docker-compose run --rm jarbas python manage.py companies contrib/sample-data/companies_sample.xz
-$ docker-compose run --rm jarbas python manage.py suspicions contrib/sample-data/suspicions_sample.xz
+$ docker-compose run --rm django python manage.py reimbursements /mnt/data/reimbursements_sample.xz
+$ docker-compose run --rm django python manage.py companies /mnt/data/companies_sample.xz
+$ docker-compose run --rm django python manage.py suspicions /mnt/data/suspicions_sample.xz
+$ docker-compose run --rm django python manage.py tweets
 ```
 
-You can get the datasets running [Rosie](https://github.com/datasciencebr/rosie) or directly with the [toolbox](https://github.com/datasciencebr/rosie).
+If you're interesting in having a database full of data you can get the datasets running [Rosie](https://github.com/datasciencebr/rosie). 
+To add a fresh new `reimbursements.xz` or `suspicions.xz` brewed by [Rosie](https://github.com/datasciencebr/rosie), or a `companies.xz` you've got from the [toolbox](https://github.com/datasciencebr/serenata-toolbox), you just need copy these files to `contrib/data` and refer to them inside the container from the path `/mnt/data/`.
 
-To add a fresh new `reimbursements.xz` brewed by [Rosie](https://github.com/datasciencebr/rosie) or made with our [toolbox](https://github.com/datasciencebr/rosie), you just need to have this file **inside project folder** and give the path at the end of the command, as bellow:
+#### Acessing Jabas
 
-```console
-$ docker-compose run --rm jarbas python manage.py reimbursements path/to/my/fresh_new_reimbursements.xz
-```
-
-To change any of the default environment variables defined in the `docker-compose.yml` just export it in a local environment variable, so when you run Jarbas it will get them.
+You can access it at [`localhost:8000`](http://localhost:8000/) in development mode or [`localhost`](http://localhost:80/) in production mode. 
 
-Finally if you would like to access the Django Admin for an alternative view of the reimbursements, you can access it at [`localhost:8000/admin/`](http://localhost:8000/admin/) creating an user with:
 
 ```console
-$ docker-compose run --rm jarbas python manage.py createsuperuser
+$ docker-compose run --rm django python manage.py reimbursements path/to/my/fresh_new_reimbursements.xz
 ```
 
+To change any of the default environment variables defined in the `docker-compose.yml` just export it in a local environment variable, so when you run Jarbas it will get them.
+
 ### Local install
 
 #### Requirements
 
-Jarbas requires [Python 3.5](http://python.org), [Node.js 6+](https://nodejs.org/en/), [Yarn](https://yarnpkg.com), and [PostgreSQL 9.4+](https://www.postgresql.org). Once you have `pip` and `yarn` available install the dependencies:
+Jarbas requires [Python 3.5](http://python.org), [Node.js 8](https://nodejs.org/en/), and [PostgreSQL 9.6](https://www.postgresql.org). Once you have `pip` and `npm` available install the dependencies:
 
 ```console
-$ yarn install
+$ npm install
+$ ./node_modules/.bin/elm-package install --yes  # this might not be necessary https://github.com/npm/npm/issues/17316
 $ python -m pip install -r requirements-dev.txt
 ```
 
@@ -220,14 +267,14 @@ $ python manage.py tweets
 $ python manage.py ceapdatasets
 ```
 
-You can get the datasets running [Rosie](https://github.com/datasciencebr/rosie) or directly with the [toolbox](https://github.com/datasciencebr/serenata-toolbox).
+There are sample files to seed yout database inside `contrib/data/`. You can get full datasets running [Rosie](https://github.com/datasciencebr/rosie) or directly with the [toolbox](https://github.com/datasciencebr/serenata-toolbox).
 
 #### Generate static files
 
 We generate assets through NodeJS, so run it before Django collecting static files:
 
 ```console
-$ yarn assets
+$ npm run assets
 $ python manage.py collectstatic
 
 ```
@@ -239,7 +286,6 @@ Not sure? Test it!
 ```
 $ python manage.py check
 $ python manage.py test
-$ yarn test
 ```
 
 #### Ready!
@@ -254,43 +300,3 @@ If you would like to access the Django Admin for an alternative view of the reim
 ```console
 $ python manage.py createsuperuser
 ```
-
-
-### Settings
-
-If **you are not** using Docker copy `contrib/.env.sample` as `.env` in the project's root folder and adjust your settings. These are the main variables:
-
-##### Django settings
-
-* `DEBUG` (_bool_) enable or disable [Django debug mode](https://docs.djangoproject.com/en/1.10/ref/settings/#debug)
-* `SECRET_KEY` (_str_) [Django's secret key](https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-SECRET_KEY)
-* `ALLOWED_HOSTS` (_str_) [Django's allowed hosts](https://docs.djangoproject.com/en/1.10/ref/settings/#allowed-hosts)
-* `USE_X_FORWARDED_HOST` (_bool_) [Whether to use the `X-Forwarded-Host` header](https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-USE_X_FORWARDED_HOST)
-* `CACHE_BACKEND` (_str_) [Cache backend](https://docs.djangoproject.com/en/1.10/ref/settings/#std:setting-CACHES-BACKEND) (e.g. `django.core.cache.backends.memcached.MemcachedCache`)
-* `CACHE_LOCATION` (_str_) [Cache location](https://docs.djangoproject.com/en/1.10/ref/settings/#location) (e.g. `localhost:11211`)
-* `SECURE_PROXY_SSL_HEADER` _(str)_ [Django secure proxy SSL header](https://docs.djangoproject.com/en/1.10/ref/settings/#secure-proxy-ssl-header) (e.g. `HTTP_X_FORWARDED_PROTO,https` transforms in tuple `('HTTP_X_FORWARDED_PROTO', 'https')`)
-
-##### Database
-
-* `DATABASE_URL` (_string_) [Database URL](https://github.com/kennethreitz/dj-database-url#url-schema), must be [PostgreSQL](https://www.postgresql.org) since Jarbas uses [JSONField](https://docs.djangoproject.com/en/1.10/ref/contrib/postgres/fields/#jsonfield).
-
-##### Amazon S3 settings
-
-* `AMAZON_S3_BUCKET` (_str_) Name of the Amazon S3 bucket to look for datasets (e.g. `serenata-de-amor-data`)
-* `AMAZON_S3_REGION` (_str_) Region of the Amazon S3 (e.g. `s3-sa-east-1`)
-* `AMAZON_S3_CEAPTRANSLATION_DATE` (_str_) File name prefix for dataset guide (e.g. `2016-08-08` for `2016-08-08-ceap-datasets.md`)
-
-##### Google settings
-
-* `GOOGLE_ANALYTICS` (_str_) Google Analytics tracking code (e.g. `UA-123456-7`)
-* `GOOGLE_STREET_VIEW_API_KEY` (_str_) Google Street View Image API key
-
-##### Twitter settings
-
-* `TWITTER_CONSUMER_KEY` (_str_) Twitter API key
-* `TWITTER_CONSUMER_SECRET` (_str_) Twitter API secret
-* `TWITTER_ACCESS_TOKEN` (_str_) Twitter access token
-* `TWITTER_ACCESS_SECRET` (_str_) Twitter access token secret
-
-To get this credentials follow [`python-twitter`
-instructions](https://python-twitter.readthedocs.io/en/latest/getting_started.html#getting-your-application-tokens).

contrib/.env.sample

@@ -1,6 +1,6 @@
 SECRET_KEY=jaa-aaa-ar-bas
 DEBUG=False
-ALLOWED_HOSTS=localhost,127.0.0.1
+ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0
 
 DATABASE_URL=postgres://jarbas:mysecretpassword@localhost/jarbas