first commit

.gitignore

@@ -0,0 +1,7 @@
+tests/cache
+tests/playbook.retry
+tests/__pycache__
+.molecule
+.cache
+.vagrant
+tests/roles

CHANGELOG.md

@@ -0,0 +1,9 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a changelog](https://github.com/olivierlacan/keep-a-changelog).
+
+## [Unreleased](https://github.com/idealista/airflow-role/tree/develop)
+
+## [1.0.0](https://github.com/idealista/airflow-role/tree/1.0.0)
+### Added
+- *First release*

README.md

@@ -0,0 +1,97 @@
+![Logo](https://raw.githubusercontent.com/idealista/airflow-role/master/logo.gif)
+
+# Apache Airflow Ansible role
+
+This ansible role installs a Apache Airflow server in a debian environment.
+
+- [Getting Started](#getting-started)
+	- [Prerequisities](#prerequisities)
+	- [Installing](#installing)
+- [Usage](#usage)
+- [Testing](#testing)
+- [Built With](#built-with)
+- [Versioning](#versioning)
+- [Authors](#authors)
+- [License](#license)
+- [Contributing](#contributing)
+
+## Getting Started
+
+These instructions will get you a copy of the role for your ansible playbook. Once launched, it will install a [Apache Airflow](https://airflow.incubator.apache.org/) server in a Debian system.
+
+### Prerequisities
+
+Ansible 2.2.1.0 version installed.
+Inventory destination should be a Debian environment.
+
+For testing purposes, [Molecule](https://molecule.readthedocs.io/) with [Vagrant](https://www.vagrantup.com/) as driver (with [landrush](https://github.com/vagrant-landrush/landrush) plugin) and [VirtualBox](https://www.virtualbox.org/) as provider.
+
+### Installing
+
+Create or add to your roles dependency file (e.g requirements.yml) from GitHub:
+
+```
+- src: http://github.com/idealista/airflow-role.git
+  scm: git
+  version: 1.0.0
+  name: airflow
+```
+
+or using [Ansible Galaxy](https://galaxy.ansible.com/idealista/airflow-role/) as origin if you prefer:
+
+```
+- src: idealista.airflow-role
+```
+
+Install the role with ansible-galaxy command:
+
+```
+ansible-galaxy install -p roles -r requirements.yml -f
+```
+
+Use in a playbook:
+
+```
+---
+- hosts: someserver
+  roles:
+    - { role: airflow }
+```
+
+## Usage
+
+Look to the defaults properties file to see the possible configuration properties.
+
+## Testing
+
+```
+molecule test --platform=Debian9
+```
+
+See molecule.yml to check possible testing platforms.
+
+## Built With
+
+![Ansible](https://img.shields.io/badge/ansible-2.2.1.0-green.svg)
+
+## Versioning
+
+For the versions available, see the [tags on this repository](https://github.com/idealista/airflow-role/tags).
+
+Additionaly you can see what change in each version in the [CHANGELOG.md](CHANGELOG.md) file.
+
+## Authors
+
+* **Idealista** - *Work with* - [idealista](https://github.com/idealista)
+
+See also the list of [contributors](https://github.com/idealista/airflow-role/contributors) who participated in this project.
+
+## License
+
+![Apache 2.0 Licence](https://img.shields.io/hexpm/l/plug.svg)
+
+This project is licensed under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license - see the [LICENSE.txt](LICENSE.txt) file for details.
+
+## Contributing
+
+Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.

defaults/main.yml

@@ -0,0 +1,196 @@
+---
+
+## General
+airflow_version: 1.8.1
+airflow_extra_packages:
+
+airflow_required_libs:
+  - python-pip
+  - acl
+
+# Owner
+airflow_user: airflow
+airflow_group: airflow
+
+## Service options
+
+airflow_scheduler_runs: 5
+
+airflow_services:
+  airflow-webserver:
+    enabled: yes
+    state: started
+  airflow-scheduler:
+    enabled: yes
+    state: started
+  airflow-worker:
+    enabled: no
+    state: started
+
+# Files & Paths
+airflow_home: /etc/airflow
+airflow_dags_folder: "{{ airflow_home }}/dags"
+airflow_logs_folder: /var/log/airflow
+airflow_child_process_log_folder: "{{ airflow_logs_folder }}/scheduler"
+airflow_environment_file_folder: /etc/sysconfig
+
+# Allowing playbooks to provide external config files&templates
+airflow_extra_conf_path: "{{ playbook_dir }}/files/airflow"
+airflow_extra_conf_template_path: "{{ playbook_dir }}/templates/airflow"
+
+# AIRFLOW CONFIGURATION
+# ---------------------
+
+airflow_load_examples: True
+
+# The executor class that airflow should use. Choices include
+# SequentialExecutor, LocalExecutor, CeleryExecutor
+airflow_executor: SequentialExecutor
+
+airflow_parallelism: 32
+airflow_dag_concurrency: 16
+airflow_dags_are_paused_at_creation: True
+airflow_non_pooled_task_slot_count: 128
+airflow_max_active_runs_per_dag: 16
+
+airflow_fernet_key:
+
+airflow_donot_pickle: False
+airflow_dagbag_import_timeout:
+
+airflow_task_runner: BashTaskRunner
+
+airflow_default_impersonation:
+airflow_unit_test_mode: False
+
+airflow_plugins_folder: "{{ airflow_home }}/plugins"
+
+# REMOTE LOGS
+airflow_remote_base_log_folder:
+airflow_remote_log_conn_id:
+airflow_encrypt_s3_logs: False
+airflow_s3_log_folder: # DEPRECATED
+
+# DB
+airflow_database_conn: sqlite:////etc/airflow/airflow.db
+airflow_database_pool_size: 5
+airflow_database_pool_recycle: 2000
+
+# CLI
+airflow_cli_api_client: airflow.api.client.local_client
+airflow_cli_api_endpoint_url: http://localhost:8080
+
+# API
+airflow_auth_backend: airflow.api.auth.backend.default
+
+# WEBSERVER
+airflow_webserver_base_url: http://localhost:8080
+airflow_webserver_host: 0.0.0.0
+airflow_webserver_port: 8080
+airflow_webserver_workers: 4
+airflow_webserver_worker_timeout: 120
+airflow_webserver_ssl_cert:
+airflow_webserver_ssl_key:
+airflow_webserver_worker_refresh_batch_size: 1
+airflow_webserver_worker_refresh_interval: 30
+airflow_webserver_secret_key: temporary_key
+airflow_webserver_worker_class: sync
+airflow_webserver_expose_config: False
+airflow_webserver_filter_by_owner: False
+airflow_webserver_owner_mode: user
+airflow_webserver_dag_orientation: LR
+airflow_webserver_demo_mode: False
+airflow_webserver_log_fetch_timeout_sec: 5
+airflow_webserver_hide_paused_dags_by_default: False
+
+## Webserver Authentication (http://pythonhosted.org/airflow/security.html#web-authentication)
+
+# Choices of auth_backend include:
+# - airflow.contrib.auth.backends.password_auth
+# - airflow.contrib.auth.backends.ldap_auth
+# - airflow.contrib.auth.backends.github_enterprise_auth
+# - others? :)
+airflow_webserver_authenticate: False
+airflow_webserver_auth_backend:
+
+# Operators
+airflow_operator_default_owner: Airflow
+airflow_operator_default_cpus: 1
+airflow_operator_default_ram: 512
+airflow_operator_default_disk: 512
+airflow_operator_default_gpus: 0
+
+# LDAP (only applies if airflow_webserver_auth_backend == "airflow.contrib.auth.backends.ldap_auth")
+airflow_ldap_uri:
+airflow_ldap_user_filter:
+airflow_ldap_user_name_attr:
+airflow_ldap_superuser_filter:
+airflow_ldap_data_profiler_filter:
+airflow_ldap_bind_user:
+airflow_ldap_bind_password:
+airflow_ldap_basedn:
+airflow_ldap_cacert:
+airflow_ldap_search_scope:
+
+# MAIL
+airflow_email_backend: airflow.utils.email.send_email_smtp
+
+# SMTP
+airflow_smtp_host: localhost
+airflow_smtp_starttls: True
+airflow_smtp_ssl: False
+airflow_smtp_port: 25
+airflow_smtp_mail_from: [email protected]
+airflow_smtp_user:
+airflow_smtp_passwd:
+
+# SCHEDULER
+airflow_scheduler_job_heartbeat_sec: 5
+airflow_scheduler_heartbeat_sec: 5
+airflow_scheduler_run_duration: -1
+airflow_scheduler_min_file_process_interval: 0
+airflow_scheduler_dag_dir_list_interval: 300
+airflow_scheduler_print_stats_interval: 30
+airflow_scheduler_zombie_task_threshold: 300
+airflow_scheduler_catchup_by_default: True
+airflow_scheduler_max_threads: 2
+airflow_scheduler_authenticate: False
+
+## STASTD
+airflow_statsd_on: False
+airflow_statsd_host: localhost
+airflow_statsd_port: 8125
+airflow_statsd_prefix: airflow
+
+# CELERY
+airflow_celery_app_name: airflow.executors.celery_executor
+airflow_celery_concurrency: 16
+airflow_celery_worker_log_server_port: 8793
+airflow_celery_broker_url: sqla+mysql://airflow:airflow@localhost:3306/airflow
+airflow_celery_result_backend: db+mysql://airflow:airflow@localhost:3306/airflow
+airflow_celery_default_queue: default
+
+# FLOWER
+airflow_flower_host: 0.0.0.0
+airflow_flower_port: 5555
+
+# MESOS
+airflow_mesos_master_host: localhost:5050
+airflow_mesos_framework_name: Airflow
+airflow_mesos_task_cpu: 1
+airflow_mesos_task_memory: 256
+airflow_mesos_checkpoint: False
+airflow_mesos_authenticate: False
+
+# KERBEROS
+airflow_kerberos_ccache: /tmp/airflow_krb5_ccache
+airflow_kerberos_principal: airflow
+airflow_kerberos_reinit_frequency: 3600
+airflow_kerberos_kinit_path: kinit
+airflow_kerberos_keytab: airflow.keytab
+
+# GITHUB ENTEPRISE
+airflow_github_enterprise_api_rev: v3
+
+# ADMIN
+airflow_admin_hide_sensitive_variable_fields: True

handlers/main.yml

@@ -0,0 +1,20 @@
+---
+# handlers file for airflow-role
+
+- name: restart airflow-webserver
+  systemd:
+    name: airflow-webserver
+    state: restarted
+  when: airflow_services['airflow-webserver'] and airflow_services['airflow-webserver']['enabled']
+
+- name: restart airflow-scheduler
+  systemd:
+    name: airflow-scheduler
+    state: restarted
+  when: airflow_services['airflow-scheduler'] and airflow_services['airflow-scheduler']['enabled']
+
+- name: restart airflow-worker
+  systemd:
+    name: airflow-worker
+    state: restarted
+  when: airflow_services['airflow-worker'] and airflow_services['airflow-worker']['enabled']

meta/main.yml

@@ -0,0 +1,10 @@
+---
+galaxy_info:
+  company: Idealista S.A.
+  description: Apache Airflow role
+  min_ansible_version: 2.2.1.0
+  license: Apache 2.0
+  platforms:
+    - name: Debian
+      versions:
+        - stretch