Merge pull request #108 from idealista/develop

Update role version and more

.gitignore

@@ -4,5 +4,6 @@ tests/__pycache__
 .molecule
 .cache
 tests/roles
-.vscode/*
-*.vt100
+/.vscode
+*.vt100
+/logs

CHANGELOG.md

@@ -7,6 +7,27 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch
 
 <!-- [Full Changelog](https://github.com/idealista/airflow-role/compare/2.0.1...bugfix/wrong-task-handler) -->
 
+## [2.0.3](https://github.com/idealista/airflow-role/tree/2.0.3)
+
+[Full Changelog](https://github.com/idealista/airflow-role/compare/2.0.2...2.0.3)
+
+### Changed
+
+- :arrows_clockwise: Bump default airflow version to 2.1.1
+- :arrows_clockwise: Bump requirements versions
+- :arrows_clockwise: Some default values has been updated
+
+### Added
+
+- :heavy_plus_sign: Add regular users task
+- :heavy_plus_sign: molecule config flower port exposed
+- :heavy_plus_sign: molecule default scenario create admin and regular users
+- :heavy_plus_sign: create home options
+
+### Fixed
+
+- :hammer_and_wrench: become configured user when check for admin users
+
 ## [2.0.2](https://github.com/idealista/airflow-role/tree/2.0.2)
 
 [Full Changelog](https://github.com/idealista/airflow-role/compare/2.0.1...2.0.2)

Pipfile

@@ -6,8 +6,8 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-ansible = "==2.9.9"
-molecule = "==3.0.6"
+ansible = "==2.9.21"
+molecule = "==3.0.8"
 docker = "==4.1.0"
 ansible-lint = "==4.2.0"
 

defaults/main/airflow-cfg.yml

@@ -44,13 +44,15 @@ airflow_dag_discovery_safe_mode: True
 airflow_default_task_retries: 0
 airflow_min_serialized_dag_update_interval: 30
 airflow_min_serialized_dag_fetch_interval: 10
-airflow_store_dag_code: False
+# airflow_store_dag_code: True
 airflow_max_num_rendered_ti_fields_per_task: 30
 airflow_check_slas: True
 airflow_xcom_backend: airflow.models.xcom.BaseXCom
 airflow_lazy_load_plugins: True
 airflow_lazy_discover_providers: True
 airflow_max_db_retries: 3
+airflow_hide_sensitive_var_conn_fields: True
+airflow_sensitive_var_conn_names:
 
 airflow_non_pooled_task_slot_count: 128
 
@@ -59,6 +61,9 @@ airflow_logs_folder: /var/log/airflow
 airflow_remote_logging: False
 airflow_remote_log_conn_id:
 airflow_google_key_path:
+airflow_access_control_allow_headers:
+airflow_access_control_allow_methods:
+airflow_access_control_allow_origin:
 airflow_remote_base_log_folder:
 airflow_encrypt_s3_logs: False
 airflow_logging_level: INFO
@@ -123,6 +128,7 @@ airflow_operator_default_cpus: 1
 airflow_operator_default_ram: 512
 airflow_operator_default_disk: 512
 airflow_operator_default_gpus: 0
+airflow_default_queue: default
 airflow_allow_illegal_arguments: False
 
 # [hive]
@@ -139,7 +145,7 @@ airflow_webserver_ssl_key:
 airflow_webserver_master_timeout: 120
 airflow_webserver_worker_timeout: 120
 airflow_webserver_worker_refresh_batch_size: 1
-airflow_webserver_worker_refresh_interval: 30
+airflow_webserver_worker_refresh_interval: 6000
 airflow_webserver_reload_on_plugin_change: False
 airflow_webserver_secret_key:
 airflow_webserver_workers: 4
@@ -152,7 +158,6 @@ airflow_webserver_expose_hostname: True
 airflow_webserver_expose_stacktrace: True
 airflow_webserver_dag_default_view: tree # Valid values are: tree, graph, duration, gantt, landing_times
 airflow_webserver_dag_orientation: LR
-airflow_webserver_demo_mode: False
 airflow_webserver_log_fetch_timeout_sec: 5
 airflow_webserver_log_fetch_delay_sec: 2
 airflow_webserver_log_auto_tailing_offset: 30
@@ -177,24 +182,28 @@ airflow_webserver_analytics_id:
 airflow_webserver_show_recent_stats_for_completed_runs: True
 airflow_webserver_update_fab_perms: True
 airflow_webserver_session_lifetime_minutes: 43200
+# airflow_instance_name:
 
 airflow_webserver_filter_by_owner: False
 airflow_webserver_owner_mode: user
 airflow_webserver_rbac: False
 
 # [email]
 airflow_email_backend: airflow.utils.email.send_email_smtp
+airflow_email_conn_id: smtp_default
 airflow_email_default_email_on_retry: True
 airflow_email_default_email_on_failure: True
 airflow_email_subject_template:
 airflow_email_html_content_template:
 
 # [smtp]
-airflow_smtp_host: localhost
+airflow_smtp_host:
 airflow_smtp_starttls: True
 airflow_smtp_ssl: True
-airflow_smtp_port: 25
-airflow_smtp_mail_from: [email protected]
+airflow_smtp_port:
+airflow_smtp_mail_from:
+airflow_smtp_timeout: 30
+airflow_smtp_retry_limit: 5
 airflow_smtp_user:
 airflow_smtp_passwd:
 
@@ -214,7 +223,6 @@ airflow_celery_worker_log_server_port: 8793
 airflow_celery_worker_umask: 0o077
 airflow_celery_broker_url: sqla+mysql://airflow:airflow@localhost:3306/airflow
 airflow_celery_result_backend: db+mysql://airflow:airflow@localhost:3306/airflow
-airflow_celery_default_queue: default
 airflow_celery_sync_parallelism: 0
 airflow_celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
 airflow_celery_ssl_active: False
@@ -264,8 +272,10 @@ airflow_scheduler_max_dagruns_to_create_per_loop: 10
 airflow_scheduler_max_dagruns_per_loop_to_schedule: 20
 airflow_scheduler_schedule_after_task_execution: True
 airflow_scheduler_parsing_processes: 2
+airflow_file_parsing_sort_mode: modified_time
 airflow_scheduler_use_job_schedule: True
 airflow_scheduler_allow_trigger_in_future: False
+airflow_dependency_detector: "airflow.serialization.serialized_objects.DependencyDetector"
 
 airflow_scheduler_run_duration: -1
 airflow_scheduler_min_file_parsing_loop_time: 1
@@ -303,6 +313,8 @@ airflow_elasticsearch_frontend:
 airflow_elasticsearch_write_stdout: False
 airflow_elasticsearch_json_format: False
 airflow_elasticsearch_json_fields: asctime, filename, lineno, levelname, message
+airflow_host_field: host
+airflow_offset_field: offset
 
 # [elasticsearch_configs]
 airflow_elasticsearch_configs_use_ssl: False
@@ -322,10 +334,14 @@ airflow_kubernetes_cluster_context:
 airflow_kubernetes_config_file:
 airflow_kubernetes_kube_client_request_args:
 airflow_kubernetes_delete_option_kwargs:
-airflow_kubernetes_enable_tcp_keepalive: False
+airflow_kubernetes_enable_tcp_keepalive: True
 airflow_kubernetes_tcp_keep_idle: 120
 airflow_kubernetes_tcp_keep_intvl: 30
 airflow_kubernetes_tcp_keep_cnt: 6
+airflow_verify_ssl: True
+airflow_worker_pods_pending_timeout: 300
+airflow_worker_pods_pending_timeout_check_interval: 120
+airflow_worker_pods_pending_timeout_batch_size: 100
 
 airflow_kubernetes_airflow_configmap:
 airflow_kubernetes_dags_volume_subpath:

defaults/main/main.yml

@@ -1,9 +1,15 @@
 ---
 # [General]
 airflow_app_name: airflow
-airflow_version: 2.0.1
+airflow_version: 2.1.1
 airflow_package: apache-airflow
 
+# Owner
+airflow_user: airflow
+airflow_group: airflow
+airflow_shell: /usr/sbin/nologin
+airflow_createhome: yes
+
 # The default buster's python version, if you want other version you should make sure it is
 # installed before
 airflow_python_version:
@@ -132,10 +138,6 @@ airflow_required_libs: "{{ airflow_default_required_libs + airflow_additional_re
 #   Version is not mandatory
 #   - {name: pyasn1, version: 0.4.4}
 
-# Owner
-airflow_user: airflow
-airflow_group: airflow
-
 # Admin user
 airflow_admin_users:
   - name:
@@ -146,6 +148,16 @@ airflow_admin_users:
     lastname:
     email:
 
+# Admin user
+# airflow_regular_users:
+#   - name:
+#     username:
+#     password:
+#     role:
+#     firstname:
+#     lastname:
+#     email:
+
 # Service options
 airflow_scheduler_runs: 1000
 airflow_private_tmp: false
@@ -179,6 +191,7 @@ airflow_services:
 # Files & Paths
 airflow_executable: "{{ airflow_app_home }}/bin/airflow"
 airflow_pip_executable: "pip3"
+airflow_user_home: "/var/lib/{{ airflow_user }}"
 airflow_app_home: "/opt/{{ airflow_app_name }}"
 airflow_conf_path: "/etc/{{ airflow_app_name }}"
 airflow_pidfile_folder: "/run/{{ airflow_app_name }}"

defaults/main/webserver-config-py.yml

@@ -40,7 +40,7 @@ airflow_AUTH_ROLE_PUBLIC: Public
 airflow_AUTH_USER_REGISTRATION: False
 
 # If we should replace ALL the user's roles each login, or only on registration
-airflow_AUTH_ROLES_SYNC_AT_LOGIN: True
+airflow_AUTH_ROLES_SYNC_AT_LOGIN: False
 
 # Force users to re-auth after 30min of inactivity (to keep roles in sync)
 airflow_PERMANENT_SESSION_LIFETIME: 1800

molecule/default/group_vars/airflow_group/main.yml

@@ -1,17 +1,45 @@
 ---
+
+airflow_load_examples: true
+
 # Example keys
 airflow_fernet_key: xKy13nPFfDflJ0DYGVTwf_DEmbItfURHlEDxrt-bBQw=
 airflow_webserver_secret_key: "ihadsf908auw0340684"
 
 airflow_admin_users:
-  - name: default
-    username: admin
-    password: admin
+  - name: Erwin
+    username: erwin
+    password: 5urv3yC0rpsFTW
     role: Admin
-    firstname: Admin
-    lastname: Admin
-    email: [email protected]
+    firstname: Erwin
+    lastname: Smith
+    email: [email protected]
+
+airflow_regular_users:
+  - name: Levi
+    username: levi
+    password: strongerThanU
+    role: Op
+    firstname: Levi
+    lastname: Ackerman
+    email: [email protected]
+  - name: Eren
+    username: eren
+    password: 104thGraduated
+    role: User
+    firstname: Eren
+    lastname: Yeager
+    email: [email protected]
 
 airflow_required_python_packages:
   - { name: SQLAlchemy, version: 1.3.23 }
-  - { name: psycopg2 }
+  - { name: psycopg2 }
+
+# [smtp]
+airflow_smtp_host: localhost
+airflow_smtp_starttls: True
+airflow_smtp_ssl: True
+airflow_smtp_port: 25
+airflow_smtp_mail_from: [email protected]
+airflow_smtp_user: user
+airflow_smtp_passwd: password

molecule/default/molecule.yml

@@ -30,11 +30,13 @@ platforms:
       - 8080/tcp
       - 8081/tcp
       - 5000/tcp
+      - 5555/tcp
     published_ports:
       - 0.0.0.0:8088:8088/tcp
       - 0.0.0.0:8080:8080/tcp
       - 0.0.0.0:8081:8081/tcp
       - 0.0.0.0:5000:5000/tcp
+      - 0.0.0.0:5555:5555/tcp
 
 provisioner:
   name: ansible

tasks/install.yml

@@ -10,8 +10,9 @@
     name: "{{ airflow_user }}"
     group: "{{ airflow_group }}"
     system: true
-    shell: /usr/sbin/nologin
-    createhome: false
+    shell: "{{ airflow_shell }}"
+    createhome: "{{ airflow_createhome }}"
+    home: "{{ airflow_user_home }}"
   become: true
 
 - name: Airflow | Ensure airflow skeleton paths

tasks/users.yml

@@ -1,13 +1,17 @@
 ---
 
-- name: Airflow | Check Admin user (> 2.0)
+- name: Airflow | Check users (> 2.0)
+  become: true
+  become_user: "{{ airflow_user }}"
   command: "{{ airflow_executable }} users list"
-  register: airflow_check_admin
+  register: airflow_check_users
   changed_when: false
   when: airflow_version is version( '2.0.0', '>=')
   no_log: true
 
-- name: Airflow | Create Admin user AUTH_DB (> 2.0)
+- name: Airflow | Create Admin users (> 2.0)
+  become: true
+  become_user: "{{ airflow_user }}"
   command:
     argv:
       - "{{ airflow_executable }}"
@@ -29,10 +33,11 @@
   with_items: "{{ airflow_admin_users }}"
   when:
     - airflow_version is version( '2.0.0', '>=')
-    - "item.email not in airflow_check_admin.stdout"
-    - "airflow_AUTH_TYPE == 'AUTH_DB'"
+    - "item.email not in airflow_check_users.stdout"
 
-- name: Airflow | Create Admin users AUTH_LDAP (> 2.0)
+- name: Airflow | Create regular users (> 2.0)
+  become: true
+  become_user: "{{ airflow_user }}"
   command:
     argv:
       - "{{ airflow_executable }}"
@@ -51,8 +56,8 @@
       - --email
       - "{{ item.email }}"
   no_log: true
-  with_items: "{{ airflow_admin_users }}"
+  with_items: "{{ airflow_regular_users }}"
   when:
     - airflow_version is version( '2.0.0', '>=')
-    - "item.email not in airflow_check_admin.stdout"
-    - "airflow_AUTH_TYPE == 'AUTH_LDAP'"
+    - airflow_regular_users is defined
+    - "item.email not in airflow_check_users.stdout"