From f1ee37cad157d71f1d8cf9381cecaf4e35ddde23 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Tue, 19 Nov 2024 18:23:05 +0530 Subject: [PATCH] Add example DAG for task group --- dev/dags/example_task_group.py | 17 +++++++++++++++++ dev/dags/example_task_group.yml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 dev/dags/example_task_group.py create mode 100644 dev/dags/example_task_group.yml diff --git a/dev/dags/example_task_group.py b/dev/dags/example_task_group.py new file mode 100644 index 0000000..5f81eab --- /dev/null +++ b/dev/dags/example_task_group.py @@ -0,0 +1,17 @@ +import os +from pathlib import Path + +# The following import is here so Airflow parses this file +# from airflow import DAG +import dagfactory + +DEFAULT_CONFIG_ROOT_DIR = "/usr/local/airflow/dags/" +CONFIG_ROOT_DIR = Path(os.getenv("CONFIG_ROOT_DIR", DEFAULT_CONFIG_ROOT_DIR)) + +config_file = str(CONFIG_ROOT_DIR / "example_task_group.yml") + +example_dag_factory = dagfactory.DagFactory(config_file) + +# Creating task dependencies +example_dag_factory.clean_dags(globals()) +example_dag_factory.generate_dags(globals()) diff --git a/dev/dags/example_task_group.yml b/dev/dags/example_task_group.yml new file mode 100644 index 0000000..b5b61a2 --- /dev/null +++ b/dev/dags/example_task_group.yml @@ -0,0 +1,30 @@ +default: + default_args: + owner: default_owner + retries: 1 + retry_delay_sec: 300 + start_date: 2024-01-01 + default_view: tree + max_active_runs: 1 + schedule_interval: 0 1 * * * +example_task_group: + description: "this dag uses task groups" + task_groups: + task_group_1: + tooltip: "this is a task group" + dependencies: [task_1] + task_group_2: + tooltip: "this is a task group" + parent_group_name: task_group_1 + tasks: + task_1: + operator: airflow.operators.bash_operator.BashOperator + bash_command: "echo 1" + task_2: + operator: airflow.operators.bash_operator.BashOperator + bash_command: "echo 2" + task_group_name: task_group_1 + task_4: + operator: airflow.operators.bash_operator.BashOperator + bash_command: "echo 4" + task_group_name: task_group_2