From f2b0473d01a6865190c1c1116e9029aa236e0a1e Mon Sep 17 00:00:00 2001 From: Jens Scheffler Date: Fri, 16 Feb 2024 23:35:36 +0100 Subject: [PATCH 1/2] Allow pre-population of trigger form values via URL parameters --- airflow/www/templates/airflow/trigger.html | 4 +- airflow/www/views.py | 72 +++++++++++--------- docs/apache-airflow/core-concepts/params.rst | 3 + docs/apache-airflow/ui.rst | 19 ++++++ 4 files changed, 63 insertions(+), 35 deletions(-) diff --git a/airflow/www/templates/airflow/trigger.html b/airflow/www/templates/airflow/trigger.html index cff3ea4de20be..6d60d04383167 100644 --- a/airflow/www/templates/airflow/trigger.html +++ b/airflow/www/templates/airflow/trigger.html @@ -251,7 +251,7 @@

- + @@ -277,7 +277,7 @@

- +
diff --git a/airflow/www/views.py b/airflow/www/views.py index b99062c551b60..2cafe291fc879 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -131,6 +131,7 @@ from airflow.utils.strings import to_boolean from airflow.utils.task_group import TaskGroup, task_group_to_dict from airflow.utils.timezone import td_format, utcnow +from airflow.utils.types import NOTSET from airflow.version import version from airflow.www import auth, utils as wwwutils from airflow.www.decorators import action_logging, gzipped @@ -1943,6 +1944,23 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): form_field_schema.pop("custom_html_form") if "description_md" in form_field_schema: form_field["description"] = wwwutils.wrapped_markdown(form_field_schema["description_md"]) + # Check for default values and pre-populate + if k in request.values: + if form_field_schema.get("type", None) in [ + "boolean", + "array", + ["array", "null"], + "object", + ["object", "null"], + ]: + try: + form_field["value"] = json.loads(request.values.get(k, "")) + except JSONDecodeError: + flash( + f'Could not pre-populate field "{k}" due to parsing error of value "{request.values.get(k)}"' + ) + else: + form_field["value"] = request.values.get(k) if form_trust_problems: flash( Markup( @@ -2000,6 +2018,15 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): for run_id, run_conf in ((run.run_id, run.conf) for run in recent_runs) if isinstance(run_conf, dict) and any(run_conf) } + render_params = { + "dag": dag, + "dag_id": dag_id, + "run_id": run_id, + "origin": origin, + "doc_md": wwwutils.wrapped_markdown(getattr(dag, "doc_md", None)), + "recent_confs": recent_confs, + "is_dag_run_conf_overrides_params": is_dag_run_conf_overrides_params, + } if request.method == "GET" or ( not request_conf and (ui_fields_defined or show_trigger_form_if_no_params) @@ -2007,7 +2034,6 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): # Populate conf textarea with conf requests parameter, or dag.params default_conf = "" - doc_md = wwwutils.wrapped_markdown(getattr(dag, "doc_md", None)) form = DateTimeForm(data={"execution_date": request_execution_date}) if request_conf: @@ -2015,7 +2041,12 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): else: try: default_conf = json.dumps( - {str(k): v.resolve(suppress_exception=True) for k, v in dag.params.items()}, + { + str(k): v.resolve( + value=request.values.get(k, default=NOTSET), suppress_exception=True + ) + for k, v in dag.params.items() + }, indent=4, ensure_ascii=False, ) @@ -2024,14 +2055,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return self.render_template( "airflow/trigger.html", form_fields=form_fields, - dag=dag, - dag_id=dag_id, - origin=origin, + **render_params, conf=default_conf, - doc_md=doc_md, form=form, - is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, - recent_confs=recent_confs, ) try: @@ -2042,13 +2068,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return self.render_template( "airflow/trigger.html", form_fields=form_fields, - dag=dag, - dag_id=dag_id, - origin=origin, + **render_params, conf=request_conf or {}, form=form, - is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, - recent_confs=recent_confs, ) dr = DagRun.find_duplicate(dag_id=dag_id, run_id=run_id, execution_date=execution_date) @@ -2073,13 +2095,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return self.render_template( "airflow/trigger.html", form_fields=form_fields, - dag=dag, - dag_id=dag_id, - origin=origin, + **render_params, conf=request_conf, form=form, - is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, - recent_confs=recent_confs, ) run_conf = {} @@ -2092,13 +2110,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return self.render_template( "airflow/trigger.html", form_fields=form_fields, - dag=dag, - dag_id=dag_id, - origin=origin, + **render_params, conf=request_conf, form=form, - is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, - recent_confs=recent_confs, ) except json.decoder.JSONDecodeError: flash("Invalid JSON configuration, not parseable", "error") @@ -2106,13 +2120,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return self.render_template( "airflow/trigger.html", form_fields=form_fields, - dag=dag, - dag_id=dag_id, - origin=origin, + **render_params, conf=request_conf, form=form, - is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, - recent_confs=recent_confs, ) if dag.get_is_paused(): @@ -2148,12 +2158,8 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return self.render_template( "airflow/trigger.html", form_fields=form_fields, - dag=dag, - dag_id=dag_id, - origin=origin, + **render_params, conf=request_conf, - form=form, - is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, ) flash(f"Triggered {dag_id}, it should start any moment now.") diff --git a/docs/apache-airflow/core-concepts/params.rst b/docs/apache-airflow/core-concepts/params.rst index 8d171d13e97bb..5263e361075a5 100644 --- a/docs/apache-airflow/core-concepts/params.rst +++ b/docs/apache-airflow/core-concepts/params.rst @@ -315,6 +315,9 @@ The following features are supported in the Trigger UI Form: The ``const`` value must match the default value to pass `JSON Schema validation `_. - On the bottom of the form the generated JSON configuration can be expanded. If you want to change values manually, the JSON configuration can be adjusted. Changes are overridden when form fields change. +- To pre-populate values in the form when publishing a link to the trigger form you can call the trigger URL ``/dags//trigger`` + and add query parameter to the URL in the form ``name=value``, for example ``/dags/example_params_ui_tutorial/trigger?required_field=some%20text``. + To pre-define the run id of the DAG run, use the URL parameter ``run_id``. .. note:: If the field is required the default value must be valid according to the schema as well. If the DAG is defined with diff --git a/docs/apache-airflow/ui.rst b/docs/apache-airflow/ui.rst index 7afe26cf3d719..26eb5e7b87ecf 100644 --- a/docs/apache-airflow/ui.rst +++ b/docs/apache-airflow/ui.rst @@ -26,6 +26,7 @@ can find in the Airflow UI. DAGs View ......... + List of the DAGs in your environment, and a set of shortcuts to useful pages. You can see exactly how many tasks succeeded, failed, or are currently running at a glance. To hide completed tasks set ``show_recent_stats_for_completed_runs = False`` @@ -50,6 +51,7 @@ For example: Datasets View ............. + A combined listing of the current datasets and a graph illustrating how they are produced and consumed by DAGs. Clicking on any dataset in either the list or the graph will highlight it and its relationships, and filter the list to show the recent history of task instances that have updated that dataset and whether it has triggered further DAG runs. @@ -63,6 +65,7 @@ Clicking on any dataset in either the list or the graph will highlight it and it Grid View ......... + A bar chart and grid representation of the DAG that spans across time. The top row is a chart of DAG Runs by duration, and below, task instances. If a pipeline is late, @@ -102,6 +105,7 @@ Mapped Tasks are indicated by square brackets and will show a table of each mapp Graph View .......... + The graph view is perhaps the most comprehensive. Visualize your DAG's dependencies and their current status for a specific run. @@ -113,6 +117,7 @@ dependencies and their current status for a specific run. Calendar View ............. + The calendar view gives you an overview of your entire DAG's history over months, or even years. Letting you quickly see trends of the overall success/failure rate of runs over time. @@ -124,6 +129,7 @@ Letting you quickly see trends of the overall success/failure rate of runs over Variable View ............. + The variable view allows you to list, create, edit or delete the key-value pair of a variable used during jobs. Value of a variable will be hidden if the key contains any words in ('password', 'secret', 'passwd', 'authorization', 'api_key', 'apikey', 'access_token') @@ -137,6 +143,7 @@ by default, but can be configured to show in cleartext. See :ref:`security:mask- Gantt Chart ........... + The Gantt chart lets you analyse task duration and overlap. You can quickly identify bottlenecks and where the bulk of the time is spent for specific DAG runs. @@ -151,6 +158,7 @@ DAG runs. Task Duration ............. + The duration of your different tasks over the past N runs. This view lets you find outliers and quickly understand where the time is spent in your DAG over many runs. @@ -178,6 +186,7 @@ The landing time for a task instance is the delta between the dag run's data int Code View ......... + Transparency is everything. While the code for your pipeline is in source control, this is a quick way to get to the code that generates the DAG and provide yet more context. @@ -185,3 +194,13 @@ provide yet more context. ------------ .. image:: img/code.png + +Trigger Form +............ + +If you trigger a manual DAG run with the arrow-button, a form is displayed. +The form display is based on the DAG Parameters as described in :ref:`_concepts:params`. + +------------ + +.. image:: img/trigger-dag-tutorial-form.png From be985901b00ca43cb421e41adee53b2627c31880 Mon Sep 17 00:00:00 2001 From: Jens Scheffler Date: Sat, 17 Feb 2024 15:40:06 +0100 Subject: [PATCH 2/2] Fix docs reference --- docs/apache-airflow/ui.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/ui.rst b/docs/apache-airflow/ui.rst index 26eb5e7b87ecf..dfe9b512c1234 100644 --- a/docs/apache-airflow/ui.rst +++ b/docs/apache-airflow/ui.rst @@ -199,7 +199,7 @@ Trigger Form ............ If you trigger a manual DAG run with the arrow-button, a form is displayed. -The form display is based on the DAG Parameters as described in :ref:`_concepts:params`. +The form display is based on the DAG Parameters as described in :doc:`core-concepts/params`. ------------