forked from apache/airflow
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.pre-commit-config.yaml
1030 lines (1030 loc) · 44.4 KB
/
.pre-commit-config.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
default_stages: [commit, push]
default_language_version:
python: python3
node: 18.6.0
minimum_pre_commit_version: '2.0.0'
repos:
- repo: meta
hooks:
- id: identity
name: Print input to the static check hooks for troubleshooting
- id: check-hooks-apply
name: Check if all hooks apply to the repository
- repo: https://github.com/thlorenz/doctoc.git
rev: v2.2.0
hooks:
- id: doctoc
name: Add TOC for Markdown and RST files
files:
^CONTRIBUTING\.md$|^README\.md$|^UPDATING.*\.md$|^chart/UPDATING.*\.md$|^dev/.*\.md$|^dev/.*\.rst$
exclude: ^.*/.*_vendor/
args:
- "--maxlevel"
- "2"
- repo: https://github.com/Lucas-C/pre-commit-hooks
rev: v1.3.1
hooks:
- id: insert-license
name: Add license for all SQL files
files: \.sql$
exclude: |
(?x)
^\.github/|
^.*/.*_vendor/
args:
- --comment-style
- "/*||*/"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all RST files
exclude: ^\.github/.*$|^.*/.*_vendor/|newsfragments/.*\.rst$
args:
- --comment-style
- "||"
- --license-filepath
- scripts/ci/license-templates/LICENSE.rst
- --fuzzy-match-generates-todo
files: \.rst$
- id: insert-license
name: Add license for all CSS/JS/JSX/PUML/TS/TSX files
files: \.(css|jsx?|puml|tsx?)$
exclude: ^\.github/.*$|^.*/.*_vendor/
args:
- --comment-style
- "/*!| *| */"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all JINJA template files
files: ^airflow/www/templates/.*\.html$
exclude: ^\.github/.*$|^.*/.*_vendor/
args:
- --comment-style
- "{#||#}"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all Shell files
exclude: ^\.github/.*$|^.*/.*_vendor/|^dev/breeze/autocomplete/.*$
files: \.bash$|\.sh$
args:
- --comment-style
- "|#|"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all Python files
exclude: ^\.github/.*$|^.*/.*_vendor/
files: \.py$|\.pyi$
args:
- --comment-style
- "|#|"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all XML files
exclude: ^\.github/.*$|^.*/.*_vendor/
files: \.xml$
args:
- --comment-style
- "<!--||-->"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all Helm template files
files: ^chart/templates/.*
args:
- --comment-style
- "{{/*||*/}}"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all YAML files except Helm templates
exclude: ^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*
types: [yaml]
files: \.ya?ml$
args:
- --comment-style
- "|#|"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all Markdown files
files: \.md$
exclude: PROVIDER_CHANGES.*\.md$|^.*/.*_vendor/
args:
- --comment-style
- "<!--|| -->"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all other files
exclude: ^\.github/.*$|^.*/.*_vendor/
args:
- --comment-style
- "|#|"
- --license-filepath
- scripts/ci/license-templates/LICENSE.txt
- --fuzzy-match-generates-todo
files: >
\.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
- repo: https://github.com/psf/black
rev: 23.10.0
hooks:
- id: black
name: Run black (Python formatter)
args: [--config=./pyproject.toml]
exclude: ^.*/.*_vendor/|^airflow/contrib/
- repo: local
hooks:
- id: update-common-sql-api-stubs
name: Check and update common.sql API stubs
entry: ./scripts/ci/pre_commit/pre_commit_update_common_sql_api_stubs.py
language: python
files: ^scripts/ci/pre_commit/pre_commit_update_common_sql_api\.py|^airflow/providers/common/sql/.*\.pyi?$
additional_dependencies: ['rich>=12.4.4', 'mypy==1.2.0', 'black==23.10.0', 'jinja2']
pass_filenames: false
require_serial: true
- id: update-black-version
name: Update black versions everywhere
entry: ./scripts/ci/pre_commit/pre_commit_update_black_version.py
language: python
files: ^.pre-commit-config.yaml$
additional_dependencies: ['pyyaml']
pass_filenames: false
require_serial: true
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.0.292
hooks:
# Since ruff makes use of multiple cores we _purposefully_ don't run this in docker so it can use the
# host CPU to it's fullest
- id: ruff
name: ruff
args: [ --fix ]
exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py
- repo: https://github.com/asottile/blacken-docs
rev: 1.16.0
hooks:
- id: blacken-docs
name: Run black on Python code blocks in documentation files
args:
- --line-length=110
- --target-version=py37
- --target-version=py38
- --target-version=py39
- --target-version=py310
alias: black
additional_dependencies: [black==23.10.0]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: check-merge-conflict
name: Check that merge conflicts are not being committed
- id: debug-statements
name: Detect accidentally committed debug statements
- id: check-builtin-literals
name: Require literal syntax when initializing builtin types
exclude: ^.*/.*_vendor/
- id: detect-private-key
name: Detect if private key is added to the repository
exclude: ^docs/apache-airflow-providers-ssh/connections/ssh.rst$
- id: end-of-file-fixer
name: Make sure that there is an empty line at the end
exclude: ^.*/.*_vendor/|^docs/apache-airflow/img/.*\.dot|^docs/apache-airflow/img/.*\.sha256
- id: mixed-line-ending
name: Detect if mixed line ending is used (\r vs. \r\n)
exclude: ^.*/.*_vendor/
- id: check-executables-have-shebangs
name: Check that executables have shebang
exclude: ^.*/.*_vendor/
- id: check-xml
name: Check XML files with xmllint
exclude: ^.*/.*_vendor/
- id: trailing-whitespace
name: Remove trailing whitespace at end of line
exclude: ^.*/.*_vendor/|^docs/apache-airflow/img/.*\.dot|^images/breeze/output.*$
- id: fix-encoding-pragma
name: Remove encoding header from Python files
exclude: ^.*/.*_vendor/
args:
- --remove
- id: pretty-format-json
name: Format JSON files
args:
- --autofix
- --no-sort-keys
- --indent
- "4"
files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
pass_filenames: true
- repo: https://github.com/pre-commit/pygrep-hooks
rev: v1.10.0
hooks:
- id: rst-backticks
name: Check if RST files use double backticks for code
exclude: ^.*/.*_vendor/
- id: python-no-log-warn
name: Check if there are no deprecate log warn
exclude: ^.*/.*_vendor/
- repo: https://github.com/adrienverge/yamllint
rev: v1.32.0
hooks:
- id: yamllint
name: Check YAML files with yamllint
entry: yamllint -c yamllint-config.yml --strict
types: [yaml]
exclude: ^.*airflow\.template\.yaml$|^.*init_git_sync\.template\.yaml$|^.*/.*_vendor/|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml$|^\.pre-commit-config\.yaml$
- repo: https://github.com/ikamensh/flynt
rev: '1.0.1'
hooks:
- id: flynt
name: Run flynt string format converter for Python
exclude: |
(?x)
^.*/.*_vendor/
args:
# If flynt detects too long text it ignores it. So we set a very large limit to make it easy
# to split the text by hand. Too long lines are detected by flake8 (below),
# so the user is informed to take action.
- --line-length
- '99999'
- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
hooks:
- id: codespell
name: Run codespell to check for common misspellings in files
entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s)
to the codespell dictionary at docs/spelling_wordlist.txt.
The word(s) should be in lowercase." && exec codespell "$@"' --
language: python
types: [text]
exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$
args:
- --ignore-words=docs/spelling_wordlist.txt
- --skip=airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md
- --exclude-file=.codespellignorelines
- repo: https://github.com/abravalheri/validate-pyproject
rev: v0.15
hooks:
- id: validate-pyproject
name: Validate pyproject.toml
- repo: local
# Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because
# we try to minimise the number of passes that must happen in order to apply some of the changes
# done by pre-commits. Some of the pre-commits not only check for errors but also fix them. This means
# that output from an earlier pre-commit becomes input to another pre-commit. Splitting the local
# scripts of our and adding some other non-local pre-commit in-between allows us to handle such
# changes quickly - especially when we want the early modifications from the first local group
# to be applied before the non-local pre-commits are run
hooks:
- id: replace-bad-characters
name: Replace bad characters
entry: ./scripts/ci/pre_commit/pre_commit_replace_bad_characters.py
language: python
types: [file, text]
exclude: ^.*/.*_vendor/|^clients/gen/go\.sh$|^\.gitmodules$
additional_dependencies: ['rich>=12.4.4']
- id: lint-openapi
name: Lint OpenAPI using spectral
language: docker_image
entry: stoplight/spectral lint -r ./scripts/ci/spectral_rules/connexion.yml
files: ^airflow/api_connexion/openapi/
- id: lint-openapi
name: Lint OpenAPI using openapi-spec-validator
entry: openapi-spec-validator --schema 3.0.0
language: python
additional_dependencies: ['openapi-spec-validator>=0.6.0']
files: ^airflow/api_connexion/openapi/
- id: lint-dockerfile
name: Lint Dockerfile
language: python
entry: ./scripts/ci/pre_commit/pre_commit_lint_dockerfile.py
files: Dockerfile.*$
pass_filenames: true
require_serial: true
- id: check-setup-order
name: Check order of dependencies in setup.cfg and setup.py
language: python
files: ^setup\.cfg$|^setup\.py$
pass_filenames: false
entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
additional_dependencies: ['rich>=12.4.4']
- id: check-airflow-k8s-not-used
name: Check airflow.kubernetes imports are not used
language: python
files: ^airflow/.*\.py$
require_serial: true
exclude: ^airflow/kubernetes/
entry: ./scripts/ci/pre_commit/pre_commit_check_airflow_k8s_not_used.py
additional_dependencies: ['rich>=12.4.4']
- id: check-cncf-k8s-only-for-executors
name: Check cncf.kubernetes imports used for executors only
language: python
files: ^airflow/.*\.py$
require_serial: true
exclude: ^airflow/kubernetes/|^airflow/providers/
entry: ./scripts/ci/pre_commit/pre_commit_check_cncf_k8s_used_for_k8s_executor_only.py
additional_dependencies: ['rich>=12.4.4']
- id: check-extra-packages-references
name: Checks setup extra packages
description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file
language: python
files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$|^airflow/providers/.*/provider\.yaml$
pass_filenames: false
entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
additional_dependencies: ['rich>=12.4.4']
- id: check-airflow-provider-compatibility
name: Check compatibility of Providers with Airflow
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_airflow_compatibility.py
language: python
pass_filenames: true
files: ^airflow/providers/.*\.py$
additional_dependencies: ['rich>=12.4.4']
- id: check-google-re2-as-dependency
name: Check google-re2 is declared as dependency when needed
entry: ./scripts/ci/pre_commit/pre_commit_check_google_re2_imports.py
language: python
pass_filenames: true
require_serial: true
files: ^airflow/providers/.*\.py$
additional_dependencies: ['rich>=12.4.4']
- id: update-local-yml-file
name: Update mounts in the local yml file
entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
language: python
files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
pass_filenames: false
additional_dependencies: ['rich>=12.4.4']
- id: update-providers-dependencies
name: Update cross-dependencies for providers packages
entry: ./scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
language: python
files: ^airflow/providers/.*\.py$|^airflow/providers/.*/provider\.yaml$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$
pass_filenames: false
additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml']
- id: update-extras
name: Update extras in documentation
entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
language: python
files: ^setup\.py$|^CONTRIBUTING\.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$
pass_filenames: false
additional_dependencies: ['rich>=12.4.4']
- id: check-extras-order
name: Check order of extras in Dockerfile
entry: ./scripts/ci/pre_commit/pre_commit_check_order_dockerfile_extras.py
language: python
files: ^Dockerfile$
pass_filenames: false
additional_dependencies: ['rich>=12.4.4']
- id: update-supported-versions
name: Updates supported versions in documentation
entry: ./scripts/ci/pre_commit/pre_commit_supported_versions.py
language: python
files: ^docs/apache-airflow/installation/supported-versions\.rst$|^scripts/ci/pre_commit/pre_commit_supported_versions\.py$|^README\.md$
pass_filenames: false
additional_dependencies: ['tabulate']
- id: check-revision-heads-map
name: Check that the REVISION_HEADS_MAP is up-to-date
language: python
entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
pass_filenames: false
files: >
(?x)
^scripts/ci/pre_commit/pre_commit_version_heads_map\.py$|
^airflow/migrations/versions/.*$|^airflow/migrations/versions|
^airflow/utils/db.py$
additional_dependencies: ['packaging','google-re2']
- id: update-version
name: Update version to the latest version in the documentation
entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
language: python
files: ^docs
pass_filenames: false
- id: check-pydevd-left-in-code
language: pygrep
name: Check for pydevd debug statements accidentally left
entry: "pydevd.*settrace\\("
pass_filenames: true
files: \.py$
- id: check-links-to-example-dags-do-not-use-hardcoded-versions
name: Verify example dags do not use hard-coded version numbers
description: The links to example dags should use |version| as version specification
language: pygrep
entry: >
(?i)
.*https://github.*[0-9]/tests/system/providers|
.*https://github.*/main/tests/system/providers|
.*https://github.*/master/tests/system/providers|
.*https://github.*/main/airflow/providers/.*/example_dags/|
.*https://github.*/master/airflow/providers/.*/example_dags/
pass_filenames: true
files: ^docs/apache-airflow-providers-.*\.rst
- id: check-safe-filter-usage-in-html
language: pygrep
name: Don't use safe in templates
description: the Safe filter is error-prone, use Markup() in code instead
entry: "\\|\\s*safe"
files: \.html$
pass_filenames: true
- id: check-no-providers-in-core-examples
language: pygrep
name: No providers imports in core example DAGs
description: The core example DAGs have no dependencies other than core Airflow
entry: "^\\s*from airflow\\.providers.*"
pass_filenames: true
files: ^airflow/example_dags/.*\.py$
- id: check-no-airflow-deprecation-in-providers
language: pygrep
name: Do not use DeprecationWarning in providers
description: Use AirflowProviderDeprecationWarning in providers
entry: "^\\s*DeprecationWarning*"
pass_filenames: true
files: ^airflow/providers/.*\.py$
- id: check-no-relative-imports
language: pygrep
name: No relative imports
description: Airflow style is to use absolute imports only (except docs building)
entry: "^\\s*from\\s+\\."
pass_filenames: true
files: \.py$
exclude: ^.*/.*_vendor/|^docs/|^tests/
- id: check-urlparse-usage-in-code
language: pygrep
name: Don't use urlparse in code
description: urlparse is not recommended, use urlsplit() in code instead
entry: "^\\s*from urllib\\.parse import ((\\|, )(urlparse\\|urlunparse))+$"
pass_filenames: true
files: \.py$
- id: check-only-new-session-with-provide-session
name: Check NEW_SESSION is only used with @provide_session
language: python
entry: ./scripts/ci/pre_commit/pre_commit_new_session_in_provide_session.py
pass_filenames: true
files: ^airflow/.+\.py$
- id: check-for-inclusive-language
language: pygrep
name: Check for language that we do not accept as community
description: Please use more appropriate words for community documentation.
entry: >
(?ix)
(black|white)[_-]?list|
\bshe\b|
\bhe\b|
\bher\b|
\bhis\b|
\bmaster\b|
\bslave\b|
\bsanity\b|
\bdummy\b
pass_filenames: true
exclude: >
(?x)
^airflow/api_connexion/openapi/v1.yaml$|
^airflow/auth/managers/fab/security_manager/|
^airflow/cli/commands/webserver_command.py$|
^airflow/config_templates/|
^airflow/models/baseoperator.py$|
^airflow/operators/__init__.py$|
^airflow/providers/amazon/aws/hooks/emr.py$|
^airflow/providers/amazon/aws/operators/emr.py$|
^airflow/providers/apache/cassandra/hooks/cassandra.py$|
^airflow/providers/apache/hive/operators/hive_stats.py$|
^airflow/providers/apache/hive/transfers/vertica_to_hive.py$|
^airflow/providers/apache/spark/hooks/|
^airflow/providers/apache/spark/operators/|
^airflow/providers/exasol/hooks/exasol.py$|
^airflow/providers/google/cloud/hooks/bigquery.py$|
^airflow/providers/google/cloud/operators/cloud_build.py$|
^airflow/providers/google/cloud/operators/dataproc.py$|
^airflow/providers/google/cloud/operators/mlengine.py$|
^airflow/providers/microsoft/azure/hooks/cosmos.py$|
^airflow/providers/microsoft/winrm/hooks/winrm.py$|
^airflow/www/fab_security/manager.py$|
^docs/.*commits.rst$|
^docs/apache-airflow/security/webserver.rst$|
^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$|
^airflow/providers/microsoft/winrm/operators/winrm.py$|
^airflow/providers/opsgenie/hooks/opsgenie.py$|
^airflow/providers/redis/provider.yaml$|
^airflow/serialization/serialized_objects.py$|
^airflow/utils/db.py$|
^airflow/utils/trigger_rule.py$|
^airflow/www/static/css/bootstrap-theme.css$|
^airflow/www/static/js/types/api-generated.ts$|
^airflow/www/templates/appbuilder/flash.html$|
^.*/.*_vendor/|
^dev/|
^docs/README.rst$|
^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store.rst$|
^docs/apache-airflow-providers-apache-hdfs/connections.rst$|
^docs/apache-airflow-providers-apache-kafka/connections/kafka.rst$|
^docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst$|
^docs/apache-airflow-providers-microsoft-azure/connections/azure_cosmos.rst$|
^docs/conf.py$|
^docs/exts/removemarktransform.py$|
^scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py$|
^tests/|
^.pre-commit-config\.yaml$|
^.*CHANGELOG\.(rst|txt)$|
^.*RELEASE_NOTES\.rst$|
^CONTRIBUTORS_QUICK_START.rst$|
^.*\.(png|gif|jp[e]?g|tgz|lock)$|
git
- id: check-base-operator-partial-arguments
name: Check BaseOperator and partial() arguments
language: python
entry: ./scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py
pass_filenames: false
files: ^airflow/models/(?:base|mapped)operator\.py$
- id: check-init-decorator-arguments
name: Check model __init__ and decorator arguments are in sync
language: python
entry: ./scripts/ci/pre_commit/pre_commit_sync_init_decorator.py
pass_filenames: false
files: ^airflow/models/dag\.py$|^airflow/(?:decorators|utils)/task_group\.py$
- id: check-base-operator-usage
language: pygrep
name: Check BaseOperator[Link] core imports
description: Make sure BaseOperator[Link] is imported from airflow.models.baseoperator in core
entry: "from airflow\\.models import.* BaseOperator"
files: \.py$
pass_filenames: true
exclude: >
(?x)
^airflow/decorators/.*$|
^airflow/hooks/.*$|
^airflow/operators/.*$|
^airflow/providers/.*$|
^airflow/sensors/.*$|
^dev/provider_packages/.*$
- id: check-base-operator-usage
language: pygrep
name: Check BaseOperator[Link] other imports
description: Make sure BaseOperator[Link] is imported from airflow.models outside of core
entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
pass_filenames: true
files: >
(?x)
^airflow/providers/.*\.py$
exclude: ^.*/.*_vendor/
- id: check-decorated-operator-implements-custom-name
name: Check @task decorator implements custom_operator_name
language: python
entry: ./scripts/ci/pre_commit/pre_commit_decorator_operator_implements_custom_name.py
pass_filenames: true
files: ^airflow/.*\.py$
- id: check-core-deprecation-classes
language: pygrep
name: Verify usage of Airflow deprecation classes in core
entry: category=DeprecationWarning|category=PendingDeprecationWarning
files: \.py$
exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$
pass_filenames: true
- id: check-provide-create-sessions-imports
language: pygrep
name: Check provide_session and create_session imports
description: provide_session and create_session should be imported from airflow.utils.session
to avoid import cycles.
entry: "from airflow\\.utils\\.db import.* (provide_session|create_session)"
files: \.py$
exclude: ^.*/.*_vendor/
pass_filenames: true
- id: check-incorrect-use-of-LoggingMixin
language: pygrep
name: Make sure LoggingMixin is not used alone
entry: "LoggingMixin\\(\\)"
files: \.py$
exclude: ^.*/.*_vendor/
pass_filenames: true
- id: check-daysago-import-from-utils
language: pygrep
name: Make sure days_ago is imported from airflow.utils.dates
entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago"
files: \.py$
exclude: ^.*/.*_vendor/
pass_filenames: true
- id: check-start-date-not-used-in-defaults
language: pygrep
name: start_date not to be defined in default_args in example_dags
entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
files: \.*example_dags.*\.py$
exclude: ^.*/.*_vendor/
pass_filenames: true
- id: check-apache-license-rat
name: Check if licenses are OK for Apache
entry: ./scripts/ci/pre_commit/pre_commit_check_license.py
language: python
files: ^.*LICENSE.*$|^.*LICENCE.*$
pass_filenames: false
- id: check-aiobotocore-optional
name: Check if aiobotocore is an optional dependency only
entry: ./scripts/ci/pre_commit/pre_commit_check_aiobotocore_optional.py
language: python
files: ^airflow/providers/.*/provider\.yaml$
pass_filenames: true
additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml']
require_serial: true
- id: check-boring-cyborg-configuration
name: Checks for Boring Cyborg configuration consistency
language: python
entry: ./scripts/ci/pre_commit/pre_commit_boring_cyborg.py
pass_filenames: false
require_serial: true
additional_dependencies: ['pyyaml', 'termcolor==1.1.0', 'wcmatch==8.2']
- id: update-in-the-wild-to-be-sorted
name: Sort INTHEWILD.md alphabetically
entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.py
language: python
files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$
pass_filenames: false
require_serial: true
- id: update-installed-providers-to-be-sorted
name: Sort alphabetically and uniquify installed_providers.txt
entry: ./scripts/ci/pre_commit/pre_commit_sort_installed_providers.py
language: python
files: ^\.pre-commit-config\.yaml$|^airflow/providers/installed_providers\.txt$
pass_filenames: false
require_serial: true
- id: update-spelling-wordlist-to-be-sorted
name: Sort alphabetically and uniquify spelling_wordlist.txt
entry: ./scripts/ci/pre_commit/pre_commit_sort_spelling_wordlist.py
language: python
files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$
require_serial: true
pass_filenames: false
- id: lint-helm-chart
name: Lint Helm Chart
entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.py
language: python
pass_filenames: false
files: ^chart
require_serial: true
additional_dependencies: ['rich>=12.4.4','requests']
- id: shellcheck
name: Check Shell scripts syntax correctness
language: docker_image
entry: koalaman/shellcheck:v0.8.0 -x -a
files: \.(bash|sh)$|^hooks/build$|^hooks/push$
exclude: ^dev/breeze/autocomplete/.*$
- id: lint-css
name: stylelint
entry: "stylelint"
language: node
files: ^airflow/www/.*\.(css|sass|scss)$
# Keep dependency versions in sync w/ airflow/www/package.json
- id: compile-www-assets
name: Compile www assets
language: node
stages: ['manual']
'types_or': [javascript, ts, tsx]
files: ^airflow/www/
entry: ./scripts/ci/pre_commit/pre_commit_compile_www_assets.py
pass_filenames: false
additional_dependencies: ['[email protected]']
- id: compile-www-assets-dev
name: Compile www assets in dev mode
language: node
stages: ['manual']
'types_or': [javascript, ts, tsx]
files: ^airflow/www/
entry: ./scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py
pass_filenames: false
additional_dependencies: ['[email protected]']
- id: check-providers-init-file-missing
name: Provider init file is missing
pass_filenames: false
always_run: true
entry: ./scripts/ci/pre_commit/pre_commit_check_providers_init.py
language: python
- id: check-providers-subpackages-init-file-exist
name: Provider subpackage init files are there
pass_filenames: false
always_run: true
entry: ./scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py
language: python
require_serial: true
- id: check-pre-commit-information-consistent
name: Validate hook IDs & names and sync with docs
entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
args:
- --max-length=60
language: python
files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names\.py$
additional_dependencies: ['pyyaml', 'jinja2', 'black==23.10.0', 'tabulate', 'rich>=12.4.4']
require_serial: true
pass_filenames: false
- id: update-breeze-readme-config-hash
name: Update Breeze README.md with config files hash
language: python
entry: ./scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py
files: ^dev/breeze/pyproject\.toml$|^dev/breeze/setup\.cfg$|^dev/breeze/setup\.py$|^dev/breeze/README\.md$
pass_filenames: false
require_serial: true
- id: check-breeze-top-dependencies-limited
name: Breeze should have small number of top-level dependencies
language: python
entry: ./scripts/tools/check_if_limited_dependencies.py
files: ^dev/breeze/.*$
pass_filenames: false
require_serial: true
additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml']
- id: check-system-tests-present
name: Check if system tests have required segments of code
entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py
language: python
files: ^tests/system/.*/example_[^/]*\.py$
exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$
pass_filenames: true
additional_dependencies: ['rich>=12.4.4']
- id: generate-pypi-readme
name: Generate PyPI README
entry: ./scripts/ci/pre_commit/pre_commit_generate_pypi_readme.py
language: python
files: ^README\.md$
pass_filenames: false
- id: lint-markdown
name: Run markdownlint
description: Checks the style of Markdown files.
entry: markdownlint
language: node
types: [markdown]
files: \.(md|mdown|markdown)$
additional_dependencies: ['markdownlint-cli']
- id: lint-json-schema
name: Lint JSON Schema files with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
- --spec-url
- https://json-schema.org/draft-07/schema
language: python
pass_filenames: true
files: .*\.schema\.json$
exclude: ^.*/.*_vendor/
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
- id: lint-json-schema
name: Lint NodePort Service with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
- --spec-url
- https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json
language: python
pass_filenames: true
files: ^scripts/ci/kubernetes/nodeport\.yaml$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
- id: lint-json-schema
name: Lint Docker compose files with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
- --spec-url
- https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json
language: python
pass_filenames: true
files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$
exclude: >
(?x)
^scripts/ci/docker-compose/grafana/.|
^scripts/ci/docker-compose/.+-config\.ya?ml
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
- id: lint-json-schema
name: Lint chart/values.schema.json file with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
- --spec-file
- chart/values_schema.schema.json
- chart/values.schema.json
language: python
pass_filenames: false
files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
- id: update-vendored-in-k8s-json-schema
name: Vendor k8s definitions into values.schema.json
entry: ./scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
language: python
files: ^chart/values\.schema\.json$
additional_dependencies: ['requests==2.25.0']
- id: lint-json-schema
name: Lint chart/values.yaml file with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
- --enforce-defaults
- --spec-file
- chart/values.schema.json
- chart/values.yaml
language: python
pass_filenames: false
files: ^chart/values\.yaml$|^chart/values\.schema\.json$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
- id: lint-json-schema
name: Lint config_templates/config.yml file with JSON Schema
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
args:
- --spec-file
- airflow/config_templates/config.yml.schema.json
language: python
pass_filenames: true
files: ^airflow/config_templates/config\.yml$
require_serial: true
additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
- id: check-persist-credentials-disabled-in-github-workflows
name: Check that workflow files have persist-credentials disabled
entry: ./scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
language: python
pass_filenames: true
files: ^\.github/workflows/.*\.yml$
additional_dependencies: ['PyYAML', 'rich>=12.4.4']
- id: check-docstring-param-types
name: Check that docstrings do not specify param types
entry: ./scripts/ci/pre_commit/pre_commit_docstring_param_type.py
language: python
pass_filenames: true
files: \.py$
exclude: ^.*/.*_vendor/
additional_dependencies: ['rich>=12.4.4']
- id: lint-chart-schema
name: Lint chart/values.schema.json file
entry: ./scripts/ci/pre_commit/pre_commit_chart_schema.py
language: python
pass_filenames: false
files: ^chart/values\.schema\.json$
require_serial: true
- id: update-inlined-dockerfile-scripts
name: Inline Dockerfile and Dockerfile.ci scripts
entry: ./scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
language: python
pass_filenames: false
files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$
require_serial: true
- id: check-changelog-has-no-duplicates
name: Check changelogs for duplicate entries
language: python
files: CHANGELOG\.(rst|txt)$
entry: ./scripts/ci/pre_commit/pre_commit_changelog_duplicates.py
pass_filenames: true
- id: check-newsfragments-are-valid
name: Check newsfragments are valid
language: python
files: newsfragments/.*\.rst
entry: ./scripts/ci/pre_commit/pre_commit_newsfragments.py
pass_filenames: true
# We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes
# This is fast, so not too much downside
always_run: true
- id: update-breeze-cmd-output
name: Update output of breeze commands in BREEZE.rst
entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
language: python
files: >
(?x)
^BREEZE\.rst$|^dev/breeze/.*$|
^\.pre-commit-config\.yaml$|
^scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py$|
^generated/provider_dependencies.json$
require_serial: true
pass_filenames: false
additional_dependencies: ['rich>=12.4.4', 'rich-click>=1.7.0', 'inputimeout', 'pyyaml', 'packaging']
- id: check-example-dags-urls
name: Check that example dags url include provider versions
entry: ./scripts/ci/pre_commit/pre_commit_update_example_dags_paths.py
language: python
pass_filenames: true
files: ^docs/.*example-dags\.rst$|^docs/.*index\.rst$
additional_dependencies: ['rich>=12.4.4', 'pyyaml']
always_run: true
- id: check-system-tests-tocs
name: Check that system tests is properly added
entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests_hidden_in_index.py
language: python
pass_filenames: true
files: ^docs/apache-airflow-providers-[^/]*/index\.rst$
additional_dependencies: ['rich>=12.4.4', 'pyyaml']
- id: check-lazy-logging
name: Check that all logging methods are lazy
entry: ./scripts/ci/pre_commit/pre_commit_check_lazy_logging.py
language: python
pass_filenames: true
files: \.py$
exclude: ^.*/.*_vendor/
additional_dependencies: ['rich>=12.4.4', 'astor']
- id: create-missing-init-py-files-tests
name: Create missing init.py files in tests
entry: ./scripts/ci/pre_commit/pre_commit_check_init_in_tests.py
language: python
additional_dependencies: ['rich>=12.4.4']
pass_filenames: false
files: ^tests/.*\.py$
- id: ts-compile-format-lint-www
name: TS types generation / ESLint / Prettier against UI files
language: node
'types_or': [javascript, ts, tsx, yaml, css, json]
files: ^airflow/www/static/js/|^airflow/api_connexion/openapi/v1\.yaml$
entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py
additional_dependencies: ['[email protected]']
pass_filenames: false
- id: check-tests-unittest-testcase
name: Check that unit tests do not inherit from unittest.TestCase
entry: ./scripts/ci/pre_commit/pre_commit_unittest_testcase.py
language: python
pass_filenames: true
files: ^tests/.*\.py$
- id: check-usage-of-re2-over-re
language: pygrep
name: Use re2 module instead of re
description: Use re2 module instead of re
entry: "^\\s*from re\\s|^\\s*import re\\s"
pass_filenames: true
files: \.py$
exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^\w+_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$
- id: check-deferrable-default-value
name: Check default value of deferrable attribute
language: python
entry: ./scripts/ci/pre_commit/pre_commit_check_deferrable_default.py
pass_filenames: false
files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$
## ADD MOST PRE-COMMITS ABOVE THAT LINE
# The below pre-commits are those requiring CI image to be built
- id: mypy-dev
name: Run mypy for dev
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
files: ^dev/.*\.py$
require_serial: true
additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']
- id: mypy-core
name: Run mypy for core
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
files: \.py$
exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
require_serial: true
additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']
- id: mypy-providers
name: Run mypy for providers
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
files: ^airflow/providers/.*\.py$|^tests/providers/\*\.py$|^tests/system/providers/\*\.py$
exclude: ^.*/.*_vendor/
require_serial: true
additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']