diff --git a/.gitignore b/.gitignore index f6d6ae3..baa38da 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ tests/__pycache__ .molecule .cache tests/roles -.vscode/* -*.vt100 \ No newline at end of file +/.vscode +*.vt100 +/logs \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0393776..4372d0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch +## [2.0.3](https://github.com/idealista/airflow-role/tree/2.0.3) + +[Full Changelog](https://github.com/idealista/airflow-role/compare/2.0.2...2.0.3) + +### Changed + +- :arrows_clockwise: Bump default airflow version to 2.1.1 +- :arrows_clockwise: Bump requirements versions +- :arrows_clockwise: Some default values has been updated + +### Added + +- :heavy_plus_sign: Add regular users task +- :heavy_plus_sign: molecule config flower port exposed +- :heavy_plus_sign: molecule default scenario create admin and regular users +- :heavy_plus_sign: create home options + +### Fixed + +- :hammer_and_wrench: become configured user when check for admin users + ## [2.0.2](https://github.com/idealista/airflow-role/tree/2.0.2) [Full Changelog](https://github.com/idealista/airflow-role/compare/2.0.1...2.0.2) diff --git a/Pipfile b/Pipfile index d7c1798..3aa2ba5 100644 --- a/Pipfile +++ b/Pipfile @@ -6,8 +6,8 @@ verify_ssl = true [dev-packages] [packages] -ansible = "==2.9.9" -molecule = "==3.0.6" +ansible = "==2.9.21" +molecule = "==3.0.8" docker = "==4.1.0" ansible-lint = "==4.2.0" diff --git a/Pipfile.lock b/Pipfile.lock index 7842859..f021ccf 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7e72e72badb6e46480fded13334a26f54157e8fdcf39dab3553b0230d64ab12e" + "sha256": "eeaaac32e974eec05124b2e529a2a85825988bc09110c792f711d9bc90cb2bfb" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "ansible": { "hashes": [ - "sha256:e83d84ae8bf131c0499d8a4c0e1144bf969454c43086e61cca3c224227df29d1" + "sha256:4098246b67aa143e1e3af79d99346419e0545d5405d1cdf6e7fd389beab6de5a" ], "index": "pypi", - "version": "==2.9.9" + "version": "==2.9.21" }, "ansible-lint": { "hashes": [ @@ -33,10 +33,10 @@ }, "arrow": { "hashes": [ - "sha256:3515630f11a15c61dcb4cdd245883270dd334c83f3e639824e65a4b79cc48543", - "sha256:399c9c8ae732270e1aa58ead835a79a40d7be8aa109c579898eb41029b5a231d" + "sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510", + "sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a" ], - "version": "==1.0.3" + "version": "==1.1.1" }, "bcrypt": { "hashes": [ @@ -59,29 +59,37 @@ }, "cerberus": { "hashes": [ - "sha256:7aff49bc793e58a88ac14bffc3eca0f67e077881d3c62c621679a621294dd174", - "sha256:eec10585c33044fb7c69650bc5b68018dac0443753337e2b07684ee0f3c83329" + "sha256:d1b21b3954b2498d9a79edf16b3170a3ac1021df88d197dc2ce5928ba519237c" ], - "version": "==1.3.3" + "version": "==1.3.4" }, "certifi": { "hashes": [ - "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", - "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" ], - "version": "==2020.12.5" + "version": "==2021.5.30" }, "cffi": { "hashes": [ "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813", + "sha256:04c468b622ed31d408fea2346bec5bbffba2cc44226302a0de1ade9f5ea3d373", + "sha256:06d7cd1abac2ffd92e65c0609661866709b4b2d82dd15f611e602b9b188b0b69", + "sha256:06db6321b7a68b2bd6df96d08a5adadc1fa0e8f419226e25b2a5fbf6ccc7350f", "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06", + "sha256:0f861a89e0043afec2a51fd177a567005847973be86f709bbb044d7f42fc4e05", "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea", "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee", + "sha256:1bf1ac1984eaa7675ca8d5745a8cb87ef7abecb5592178406e55858d411eadc0", "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396", + "sha256:24a570cd11895b60829e941f2613a4f79df1a27344cbbb82164ef2e0116f09c7", + "sha256:24ec4ff2c5c0c8f9c6b87d5bb53555bf267e1e6f70e52e5a9740d32861d36b6f", "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73", "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315", + "sha256:293e7ea41280cb28c6fcaaa0b1aa1f533b8ce060b9e701d78511e1e6c4a1de76", "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1", "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49", + "sha256:3c3f39fa737542161d8b0d680df2ec249334cd70a8f420f71c9304bd83c3cbed", "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892", "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482", "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058", @@ -89,6 +97,7 @@ "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53", "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045", "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3", + "sha256:681d07b0d1e3c462dd15585ef5e33cb021321588bebd910124ef4f4fb71aef55", "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5", "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e", "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c", @@ -106,8 +115,10 @@ "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e", "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991", "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6", + "sha256:cc5a8e069b9ebfa22e26d0e6b97d6f9781302fe7f4f2b8776c3e1daea35f1adc", "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1", "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406", + "sha256:df5052c5d867c1ea0b311fb7c3cd28b19df469c056f7fdcfe88c7473aa63e333", "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d", "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c" ], @@ -122,10 +133,10 @@ }, "click": { "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", + "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" ], - "version": "==7.1.2" + "version": "==8.0.1" }, "click-completion": { "hashes": [ @@ -149,10 +160,10 @@ }, "cookiecutter": { "hashes": [ - "sha256:430eb882d028afb6102c084bab6cf41f6559a77ce9b18dc6802e3bc0cc5f4a30", - "sha256:efb6b2d4780feda8908a873e38f0e61778c23f6a2ea58215723bcceb5b515dac" + "sha256:6b9a4d72882e243be077a7397d0f1f76fe66cf3df91f3115dbb5330e214fa457", + "sha256:f8671531fa96ab14339d0c59b4f662a4f12a2ecacd94a0f70a3500843da588e2" ], - "version": "==1.7.2" + "version": "==1.7.3" }, "cryptography": { "hashes": [ @@ -188,10 +199,10 @@ }, "fasteners": { "hashes": [ - "sha256:74b6847e0a6bb3b56c8511af8e24c40e4cf7a774dfff5b251c260ed338096a4b", - "sha256:c995d8c26b017c5d6a6de9ad29a0f9cdd57de61ae1113d28fac26622b06a0933" + "sha256:8408e52656455977053871990bd25824d85803b9417aa348f10ba29ef0c751f7", + "sha256:b1ab4e5adfbc28681ce44b3024421c4f567e705cc3963c732bf1cba3348307de" ], - "version": "==0.16" + "version": "==0.16.3" }, "idna": { "hashes": [ @@ -202,18 +213,18 @@ }, "importlib-metadata": { "hashes": [ - "sha256:2ec0faae539743ae6aaa84b49a169670a465f7f5d64e6add98388cc29fd1f2f6", - "sha256:c9356b657de65c53744046fa8f7358afe0714a1af7d570c00c3835c2d724a7c1" + "sha256:079ada16b7fc30dfbb5d13399a5113110dab1aa7c2bc62f66af75f0b717c8cac", + "sha256:9f55f560e116f8643ecf2922d9cd3e1c7e8d52e683178fecd9d08f6aa357e11e" ], "markers": "python_version < '3.8'", - "version": "==3.10.1" + "version": "==4.6.1" }, "jinja2": { "hashes": [ - "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", - "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" + "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4", + "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4" ], - "version": "==2.11.3" + "version": "==3.0.1" }, "jinja2-time": { "hashes": [ @@ -224,68 +235,50 @@ }, "markupsafe": { "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", - "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", - "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", - "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", - "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", - "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", - "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", - "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", - "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", - "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", - "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" - ], - "version": "==1.1.1" + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "version": "==2.0.1" }, "molecule": { "hashes": [ - "sha256:b83d73b3e34ad9bc7cc62f1869a2c9e4c4cc5e5c5a2d9de963e838d5f37fb47b", - "sha256:e3ba0fb0bab1f60b6f0aef768a3759b383267773be47ee942adadf27d9f6fa35" + "sha256:42d0c661b52074b00a620466df367ddab9c3682875e6d685bfc93487ef0479cc", + "sha256:6fb202099ff52bc427c6bd8f46f13b49b3d5812daa6880960a0fd0562e8a9be6" ], "index": "pypi", - "version": "==3.0.6" + "version": "==3.0.8" }, "paramiko": { "hashes": [ @@ -374,9 +367,10 @@ }, "python-slugify": { "hashes": [ - "sha256:69a517766e00c1268e5bbfc0d010a0a8508de0b18d30ad5a1ff357f8ae724270" + "sha256:6d8c5df75cd4a7c3a2d21e257633de53f52ab0265cd2d1dc62a730e8194a7380", + "sha256:f13383a0b9fcbe649a1892b9c8eb4f8eab1d6d84b84bb7a624317afa98159cab" ], - "version": "==4.0.1" + "version": "==5.0.2" }, "pyyaml": { "hashes": [ @@ -421,48 +415,38 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:44bc6b54fddd45e4bc0619059196679f9e8b79c027f4131bb072e6a22f4d5e28", - "sha256:ac79fb25f5476e8e9ed1c53b8a2286d2c3f5dde49eb37dbcee5c7eb6a8415a22" + "sha256:106bc8d6dc6a0ff7c9196a47570432036f41d556b779c6b4e618085f57e39e67", + "sha256:ffb9b703853e9e8b7861606dfdab1026cf02505bade0653d1880f4b2db47f815" ], "markers": "python_version >= '3.7'", - "version": "==0.17.4" + "version": "==0.17.10" }, "ruamel.yaml.clib": { "hashes": [ - "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b", - "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f", - "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c", - "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91", - "sha256:28116f204103cb3a108dfd37668f20abe6e3cafd0d3fd40dba126c732457b3cc", - "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7", - "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3", - "sha256:30dca9bbcbb1cc858717438218d11eafb78666759e5094dd767468c0d577a7e7", - "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6", - "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6", - "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd", - "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0", - "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62", - "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99", - "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5", - "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026", - "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb", - "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2", - "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1", - "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4", - "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b", - "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923", - "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e", - "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c", - "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988", - "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f", - "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5", - "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a", - "sha256:daf21aa33ee9b351f66deed30a3d450ab55c14242cfdfcd377798e2c0d25c9f1", - "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2", - "sha256:f6061a31880c1ed6b6ce341215336e2f3d0c1deccd84957b6fa8ca474b41e89f" + "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd", + "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0", + "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277", + "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104", + "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd", + "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78", + "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99", + "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527", + "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84", + "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7", + "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468", + "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b", + "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94", + "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233", + "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb", + "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5", + "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe", + "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751", + "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502", + "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed", + "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c" ], "markers": "platform_python_implementation == 'CPython' and python_version < '3.10'", - "version": "==0.2.2" + "version": "==0.2.6" }, "selinux": { "hashes": [ @@ -488,10 +472,10 @@ }, "six": { "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "version": "==1.15.0" + "version": "==1.16.0" }, "tabulate": { "hashes": [ @@ -516,26 +500,26 @@ }, "typing-extensions": { "hashes": [ - "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", - "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", - "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "markers": "python_version < '3.8'", - "version": "==3.7.4.3" + "version": "==3.10.0.0" }, "urllib3": { "hashes": [ - "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", - "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" + "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", + "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "version": "==1.26.4" + "version": "==1.26.6" }, "websocket-client": { "hashes": [ - "sha256:44b5df8f08c74c3d82d28100fdc81f4536809ce98a17f0757557813275fbb663", - "sha256:63509b41d158ae5b7f67eb4ad20fecbb4eee99434e73e140354dc3ff8e09716f" + "sha256:b68e4959d704768fa20e35c9d508c8dc2bbc041fd8d267c0d7345cffe2824568", + "sha256:e5c333bfa9fa739538b652b6f8c8fc2559f1d364243c8a689d7c0e1d41c2e611" ], - "version": "==0.58.0" + "version": "==1.1.0" }, "yamllint": { "hashes": [ @@ -545,10 +529,10 @@ }, "zipp": { "hashes": [ - "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", - "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" + "sha256:957cfda87797e389580cb8b9e3870841ca991e2125350677b2ca83a0e99390a3", + "sha256:f5812b1e007e48cff63449a5e9f4e7ebea716b4111f9c4f9a645f91d579bf0c4" ], - "version": "==3.4.1" + "version": "==3.5.0" } }, "develop": {} diff --git a/defaults/main/airflow-cfg.yml b/defaults/main/airflow-cfg.yml index 5798d76..677a9f8 100644 --- a/defaults/main/airflow-cfg.yml +++ b/defaults/main/airflow-cfg.yml @@ -44,13 +44,15 @@ airflow_dag_discovery_safe_mode: True airflow_default_task_retries: 0 airflow_min_serialized_dag_update_interval: 30 airflow_min_serialized_dag_fetch_interval: 10 -airflow_store_dag_code: False +# airflow_store_dag_code: True airflow_max_num_rendered_ti_fields_per_task: 30 airflow_check_slas: True airflow_xcom_backend: airflow.models.xcom.BaseXCom airflow_lazy_load_plugins: True airflow_lazy_discover_providers: True airflow_max_db_retries: 3 +airflow_hide_sensitive_var_conn_fields: True +airflow_sensitive_var_conn_names: airflow_non_pooled_task_slot_count: 128 @@ -59,6 +61,9 @@ airflow_logs_folder: /var/log/airflow airflow_remote_logging: False airflow_remote_log_conn_id: airflow_google_key_path: +airflow_access_control_allow_headers: +airflow_access_control_allow_methods: +airflow_access_control_allow_origin: airflow_remote_base_log_folder: airflow_encrypt_s3_logs: False airflow_logging_level: INFO @@ -123,6 +128,7 @@ airflow_operator_default_cpus: 1 airflow_operator_default_ram: 512 airflow_operator_default_disk: 512 airflow_operator_default_gpus: 0 +airflow_default_queue: default airflow_allow_illegal_arguments: False # [hive] @@ -139,7 +145,7 @@ airflow_webserver_ssl_key: airflow_webserver_master_timeout: 120 airflow_webserver_worker_timeout: 120 airflow_webserver_worker_refresh_batch_size: 1 -airflow_webserver_worker_refresh_interval: 30 +airflow_webserver_worker_refresh_interval: 6000 airflow_webserver_reload_on_plugin_change: False airflow_webserver_secret_key: airflow_webserver_workers: 4 @@ -152,7 +158,6 @@ airflow_webserver_expose_hostname: True airflow_webserver_expose_stacktrace: True airflow_webserver_dag_default_view: tree # Valid values are: tree, graph, duration, gantt, landing_times airflow_webserver_dag_orientation: LR -airflow_webserver_demo_mode: False airflow_webserver_log_fetch_timeout_sec: 5 airflow_webserver_log_fetch_delay_sec: 2 airflow_webserver_log_auto_tailing_offset: 30 @@ -177,6 +182,7 @@ airflow_webserver_analytics_id: airflow_webserver_show_recent_stats_for_completed_runs: True airflow_webserver_update_fab_perms: True airflow_webserver_session_lifetime_minutes: 43200 +# airflow_instance_name: airflow_webserver_filter_by_owner: False airflow_webserver_owner_mode: user @@ -184,17 +190,20 @@ airflow_webserver_rbac: False # [email] airflow_email_backend: airflow.utils.email.send_email_smtp +airflow_email_conn_id: smtp_default airflow_email_default_email_on_retry: True airflow_email_default_email_on_failure: True airflow_email_subject_template: airflow_email_html_content_template: # [smtp] -airflow_smtp_host: localhost +airflow_smtp_host: airflow_smtp_starttls: True airflow_smtp_ssl: True -airflow_smtp_port: 25 -airflow_smtp_mail_from: airflow@airflow.com +airflow_smtp_port: +airflow_smtp_mail_from: +airflow_smtp_timeout: 30 +airflow_smtp_retry_limit: 5 airflow_smtp_user: airflow_smtp_passwd: @@ -214,7 +223,6 @@ airflow_celery_worker_log_server_port: 8793 airflow_celery_worker_umask: 0o077 airflow_celery_broker_url: sqla+mysql://airflow:airflow@localhost:3306/airflow airflow_celery_result_backend: db+mysql://airflow:airflow@localhost:3306/airflow -airflow_celery_default_queue: default airflow_celery_sync_parallelism: 0 airflow_celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG airflow_celery_ssl_active: False @@ -264,8 +272,10 @@ airflow_scheduler_max_dagruns_to_create_per_loop: 10 airflow_scheduler_max_dagruns_per_loop_to_schedule: 20 airflow_scheduler_schedule_after_task_execution: True airflow_scheduler_parsing_processes: 2 +airflow_file_parsing_sort_mode: modified_time airflow_scheduler_use_job_schedule: True airflow_scheduler_allow_trigger_in_future: False +airflow_dependency_detector: "airflow.serialization.serialized_objects.DependencyDetector" airflow_scheduler_run_duration: -1 airflow_scheduler_min_file_parsing_loop_time: 1 @@ -303,6 +313,8 @@ airflow_elasticsearch_frontend: airflow_elasticsearch_write_stdout: False airflow_elasticsearch_json_format: False airflow_elasticsearch_json_fields: asctime, filename, lineno, levelname, message +airflow_host_field: host +airflow_offset_field: offset # [elasticsearch_configs] airflow_elasticsearch_configs_use_ssl: False @@ -322,10 +334,14 @@ airflow_kubernetes_cluster_context: airflow_kubernetes_config_file: airflow_kubernetes_kube_client_request_args: airflow_kubernetes_delete_option_kwargs: -airflow_kubernetes_enable_tcp_keepalive: False +airflow_kubernetes_enable_tcp_keepalive: True airflow_kubernetes_tcp_keep_idle: 120 airflow_kubernetes_tcp_keep_intvl: 30 airflow_kubernetes_tcp_keep_cnt: 6 +airflow_verify_ssl: True +airflow_worker_pods_pending_timeout: 300 +airflow_worker_pods_pending_timeout_check_interval: 120 +airflow_worker_pods_pending_timeout_batch_size: 100 airflow_kubernetes_airflow_configmap: airflow_kubernetes_dags_volume_subpath: diff --git a/defaults/main/main.yml b/defaults/main/main.yml index cda464b..1c8b4a2 100644 --- a/defaults/main/main.yml +++ b/defaults/main/main.yml @@ -1,9 +1,15 @@ --- # [General] airflow_app_name: airflow -airflow_version: 2.0.1 +airflow_version: 2.1.1 airflow_package: apache-airflow +# Owner +airflow_user: airflow +airflow_group: airflow +airflow_shell: /usr/sbin/nologin +airflow_createhome: yes + # The default buster's python version, if you want other version you should make sure it is # installed before airflow_python_version: @@ -132,10 +138,6 @@ airflow_required_libs: "{{ airflow_default_required_libs + airflow_additional_re # Version is not mandatory # - {name: pyasn1, version: 0.4.4} -# Owner -airflow_user: airflow -airflow_group: airflow - # Admin user airflow_admin_users: - name: @@ -146,6 +148,16 @@ airflow_admin_users: lastname: email: +# Admin user +# airflow_regular_users: +# - name: +# username: +# password: +# role: +# firstname: +# lastname: +# email: + # Service options airflow_scheduler_runs: 1000 airflow_private_tmp: false @@ -179,6 +191,7 @@ airflow_services: # Files & Paths airflow_executable: "{{ airflow_app_home }}/bin/airflow" airflow_pip_executable: "pip3" +airflow_user_home: "/var/lib/{{ airflow_user }}" airflow_app_home: "/opt/{{ airflow_app_name }}" airflow_conf_path: "/etc/{{ airflow_app_name }}" airflow_pidfile_folder: "/run/{{ airflow_app_name }}" diff --git a/defaults/main/webserver-config-py.yml b/defaults/main/webserver-config-py.yml index a6c9dae..72a751f 100644 --- a/defaults/main/webserver-config-py.yml +++ b/defaults/main/webserver-config-py.yml @@ -40,7 +40,7 @@ airflow_AUTH_ROLE_PUBLIC: Public airflow_AUTH_USER_REGISTRATION: False # If we should replace ALL the user's roles each login, or only on registration -airflow_AUTH_ROLES_SYNC_AT_LOGIN: True +airflow_AUTH_ROLES_SYNC_AT_LOGIN: False # Force users to re-auth after 30min of inactivity (to keep roles in sync) airflow_PERMANENT_SESSION_LIFETIME: 1800 diff --git a/molecule/default/group_vars/airflow_group/main.yml b/molecule/default/group_vars/airflow_group/main.yml index f7ca69f..cdcdb63 100644 --- a/molecule/default/group_vars/airflow_group/main.yml +++ b/molecule/default/group_vars/airflow_group/main.yml @@ -1,17 +1,45 @@ --- + +airflow_load_examples: true + # Example keys airflow_fernet_key: xKy13nPFfDflJ0DYGVTwf_DEmbItfURHlEDxrt-bBQw= airflow_webserver_secret_key: "ihadsf908auw0340684" airflow_admin_users: - - name: default - username: admin - password: admin + - name: Erwin + username: erwin + password: 5urv3yC0rpsFTW role: Admin - firstname: Admin - lastname: Admin - email: admin@email.com + firstname: Erwin + lastname: Smith + email: erwin@attack.com + +airflow_regular_users: + - name: Levi + username: levi + password: strongerThanU + role: Op + firstname: Levi + lastname: Ackerman + email: levi@attack.com + - name: Eren + username: eren + password: 104thGraduated + role: User + firstname: Eren + lastname: Yeager + email: eren@attack.com airflow_required_python_packages: - { name: SQLAlchemy, version: 1.3.23 } - - { name: psycopg2 } \ No newline at end of file + - { name: psycopg2 } + +# [smtp] +airflow_smtp_host: localhost +airflow_smtp_starttls: True +airflow_smtp_ssl: True +airflow_smtp_port: 25 +airflow_smtp_mail_from: airflow@attack.com +airflow_smtp_user: user +airflow_smtp_passwd: password \ No newline at end of file diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 6fa135f..d00ef02 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -30,11 +30,13 @@ platforms: - 8080/tcp - 8081/tcp - 5000/tcp + - 5555/tcp published_ports: - 0.0.0.0:8088:8088/tcp - 0.0.0.0:8080:8080/tcp - 0.0.0.0:8081:8081/tcp - 0.0.0.0:5000:5000/tcp + - 0.0.0.0:5555:5555/tcp provisioner: name: ansible diff --git a/tasks/install.yml b/tasks/install.yml index 220092d..58a33d2 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -10,8 +10,9 @@ name: "{{ airflow_user }}" group: "{{ airflow_group }}" system: true - shell: /usr/sbin/nologin - createhome: false + shell: "{{ airflow_shell }}" + createhome: "{{ airflow_createhome }}" + home: "{{ airflow_user_home }}" become: true - name: Airflow | Ensure airflow skeleton paths diff --git a/tasks/users.yml b/tasks/users.yml index 7eba0b5..95ed4e3 100644 --- a/tasks/users.yml +++ b/tasks/users.yml @@ -1,13 +1,17 @@ --- -- name: Airflow | Check Admin user (> 2.0) +- name: Airflow | Check users (> 2.0) + become: true + become_user: "{{ airflow_user }}" command: "{{ airflow_executable }} users list" - register: airflow_check_admin + register: airflow_check_users changed_when: false when: airflow_version is version( '2.0.0', '>=') no_log: true -- name: Airflow | Create Admin user AUTH_DB (> 2.0) +- name: Airflow | Create Admin users (> 2.0) + become: true + become_user: "{{ airflow_user }}" command: argv: - "{{ airflow_executable }}" @@ -29,10 +33,11 @@ with_items: "{{ airflow_admin_users }}" when: - airflow_version is version( '2.0.0', '>=') - - "item.email not in airflow_check_admin.stdout" - - "airflow_AUTH_TYPE == 'AUTH_DB'" + - "item.email not in airflow_check_users.stdout" -- name: Airflow | Create Admin users AUTH_LDAP (> 2.0) +- name: Airflow | Create regular users (> 2.0) + become: true + become_user: "{{ airflow_user }}" command: argv: - "{{ airflow_executable }}" @@ -51,8 +56,8 @@ - --email - "{{ item.email }}" no_log: true - with_items: "{{ airflow_admin_users }}" + with_items: "{{ airflow_regular_users }}" when: - airflow_version is version( '2.0.0', '>=') - - "item.email not in airflow_check_admin.stdout" - - "airflow_AUTH_TYPE == 'AUTH_LDAP'" \ No newline at end of file + - airflow_regular_users is defined + - "item.email not in airflow_check_users.stdout" diff --git a/templates/airflow2.cfg.j2 b/templates/airflow2.cfg.j2 index be974c4..b709f3e 100644 --- a/templates/airflow2.cfg.j2 +++ b/templates/airflow2.cfg.j2 @@ -24,8 +24,9 @@ default_timezone = {{ airflow_default_timezone }} executor = {{ airflow_executor }} # The SqlAlchemy connection string to the metadata database. -# SqlAlchemy supports many different database engine, more information -# their website +# SqlAlchemy supports many different database engines. +# More information here: +# http://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri sql_alchemy_conn = {{ airflow_database_conn }} # The encoding for the databases @@ -79,19 +80,23 @@ sql_alchemy_schema = {{ airflow_database_schema }} # sql_alchemy_connect_args = {% if airflow_database_connect_args %}sql_alchemy_connect_args = {{ airflow_database_connect_args }}{% endif %} -# The amount of parallelism as a setting to the executor. This defines -# the max number of task instances that should run simultaneously -# on this airflow installation +# This defines the maximum number of task instances that can run concurrently in Airflow +# regardless of scheduler count and worker count. Generally, this value is reflective of +# the number of task instances with the running state in the metadata database. parallelism = {{ airflow_parallelism }} -# The number of task instances allowed to run concurrently by the scheduler -# in one DAG. Can be overridden by ``concurrency`` on DAG level. +# The maximum number of task instances allowed to run concurrently in each DAG. To calculate +# the number of tasks that is running concurrently for a DAG, add up the number of running +# tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``concurrency``, +# which is defaulted as ``dag_concurrency``. dag_concurrency = {{ airflow_dag_concurrency }} # Are DAGs paused by default at creation dags_are_paused_at_creation = {{ airflow_dags_are_paused_at_creation }} -# The maximum number of active DAG runs per DAG +# The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs +# if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``, +# which is defaulted as ``max_active_runs_per_dag``. max_active_runs_per_dag = {{ airflow_max_active_runs_per_dag }} # Whether to load the DAG examples that ship with Airflow. It's good to @@ -176,8 +181,9 @@ min_serialized_dag_fetch_interval = {{ airflow_min_serialized_dag_fetch_interval # Whether to persist DAG files code in DB. # If set to True, Webserver reads file contents from DB instead of # trying to access files in a DAG folder. -# Example: store_dag_code = False -{% if airflow_store_dag_code %}store_dag_code = {{ airflow_store_dag_code }}{% endif %} +# (Default is ``True``) +# Example: store_dag_code = True +{% if airflow_store_dag_code is defined %}store_dag_code = {{ airflow_store_dag_code }}{% endif %} # Maximum number of Rendered Task Instance Fields (Template Fields) per task to store # in the Database. @@ -207,6 +213,15 @@ lazy_discover_providers = {{ airflow_lazy_discover_providers }} # Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. max_db_retries = {{ airflow_max_db_retries }} +# Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True +# +# (Connection passwords are always hidden in logs) +hide_sensitive_var_conn_fields = {{ airflow_hide_sensitive_var_conn_fields }} + +# A comma-separated list of extra sensitive keywords to look for in variables names or connection's +# extra JSON. +sensitive_var_conn_names = {{ airflow_sensitive_var_conn_names }} + [logging] # The folder where airflow should store its log files # This path must be absolute @@ -238,9 +253,11 @@ remote_base_log_folder = {{ airflow_remote_base_log_folder }} encrypt_s3_logs = {{ airflow_encrypt_s3_logs }} # Logging level +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. logging_level = {{ airflow_logging_level }} # Logging level for Flask-appbuilder UI +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. fab_logging_level = {{ airflow_fab_logging_level }} # Logging class @@ -350,11 +367,11 @@ fail_fast = {{ airflow_fail_fast }} # deprecated since version 2.0. Please consider using # `the Stable REST API `__. # For more information on migration, see -# `UPDATING.md `_ +# `UPDATING.md `_ enable_experimental_api = {{ airflow_enable_experimental_api }} # How to authenticate users of the API. See -# https://airflow.apache.org/docs/stable/security.html for possible values. +# https://airflow.apache.org/docs/apache-airflow/stable/security.html for possible values. # ("airflow.api.auth.backend.default" allows all requests for historic reasons) auth_backend = {{ airflow_auth_backend }} @@ -378,6 +395,18 @@ google_oauth2_audience = {{ airflow_google_oauth2_audience }} # Example: google_key_path = /files/service-account-json google_key_path = {{ airflow_google_key_path}} +# Used in response to a preflight request to indicate which HTTP +# headers can be used when making the actual request. This header is +# the server side response to the browser's +# Access-Control-Request-Headers header. +access_control_allow_headers = {{ airflow_access_control_allow_headers }} + +# Specifies the method or methods allowed when accessing the resource. +access_control_allow_methods = {{ airflow_access_control_allow_methods }} + +# Indicates whether the response can be shared with requesting code from the given origin. +access_control_allow_origin = {{ airflow_access_control_allow_origin }} + [lineage] # what lineage backend to use backend = {{ airflow_lineage_backend }} @@ -398,6 +427,9 @@ default_ram = {{ airflow_operator_default_ram }} default_disk = {{ airflow_operator_default_disk }} default_gpus = {{ airflow_operator_default_gpus }} +# Default queue that tasks get assigned to and that worker listen on. +default_queue = {{ airflow_default_queue }} + # Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. # If set to False, an exception will be thrown, otherwise only the console message will be displayed. allow_illegal_arguments = {{ airflow_allow_illegal_arguments }} @@ -454,8 +486,9 @@ worker_refresh_interval = {{ airflow_webserver_worker_refresh_interval }} # then reload the gunicorn. reload_on_plugin_change = {{ airflow_webserver_reload_on_plugin_change }} -# Secret key used to run your flask app -# It should be as random as possible +# Secret key used to run your flask app. It should be as random as possible. However, when running +# more than 1 instances of webserver, make sure all of them use the same ``secret_key`` otherwise +# one of them will error with "CSRF session token is missing". secret_key = {{ airflow_webserver_secret_key }} # Number of workers to run the Gunicorn web server @@ -492,10 +525,6 @@ dag_default_view = {{ airflow_webserver_dag_default_view }} # ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) dag_orientation = {{ airflow_webserver_dag_orientation }} -# Puts the webserver in demonstration mode; blurs the names of Operators for -# privacy. -demo_mode = {{ airflow_webserver_demo_mode }} - # The amount of time (in secs) webserver will wait for initial handshake # while fetching logs from other worker machine log_fetch_timeout_sec = {{ airflow_webserver_log_fetch_timeout_sec }} @@ -571,6 +600,10 @@ update_fab_perms = {{ airflow_webserver_update_fab_perms }} # ``session_lifetime_minutes`` of non-activity session_lifetime_minutes = {{ airflow_webserver_session_lifetime_minutes }} +# Sets a custom page title for the DAGs overview page and site title for all pages +# instance_name = +{% if airflow_instance_name is defined %}instance_name = {{ airflow_instance_name }}{% endif %} + [email] # Configuration email backend and whether to @@ -578,6 +611,9 @@ session_lifetime_minutes = {{ airflow_webserver_session_lifetime_minutes }} # Email backend to use email_backend = {{ airflow_email_backend }} +# Email connection to use +email_conn_id = {{ airflow_email_conn_id }} + # Whether email alerts should be sent when a task is retried default_email_on_retry = {{ airflow_email_default_email_on_retry }} @@ -602,10 +638,10 @@ default_email_on_failure = {{ airflow_email_default_email_on_failure }} smtp_host = {{ airflow_smtp_host }} smtp_starttls = {{ airflow_smtp_starttls }} smtp_ssl = {{ airflow_smtp_ssl }} -smtp_port = 25 -smtp_mail_from = airflow@example.com -smtp_timeout = 30 -smtp_retry_limit = 5 +smtp_port = {{ airflow_smtp_port }} +smtp_mail_from = {{ airflow_smtp_mail_from }} +smtp_timeout = {{ airflow_smtp_timeout }} +smtp_retry_limit = {{ airflow_smtp_retry_limit }} {% if airflow_smtp_user %} smtp_user = {{ airflow_smtp_user }} {% endif %} @@ -706,9 +742,6 @@ flower_port = {{ airflow_flower_port }} # Example: flower_basic_auth = user1:password1,user2:password2 flower_basic_auth = {{ airflow_flower_basic_auth }} -# Default queue that tasks get assigned to and that worker listen on. -default_queue = {{ airflow_celery_default_queue }} - # How many processes CeleryExecutor uses to sync task state. # 0 means to use max(1, number of cores - 1) processes. sync_parallelism = {{ airflow_celery_sync_parallelism }} @@ -870,6 +903,17 @@ schedule_after_task_execution = {{ airflow_scheduler_schedule_after_task_executi # This defines how many processes will run. parsing_processes = {{ airflow_scheduler_parsing_processes }} +# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. +# The scheduler will list and sort the dag files to decide the parsing order. +# +# * ``modified_time``: Sort by modified time of the files. This is useful on large scale to parse the +# recently modified DAGs first. +# * ``random_seeded_by_host``: Sort randomly across multiple Schedulers but with same order on the +# same host. This is useful when running with Scheduler in HA mode where each scheduler can +# parse different DAG files. +# * ``alphabetical``: Sort by filename +file_parsing_sort_mode = {{ airflow_file_parsing_sort_mode }} + # Turn off scheduler use of cron intervals by setting this to False. # DAGs submitted manually in the web UI or with trigger_dag will still run. use_job_schedule = {{ airflow_scheduler_use_job_schedule }} @@ -878,6 +922,9 @@ use_job_schedule = {{ airflow_scheduler_use_job_schedule }} # Only has effect if schedule_interval is set to None in DAG allow_trigger_in_future = {{ airflow_scheduler_allow_trigger_in_future }} +# DAG dependency detector class to use +dependency_detector = {{ airflow_dependency_detector }} + [kerberos] ccache = {{ airflow_kerberos_ccache }} @@ -890,13 +937,6 @@ keytab = {{ airflow_kerberos_keytab }} [github_enterprise] api_rev = {{ airflow_github_enterprise_api_rev }} -[admin] -# UI to hide sensitive variable fields when set to True -hide_sensitive_variable_fields = {{ airflow_admin_hide_sensitive_variable_fields }} - -# A comma-separated list of sensitive keywords to look for in variables names. -sensitive_variable_fields = {{ airflow_admin_sensitive_variable_fields }} - [elasticsearch] # Elasticsearch host host = {{ airflow_elasticsearch_host }} @@ -921,6 +961,12 @@ json_format = {{ airflow_elasticsearch_json_format }} # Log fields to also attach to the json output, if enabled json_fields = {{ airflow_elasticsearch_json_fields }} +# The field where host name is stored (normally either `host` or `host.name`) +host_field = {{ airflow_host_field }} + +# The field where offset is stored (normally either `offset` or `log.offset`) +offset_field = {{ airflow_offset_field }} + [elasticsearch_configs] use_ssl = {{ airflow_elasticsearch_configs_use_ssl }} verify_certs = {{ airflow_elasticsearch_configs_verify_certs }} @@ -943,6 +989,8 @@ delete_worker_pods = {{ airflow_kubernetes_delete_worker_pods }} # If False (and delete_worker_pods is True), # failed worker pods will not be deleted so users can investigate them. +# This only prevents removal of worker pods where the worker itself failed, +# not when the task it ran failed. delete_worker_pods_on_failure = {{ airlfow_kubernetes_delete_worker_pods_on_failure }} # Number of Kubernetes Worker Pod creation calls per scheduler loop. @@ -999,6 +1047,19 @@ tcp_keep_intvl = {{ airflow_kubernetes_tcp_keep_intvl }} # a connection is considered to be broken. tcp_keep_cnt = {{ airflow_kubernetes_tcp_keep_cnt }} +# Set this to false to skip verifying SSL certificate of Kubernetes python client. +verify_ssl = {{ airflow_verify_ssl }} + +# How long in seconds a worker can be in Pending before it is considered a failure +worker_pods_pending_timeout = {{ airflow_worker_pods_pending_timeout }} + +# How often in seconds to check if Pending workers have exceeded their timeouts +worker_pods_pending_timeout_check_interval = {{ airflow_worker_pods_pending_timeout_check_interval }} + +# How many pending pods to check for timeout violations in each check interval. +# You may want this higher if you have a very large cluster and/or use ``multi_namespace_mode``. +worker_pods_pending_timeout_batch_size = {{ airflow_worker_pods_pending_timeout_batch_size }} + [smart_sensor] # When `use_smart_sensor` is True, Airflow redirects multiple qualified sensor tasks to # smart sensor task. diff --git a/test-requirements.txt b/test-requirements.txt index 357a06a..75df71e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -ansible==2.9.9 -molecule==3.0.6 +ansible==2.9.21 +molecule==3.0.8 docker==4.1.0 ansible-lint==4.2.0 \ No newline at end of file