From 16ade52c91e6711613e08aaec5836c69d104734c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 02:52:54 -0600 Subject: [PATCH 1/4] test_ambassador.py: Fuss with comment formatting This is essentially so that 'black' and 'isort' in the coming commits don't mess up its grouping. Signed-off-by: Luke Shumaker --- python/tests/kat/test_ambassador.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/python/tests/kat/test_ambassador.py b/python/tests/kat/test_ambassador.py index f1c1a9541b..2215cbd4c8 100644 --- a/python/tests/kat/test_ambassador.py +++ b/python/tests/kat/test_ambassador.py @@ -6,7 +6,6 @@ from abstract_tests import AmbassadorTest # Import all the real tests from other files, to make it easier to pick and choose during development. - import t_basics import t_bufferlimitbytes import t_chunked_length @@ -34,11 +33,8 @@ import t_lua_scripts import t_max_req_header_kb import t_no_ui -# mapping tests executed in the default namespace -import t_mappingtests_default -# t_plain include t_mappingtests_plain and t_optiontests as imports -# these tests require each other and need to be executed as a set -import t_plain +import t_mappingtests_default # mapping tests executed in the default namespace +import t_plain # t_plain include t_mappingtests_plain and t_optiontests as imports; these tests require each other and need to be executed as a set import t_queryparameter_routing import t_ratelimit import t_redirect @@ -46,8 +42,7 @@ import t_request_header import t_retrypolicy #import t_shadow -# t_stats has tests for statsd and dogstatsd. It's too flaky to run all the time. -# import t_stats +#import t_stats # t_stats has tests for statsd and dogstatsd. It's too flaky to run all the time. import t_tcpmapping import t_tls import t_tracing From f51e0090b990efd8a35cbed31fd22add09268b77 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 19:21:20 -0600 Subject: [PATCH 2/4] selfsigned.py.gen: Fuss with whitespace and formatting Signed-off-by: Luke Shumaker --- python/tests/selfsigned.py | 131 +++++++++++++++++++------------- python/tests/selfsigned.py.gen | 133 ++++++++++++++++++++------------- 2 files changed, 163 insertions(+), 101 deletions(-) diff --git a/python/tests/selfsigned.py b/python/tests/selfsigned.py index 2c6406b117..01321421a6 100644 --- a/python/tests/selfsigned.py +++ b/python/tests/selfsigned.py @@ -3,6 +3,7 @@ from base64 import b64encode from typing import Dict, List, NamedTuple, Optional + class Cert(NamedTuple): names: List[str] pubcert: str @@ -10,21 +11,24 @@ class Cert(NamedTuple): @property def k8s_crt(self) -> str: - return b64encode((self.pubcert+"\n").encode('utf-8')).decode('utf-8') + return b64encode((self.pubcert + "\n").encode("utf-8")).decode("utf-8") @property def k8s_key(self) -> str: - return b64encode((self.privkey+"\n").encode('utf-8')).decode('utf-8') + return b64encode((self.privkey + "\n").encode("utf-8")).decode("utf-8") + def strip(s: str) -> str: return "\n".join(l.strip() for l in s.split("\n") if l.strip()) + _TLSCerts: List[Cert] = [ Cert( names=["master.datawire.io"], # Note: This cert is also used to sign several other certs in # this file (as the issuer). - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIID8zCCAtugAwIBAgIRAIBtMsh/xwUcw6m3hSPuJP4wDQYJKoZIhvcNAQELBQAw eDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -49,8 +53,10 @@ def strip(s: str) -> str: 6JVnpZi+8XyWWHV0LUtLYx7ZRFUMTf0QdBeh4jLHcKozFSXBk52AFJTfd1wVs7EX 1hOoYbcTbQ== -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC/gqEHyomZn0Xd 0pLj1FuZ4fwlNuqMM7lIwu+T8bjrjs2/1A/oJpdp5Uhp0YtPT3zbKmywV3TjfWAz @@ -79,9 +85,9 @@ def strip(s: str) -> str: SVx+QqH6cMtBK6tSdjJT2KuOfqkzA7tK6SzmApaZTZiz8XM3MooYDj86EJ93l8EH xvKPa17iyHPUWNTHy9NuTA== -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=["presto.example.com"], # Note: @@ -89,7 +95,8 @@ def strip(s: str) -> str: # (rather than being self-signed). # 2. This cert is a client cert (rather than being a server # cert). - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIID8TCCAtmgAwIBAgIQHjsjEOZ4SEEVcrnClDnBPjANBgkqhkiG9w0BAQsFADB4 MQswCQYDVQQGEwJVUzELMAkGA1UECBMCTUExDzANBgNVBAcTBkJvc3RvbjEYMBYG @@ -114,8 +121,10 @@ def strip(s: str) -> str: 4gpi/Fe5t6hpjGu2AiUQ4CGxUeAYVMl62WDefAFMjOL1nj0KeszCh9BcVKEg2LaD MbHj/1Q= -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDjKEqZ9hpEasow ovCgiL59bFxxFeMFUOMvvn2PjFjZaZ1mwDQ4aS6E+dT3Znu8ysRbYiivUN4mQlRG @@ -144,12 +153,13 @@ def strip(s: str) -> str: fNnQ+OIW34GWz7pojjvVgirlFVmrT9gV6OXJBmh9aGvQN0En8qxZVQouo/UUkwoO 4gpH/If2ar0U3JDIU+d87o0= -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=["ratelimit.datawire.io"], - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIID2jCCAsKgAwIBAgIRAKlRg3DeRR97bt/PNtG2qw0wDQYJKoZIhvcNAQELBQAw ezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -173,8 +183,10 @@ def strip(s: str) -> str: 5+mskjzbXGkjzsicklr7Nkji2VlsfOTCQudacamYj11D8YhYOzrqd0wh9DtAQD5a 44RUfY8jf7fdqK4ZkaDhFXfQzE/iZ5WjVD6h5aAD -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC8qWCqEKRCm0sh t61TNnXQ7d16g62up3wZmfGKltN3SxNF2PIRA9ij67l9vNUiklOb4BpgNCEVH3pc @@ -203,14 +215,15 @@ def strip(s: str) -> str: DyFlHyWTrFwoDBYy4d7cNgDJnaIN2qBY36GDyL2x7/DyKd5+CQN07XWuOmnGrzIo 6ABc+KN1kmXbr9VteFRagAI= -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=["ambassador.example.com"], # Note: This cert is signed by the "master.datawire.io" cert # (rather than being self-signed). - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIID+jCCAuKgAwIBAgIRAMQO4rSR9giQAjULkZVYJd0wDQYJKoZIhvcNAQELBQAw eDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -235,8 +248,10 @@ def strip(s: str) -> str: 1r/e4SLxS8diOQyrwXwn5dqPH713qjnuhk7fFRSa1aa6aFZvsBmqBEykBeVJtErI BUI4H1BdGgQsOuX/nbk= -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC7IW+b05L3KzjY 3p4OPsynN/8OGztOANqVOXgjQfoe0eQ9KXSMRLccirWOgQHDUlnpuArPpwxmFx8a @@ -265,12 +280,13 @@ def strip(s: str) -> str: f/0OyEC+Y9lQB1/ka8BSjpIfM2L0FWDHJUsGOKZ+C+KlGZpRgF8YDFZHdwmWOokL GhN8OjQwTBEo9hm21z14qvAC -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=["tls-context-host-2"], - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIID0TCCArmgAwIBAgIRAL/XZdKaUkBM0tuy0P4+UNEwDQYJKoZIhvcNAQELBQAw eDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -294,8 +310,10 @@ def strip(s: str) -> str: r6cDMXyBph9J29kPgu4QuAcGInwYqmB5jLJB2FW1mOzg5eG4WW7ZFLDBvA9SnUQP LCNz4+dspzQ8mkD5b+lhuHmY0ywd -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCvElDztAzQnaMM RON6bUrHF16J+WpbST37tGUqN/HtpY0SnwQw9n/1otvshAHWfcwf08z6EsjGbdWF @@ -324,12 +342,13 @@ def strip(s: str) -> str: Nwq6gHnTGSbrM+WAa78lvYD6arIW6fD8MvCQXWMN4J1FVk86H2vXhaSObOTls9+D OGSk6BLtLUX79aY4TfNg0yk= -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=["tls-context-host-1"], - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIID0TCCArmgAwIBAgIRAKLpGzzKxkWzv1M5uTQKopwwDQYJKoZIhvcNAQELBQAw eDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -353,8 +372,10 @@ def strip(s: str) -> str: 39dj1MiqRdOoc7R6oKwyFwASd922pLwTXmovV93gg4hJAXCw7m8a8GB1gPt710j0 GTueVtWpvLIGu5TB23vCmQBgcmJP -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCx8cmKaWjjSlUB ayOHdSO/sZ5idQ4+IyOiHLraBCk5Sy2cRbSPyo4/kuHD31JldwK5tr7Bt2o2eQDI @@ -383,12 +404,13 @@ def strip(s: str) -> str: /DlPdOnqrlBhY3TdmVk60j7IM8182GKjcMImfFoMjot/oZlKxrq+PcsVoIHeOQL6 b6gCAPn1RvigtZtc5EUILDfG -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=["localhost"], - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIIDtTCCAp2gAwIBAgIQBmwf1lv4+/h6sWcQkb0rdzANBgkqhkiG9w0BAQsFADBv MQswCQYDVQQGEwJVUzELMAkGA1UECBMCTUExDzANBgNVBAcTBkJvc3RvbjEYMBYG @@ -411,8 +433,10 @@ def strip(s: str) -> str: eNA6OuaIk6RTwTSDfGP0aLOUs9YTITez2zG+bIDxL1jgULj2z5APVjsq97BJpAxP zXNRz2CDAv3axyY2uO63k1NNSbJaFx4t+9QhfyzfBIEsmvx1k5W9k1g= -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCepF6M43h8NXG8 kW++17KVQENYCMqZAZfe1rT2O9x1anp+9i8gEmBL8DTfobFAIOKNmOb2vRG6r5K1 @@ -441,21 +465,22 @@ def strip(s: str) -> str: dYfI6Lw45AhpQsANPM2dB5ak+3TJGr2RvpSIF6M0+CA69A+VL7NOeBt0acXZv9p7 rSbwm9Ewfog/UaoT+fUUfcs= -----END PRIVATE KEY----- - """) + """ + ), ), - Cert( names=[ "a.domain.com", "b.domain.com", "*.domain.com", - #"localhost", # don't clash with the other "localhost" cert + # "localhost", # don't clash with the other "localhost" cert "127.0.0.1", - "0:0:0:0:0:0:0:1" + "0:0:0:0:0:0:0:1", ], # Note: This cert is signed by a cert not present in this file # (rather than being self-signed). - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIIEADCCAuigAwIBAgIRAJ3dtx28bAfwdex3/R/ETrAwDQYJKoZIhvcNAQELBQAw cjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -480,8 +505,10 @@ def strip(s: str) -> str: bZNZImIIvzD6Osl7TwQf8AziWDlgEKhLrxEuRUeb15+HVXUx7XSPxPtHByRCozxA PZHq6TsBZAaoCzAOP9uxC9JcQJA= -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDgabCrcf2+DxEX qNp2Y+mml1u5XjbaWZX8t14H0yp7hrB4nLg63z4HGqapZZvaPR/xemPTYaQZYiSZ @@ -510,12 +537,13 @@ def strip(s: str) -> str: OqTouIFDHW+jluzNa++cT2FVM2Tj5G9uKwVxS3SfITfZEHRmfoxwWmrmn+wgz+do NuVfL7r0erO8V2bLW2ASEDs= -----END PRIVATE KEY----- - """) - ), - + """ + ), + ), Cert( names=["acook"], - pubcert=strip(""" + pubcert=strip( + """ -----BEGIN CERTIFICATE----- MIIDqjCCApKgAwIBAgIRAKcrynhenFaTcxKOdiGvT6QwDQYJKoZIhvcNAQELBQAw azELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1BMQ8wDQYDVQQHEwZCb3N0b24xGDAW @@ -538,8 +566,10 @@ def strip(s: str) -> str: cTXNk8q9anHFFASQ2o4RFMUaKTMXf5OtrZ41x/aGznJ6LM3OpPhRSITPD9Nhosja uKD5HptKkoH+RW74nmDZbUPoOWl4OcG9EXvTm1PG -----END CERTIFICATE----- - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDAg4ssHk+na8yg Sofzl8O4hIcISKs61iFX002Vo4MxyLPbydxvKvekYHrA/CKbzJfNsECX0V1+waFP @@ -568,8 +598,9 @@ def strip(s: str) -> str: Kwa//xmc5pgnSmGaUE1XKN59iMJBYzC7CaljdxtuZSipLhZhZfWS3IZqnJYQyjI8 mBtaOgOj5lu1xVKdSm/8Gp4= -----END PRIVATE KEY----- - """) + """ ), + ), ] -TLSCerts: Dict[str, Cert] = { k: v for v in _TLSCerts for k in v.names } +TLSCerts: Dict[str, Cert] = {k: v for v in _TLSCerts for k in v.names} diff --git a/python/tests/selfsigned.py.gen b/python/tests/selfsigned.py.gen index fe09ea4fd1..384c1cae14 100755 --- a/python/tests/selfsigned.py.gen +++ b/python/tests/selfsigned.py.gen @@ -1,4 +1,4 @@ - #!/usr/bin/env bash +#!/usr/bin/env bash set -euE -o pipefail if [[ $# -gt 0 ]]; then @@ -25,6 +25,7 @@ cat <<_EOF_ from base64 import b64encode from typing import Dict, List, NamedTuple, Optional + class Cert(NamedTuple): names: List[str] pubcert: str @@ -32,28 +33,33 @@ class Cert(NamedTuple): @property def k8s_crt(self) -> str: - return b64encode((self.pubcert+"\n").encode('utf-8')).decode('utf-8') + return b64encode((self.pubcert + "\n").encode("utf-8")).decode("utf-8") @property def k8s_key(self) -> str: - return b64encode((self.privkey+"\n").encode('utf-8')).decode('utf-8') + return b64encode((self.privkey + "\n").encode("utf-8")).decode("utf-8") + def strip(s: str) -> str: return "\n".join(l.strip() for l in s.split("\n") if l.strip()) + _TLSCerts: List[Cert] = [ Cert( names=["master.datawire.io"], # Note: This cert is also used to sign several other certs in # this file (as the issuer). - pubcert=strip(""" + pubcert=strip( + """ $(master.datawire.io cert | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(master.datawire.io key | indent) - """) + """ + ), ), - Cert( names=["presto.example.com"], # Note: @@ -61,95 +67,120 @@ _TLSCerts: List[Cert] = [ # (rather than being self-signed). # 2. This cert is a client cert (rather than being a server # cert). - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --is-client=true --is-server=false --hosts=presto.example.com --signed-by=<(master.datawire.io cert),<(master.datawire.io key) | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --is-client=true --is-server=false --hosts=presto.example.com --signed-by=<(master.datawire.io cert),<(master.datawire.io key) | indent) - """) + """ + ), ), - Cert( names=["ratelimit.datawire.io"], - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts=ratelimit.datawire.io | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts=ratelimit.datawire.io | indent) - """) + """ + ), ), - Cert( names=["ambassador.example.com"], # Note: This cert is signed by the "master.datawire.io" cert # (rather than being self-signed). - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts=ambassador.example.com --signed-by=<(master.datawire.io cert),<(master.datawire.io key) | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts=ambassador.example.com --signed-by=<(master.datawire.io cert),<(master.datawire.io key) | indent) - """) + """ + ), ), - Cert( names=["tls-context-host-2"], - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts=tls-context-host-2 | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts=tls-context-host-2 | indent) - """) + """ + ), ), - Cert( names=["tls-context-host-1"], - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts=tls-context-host-1 | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts=tls-context-host-1 | indent) - """) + """ + ), ), - Cert( names=["localhost"], - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts=localhost | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts=localhost | indent) - """) + """ + ), ), - Cert( names=[ "a.domain.com", "b.domain.com", "*.domain.com", - #"localhost", # don't clash with the other "localhost" cert + # "localhost", # don't clash with the other "localhost" cert "127.0.0.1", - "0:0:0:0:0:0:0:1" + "0:0:0:0:0:0:0:1", ], # Note: This cert is signed by a cert not present in this file # (rather than being self-signed). - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts='a.domain.com,b.domain.com,*.domain.com,localhost,127.0.0.1,::1' | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts='a.domain.com,b.domain.com,*.domain.com,localhost,127.0.0.1,::1' | indent) - """) - ), - + """ + ), + ), Cert( names=["acook"], - pubcert=strip(""" + pubcert=strip( + """ $(testcert-gen ${cert} --hosts=acook | indent) - """), - privkey=strip(""" + """ + ), + privkey=strip( + """ $(testcert-gen ${key} --hosts=acook | indent) - """) + """ ), + ), ] -TLSCerts: Dict[str, Cert] = { k: v for v in _TLSCerts for k in v.names } +TLSCerts: Dict[str, Cert] = {k: v for v in _TLSCerts for k in v.names} _EOF_ From ab9d10931fa79cdb39a70e9ab274423f4eefd12d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 02:15:42 -0600 Subject: [PATCH 3/4] Add the 'black' Python code formatter [ci-skip] Black has been the standard Python formatter for a couple of years now. Use it the way we use gofmt. Signed-off-by: Luke Shumaker --- build-aux/lint.mk | 11 +++++++++++ pyproject.toml | 2 ++ python/requirements-dev.txt | 1 + 3 files changed, 14 insertions(+) create mode 100644 pyproject.toml diff --git a/build-aux/lint.mk b/build-aux/lint.mk index 9b4bc7d7cf..fcd73ce3a8 100644 --- a/build-aux/lint.mk +++ b/build-aux/lint.mk @@ -18,6 +18,7 @@ format/go: $(tools/golangci-lint) # Python lint-deps += $(OSS_HOME)/venv + lint-goals += lint/mypy lint/mypy: $(OSS_HOME)/venv set -e; { \ @@ -31,6 +32,16 @@ lint/mypy: $(OSS_HOME)/venv .PHONY: lint/mypy clean: .dmypy.json.rm .mypy_cache.rm-r +lint-goals += lint/black +lint/black: $(OSS_HOME)/venv + . $(OSS_HOME)/venv/bin/activate && black --check ./python/ +.PHONY: lint/black + +format-goals += format/black +format/black: $(OSS_HOME)/venv + . $(OSS_HOME)/venv/bin/activate && black ./python/ +.PHONY: format/black + # # Helm diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..aa4949aa1c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 100 diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index 680968b184..e734fb857f 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -9,6 +9,7 @@ pytest==6.2.5 pytest-cov pytest-rerunfailures retry +black==22.6.0 # Type stubs types-orjson From e110ebf3fc24145448f3739b54a2b1a9de06c2d2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 02:43:42 -0600 Subject: [PATCH 4/4] make format/black Signed-off-by: Luke Shumaker --- python/ambassador/VERSION.py | 2 +- python/ambassador/ambscout.py | 158 ++- python/ambassador/cache.py | 17 +- python/ambassador/compile.py | 19 +- python/ambassador/config/acmapping.py | 77 +- python/ambassador/config/acpragma.py | 34 +- python/ambassador/config/acresource.py | 99 +- python/ambassador/config/config.py | 230 ++-- python/ambassador/diagnostics/diagnostics.py | 312 ++--- python/ambassador/diagnostics/envoy_stats.py | 151 ++- python/ambassador/envoy/common.py | 33 +- python/ambassador/envoy/v3/__init__.py | 1 - .../envoy/v3/v3_static_resources.py | 16 +- python/ambassador/envoy/v3/v3admin.py | 21 +- python/ambassador/envoy/v3/v3bootstrap.py | 263 ++-- python/ambassador/envoy/v3/v3cidrrange.py | 14 +- python/ambassador/envoy/v3/v3cluster.py | 194 +-- python/ambassador/envoy/v3/v3config.py | 36 +- python/ambassador/envoy/v3/v3httpfilter.py | 380 +++--- python/ambassador/envoy/v3/v3listener.py | 620 +++++---- python/ambassador/envoy/v3/v3ratelimit.py | 20 +- .../ambassador/envoy/v3/v3ratelimitaction.py | 92 +- python/ambassador/envoy/v3/v3route.py | 448 ++++--- python/ambassador/envoy/v3/v3tls.py | 57 +- python/ambassador/envoy/v3/v3tracing.py | 43 +- python/ambassador/fetch/ambassador.py | 39 +- python/ambassador/fetch/dependency.py | 44 +- python/ambassador/fetch/fetcher.py | 219 ++-- python/ambassador/fetch/ingress.py | 175 +-- python/ambassador/fetch/k8sobject.py | 56 +- python/ambassador/fetch/k8sprocessor.py | 8 +- python/ambassador/fetch/knative.py | 108 +- python/ambassador/fetch/location.py | 6 +- python/ambassador/fetch/resource.py | 75 +- python/ambassador/fetch/secret.py | 57 +- python/ambassador/fetch/service.py | 137 +- python/ambassador/ir/__init__.py | 1 - python/ambassador/ir/ir.py | 621 +++++---- python/ambassador/ir/irambassador.py | 378 +++--- python/ambassador/ir/irauth.py | 118 +- python/ambassador/ir/irbasemapping.py | 176 ++- python/ambassador/ir/irbasemappinggroup.py | 51 +- python/ambassador/ir/irbuffer.py | 27 +- python/ambassador/ir/ircluster.py | 246 ++-- python/ambassador/ir/ircors.py | 74 +- python/ambassador/ir/irerrorresponse.py | 175 ++- python/ambassador/ir/irfilter.py | 31 +- python/ambassador/ir/irgzip.py | 47 +- python/ambassador/ir/irhost.py | 341 +++-- python/ambassador/ir/irhttpmapping.py | 304 +++-- python/ambassador/ir/irhttpmappinggroup.py | 314 +++-- python/ambassador/ir/iripallowdeny.py | 63 +- python/ambassador/ir/irlistener.py | 178 +-- python/ambassador/ir/irlogservice.py | 79 +- python/ambassador/ir/irmappingfactory.py | 28 +- python/ambassador/ir/irratelimit.py | 56 +- python/ambassador/ir/irresource.py | 94 +- python/ambassador/ir/irretrypolicy.py | 49 +- python/ambassador/ir/irserviceresolver.py | 270 ++-- python/ambassador/ir/irtcpmapping.py | 78 +- python/ambassador/ir/irtcpmappinggroup.py | 146 ++- python/ambassador/ir/irtls.py | 111 +- python/ambassador/ir/irtlscontext.py | 280 +++-- python/ambassador/ir/irtracing.py | 73 +- python/ambassador/ir/irutils.py | 24 +- python/ambassador/reconfig_stats.py | 48 +- python/ambassador/resource.py | 91 +- python/ambassador/scout.py | 93 +- python/ambassador/utils.py | 472 ++++--- python/ambassador_cli/ambassador.py | 210 +++- python/ambassador_cli/ert.py | 16 +- python/ambassador_cli/grab_snapshots.py | 65 +- python/ambassador_cli/madness.py | 36 +- python/ambassador_cli/mockery.py | 263 ++-- python/ambassador_diag/diagd.py | 1007 +++++++++------ python/kat/harness.py | 729 ++++++----- python/kat/parser.py | 52 +- python/kat/utils.py | 17 +- python/kubewatch.py | 202 +-- python/post_update.py | 41 +- python/setup.py | 36 +- python/tests/integration/manifests.py | 69 +- python/tests/integration/test_docker.py | 69 +- .../integration/test_header_case_overrides.py | 206 +-- python/tests/integration/test_knative.py | 153 ++- python/tests/integration/test_scout.py | 152 +-- python/tests/integration/test_watt_scaling.py | 32 +- python/tests/integration/utils.py | 92 +- python/tests/kat/abstract_tests.py | 181 ++- python/tests/kat/t_basics.py | 107 +- python/tests/kat/t_bufferlimitbytes.py | 9 +- python/tests/kat/t_chunked_length.py | 22 +- python/tests/kat/t_circuitbreaker.py | 156 ++- python/tests/kat/t_cluster_tag.py | 17 +- python/tests/kat/t_consul.py | 159 ++- python/tests/kat/t_cors.py | 18 +- python/tests/kat/t_dns_type.py | 34 +- python/tests/kat/t_envoy_logs.py | 28 +- python/tests/kat/t_error_response.py | 654 ++++++---- python/tests/kat/t_extauth.py | 671 +++++++--- python/tests/kat/t_grpc.py | 90 +- python/tests/kat/t_grpc_bridge.py | 33 +- python/tests/kat/t_grpc_stats.py | 203 +-- python/tests/kat/t_grpc_web.py | 47 +- python/tests/kat/t_gzip.py | 40 +- python/tests/kat/t_headerrouting.py | 60 +- .../kat/t_headerswithunderscoresaction.py | 24 +- python/tests/kat/t_hosts.py | 1120 ++++++++++++----- python/tests/kat/t_ingress.py | 351 ++++-- python/tests/kat/t_ip_allow_deny.py | 64 +- python/tests/kat/t_listeneridletimeout.py | 76 +- python/tests/kat/t_loadbalancer.py | 261 ++-- python/tests/kat/t_logservice.py | 78 +- python/tests/kat/t_lua_scripts.py | 11 +- python/tests/kat/t_mappingtests_default.py | 135 +- python/tests/kat/t_mappingtests_plain.py | 269 ++-- python/tests/kat/t_max_req_header_kb.py | 50 +- python/tests/kat/t_no_ui.py | 15 +- python/tests/kat/t_no_ui_allow_non_local.py | 15 +- python/tests/kat/t_optiontests.py | 64 +- python/tests/kat/t_plain.py | 14 +- python/tests/kat/t_queryparameter_routing.py | 43 +- python/tests/kat/t_ratelimit.py | 205 +-- python/tests/kat/t_redirect.py | 126 +- python/tests/kat/t_regexrewrite_forwarding.py | 28 +- python/tests/kat/t_request_header.py | 38 +- python/tests/kat/t_retrypolicy.py | 72 +- python/tests/kat/t_shadow.py | 35 +- python/tests/kat/t_stats.py | 88 +- python/tests/kat/t_tcpmapping.py | 138 +- python/tests/kat/t_tls.py | 1110 ++++++++++------ python/tests/kat/t_tracing.py | 257 ++-- python/tests/kat/test_ambassador.py | 9 +- python/tests/kubeutils.py | 11 +- python/tests/manifests.py | 4 +- python/tests/runutils.py | 14 +- .../unit/test_acme_privatekey_secrets.py | 91 +- .../unit/test_ambassador_module_validation.py | 114 +- python/tests/unit/test_bootstrap.py | 43 +- python/tests/unit/test_buffer_limit_bytes.py | 73 +- python/tests/unit/test_cache.py | 236 ++-- python/tests/unit/test_cidrrange.py | 62 +- python/tests/unit/test_cluster_options.py | 25 +- .../unit/test_common_http_protocol_options.py | 74 +- python/tests/unit/test_envoy_stats.py | 165 ++- python/tests/unit/test_envvar_expansion.py | 8 +- python/tests/unit/test_error_response.py | 340 ++--- python/tests/unit/test_fetch.py | 184 +-- python/tests/unit/test_hcm.py | 54 +- .../tests/unit/test_host_redirect_errors.py | 109 +- python/tests/unit/test_hostglob_matches.py | 62 +- python/tests/unit/test_ir.py | 88 +- python/tests/unit/test_irauth.py | 37 +- python/tests/unit/test_irlogservice.py | 210 ++-- python/tests/unit/test_irmapping.py | 9 +- python/tests/unit/test_irratelimit.py | 101 +- python/tests/unit/test_listener.py | 198 +-- ...t_listener_common_http_protocol_options.py | 30 +- .../test_listener_http_protocol_options.py | 26 +- .../tests/unit/test_listener_statsprefix.py | 35 +- python/tests/unit/test_lookup.py | 146 ++- python/tests/unit/test_mapping.py | 41 +- python/tests/unit/test_max_request_header.py | 42 +- python/tests/unit/test_qualify_service.py | 747 ++++++++--- python/tests/unit/test_reconfig_stats.py | 10 +- python/tests/unit/test_route.py | 27 +- python/tests/unit/test_router.py | 43 +- python/tests/unit/test_shadow.py | 60 +- python/tests/unit/test_statsd.py | 133 +- python/tests/unit/test_timer.py | 3 +- python/tests/unit/test_tracing.py | 58 +- python/tests/utils.py | 124 +- python/watch_hook.py | 254 ++-- 173 files changed, 14901 insertions(+), 9260 deletions(-) diff --git a/python/ambassador/VERSION.py b/python/ambassador/VERSION.py index 496aa999eb..19ebd9983b 100644 --- a/python/ambassador/VERSION.py +++ b/python/ambassador/VERSION.py @@ -24,7 +24,7 @@ Commit = "MISSING(FILE)" try: with open(os.path.join(os.path.dirname(__file__), "..", "ambassador.version")) as version: - info = version.read().split('\n') + info = version.read().split("\n") while len(info) < 2: info.append("MISSING(VAL)") diff --git a/python/ambassador/ambscout.py b/python/ambassador/ambscout.py index f3dc5a3602..623b1768bf 100644 --- a/python/ambassador/ambscout.py +++ b/python/ambassador/ambscout.py @@ -26,20 +26,20 @@ def __init__(self, logger, app: str, version: str, install_id: str) -> None: self.events: List[Dict[str, Any]] = [] - self.logger.info(f'LocalScout: initialized for {app} {version}: ID {install_id}') + self.logger.info(f"LocalScout: initialized for {app} {version}: ID {install_id}") def report(self, **kwargs) -> dict: self.events.append(kwargs) - mode = kwargs['mode'] - action = kwargs['action'] + mode = kwargs["mode"] + action = kwargs["action"] now = datetime.datetime.now().timestamp() - kwargs['local_scout_timestamp'] = now + kwargs["local_scout_timestamp"] = now - if 'timestamp' not in kwargs: - kwargs['timestamp'] = now + if "timestamp" not in kwargs: + kwargs["timestamp"] = now self.logger.info(f"LocalScout: mode {mode}, action {action} ({kwargs})") @@ -47,16 +47,17 @@ def report(self, **kwargs) -> dict: "latest_version": self.version, "application": self.app, "cached": False, - "notices": [ { "level": "WARNING", "message": "Using LocalScout, result is faked!" } ], - "timestamp": now + "notices": [{"level": "WARNING", "message": "Using LocalScout, result is faked!"}], + "timestamp": now, } def reset_events(self) -> None: self.events = [] + class AmbScout: - reTaggedBranch: ClassVar = re.compile(r'^v?(\d+\.\d+\.\d+)(-[a-zA-Z][a-zA-Z]\.\d+)?$') - reGitDescription: ClassVar = re.compile(r'-(\d+)-g([0-9a-f]+)$') + reTaggedBranch: ClassVar = re.compile(r"^v?(\d+\.\d+\.\d+)(-[a-zA-Z][a-zA-Z]\.\d+)?$") + reGitDescription: ClassVar = re.compile(r"-(\d+)-g([0-9a-f]+)$") install_id: str runtime: str @@ -76,19 +77,25 @@ class AmbScout: _latest_version: Optional[str] = None _latest_semver: Optional[semantic_version.Version] = None - def __init__(self, install_id=None, update_frequency=datetime.timedelta(hours=12), local_only=False) -> None: + def __init__( + self, install_id=None, update_frequency=datetime.timedelta(hours=12), local_only=False + ) -> None: if not install_id: - install_id = os.environ.get('AMBASSADOR_CLUSTER_ID', - os.environ.get('AMBASSADOR_SCOUT_ID', "00000000-0000-0000-0000-000000000000")) + install_id = os.environ.get( + "AMBASSADOR_CLUSTER_ID", + os.environ.get("AMBASSADOR_SCOUT_ID", "00000000-0000-0000-0000-000000000000"), + ) self.install_id = install_id - self.runtime = "kubernetes" if os.environ.get('KUBERNETES_SERVICE_HOST', None) else "docker" - self.namespace = os.environ.get('AMBASSADOR_NAMESPACE', 'default') + self.runtime = "kubernetes" if os.environ.get("KUBERNETES_SERVICE_HOST", None) else "docker" + self.namespace = os.environ.get("AMBASSADOR_NAMESPACE", "default") # Allow an environment variable to state whether we're in Edge Stack. But keep the # existing condition as sufficient, so that there is less of a chance of breaking # things running in a container with this file present. - self.is_edge_stack = parse_bool(os.environ.get('EDGE_STACK', 'false')) or os.path.exists('/ambassador/.edge_stack') + self.is_edge_stack = parse_bool(os.environ.get("EDGE_STACK", "false")) or os.path.exists( + "/ambassador/.edge_stack" + ) self.app = "aes" if self.is_edge_stack else "ambassador" self.version = Version self.semver = self.get_semver(self.version) @@ -97,7 +104,9 @@ def __init__(self, install_id=None, update_frequency=datetime.timedelta(hours=12 # self.logger.setLevel(logging.DEBUG) self.logger.debug("Ambassador version %s built from %s" % (Version, Commit)) - self.logger.debug("Scout version %s%s" % (self.version, " - BAD SEMVER" if not self.semver else "")) + self.logger.debug( + "Scout version %s%s" % (self.version, " - BAD SEMVER" if not self.semver else "") + ) self.logger.debug("Runtime %s" % self.runtime) self.logger.debug("Namespace %s" % self.namespace) @@ -119,33 +128,43 @@ def reset_cache_time(self) -> None: def reset_events(self) -> None: if self._local_only: - assert(self._scout) + assert self._scout typecast(LocalScout, self._scout).reset_events() def __str__(self) -> str: - return ("%s: %s" % ("OK" if self._scout else "??", - self._scout_error if self._scout_error else "OK")) + return "%s: %s" % ( + "OK" if self._scout else "??", + self._scout_error if self._scout_error else "OK", + ) @property def scout(self) -> Optional[Union[Scout, LocalScout]]: if not self._scout: if self._local_only: - self._scout = LocalScout(logger=self.logger, - app=self.app, version=self.version, install_id=self.install_id) + self._scout = LocalScout( + logger=self.logger, + app=self.app, + version=self.version, + install_id=self.install_id, + ) self.logger.debug("LocalScout initialized") else: try: - self._scout = Scout(app=self.app, version=self.version, install_id=self.install_id) + self._scout = Scout( + app=self.app, version=self.version, install_id=self.install_id + ) self._scout_error = None self.logger.debug("Scout connection established") except OSError as e: self._scout = None self._scout_error = str(e) - self.logger.debug("Scout connection failed, will retry later: %s" % self._scout_error) + self.logger.debug( + "Scout connection failed, will retry later: %s" % self._scout_error + ) return self._scout - def report(self, force_result: Optional[dict]=None, no_cache=False, **kwargs) -> dict: + def report(self, force_result: Optional[dict] = None, no_cache=False, **kwargs) -> dict: _notices: List[ScoutNotice] = [] # Silly, right? @@ -163,11 +182,11 @@ def report(self, force_result: Optional[dict]=None, no_cache=False, **kwargs) -> result_was_cached: bool = False if not result: - if 'runtime' not in kwargs: - kwargs['runtime'] = self.runtime + if "runtime" not in kwargs: + kwargs["runtime"] = self.runtime - if 'commit' not in kwargs: - kwargs['commit'] = Commit + if "commit" not in kwargs: + kwargs["commit"] = Commit # How long since the last Scout update? If it's been more than an hour, # check Scout again. @@ -179,7 +198,7 @@ def report(self, force_result: Optional[dict]=None, no_cache=False, **kwargs) -> if use_cache: if self._last_update: since_last_update = now - typecast(datetime.datetime, self._last_update) - needs_update = (since_last_update > self._update_frequency) + needs_update = since_last_update > self._update_frequency if needs_update: if self.scout: @@ -188,79 +207,88 @@ def report(self, force_result: Optional[dict]=None, no_cache=False, **kwargs) -> self._last_update = now self._last_result = dict(**typecast(dict, result)) if result else None else: - result = { "scout": "unavailable: %s" % self._scout_error } - _notices.append({ "level": "DEBUG", - "message": "scout temporarily unavailable: %s" % self._scout_error }) + result = {"scout": "unavailable: %s" % self._scout_error} + _notices.append( + { + "level": "DEBUG", + "message": "scout temporarily unavailable: %s" % self._scout_error, + } + ) # Whether we could talk to Scout or not, update the timestamp so we don't # try again too soon. result_timestamp = datetime.datetime.now() else: - _notices.append({ "level": "DEBUG", "message": "Returning cached result" }) + _notices.append({"level": "DEBUG", "message": "Returning cached result"}) result = dict(**typecast(dict, self._last_result)) if self._last_result else None result_was_cached = True # We can't get here unless self._last_update is set. result_timestamp = typecast(datetime.datetime, self._last_update) else: - _notices.append({ "level": "INFO", "message": "Returning forced Scout result" }) + _notices.append({"level": "INFO", "message": "Returning forced Scout result"}) result_timestamp = datetime.datetime.now() if not self.semver: - _notices.append({ - "level": "WARNING", - "message": "Ambassador has invalid version '%s'??!" % self.version - }) + _notices.append( + { + "level": "WARNING", + "message": "Ambassador has invalid version '%s'??!" % self.version, + } + ) if result: - result['cached'] = result_was_cached + result["cached"] = result_was_cached else: - result = { 'cached': False } + result = {"cached": False} - result['timestamp'] = result_timestamp.timestamp() + result["timestamp"] = result_timestamp.timestamp() # Do version & notices stuff. - if 'latest_version' in result: - latest_version = result['latest_version'] + if "latest_version" in result: + latest_version = result["latest_version"] latest_semver = self.get_semver(latest_version) if latest_semver: self._latest_version = latest_version self._latest_semver = latest_semver else: - _notices.append({ - "level": "WARNING", - "message": "Scout returned invalid version '%s'??!" % latest_version - }) - - if (self._latest_semver and ((not self.semver) or - (self._latest_semver > self.semver))): - _notices.append({ - "level": "INFO", - "message": "Upgrade available! to Ambassador version %s" % self._latest_semver - }) - - if 'notices' in result: - rnotices = typecast(List[Union[str, ScoutNotice]], result['notices']) + _notices.append( + { + "level": "WARNING", + "message": "Scout returned invalid version '%s'??!" % latest_version, + } + ) + + if self._latest_semver and ((not self.semver) or (self._latest_semver > self.semver)): + _notices.append( + { + "level": "INFO", + "message": "Upgrade available! to Ambassador version %s" % self._latest_semver, + } + ) + + if "notices" in result: + rnotices = typecast(List[Union[str, ScoutNotice]], result["notices"]) for notice in rnotices: if isinstance(notice, str): - _notices.append({ "level": "WARNING", "message": notice }) + _notices.append({"level": "WARNING", "message": notice}) elif isinstance(notice, dict): - lvl = notice.get('level', 'WARNING').upper() - msg = notice.get('message', None) + lvl = notice.get("level", "WARNING").upper() + msg = notice.get("message", None) if msg: - _notices.append({ "level": lvl, "message": msg }) + _notices.append({"level": lvl, "message": msg}) else: - _notices.append({ "level": "WARNING", "message": dump_json(notice) }) + _notices.append({"level": "WARNING", "message": dump_json(notice)}) self._notices = _notices if self._notices: - result['notices'] = self._notices + result["notices"] = self._notices else: - result.pop('notices', None) + result.pop("notices", None) return result diff --git a/python/ambassador/cache.py b/python/ambassador/cache.py index be6eebc215..422df1500b 100644 --- a/python/ambassador/cache.py +++ b/python/ambassador/cache.py @@ -2,6 +2,7 @@ import logging + class Cacheable(dict): """ A dictionary that is specifically cacheable, by way of its added @@ -27,7 +28,7 @@ def cache_key(self, cache_key: str) -> None: CacheLink = Set[str] -class Cache(): +class Cache: """ A cache of Cacheables, supporting add/delete/fetch and also linking an owning Cacheable to an owned Cacheable. Deletion is cascaded: if you @@ -55,8 +56,7 @@ def reset_stats(self) -> None: def fn_name(fn: Optional[Callable]) -> str: return fn.__name__ if (fn and fn.__name__) else "-none-" - def add(self, rsrc: Cacheable, - on_delete: Optional[DeletionHandler]=None) -> None: + def add(self, rsrc: Cacheable, on_delete: Optional[DeletionHandler] = None) -> None: """ Adds an entry to the cache, if it's not already present. If on_delete is not None, it will called when rsrc is removed from @@ -105,7 +105,7 @@ def link(self, owner: Cacheable, owned: Cacheable) -> None: self.logger.debug(f"CACHE: linking {owner_key} -> {owned_key}") links = self.links.setdefault(owner_key, set()) - links.update([ owned_key ]) + links.update([owned_key]) def invalidate(self, key: str) -> None: """ @@ -126,7 +126,7 @@ def invalidate(self, key: str) -> None: self.invalidate_calls += 1 - worklist = [ key ] + worklist = [key] # Under the hood, "invalidating" something from this cache is really # deleting it, so we'll use "to_delete" for the set of things we're going @@ -167,10 +167,10 @@ def invalidate(self, key: str) -> None: self.logger.debug(f"CACHE: DEL {key}: smiting!") self.invalidated_objects += 1 - del(self.cache[key]) + del self.cache[key] if key in self.links: - del(self.links[key]) + del self.links[key] rsrc, on_delete = rdh @@ -245,8 +245,7 @@ def __init__(self, logger: logging.Logger) -> None: self.reset_stats() pass - def add(self, rsrc: Cacheable, - on_delete: Optional[DeletionHandler]=None) -> None: + def add(self, rsrc: Cacheable, on_delete: Optional[DeletionHandler] = None) -> None: pass def link(self, owner: Cacheable, owned: Cacheable) -> None: diff --git a/python/ambassador/compile.py b/python/ambassador/compile.py index b21a6f0c9d..9b364f8dd1 100644 --- a/python/ambassador/compile.py +++ b/python/ambassador/compile.py @@ -25,15 +25,20 @@ from .fetch import ResourceFetcher from .utils import SecretHandler, NullSecretHandler, Timer + class _CompileResult(TypedDict): ir: IR xds: NotRequired[EnvoyConfig] -def Compile(logger: logging.Logger, input_text: str, - cache: Optional[Cache]=None, - file_checker: Optional[IRFileChecker]=None, - secret_handler: Optional[SecretHandler]=None, - k8s: bool=False) -> _CompileResult: + +def Compile( + logger: logging.Logger, + input_text: str, + cache: Optional[Cache] = None, + file_checker: Optional[IRFileChecker] = None, + secret_handler: Optional[SecretHandler] = None, + k8s: bool = False, +) -> _CompileResult: """ Compile is a helper function to take a bunch of YAML and compile it into an IR and, optionally, an Envoy config. @@ -69,9 +74,9 @@ def Compile(logger: logging.Logger, input_text: str, ir = IR(aconf, cache=cache, file_checker=file_checker, secret_handler=secret_handler) - out: _CompileResult = { "ir": ir } + out: _CompileResult = {"ir": ir} if ir: - out['xds'] = EnvoyConfig.generate(ir, cache=cache) + out["xds"] = EnvoyConfig.generate(ir, cache=cache) return out diff --git a/python/ambassador/config/acmapping.py b/python/ambassador/config/acmapping.py index e32a5772be..c7c60b1824 100644 --- a/python/ambassador/config/acmapping.py +++ b/python/ambassador/config/acmapping.py @@ -36,34 +36,36 @@ # DictOfStringOrBool = Dict[str, StringOrBool] -class ACMapping (ACResource): +class ACMapping(ACResource): """ ACMappings are ACResources with a bunch of extra stuff. TODO: moar docstring. """ - def __init__(self, rkey: str, location: str, *, - name: str, - kind: str="Mapping", - apiVersion: Optional[str]=None, - serialization: Optional[str]=None, - - service: str, - prefix: str, - prefix_regex: bool=False, - rewrite: Optional[str]="/", - case_sensitive: bool=False, - grpc: bool=False, - bypass_auth: bool=False, - bypass_error_response_overrides: bool=False, - - # We don't list "method" or "method_regex" above because if they're - # not present, we want them to be _not present_. Having them be always - # present with an optional method is too annoying for schema validation - # at this point. - - **kwargs) -> None: + def __init__( + self, + rkey: str, + location: str, + *, + name: str, + kind: str = "Mapping", + apiVersion: Optional[str] = None, + serialization: Optional[str] = None, + service: str, + prefix: str, + prefix_regex: bool = False, + rewrite: Optional[str] = "/", + case_sensitive: bool = False, + grpc: bool = False, + bypass_auth: bool = False, + bypass_error_response_overrides: bool = False, + # We don't list "method" or "method_regex" above because if they're + # not present, we want them to be _not present_. Having them be always + # present with an optional method is too annoying for schema validation + # at this point. + **kwargs + ) -> None: """ Initialize an ACMapping from the raw fields of its ACResource. """ @@ -72,17 +74,20 @@ def __init__(self, rkey: str, location: str, *, # First init our superclass... - super().__init__(rkey, location, - kind=kind, - name=name, - apiVersion=apiVersion, - serialization=serialization, - service=service, - prefix=prefix, - prefix_regex=prefix_regex, - rewrite=rewrite, - case_sensitive=case_sensitive, - grpc=grpc, - bypass_auth=bypass_auth, - bypass_error_response_overrides=bypass_error_response_overrides, - **kwargs) + super().__init__( + rkey, + location, + kind=kind, + name=name, + apiVersion=apiVersion, + serialization=serialization, + service=service, + prefix=prefix, + prefix_regex=prefix_regex, + rewrite=rewrite, + case_sensitive=case_sensitive, + grpc=grpc, + bypass_auth=bypass_auth, + bypass_error_response_overrides=bypass_error_response_overrides, + **kwargs + ) diff --git a/python/ambassador/config/acpragma.py b/python/ambassador/config/acpragma.py index 63b741e41b..2f1fb362cf 100644 --- a/python/ambassador/config/acpragma.py +++ b/python/ambassador/config/acpragma.py @@ -20,20 +20,24 @@ ## pragma.py -- the pragma configuration object for Ambassador -class ACPragma (ACResource): +class ACPragma(ACResource): """ ACPragmas are ACResources with a bunch of extra stuff. TODO: moar docstring. """ - def __init__(self, rkey: str, location: str="-pragma-", *, - name: str="-pragma-", - kind: str="Pragma", - apiVersion: Optional[str]=None, - serialization: Optional[str]=None, - - **kwargs) -> None: + def __init__( + self, + rkey: str, + location: str = "-pragma-", + *, + name: str = "-pragma-", + kind: str = "Pragma", + apiVersion: Optional[str] = None, + serialization: Optional[str] = None, + **kwargs + ) -> None: """ Initialize an ACPragma from the raw fields of its ACResource. """ @@ -42,8 +46,12 @@ def __init__(self, rkey: str, location: str="-pragma-", *, # First init our superclass... - super().__init__(rkey, location, - kind=kind, name=name, - apiVersion=apiVersion, - serialization=serialization, - **kwargs) + super().__init__( + rkey, + location, + kind=kind, + name=name, + apiVersion=apiVersion, + serialization=serialization, + **kwargs + ) diff --git a/python/ambassador/config/acresource.py b/python/ambassador/config/acresource.py index c11e7e1cbe..ebfe060248 100644 --- a/python/ambassador/config/acresource.py +++ b/python/ambassador/config/acresource.py @@ -4,10 +4,10 @@ from ..resource import Resource -R = TypeVar('R', bound=Resource) +R = TypeVar("R", bound=Resource) -class ACResource (Resource): +class ACResource(Resource): """ A resource that we're going to use as part of the Ambassador configuration. @@ -49,14 +49,19 @@ class ACResource (Resource): name: str apiVersion: str - def __init__(self, rkey: str, location: str, *, - kind: str, - name: Optional[str]=None, - namespace: Optional[str]=None, - metadata_labels: Optional[str]=None, - apiVersion: Optional[str]="getambassador.io/v0", - serialization: Optional[str]=None, - **kwargs) -> None: + def __init__( + self, + rkey: str, + location: str, + *, + kind: str, + name: Optional[str] = None, + namespace: Optional[str] = None, + metadata_labels: Optional[str] = None, + apiVersion: Optional[str] = "getambassador.io/v0", + serialization: Optional[str] = None, + **kwargs + ) -> None: if not rkey: raise Exception("ACResource requires rkey") @@ -79,28 +84,36 @@ def __init__(self, rkey: str, location: str, *, # print("ACResource __init__ (%s %s)" % (kind, name)) - super().__init__(rkey=rkey, location=location, - kind=kind, name=name, namespace=namespace, - apiVersion=typecast(str, apiVersion), - serialization=serialization, - **kwargs) + super().__init__( + rkey=rkey, + location=location, + kind=kind, + name=name, + namespace=namespace, + apiVersion=typecast(str, apiVersion), + serialization=serialization, + **kwargs + ) # XXX It kind of offends me that we need this, exactly. Meta-ize this maybe? @classmethod - def from_resource(cls: Type[R], other: R, - rkey: Optional[str]=None, - location: Optional[str]=None, - kind: Optional[str]=None, - serialization: Optional[str]=None, - name: Optional[str]=None, - namespace: Optional[str]=None, - metadata_labels: Optional[str] = None, - apiVersion: Optional[str]=None, - **kwargs) -> R: + def from_resource( + cls: Type[R], + other: R, + rkey: Optional[str] = None, + location: Optional[str] = None, + kind: Optional[str] = None, + serialization: Optional[str] = None, + name: Optional[str] = None, + namespace: Optional[str] = None, + metadata_labels: Optional[str] = None, + apiVersion: Optional[str] = None, + **kwargs + ) -> R: new_name = name or other.name new_apiVersion = apiVersion or other.apiVersion new_namespace = namespace or other.namespace - new_metadata_labels = metadata_labels or other.get('metadata_labels', None) + new_metadata_labels = metadata_labels or other.get("metadata_labels", None) # mypy 0.730 is Just Flat Wrong here. It tries to be "more strict" about # super(), which is fine, but it also flags this particular super() call @@ -111,34 +124,44 @@ def from_resource(cls: Type[R], other: R, # cls is _not_ an instance at all, it's a class, so isinstance() will # fail at runtime. So we only do the assertion if TYPE_CHECKING. Grrrr. if TYPE_CHECKING: - assert(isinstance(cls, Resource)) # pragma: no cover - - return super().from_resource(other, rkey=rkey, location=location, kind=kind, - name=new_name, apiVersion=new_apiVersion, namespace=new_namespace, - metadata_labels=new_metadata_labels, serialization=serialization, **kwargs) + assert isinstance(cls, Resource) # pragma: no cover + + return super().from_resource( + other, + rkey=rkey, + location=location, + kind=kind, + name=new_name, + apiVersion=new_apiVersion, + namespace=new_namespace, + metadata_labels=new_metadata_labels, + serialization=serialization, + **kwargs + ) # ACResource.INTERNAL is the magic ACResource we use to represent something created by # Ambassador's internals. @classmethod - def internal_resource(cls) -> 'ACResource': + def internal_resource(cls) -> "ACResource": return ACResource( - "--internal--", "--internal--", + "--internal--", + "--internal--", kind="Internal", name="Ambassador Internals", version="getambassador.io/v0", - description="The '--internal--' source marks objects created by Ambassador's internal logic." + description="The '--internal--' source marks objects created by Ambassador's internal logic.", ) # ACResource.DIAGNOSTICS is the magic ACResource we use to represent something created by # Ambassador's diagnostics logic. (We could use ACResource.INTERNAL here, but explicitly # calling out diagnostics stuff actually helps with, well, diagnostics.) @classmethod - def diagnostics_resource(cls) -> 'ACResource': + def diagnostics_resource(cls) -> "ACResource": return ACResource( - "--diagnostics--", "--diagnostics--", + "--diagnostics--", + "--diagnostics--", kind="Diagnostics", name="Ambassador Diagnostics", version="getambassador.io/v0", - description="The '--diagnostics--' source marks objects created by Ambassador to assist with diagnostic output." + description="The '--diagnostics--' source marks objects created by Ambassador to assist with diagnostic output.", ) - diff --git a/python/ambassador/config/config.py b/python/ambassador/config/config.py index 996c8372c9..52622cf307 100644 --- a/python/ambassador/config/config.py +++ b/python/ambassador/config/config.py @@ -52,28 +52,32 @@ class Config: # CLASS VARIABLES # When using multiple Ambassadors in one cluster, use AMBASSADOR_ID to distinguish them. - ambassador_id: ClassVar[str] = os.environ.get('AMBASSADOR_ID', 'default') - ambassador_namespace: ClassVar[str] = os.environ.get('AMBASSADOR_NAMESPACE', 'default') - single_namespace: ClassVar[bool] = bool(os.environ.get('AMBASSADOR_SINGLE_NAMESPACE')) - certs_single_namespace: ClassVar[bool] = bool(os.environ.get('AMBASSADOR_CERTS_SINGLE_NAMESPACE', os.environ.get('AMBASSADOR_SINGLE_NAMESPACE'))) - enable_endpoints: ClassVar[bool] = not bool(os.environ.get('AMBASSADOR_DISABLE_ENDPOINTS')) - log_resources: ClassVar[bool] = parse_bool(os.environ.get('AMBASSADOR_LOG_RESOURCES')) - envoy_bind_address: ClassVar[str] = os.environ.get('AMBASSADOR_ENVOY_BIND_ADDRESS', "0.0.0.0") + ambassador_id: ClassVar[str] = os.environ.get("AMBASSADOR_ID", "default") + ambassador_namespace: ClassVar[str] = os.environ.get("AMBASSADOR_NAMESPACE", "default") + single_namespace: ClassVar[bool] = bool(os.environ.get("AMBASSADOR_SINGLE_NAMESPACE")) + certs_single_namespace: ClassVar[bool] = bool( + os.environ.get( + "AMBASSADOR_CERTS_SINGLE_NAMESPACE", os.environ.get("AMBASSADOR_SINGLE_NAMESPACE") + ) + ) + enable_endpoints: ClassVar[bool] = not bool(os.environ.get("AMBASSADOR_DISABLE_ENDPOINTS")) + log_resources: ClassVar[bool] = parse_bool(os.environ.get("AMBASSADOR_LOG_RESOURCES")) + envoy_bind_address: ClassVar[str] = os.environ.get("AMBASSADOR_ENVOY_BIND_ADDRESS", "0.0.0.0") StorageByKind: ClassVar[Dict[str, str]] = { - 'authservice': "auth_configs", - 'consulresolver': "resolvers", - 'host': "hosts", - 'listener': "listeners", - 'mapping': "mappings", - 'kubernetesendpointresolver': "resolvers", - 'kubernetesserviceresolver': "resolvers", - 'ratelimitservice': "ratelimit_configs", - 'devportal': "devportals", - 'tcpmapping': "tcpmappings", - 'tlscontext': "tls_contexts", - 'tracingservice': "tracing_configs", - 'logservice': "log_services", + "authservice": "auth_configs", + "consulresolver": "resolvers", + "host": "hosts", + "listener": "listeners", + "mapping": "mappings", + "kubernetesendpointresolver": "resolvers", + "kubernetesserviceresolver": "resolvers", + "ratelimitservice": "ratelimit_configs", + "devportal": "devportals", + "tcpmapping": "tcpmappings", + "tlscontext": "tls_contexts", + "tracingservice": "tracing_configs", + "logservice": "log_services", } SupportedVersions: ClassVar[Dict[str, str]] = { @@ -82,9 +86,9 @@ class Config: } # INSTANCE VARIABLES - ambassador_nodename: str = "ambassador" # overridden in Config.reset + ambassador_nodename: str = "ambassador" # overridden in Config.reset - schema_dir_path: str # where to look for JSONSchema files + schema_dir_path: str # where to look for JSONSchema files current_resource: Optional[ACResource] = None helm_chart: Optional[str] @@ -101,12 +105,12 @@ class Config: # Invalid objects (currently loaded using load_invalid()) invalid: List[Dict] - errors: Dict[str, List[dict]] # errors to post to the UI - notices: Dict[str, List[str]] # notices to post to the UI + errors: Dict[str, List[dict]] # errors to post to the UI + notices: Dict[str, List[str]] # notices to post to the UI fatal_errors: int object_errors: int - def __init__(self, schema_dir_path: Optional[str]=None) -> None: + def __init__(self, schema_dir_path: Optional[str] = None) -> None: self.logger = logging.getLogger("ambassador.config") if not schema_dir_path: @@ -119,26 +123,28 @@ def __init__(self, schema_dir_path: Optional[str]=None) -> None: assert schema_dir_path is not None self.statsd: Dict[str, Any] = { - 'enabled': (os.environ.get('STATSD_ENABLED', '').lower() == 'true'), - 'dogstatsd': (os.environ.get('DOGSTATSD', '').lower() == 'true') + "enabled": (os.environ.get("STATSD_ENABLED", "").lower() == "true"), + "dogstatsd": (os.environ.get("DOGSTATSD", "").lower() == "true"), } - if self.statsd['enabled']: - self.statsd['interval'] = os.environ.get('STATSD_FLUSH_INTERVAL', '1') + if self.statsd["enabled"]: + self.statsd["interval"] = os.environ.get("STATSD_FLUSH_INTERVAL", "1") - statsd_host = os.environ.get('STATSD_HOST', 'statsd-sink') + statsd_host = os.environ.get("STATSD_HOST", "statsd-sink") try: resolved_ip = socket.gethostbyname(statsd_host) - self.statsd['ip'] = resolved_ip + self.statsd["ip"] = resolved_ip except socket.gaierror as e: self.logger.error("Unable to resolve {} to IP : {}".format(statsd_host, e)) self.logger.error("Stats will not be exported to {}".format(statsd_host)) - self.statsd['enabled'] = False + self.statsd["enabled"] = False self.schema_dir_path = schema_dir_path self.logger.debug("SCHEMA DIR %s" % os.path.abspath(self.schema_dir_path)) - self.k8s_status_updates: Dict[str, Tuple[str, str, Optional[Dict[str, Any]]]] = {} # Tuple is (name, namespace, status_json) + self.k8s_status_updates: Dict[ + str, Tuple[str, str, Optional[Dict[str, Any]]] + ] = {} # Tuple is (name, namespace, status_json) self.pod_labels: Dict[str, str] = {} self._reset() @@ -174,11 +180,13 @@ def _reset(self) -> None: # Build up the Ambassador node name. # # XXX This should be overrideable by the Ambassador module. - self.ambassador_nodename = "%s-%s" % (os.environ.get('AMBASSADOR_ID', 'ambassador'), - Config.ambassador_namespace) + self.ambassador_nodename = "%s-%s" % ( + os.environ.get("AMBASSADOR_ID", "ambassador"), + Config.ambassador_namespace, + ) def __str__(self) -> str: - s = [ " str: return "\n".join(s) def as_dict(self) -> Dict[str, Any]: - od: Dict[str, Any] = { - '_errors': self.errors, - '_notices': self.notices, - '_sources': {} - } + od: Dict[str, Any] = {"_errors": self.errors, "_notices": self.notices, "_sources": {}} if self.helm_chart: - od['_helm_chart'] = self.helm_chart + od["_helm_chart"] = self.helm_chart for k, v in self.sources.items(): - sd = dict(v) # Shallow copy + sd = dict(v) # Shallow copy - if '_errors' in v: - sd['_errors'] = [ x.as_dict() for x in v._errors ] + if "_errors" in v: + sd["_errors"] = [x.as_dict() for x in v._errors] - od['_sources'][k] = sd + od["_sources"][k] = sd for kind, configs in self.config.items(): od[kind] = {} @@ -222,23 +226,23 @@ def as_json(self): # Often good_ambassador_id will be passed an ACResource, but sometimes # just a plain old dict. def good_ambassador_id(self, resource: dict): - resource_kind = resource.get('kind', '') + resource_kind = resource.get("kind", "") # Is an ambassador_id present in this object? # # NOTE WELL: when we update the status of a Host (or a Mapping?) then reserialization # can cause the `ambassador_id` element to turn into an `ambassadorId` element. So # treat those as synonymous. - allowed_ids: StringOrList = resource.get('ambassadorId', None) + allowed_ids: StringOrList = resource.get("ambassadorId", None) if allowed_ids is None: - allowed_ids = resource.get('ambassador_id', 'default') + allowed_ids = resource.get("ambassador_id", "default") # If we find the array [ '_automatic_' ] then allow it, so that hardcoded resources # can have a useful effect. This is mostly for init-config, but could be used for # other things, too. - if allowed_ids == [ "_automatic_" ]: + if allowed_ids == ["_automatic_"]: self.logger.debug(f"ambassador_id {allowed_ids} always accepted") return True @@ -247,15 +251,17 @@ def good_ambassador_id(self, resource: dict): # but the jsonschema will allow only a string or a list, # and guess what? Strings are Iterables. if type(allowed_ids) != list: - allowed_ids = typecast(StringOrList, [ allowed_ids ]) + allowed_ids = typecast(StringOrList, [allowed_ids]) if Config.ambassador_id in allowed_ids: return True else: - rkey = resource.get('rkey', '-anonymous-yaml-') - name = resource.get('name', '-no-name-') + rkey = resource.get("rkey", "-anonymous-yaml-") + name = resource.get("name", "-no-name-") - self.logger.debug(f"{rkey}: {resource_kind} {name} has IDs {allowed_ids}, no match with {Config.ambassador_id}") + self.logger.debug( + f"{rkey}: {resource_kind} {name} has IDs {allowed_ids}, no match with {Config.ambassador_id}" + ) return False def incr_count(self, key: str) -> None: @@ -270,7 +276,7 @@ def save_source(self, resource: ACResource) -> None: """ self.sources[resource.rkey] = resource - def load_invalid(self, fetcher: 'ResourceFetcher') -> None: + def load_invalid(self, fetcher: "ResourceFetcher") -> None: """ Loads the invalid resources from a ResourceFetcher. This and load_all() should be combined. @@ -317,11 +323,13 @@ def load_all(self, resources: Iterable[ACResource]) -> None: if self.errors: self.logger.error("ERROR ERROR ERROR Starting with configuration errors") - def post_notice(self, msg: str, resource: Optional[Resource]=None, log_level=logging.DEBUG) -> None: + def post_notice( + self, msg: str, resource: Optional[Resource] = None, log_level=logging.DEBUG + ) -> None: if resource is None: resource = self.current_resource - rkey = '-global-' + rkey = "-global-" if resource is not None: rkey = resource.rkey @@ -332,22 +340,40 @@ def post_notice(self, msg: str, resource: Optional[Resource]=None, log_level=log self.logger.log(log_level, "%s: NOTICE: %s" % (rkey, msg)) @singledispatchmethod - def post_error(self, msg: Union[RichStatus, str], resource: Optional[Resource]=None, rkey: Optional[str]=None, log_level=logging.INFO) -> None: + def post_error( + self, + msg: Union[RichStatus, str], + resource: Optional[Resource] = None, + rkey: Optional[str] = None, + log_level=logging.INFO, + ) -> None: assert False @post_error.register - def post_error_string(self, msg: str, resource: Optional[Resource]=None, rkey: Optional[str]=None, log_level=logging.INFO): + def post_error_string( + self, + msg: str, + resource: Optional[Resource] = None, + rkey: Optional[str] = None, + log_level=logging.INFO, + ): rc = RichStatus.fromError(msg) self.post_error(rc, resource=resource, log_level=log_level) @post_error.register - def post_error_richstatus(self, rc: RichStatus, resource: Optional[Resource]=None, rkey: Optional[str]=None, log_level=logging.INFO): + def post_error_richstatus( + self, + rc: RichStatus, + resource: Optional[Resource] = None, + rkey: Optional[str] = None, + log_level=logging.INFO, + ): if resource is None: resource = self.current_resource if not rkey: - rkey = '-global-' + rkey = "-global-" if resource is not None: rkey = resource.rkey @@ -374,12 +400,12 @@ def process(self, resource: ACResource) -> RichStatus: return RichStatus.fromError("need kind") # Make sure this resource has a name... - if 'name' not in resource: + if "name" not in resource: return RichStatus.fromError("need name") # ...and also make sure it has a namespace. - if not resource.get('namespace', None): - resource['namespace'] = self.ambassador_namespace + if not resource.get("namespace", None): + resource["namespace"] = self.ambassador_namespace # ...it doesn't actually need a metadata_labels, so off we go. Save the source info... self.save_source(resource) @@ -404,7 +430,9 @@ def process(self, resource: ACResource) -> RichStatus: handler = getattr(self, handler_name, None) if not handler: - self.logger.warning("%s: no handler for %s, just saving" % (resource, resource.kind)) + self.logger.warning( + "%s: no handler for %s, just saving" % (resource, resource.kind) + ) handler = self.save_object # else: # self.logger.debug("%s: handling %s..." % (resource, resource.kind)) @@ -429,31 +457,31 @@ def validate_object(self, resource: ACResource) -> RichStatus: apiVersion = resource.apiVersion if apiVersion.startswith("getambassador.io/"): - version = apiVersion.split('/', 1)[1].lower() - status = Config.SupportedVersions.get(version, 'is not supported') - if status != 'ok': + version = apiVersion.split("/", 1)[1].lower() + status = Config.SupportedVersions.get(version, "is not supported") + if status != "ok": self.post_notice(f"apiVersion {apiVersion} {status}", resource=resource) - elif apiVersion.startswith('networking.internal.knative.dev'): + elif apiVersion.startswith("networking.internal.knative.dev"): # This is not an Ambassador resource, we're trying to parse Knative # here pass else: return RichStatus.fromError("apiVersion %s unsupported" % apiVersion) - ns = resource.get('namespace') or self.ambassador_namespace + ns = resource.get("namespace") or self.ambassador_namespace name = f"{resource.name} ns {ns}" # Did entrypoint.go flag errors here that we should show to the user? # # (It's still called watt_errors because our other docs talk about "watt # snapshots", and I'm OK with retaining that name for the format.) - if 'errors' in resource: + if "errors" in resource: # Pop the errors out of this resource... - errors = resource.pop('errors').split('\n') + errors = resource.pop("errors").split("\n") # ...strip any empty lines in the error list with this one weird list # comprehension... - watt_errors = '; '.join([error for error in errors if error]) + watt_errors = "; ".join([error for error in errors if error]) # ...and, assuming that we're left with any error message, post it. if watt_errors: @@ -461,7 +489,7 @@ def validate_object(self, resource: ACResource) -> RichStatus: return RichStatus.OK(msg=f"good {resource.kind}") - def safe_store(self, storage_name: str, resource: ACResource, allow_log: bool=True) -> None: + def safe_store(self, storage_name: str, resource: ACResource, allow_log: bool = True) -> None: """ Safely store a ACResource under a given storage name. The storage_name is separate because we may need to e.g. store a Module under the 'ratelimit' name or the like. @@ -475,26 +503,27 @@ def safe_store(self, storage_name: str, resource: ACResource, allow_log: bool=Tr storage = self.config.setdefault(storage_name, {}) if resource.name in storage: - if resource.namespace == storage[resource.name].get('namespace'): + if resource.namespace == storage[resource.name].get("namespace"): # If the name and namespace, both match, then it's definitely an error. # Oooops. - self.post_error("%s defines %s %s, which is already defined by %s" % - (resource, resource.kind, resource.name, storage[resource.name].location), - resource=resource) + self.post_error( + "%s defines %s %s, which is already defined by %s" + % (resource, resource.kind, resource.name, storage[resource.name].location), + resource=resource, + ) else: # Here, we deal with the case when multiple resources have the same name but they exist in different # namespaces. Our current data structure to store resources is a flat string. Till we move to # identifying resources with both, name and namespace, we change names of any subsequent resources with # the same name here. - resource.name = f'{resource.name}.{resource.namespace}' + resource.name = f"{resource.name}.{resource.namespace}" if allow_log: - self.logger.debug("%s: saving %s %s" % - (resource, resource.kind, resource.name)) + self.logger.debug("%s: saving %s %s" % (resource, resource.kind, resource.name)) storage[resource.name] = resource - def save_object(self, resource: ACResource, allow_log: bool=False) -> None: + def save_object(self, resource: ACResource, allow_log: bool = False) -> None: """ Saves a ACResource using its kind as the storage class name. Sort of the defaulted version of safe_store. @@ -523,7 +552,7 @@ def get_module(self, module_name: str) -> Optional[ACResource]: else: return None - def module_lookup(self, module_name: str, key: str, default: Any=None) -> Any: + def module_lookup(self, module_name: str, key: str, default: Any = None) -> Any: """ Look up a specific key in a given module. If the named module doesn't exist, or if the key doesn't exist in the module, return the default. @@ -562,24 +591,28 @@ def handle_secret(self, resource: ACResource) -> None: self.logger.debug(f"Handling secret resource {resource.as_dict()}") - storage = self.config.setdefault('secrets', {}) + storage = self.config.setdefault("secrets", {}) key = resource.rkey if key in storage: - self.post_error("%s defines %s %s, which is already defined by %s" % - (resource, resource.kind, key, storage[key].location), - resource=resource) + self.post_error( + "%s defines %s %s, which is already defined by %s" + % (resource, resource.kind, key, storage[key].location), + resource=resource, + ) storage[key] = resource def handle_ingress(self, resource: ACResource) -> None: - storage = self.config.setdefault('ingresses', {}) + storage = self.config.setdefault("ingresses", {}) key = resource.rkey if key in storage: - self.post_error("%s defines %s %s, which is already defined by %s" % - (resource, resource.kind, key, storage[key].location), - resource=resource) + self.post_error( + "%s defines %s %s, which is already defined by %s" + % (resource, resource.kind, key, storage[key].location), + resource=resource, + ) storage[key] = resource @@ -589,20 +622,21 @@ def handle_service(self, resource: ACResource) -> None: the rkey, not the name, and because we need to check the helm_chart attribute. """ - storage = self.config.setdefault('service', {}) + storage = self.config.setdefault("service", {}) key = resource.rkey if key in storage: - self.post_error("%s defines %s %s, which is already defined by %s" % - (resource, resource.kind, key, storage[key].location), - resource=resource) + self.post_error( + "%s defines %s %s, which is already defined by %s" + % (resource, resource.kind, key, storage[key].location), + resource=resource, + ) - self.logger.debug("%s: saving %s %s" % - (resource, resource.kind, key)) + self.logger.debug("%s: saving %s %s" % (resource, resource.kind, key)) storage[key] = resource - chart = resource.get('helm_chart', None) + chart = resource.get("helm_chart", None) if chart: self.helm_chart = chart diff --git a/python/ambassador/diagnostics/diagnostics.py b/python/ambassador/diagnostics/diagnostics.py index c24673c1ef..be31b6e548 100644 --- a/python/ambassador/diagnostics/diagnostics.py +++ b/python/ambassador/diagnostics/diagnostics.py @@ -27,11 +27,11 @@ from ..utils import dump_json -class DiagSource (dict): +class DiagSource(dict): pass -class DiagCluster (dict): +class DiagCluster(dict): """ A DiagCluster represents what Envoy thinks about the health of a cluster. DO NOT JUST PASS AN IRCluster into DiagCluster; turn it into a dict with @@ -43,19 +43,19 @@ def __init__(self, cluster) -> None: def update_health(self, other: dict) -> None: for from_key, to_key in [ - ( 'health', '_health' ), - ( 'hmetric', '_hmetric' ), - ( 'hcolor', '_hcolor' ) + ("health", "_health"), + ("hmetric", "_hmetric"), + ("hcolor", "_hcolor"), ]: if from_key in other: self[to_key] = other[from_key] def default_missing(self) -> dict: for key, default in [ - ('service', 'unknown service!'), - ('weight', 100), - ('_hmetric', 'unknown'), - ('_hcolor', 'orange') + ("service", "unknown service!"), + ("weight", 100), + ("_hmetric", "unknown"), + ("_hcolor", "orange"), ]: if not self.get(key, None): self[key] = default @@ -64,12 +64,14 @@ def default_missing(self) -> dict: @classmethod def unknown_cluster(cls): - return DiagCluster({ - 'service': 'unknown service!', - '_health': 'unknown cluster!', - '_hmetric': 'unknown', - '_hcolor': 'orange' - }) + return DiagCluster( + { + "service": "unknown service!", + "_health": "unknown cluster!", + "_hmetric": "unknown", + "_hcolor": "orange", + } + ) class DiagClusters: @@ -86,7 +88,7 @@ def __init__(self, clusters: Optional[List[dict]] = None) -> None: if clusters: for cluster in typecast(List[dict], clusters): - self[cluster['name']] = DiagCluster(cluster) + self[cluster["name"]] = DiagCluster(cluster) def __getitem__(self, key: str) -> DiagCluster: if key not in self.clusters: @@ -109,37 +111,40 @@ class DiagResult: A DiagResult is the result of a diagnostics request, whether for an overview or for a particular key. """ - def __init__(self, diag: 'Diagnostics', estat: EnvoyStats, request) -> None: + + def __init__(self, diag: "Diagnostics", estat: EnvoyStats, request) -> None: self.diag = diag self.logger = self.diag.logger self.estat = estat # Go ahead and grab Envoy cluster stats for all possible clusters. # XXX This might be a bit silly. - self.cluster_names = [ cluster.envoy_name for cluster in self.diag.clusters.values() ] - self.cstats = { name: self.estat.cluster_stats(name) for name in self.cluster_names } + self.cluster_names = [cluster.envoy_name for cluster in self.diag.clusters.values()] + self.cstats = {name: self.estat.cluster_stats(name) for name in self.cluster_names} # Save the request host and scheme. We'll need them later. - self.request_host = request.headers.get('Host', '*') - self.request_scheme = request.headers.get('X-Forwarded-Proto', 'http').lower() + self.request_host = request.headers.get("Host", "*") + self.request_scheme = request.headers.get("X-Forwarded-Proto", "http").lower() # All of these things reflect _only_ resources that are relevant to the request # we're handling -- e.g. if you ask for a particular group, you'll only get the # clusters that are part of that group. - self.clusters: Dict[str, DiagCluster] = {} # Envoy clusters - self.routes: List[dict] = [] # Envoy routes - self.element_keys: Dict[str, bool] = {} # Active element keys - self.ambassador_resources: Dict[str, str] = {} # Actually serializations of Ambassador config resources - self.envoy_resources: Dict[str, dict] = {} # Envoy config resources + self.clusters: Dict[str, DiagCluster] = {} # Envoy clusters + self.routes: List[dict] = [] # Envoy routes + self.element_keys: Dict[str, bool] = {} # Active element keys + self.ambassador_resources: Dict[ + str, str + ] = {} # Actually serializations of Ambassador config resources + self.envoy_resources: Dict[str, dict] = {} # Envoy config resources def as_dict(self) -> Dict[str, Any]: return { - 'cluster_stats': self.cstats, - 'cluster_info': self.clusters, - 'route_info': self.routes, - 'active_elements': sorted(self.element_keys.keys()), - 'ambassador_resources': self.ambassador_resources, - 'envoy_resources': self.envoy_resources + "cluster_stats": self.cstats, + "cluster_info": self.clusters, + "route_info": self.routes, + "active_elements": sorted(self.element_keys.keys()), + "ambassador_resources": self.ambassador_resources, + "envoy_resources": self.envoy_resources, } def include_element(self, key: str) -> None: @@ -163,7 +168,7 @@ def include_referenced_elements(self, obj: dict) -> None: :param obj: object for which to include referencing keys """ - for element_key in obj['_referenced_by']: + for element_key in obj["_referenced_by"]: self.include_element(element_key) def include_cluster(self, cluster: dict) -> DiagCluster: @@ -181,7 +186,7 @@ def include_cluster(self, cluster: dict) -> DiagCluster: :return: the DiagCluster for this cluster """ - c_name = cluster['name'] + c_name = cluster["name"] if c_name not in self.clusters: self.clusters[c_name] = DiagCluster(cluster) @@ -207,48 +212,50 @@ def include_httpgroup(self, group: IRHTTPMappingGroup) -> None: # self.logger.debug("GROUP %s" % group.as_json()) - prefix = group['prefix'] if 'prefix' in group else group['regex'] - rewrite = group.get('rewrite', "/") - method = '*' + prefix = group["prefix"] if "prefix" in group else group["regex"] + rewrite = group.get("rewrite", "/") + method = "*" host = None route_clusters: List[DiagCluster] = [] - for mapping in group.get('mappings', []): - cluster = mapping['cluster'] + for mapping in group.get("mappings", []): + cluster = mapping["cluster"] mapping_cluster = self.include_cluster(cluster.as_dict()) - mapping_cluster.update({'weight': mapping.get('weight', 100)}) + mapping_cluster.update({"weight": mapping.get("weight", 100)}) # self.logger.debug("GROUP %s CLUSTER %s %d%% (%s)" % # (group['group_id'], c_name, mapping['weight'], mapping_cluster)) route_clusters.append(mapping_cluster) - host_redir = group.get('host_redirect', None) + host_redir = group.get("host_redirect", None) if host_redir: # XXX Stupid hackery here. redirect_cluster should be a real # IRCluster object. - redirect_cluster = self.include_cluster({ - 'name': host_redir['name'], - 'service': host_redir['service'], - 'weight': 100, - 'type_label': 'redirect', - '_referenced_by': [ host_redir['rkey'] ] - }) + redirect_cluster = self.include_cluster( + { + "name": host_redir["name"], + "service": host_redir["service"], + "weight": 100, + "type_label": "redirect", + "_referenced_by": [host_redir["rkey"]], + } + ) route_clusters.append(redirect_cluster) self.logger.debug("host_redirect route: %s" % group) self.logger.debug("host_redirect cluster: %s" % redirect_cluster) - shadows = group.get('shadows', []) + shadows = group.get("shadows", []) for shadow in shadows: # Shadows have a real cluster object. - shadow_dict = shadow['cluster'].as_dict() - shadow_dict['type_label'] = 'shadow' + shadow_dict = shadow["cluster"].as_dict() + shadow_dict["type_label"] = "shadow" shadow_cluster = self.include_cluster(shadow_dict) route_clusters.append(shadow_cluster) @@ -258,41 +265,46 @@ def include_httpgroup(self, group: IRHTTPMappingGroup) -> None: headers = [] - for header in group.get('headers', []): - hdr_name = header.get('name', None) - hdr_value = header.get('value', None) + for header in group.get("headers", []): + hdr_name = header.get("name", None) + hdr_value = header.get("value", None) - if hdr_name == ':authority': + if hdr_name == ":authority": host = hdr_value - elif hdr_name == ':method': + elif hdr_name == ":method": method = hdr_value else: headers.append(header) sep = "" if prefix.startswith("/") else "/" - route_key = "%s://%s%s%s" % (self.request_scheme, host if host else self.request_host, sep, prefix) + route_key = "%s://%s%s%s" % ( + self.request_scheme, + host if host else self.request_host, + sep, + prefix, + ) route_info = { - '_route': group.as_dict(), - '_source': group['location'], - '_group_id': group['group_id'], - 'key': route_key, - 'prefix': prefix, - 'rewrite': rewrite, - 'method': method, - 'headers': headers, - 'clusters': [ x.default_missing() for x in route_clusters ], - 'host': host if host else '*', + "_route": group.as_dict(), + "_source": group["location"], + "_group_id": group["group_id"], + "key": route_key, + "prefix": prefix, + "rewrite": rewrite, + "method": method, + "headers": headers, + "clusters": [x.default_missing() for x in route_clusters], + "host": host if host else "*", } - if 'precedence' in group: - route_info['precedence'] = group['precedence'] + if "precedence" in group: + route_info["precedence"] = group["precedence"] - metadata_labels = group.get('metadata_labels') or {} - diag_class = metadata_labels.get('ambassador_diag_class') or None + metadata_labels = group.get("metadata_labels") or {} + diag_class = metadata_labels.get("ambassador_diag_class") or None if diag_class: - route_info['diag_class'] = diag_class + route_info["diag_class"] = diag_class self.routes.append(route_info) self.include_referenced_elements(group) @@ -308,7 +320,7 @@ def finalize(self) -> None: amb_el_info = self.diag.ambassador_elements.get(key, None) if amb_el_info: - serialization = amb_el_info.get('serialization', None) + serialization = amb_el_info.get("serialization", None) if serialization: self.ambassador_resources[key] = serialization @@ -342,12 +354,9 @@ class Diagnostics: source_map: Dict[str, Dict[str, bool]] - reKeyIndex = re.compile(r'\.(\d+)$') + reKeyIndex = re.compile(r"\.(\d+)$") - filter_map = { - 'IRAuth': 'AuthService', - 'IRRateLimit': 'RateLimitService' - } + filter_map = {"IRAuth": "AuthService", "IRRateLimit": "RateLimitService"} def __init__(self, ir: IR, econf: EnvoyConfig) -> None: self.logger = logging.getLogger("ambassador.diagnostics") @@ -383,25 +392,27 @@ def __init__(self, ir: IR, econf: EnvoyConfig) -> None: warn_ratelimit = False for filter in self.ir.filters: - if filter.kind == 'IRAuth': - proto = filter.get('proto') or 'http' + if filter.kind == "IRAuth": + proto = filter.get("proto") or "http" - if proto.lower() != 'http': + if proto.lower() != "http": warn_auth = True - if filter.kind == 'IRRateLimit': + if filter.kind == "IRRateLimit": warn_ratelimit = True things_to_warn = [] if warn_auth: - things_to_warn.append('AuthServices') + things_to_warn.append("AuthServices") if warn_ratelimit: - things_to_warn.append('RateLimitServices') + things_to_warn.append("RateLimitServices") if things_to_warn: - self.ir.aconf.post_notice(f'A future Ambassador version will change the GRPC protocol version for {" and ".join(things_to_warn)}. See the CHANGELOG for details.') + self.ir.aconf.post_notice( + f'A future Ambassador version will change the GRPC protocol version for {" and ".join(things_to_warn)}. See the CHANGELOG for details.' + ) # # Warn people about the default port change. # if self.ir.ambassador_module.service_port < 1024: @@ -443,39 +454,37 @@ def __init__(self, ir: IR, econf: EnvoyConfig) -> None: # Next up, walk the list of Ambassador sources. for key, rsrc in self.ir.aconf.sources.items(): - uqkey = key # Unqualified key, e.g. ambassador.yaml - fqkey = uqkey # Fully-qualified key, e.g. ambassador.yaml.1 + uqkey = key # Unqualified key, e.g. ambassador.yaml + fqkey = uqkey # Fully-qualified key, e.g. ambassador.yaml.1 key_index = None - if 'rkey' in rsrc: + if "rkey" in rsrc: uqkey, key_index = self.split_key(rsrc.rkey) if key_index is not None: fqkey = "%s.%s" % (uqkey, key_index) - location, _ = self.split_key(rsrc.get('location', key)) + location, _ = self.split_key(rsrc.get("location", key)) - self.logger.debug(" %s (%s): UQ %s, FQ %s, LOC %s" % (key, rsrc, uqkey, fqkey, location)) + self.logger.debug( + " %s (%s): UQ %s, FQ %s, LOC %s" % (key, rsrc, uqkey, fqkey, location) + ) self.remember_source(uqkey, fqkey, location, rsrc.rkey) ambassador_element: dict = self.ambassador_elements.setdefault( - fqkey, - { - 'location': location, - 'kind': rsrc.kind - } + fqkey, {"location": location, "kind": rsrc.kind} ) if uqkey and (uqkey != fqkey): - ambassador_element['parent'] = uqkey + ambassador_element["parent"] = uqkey - serialization = rsrc.get('serialization', None) + serialization = rsrc.get("serialization", None) if serialization: if ambassador_element["kind"] == "Secret": serialization = "kind: Secret\ndata: (elided by Ambassador)\n" - ambassador_element['serialization'] = serialization + ambassador_element["serialization"] = serialization # Next up, the Envoy elements. for kind, elements in self.econf.elements.items(): @@ -485,15 +494,21 @@ def __init__(self, ir: IR, econf: EnvoyConfig) -> None: element_dict = self.envoy_elements.setdefault(fqkey, {}) element_list = element_dict.setdefault(kind, []) - element_list.append({ k: v for k, v in envoy_element.items() if k[0] != '_' }) + element_list.append({k: v for k, v in envoy_element.items() if k[0] != "_"}) # Always generate the full group set so that we can look up groups. - self.groups = { 'grp-%s' % group.group_id: group for group in self.ir.groups.values() - if group.location != "--diagnostics--" } + self.groups = { + "grp-%s" % group.group_id: group + for group in self.ir.groups.values() + if group.location != "--diagnostics--" + } # Always generate the full cluster set so that we can look up clusters. - self.clusters = { cluster.name: cluster for cluster in self.ir.clusters.values() - if cluster.location != "--diagnostics--" } + self.clusters = { + cluster.name: cluster + for cluster in self.ir.clusters.values() + if cluster.location != "--diagnostics--" + } # Build up our Ambassador services too (auth, ratelimit, tracing). self.ambassador_services = [] @@ -506,7 +521,9 @@ def __init__(self, ir: IR, econf: EnvoyConfig) -> None: self.add_ambassador_service(filt, type_name) if self.ir.tracing: - self.add_ambassador_service(self.ir.tracing, 'TracingService (%s)' % self.ir.tracing.driver) + self.add_ambassador_service( + self.ir.tracing, "TracingService (%s)" % self.ir.tracing.driver + ) self.ambassador_resolvers = [] used_resolvers: Dict[str, List[str]] = {} @@ -535,13 +552,15 @@ def add_ambassador_service(self, svc, type_name) -> None: svc_weight = 100.0 / len(urls) for url in urls: - self.ambassador_services.append({ - 'type': type_name, - '_source': svc.location, - 'name': url, - 'cluster': cluster.envoy_name, - '_service_weight': svc_weight - }) + self.ambassador_services.append( + { + "type": type_name, + "_source": svc.location, + "name": url, + "cluster": cluster.envoy_name, + "_service_weight": svc_weight, + } + ) def add_ambassador_resolver(self, resolver, group_list) -> None: """ @@ -551,12 +570,14 @@ def add_ambassador_resolver(self, resolver, group_list) -> None: :param group_list: list of groups that use this resolver """ - self.ambassador_resolvers.append({ - 'kind': resolver.kind, - '_source': resolver.location, - 'name': resolver.name, - 'groups': group_list - }) + self.ambassador_resolvers.append( + { + "kind": resolver.kind, + "_source": resolver.location, + "name": resolver.name, + "groups": group_list, + } + ) @staticmethod def split_key(key) -> Tuple[str, Optional[str]]: @@ -573,56 +594,56 @@ def split_key(key) -> Tuple[str, Optional[str]]: m = Diagnostics.reKeyIndex.search(key) if m: - key_base = key[:m.start()] + key_base = key[: m.start()] key_index = m.group(1) return key_base, key_index def as_dict(self) -> dict: return { - 'source_map': self.source_map, - 'ambassador_services': self.ambassador_services, - 'ambassador_resolvers': self.ambassador_resolvers, - 'ambassador_elements': self.ambassador_elements, - 'envoy_elements': self.envoy_elements, - 'errors': self.errors, - 'notices': self.notices, - 'groups': { key: self.flattened(value) for key, value in self.groups.items() }, + "source_map": self.source_map, + "ambassador_services": self.ambassador_services, + "ambassador_resolvers": self.ambassador_resolvers, + "ambassador_elements": self.ambassador_elements, + "envoy_elements": self.envoy_elements, + "errors": self.errors, + "notices": self.notices, + "groups": {key: self.flattened(value) for key, value in self.groups.items()}, # 'clusters': { key: value.as_dict() for key, value in self.clusters.items() }, - 'tlscontexts': [ x.as_dict() for x in self.ir.tls_contexts.values() ] + "tlscontexts": [x.as_dict() for x in self.ir.tls_contexts.values()], } def flattened(self, group: IRBaseMappingGroup) -> dict: - flattened = { k: v for k, v in group.as_dict().items() if k != 'mappings' } + flattened = {k: v for k, v in group.as_dict().items() if k != "mappings"} flattened_mappings = [] - for m in group['mappings']: + for m in group["mappings"]: fm = { - "_active": m['_active'], - "_errored": m['_errored'], - "_rkey": m['rkey'], - "location": m['location'], - "name": m['name'], - "cluster_service": m.get('cluster', {}).get("service"), - "cluster_name": m.get('cluster', {}).get("envoy_name"), + "_active": m["_active"], + "_errored": m["_errored"], + "_rkey": m["rkey"], + "location": m["location"], + "name": m["name"], + "cluster_service": m.get("cluster", {}).get("service"), + "cluster_name": m.get("cluster", {}).get("envoy_name"), } - if flattened['kind'] == 'IRHTTPMappingGroup': - fm['prefix'] = m.get('prefix') + if flattened["kind"] == "IRHTTPMappingGroup": + fm["prefix"] = m.get("prefix") - rewrite = m.get('rewrite', None) + rewrite = m.get("rewrite", None) if rewrite: - fm['rewrite'] = rewrite + fm["rewrite"] = rewrite - host = m.get('host', None) + host = m.get("host", None) if host: - fm['host'] = host + fm["host"] = host flattened_mappings.append(fm) - flattened['mappings'] = flattened_mappings + flattened["mappings"] = flattened_mappings return flattened @@ -642,8 +663,9 @@ def _remember_source(self, src_key: str, dest_key: str) -> None: src_map = self.source_map.setdefault(src_key, {}) src_map[dest_key] = True - def remember_source(self, uqkey: str, fqkey: Optional[str], location: Optional[str], - dest_key: str) -> None: + def remember_source( + self, uqkey: str, fqkey: Optional[str], location: Optional[str], dest_key: str + ) -> None: """ Populate the source map in various ways. A mapping from uqkey to dest_key is always added; mappings for fqkey and location are added if they are unique diff --git a/python/ambassador/diagnostics/envoy_stats.py b/python/ambassador/diagnostics/envoy_stats.py index 902275b12e..8c1bc85c96 100644 --- a/python/ambassador/diagnostics/envoy_stats.py +++ b/python/ambassador/diagnostics/envoy_stats.py @@ -22,12 +22,14 @@ from dataclasses import dataclass from dataclasses import field as dc_field + def percentage(x: float, y: float) -> int: if y == 0: return 0 else: return int(((x * 100) / y) + 0.5) + @dataclass(frozen=True) class EnvoyStats: max_live_age: int = 120 @@ -77,7 +79,7 @@ def is_ready(self) -> bool: return (time.time() - epoch) <= self.max_ready_age def time_since_boot(self) -> float: - """ Return the number of seconds since Envoy booted. """ + """Return the number of seconds since Envoy booted.""" return time.time() - self.created def time_since_update(self) -> Optional[float]: @@ -95,11 +97,11 @@ def cluster_stats(self, name: str) -> Dict[str, Union[str, bool]]: if not self.last_update: # No updates. return { - 'valid': False, - 'reason': "No stats updates have succeeded", - 'health': "no stats yet", - 'hmetric': 'startup', - 'hcolor': 'grey' + "valid": False, + "reason": "No stats updates have succeeded", + "health": "no stats yet", + "hmetric": "startup", + "hcolor": "grey", } # OK, we should be OK. @@ -108,40 +110,29 @@ def cluster_stats(self, name: str) -> Dict[str, Union[str, bool]]: if name not in cstat: return { - 'valid': False, - 'reason': "Cluster %s is not defined" % name, - 'health': "undefined cluster", - 'hmetric': 'undefined cluster', - 'hcolor': 'orange', + "valid": False, + "reason": "Cluster %s is not defined" % name, + "health": "undefined cluster", + "hmetric": "undefined cluster", + "hcolor": "orange", } cstat = dict(**cstat[name]) - cstat.update({ - 'valid': True, - 'reason': "Cluster %s updated at %d" % (name, when) - }) + cstat.update({"valid": True, "reason": "Cluster %s updated at %d" % (name, when)}) - pct = cstat.get('healthy_percent', None) + pct = cstat.get("healthy_percent", None) if pct != None: - color = 'green' + color = "green" if pct < 70: - color = 'red' + color = "red" elif pct < 90: - color = 'yellow' + color = "yellow" - cstat.update({ - 'health': "%d%% healthy" % pct, - 'hmetric': int(pct), - 'hcolor': color - }) + cstat.update({"health": "%d%% healthy" % pct, "hmetric": int(pct), "hcolor": color}) else: - cstat.update({ - 'health': "no requests yet", - 'hmetric': 'waiting', - 'hcolor': 'grey' - }) + cstat.update({"health": "no requests yet", "hmetric": "waiting", "hcolor": "grey"}) return cstat @@ -149,11 +140,17 @@ def cluster_stats(self, name: str) -> Dict[str, Union[str, bool]]: LogLevelFetcher = Callable[[Optional[str]], Optional[str]] EnvoyStatsFetcher = Callable[[], Optional[str]] + class EnvoyStatsMgr: # fetch_log_levels and fetch_envoy_stats are debugging hooks - def __init__(self, logger: logging.Logger, max_live_age: int=120, max_ready_age: int=120, - fetch_log_levels: Optional[LogLevelFetcher] = None, - fetch_envoy_stats: Optional[EnvoyStatsFetcher] = None) -> None: + def __init__( + self, + logger: logging.Logger, + max_live_age: int = 120, + max_ready_age: int = 120, + fetch_log_levels: Optional[LogLevelFetcher] = None, + fetch_envoy_stats: Optional[EnvoyStatsFetcher] = None, + ) -> None: self.logger = logger self.loginfo: Dict[str, Union[str, List[str]]] = {} @@ -164,9 +161,7 @@ def __init__(self, logger: logging.Logger, max_live_age: int=120, max_ready_age: self.fetch_envoy_stats = fetch_envoy_stats or self._fetch_envoy_stats self.stats = EnvoyStats( - created=time.time(), - max_live_age=max_live_age, - max_ready_age=max_ready_age + created=time.time(), max_live_age=max_live_age, max_ready_age=max_ready_age ) def _fetch_log_levels(self, level: Optional[str]) -> Optional[str]: @@ -201,7 +196,7 @@ def _fetch_envoy_stats(self) -> Optional[str]: self.logger.warning("EnvoyStats.update failed: %s" % e) return None - def update_log_levels(self, last_attempt: float, level: Optional[str]=None) -> bool: + def update_log_levels(self, last_attempt: float, level: Optional[str] = None) -> bool: """ Heavy lifting around updating the Envoy log levels. @@ -225,11 +220,11 @@ def update_log_levels(self, last_attempt: float, level: Optional[str]=None) -> b max_ready_age=self.stats.max_ready_age, created=self.stats.created, last_update=self.stats.last_update, - last_attempt=last_attempt, # THIS IS A CHANGE - update_errors=self.stats.update_errors + 1, # THIS IS A CHANGE + last_attempt=last_attempt, # THIS IS A CHANGE + update_errors=self.stats.update_errors + 1, # THIS IS A CHANGE requests=self.stats.requests, clusters=self.stats.clusters, - envoy=self.stats.envoy + envoy=self.stats.envoy, ) self.stats = new_stats @@ -242,8 +237,8 @@ def update_log_levels(self, last_attempt: float, level: Optional[str]=None) -> b if not line: continue - if line.startswith(' '): - ( logtype, level ) = line[2:].split(": ") + if line.startswith(" "): + (logtype, level) = line[2:].split(": ") x = levels.setdefault(level, {}) x[logtype] = True @@ -253,9 +248,9 @@ def update_log_levels(self, last_attempt: float, level: Optional[str]=None) -> b loginfo: Dict[str, Union[str, List[str]]] if len(levels.keys()) == 1: - loginfo = { 'all': list(levels.keys())[0] } + loginfo = {"all": list(levels.keys())[0]} else: - loginfo = { x: list(levels[x].keys()) for x in levels.keys() } + loginfo = {x: list(levels[x].keys()) for x in levels.keys()} with self.access_lock: self.loginfo = loginfo @@ -278,11 +273,11 @@ def get_prometheus_stats(self) -> str: r = requests.get("http://127.0.0.1:8001/stats/prometheus") except OSError as e: self.logger.warning("EnvoyStats.get_prometheus_state failed: %s" % e) - return '' + return "" if r.status_code != 200: self.logger.warning("EnvoyStats.get_prometheus_state failed: %s" % r.text) - return '' + return "" return r.text def update_envoy_stats(self, last_attempt: float) -> None: @@ -306,11 +301,11 @@ def update_envoy_stats(self, last_attempt: float) -> None: max_ready_age=self.stats.max_ready_age, created=self.stats.created, last_update=self.stats.last_update, - last_attempt=last_attempt, # THIS IS A CHANGE - update_errors=self.stats.update_errors + 1, # THIS IS A CHANGE + last_attempt=last_attempt, # THIS IS A CHANGE + update_errors=self.stats.update_errors + 1, # THIS IS A CHANGE requests=self.stats.requests, clusters=self.stats.clusters, - envoy=self.stats.envoy + envoy=self.stats.envoy, ) with self.access_lock: @@ -318,7 +313,7 @@ def update_envoy_stats(self, last_attempt: float) -> None: return # Parse stats into a hierarchy. - envoy_stats: Dict[str, Any] = {} # Ew. + envoy_stats: Dict[str, Any] = {} # Ew. for line in text.split("\n"): if not line: @@ -326,7 +321,7 @@ def update_envoy_stats(self, last_attempt: float) -> None: # self.logger.info('line: %s' % line) key, value = line.split(":") - keypath = key.split('.') + keypath = key.split(".") node = envoy_stats @@ -360,8 +355,8 @@ def update_envoy_stats(self, last_attempt: float) -> None: requests_total = ingress_stats.get("downstream_rq_total", 0) - requests_4xx = ingress_stats.get('downstream_rq_4xx', 0) - requests_5xx = ingress_stats.get('downstream_rq_5xx', 0) + requests_4xx = ingress_stats.get("downstream_rq_4xx", 0) + requests_5xx = ingress_stats.get("downstream_rq_5xx", 0) requests_bad = requests_4xx + requests_5xx requests_ok = requests_total - requests_bad @@ -375,8 +370,8 @@ def update_envoy_stats(self, last_attempt: float) -> None: } if "cluster" in envoy_stats: - for cluster_name in envoy_stats['cluster']: - cluster = envoy_stats['cluster'][cluster_name] + for cluster_name in envoy_stats["cluster"]: + cluster = envoy_stats["cluster"][cluster_name] # # Toss any _%d -- that's madness with our Istio code at the moment. # cluster_name = re.sub('_\d+$', '', cluster_name) @@ -388,22 +383,22 @@ def update_envoy_stats(self, last_attempt: float) -> None: healthy_percent: Optional[int] - healthy_members = cluster['membership_healthy'] - total_members = cluster['membership_total'] + healthy_members = cluster["membership_healthy"] + total_members = cluster["membership_total"] healthy_percent = percentage(healthy_members, total_members) - update_attempts = cluster['update_attempt'] - update_successes = cluster['update_success'] + update_attempts = cluster["update_attempt"] + update_successes = cluster["update_success"] update_percent = percentage(update_successes, update_attempts) # Weird. # upstream_ok = cluster.get('upstream_rq_2xx', 0) # upstream_total = cluster.get('upstream_rq_pending_total', 0) - upstream_total = cluster.get('upstream_rq_completed', 0) + upstream_total = cluster.get("upstream_rq_completed", 0) - upstream_4xx = cluster.get('upstream_rq_4xx', 0) - upstream_5xx = cluster.get('upstream_rq_5xx', 0) - upstream_bad = upstream_5xx # used to include 4XX here, but that seems wrong. + upstream_4xx = cluster.get("upstream_rq_4xx", 0) + upstream_5xx = cluster.get("upstream_rq_5xx", 0) + upstream_bad = upstream_5xx # used to include 4XX here, but that seems wrong. upstream_ok = upstream_total - upstream_bad @@ -417,18 +412,16 @@ def update_envoy_stats(self, last_attempt: float) -> None: # self.logger.debug("cluster %s has had no requests" % cluster_name) active_clusters[cluster_name] = { - 'healthy_members': healthy_members, - 'total_members': total_members, - 'healthy_percent': healthy_percent, - - 'update_attempts': update_attempts, - 'update_successes': update_successes, - 'update_percent': update_percent, - - 'upstream_ok': upstream_ok, - 'upstream_4xx': upstream_4xx, - 'upstream_5xx': upstream_5xx, - 'upstream_bad': upstream_bad + "healthy_members": healthy_members, + "total_members": total_members, + "healthy_percent": healthy_percent, + "update_attempts": update_attempts, + "update_successes": update_successes, + "update_percent": update_percent, + "upstream_ok": upstream_ok, + "upstream_4xx": upstream_4xx, + "upstream_5xx": upstream_5xx, + "upstream_bad": upstream_bad, } # OK, we're now officially finished with all the hard stuff. @@ -439,12 +432,12 @@ def update_envoy_stats(self, last_attempt: float) -> None: max_live_age=self.stats.max_live_age, max_ready_age=self.stats.max_ready_age, created=self.stats.created, - last_update=last_update, # THIS IS A CHANGE - last_attempt=last_attempt, # THIS IS A CHANGE + last_update=last_update, # THIS IS A CHANGE + last_attempt=last_attempt, # THIS IS A CHANGE update_errors=self.stats.update_errors, - requests=requests_info, # THIS IS A CHANGE - clusters=active_clusters, # THIS IS A CHANGE - envoy=envoy_stats # THIS IS A CHANGE + requests=requests_info, # THIS IS A CHANGE + clusters=active_clusters, # THIS IS A CHANGE + envoy=envoy_stats, # THIS IS A CHANGE ) # Make sure we hold the access_lock while messing with self.stats! diff --git a/python/ambassador/envoy/common.py b/python/ambassador/envoy/common.py index 908201a06b..ff6be3da4f 100644 --- a/python/ambassador/envoy/common.py +++ b/python/ambassador/envoy/common.py @@ -22,9 +22,10 @@ from ..utils import dump_json if TYPE_CHECKING: - from ..ir import IR, IRResource # pragma: no cover - from ..ir.irhttpmappinggroup import IRHTTPMappingGroup # pragma: no cover - from ..ir.irserviceresolver import ClustermapEntry # pragma: no cover + from ..ir import IR, IRResource # pragma: no cover + from ..ir.irhttpmappinggroup import IRHTTPMappingGroup # pragma: no cover + from ..ir.irserviceresolver import ClustermapEntry # pragma: no cover + def sanitize_pre_json(input): # Removes all potential null values @@ -39,16 +40,17 @@ def sanitize_pre_json(input): sanitize_pre_json(item) return input + class EnvoyConfig: """ Base class for Envoy configuration that permits fetching configuration for various elements to show in diagnostics. """ - ir: 'IR' + ir: "IR" elements: Dict[str, Dict[str, Any]] - def __init__(self, ir: 'IR') -> None: + def __init__(self, ir: "IR") -> None: self.ir = ir self.elements = {} @@ -64,7 +66,7 @@ def pop_element(self, kind: str, key: str, default: Any) -> Optional[Any]: eldict = self.elements.get(kind, {}) return eldict.pop(key, default) - def save_element(self, kind: str, resource: 'IRResource', obj: Any): + def save_element(self, kind: str, resource: "IRResource", obj: Any): self.add_element(kind, resource.rkey, obj) self.add_element(kind, resource.location, obj) return obj @@ -74,7 +76,7 @@ def has_listeners(self) -> bool: pass @abstractmethod - def split_config(self) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, 'ClustermapEntry']]: + def split_config(self) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, "ClustermapEntry"]]: pass @abstractmethod @@ -85,22 +87,23 @@ def as_json(self): return dump_json(sanitize_pre_json(self.as_dict()), pretty=True) @classmethod - def generate(cls, ir: 'IR', cache: Optional[Cache]=None) -> 'EnvoyConfig': + def generate(cls, ir: "IR", cache: Optional[Cache] = None) -> "EnvoyConfig": from . import V3Config + return V3Config(ir, cache=cache) class EnvoyRoute: - def __init__(self, group: 'IRHTTPMappingGroup'): - self.prefix = 'prefix' - self.path = 'path' - self.regex = 'regex' + def __init__(self, group: "IRHTTPMappingGroup"): + self.prefix = "prefix" + self.path = "path" + self.regex = "regex" self.envoy_route = self._get_envoy_route(group) - def _get_envoy_route(self, group: 'IRHTTPMappingGroup') -> str: - if group.get('prefix_regex', False): + def _get_envoy_route(self, group: "IRHTTPMappingGroup") -> str: + if group.get("prefix_regex", False): return self.regex - if group.get('prefix_exact', False): + if group.get("prefix_exact", False): return self.path else: return self.prefix diff --git a/python/ambassador/envoy/v3/__init__.py b/python/ambassador/envoy/v3/__init__.py index 6f34a74ca0..a0c07891e4 100644 --- a/python/ambassador/envoy/v3/__init__.py +++ b/python/ambassador/envoy/v3/__init__.py @@ -13,4 +13,3 @@ # limitations under the License from .v3config import V3Config - diff --git a/python/ambassador/envoy/v3/v3_static_resources.py b/python/ambassador/envoy/v3/v3_static_resources.py index 9deea9e93b..10070e2517 100644 --- a/python/ambassador/envoy/v3/v3_static_resources.py +++ b/python/ambassador/envoy/v3/v3_static_resources.py @@ -15,19 +15,21 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3StaticResources(dict): - def __init__(self, config: 'V3Config') -> None: + def __init__(self, config: "V3Config") -> None: super().__init__() - self.update({ - 'listeners': [ l.as_dict() for l in config.listeners ], - 'clusters': config.clusters, - }) + self.update( + { + "listeners": [l.as_dict() for l in config.listeners], + "clusters": config.clusters, + } + ) @classmethod - def generate(cls, config: 'V3Config') -> None: + def generate(cls, config: "V3Config") -> None: # We needn't use config.save_element here -- this is just a wrapper element. config.static_resources = V3StaticResources(config) diff --git a/python/ambassador/envoy/v3/v3admin.py b/python/ambassador/envoy/v3/v3admin.py index a3ec5eaae2..b787e6842e 100644 --- a/python/ambassador/envoy/v3/v3admin.py +++ b/python/ambassador/envoy/v3/v3admin.py @@ -15,25 +15,22 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3Admin(dict): - def __init__(self, config: 'V3Config') -> None: + def __init__(self, config: "V3Config") -> None: super().__init__() aport = config.ir.ambassador_module.admin_port - self.update({ - 'access_log_path': '/tmp/admin_access_log', - 'address': { - 'socket_address': { - 'address': '127.0.0.1', - 'port_value': aport - } + self.update( + { + "access_log_path": "/tmp/admin_access_log", + "address": {"socket_address": {"address": "127.0.0.1", "port_value": aport}}, } - }) + ) @classmethod - def generate(cls, config: 'V3Config') -> None: - config.admin = config.save_element('admin', config.ir.ambassador_module, V3Admin(config)) + def generate(cls, config: "V3Config") -> None: + config.admin = config.save_element("admin", config.ir.ambassador_module, V3Admin(config)) diff --git a/python/ambassador/envoy/v3/v3bootstrap.py b/python/ambassador/envoy/v3/v3bootstrap.py index af41de5b90..a44a1db48d 100644 --- a/python/ambassador/envoy/v3/v3bootstrap.py +++ b/python/ambassador/envoy/v3/v3bootstrap.py @@ -13,87 +13,81 @@ from .v3cluster import V3Cluster if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3Bootstrap(dict): - def __init__(self, config: 'V3Config') -> None: + def __init__(self, config: "V3Config") -> None: api_version = "V3" - super().__init__(**{ - "node": { - "cluster": config.ir.ambassador_nodename, - "id": "test-id" # MUST BE test-id, see below - }, - "static_resources": {}, # Filled in later - "dynamic_resources": { - "ads_config": { - "api_type": "GRPC", - "transport_api_version": api_version, - "grpc_services": [ { - "envoy_grpc": { - "cluster_name": "xds_cluster" - } - } ] + super().__init__( + **{ + "node": { + "cluster": config.ir.ambassador_nodename, + "id": "test-id", # MUST BE test-id, see below }, - "cds_config": { - "ads": {}, - "resource_api_version": api_version + "static_resources": {}, # Filled in later + "dynamic_resources": { + "ads_config": { + "api_type": "GRPC", + "transport_api_version": api_version, + "grpc_services": [{"envoy_grpc": {"cluster_name": "xds_cluster"}}], + }, + "cds_config": {"ads": {}, "resource_api_version": api_version}, + "lds_config": {"ads": {}, "resource_api_version": api_version}, }, - "lds_config": { - "ads": {}, - "resource_api_version": api_version - } - }, - "admin": dict(config.admin), - 'layered_runtime': { - 'layers': [ - { - 'name': 'static_layer', - 'static_layer': { - 're2.max_program_size.error_level': 200, - # TODO(lance): - # the new default is that all filters are looked up using the @type which currently we exclude on a lot of - # our filters. This will ensure we do not break current config. We can migrate over - # in a minor release. see here: https://www.envoyproxy.io/docs/envoy/v1.22.0/version_history/current#minor-behavior-changes - # The biggest impact of this is ensuring that ambex imports all the types because we will need to import many more - "envoy.reloadable_features.no_extension_lookup_by_name": False + "admin": dict(config.admin), + "layered_runtime": { + "layers": [ + { + "name": "static_layer", + "static_layer": { + "re2.max_program_size.error_level": 200, + # TODO(lance): + # the new default is that all filters are looked up using the @type which currently we exclude on a lot of + # our filters. This will ensure we do not break current config. We can migrate over + # in a minor release. see here: https://www.envoyproxy.io/docs/envoy/v1.22.0/version_history/current#minor-behavior-changes + # The biggest impact of this is ensuring that ambex imports all the types because we will need to import many more + "envoy.reloadable_features.no_extension_lookup_by_name": False, + }, } - } - ] + ] + }, } - }) - - clusters = [{ - "name": "xds_cluster", - "connect_timeout": "1s", - "dns_lookup_family": "V4_ONLY", - "http2_protocol_options": {}, - "lb_policy": "ROUND_ROBIN", - "load_assignment": { - "cluster_name": "cluster_127_0_0_1_8003", - "endpoints": [ - { - "lb_endpoints": [ - { - "endpoint": { - "address": { - "socket_address": { - # this should be kept in-sync with entrypoint.sh `ambex --ads-listen-address=...` - "address": "127.0.0.1", - "port_value": 8003, - "protocol": "TCP" + ) + + clusters = [ + { + "name": "xds_cluster", + "connect_timeout": "1s", + "dns_lookup_family": "V4_ONLY", + "http2_protocol_options": {}, + "lb_policy": "ROUND_ROBIN", + "load_assignment": { + "cluster_name": "cluster_127_0_0_1_8003", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + # this should be kept in-sync with entrypoint.sh `ambex --ads-listen-address=...` + "address": "127.0.0.1", + "port_value": 8003, + "protocol": "TCP", + } } } } - } - ] - } - ] + ] + } + ], + }, } - }] + ] if config.tracing: - self['tracing'] = dict(config.tracing) + self["tracing"] = dict(config.tracing) tracing = typecast(IRTracing, config.ir.tracing) @@ -122,89 +116,92 @@ def __init__(self, config: 'V3Config') -> None: host, port = split_host_port(grpcSink) valid = True except ValueError as ex: - config.ir.logger.error("AMBASSADOR_GRPC_METRICS_SINK value %s is invalid: %s" % (grpcSink, ex)) + config.ir.logger.error( + "AMBASSADOR_GRPC_METRICS_SINK value %s is invalid: %s" % (grpcSink, ex) + ) valid = False if valid: - stats_sinks.append({ - 'name': "envoy.metrics_service", - 'typed_config': { - '@type': 'type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig', - 'transport_api_version': 'V3', - 'grpc_service': { - 'envoy_grpc': { - 'cluster_name': 'envoy_metrics_service' - } - } + stats_sinks.append( + { + "name": "envoy.metrics_service", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig", + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": {"cluster_name": "envoy_metrics_service"} + }, + }, } - }) - clusters.append({ - "name": "envoy_metrics_service", - "type": "strict_dns", - "connect_timeout": "1s", - "http2_protocol_options": {}, - "load_assignment": { - "cluster_name": "envoy_metrics_service", - "endpoints": [ - { - "lb_endpoints": [ - { - "endpoint": { - "address": { - "socket_address": { - "address": host, - "port_value": port, - "protocol": "TCP" + ) + clusters.append( + { + "name": "envoy_metrics_service", + "type": "strict_dns", + "connect_timeout": "1s", + "http2_protocol_options": {}, + "load_assignment": { + "cluster_name": "envoy_metrics_service", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": host, + "port_value": port, + "protocol": "TCP", + } } } } - } - ] - } - ] + ] + } + ], + }, } - }) + ) - if config.ir.statsd['enabled']: - if config.ir.statsd['dogstatsd']: - name = 'envoy.stat_sinks.dog_statsd' - typename = 'type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink' - dd_entity_id = os.environ.get('DD_ENTITY_ID', None) + if config.ir.statsd["enabled"]: + if config.ir.statsd["dogstatsd"]: + name = "envoy.stat_sinks.dog_statsd" + typename = "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink" + dd_entity_id = os.environ.get("DD_ENTITY_ID", None) if dd_entity_id: - stats_tags = self.setdefault('stats_config', {}).setdefault('stats_tags', []) - stats_tags.append({ - 'tag_name': 'dd.internal.entity_id', - 'fixed_value': dd_entity_id - }) + stats_tags = self.setdefault("stats_config", {}).setdefault("stats_tags", []) + stats_tags.append( + {"tag_name": "dd.internal.entity_id", "fixed_value": dd_entity_id} + ) else: - name = 'envoy.stats_sinks.statsd' - typename = 'type.googleapis.com/envoy.config.metrics.v3.StatsdSink' - - stats_sinks.append({ - 'name': name, - 'typed_config': { - '@type': typename, - 'address': { - 'socket_address': { - 'protocol': 'UDP', - 'address': config.ir.statsd['ip'], - 'port_value': 8125 - } - } + name = "envoy.stats_sinks.statsd" + typename = "type.googleapis.com/envoy.config.metrics.v3.StatsdSink" + + stats_sinks.append( + { + "name": name, + "typed_config": { + "@type": typename, + "address": { + "socket_address": { + "protocol": "UDP", + "address": config.ir.statsd["ip"], + "port_value": 8125, + } + }, + }, } - }) + ) - self['stats_flush_interval'] = { - 'seconds': config.ir.statsd['interval'] - } - self['stats_sinks'] = stats_sinks - self['static_resources']['clusters'] = clusters + self["stats_flush_interval"] = {"seconds": config.ir.statsd["interval"]} + self["stats_sinks"] = stats_sinks + self["static_resources"]["clusters"] = clusters @classmethod - def generate(cls, config: 'V3Config') -> None: + def generate(cls, config: "V3Config") -> None: config.bootstrap = V3Bootstrap(config) def split_host_port(value: str) -> Tuple[Optional[str], int]: - parsed = urlparse("//"+value) + parsed = urlparse("//" + value) return parsed.hostname, int(parsed.port or 80) diff --git a/python/ambassador/envoy/v3/v3cidrrange.py b/python/ambassador/envoy/v3/v3cidrrange.py index 2ab29c31a8..3204610dd4 100644 --- a/python/ambassador/envoy/v3/v3cidrrange.py +++ b/python/ambassador/envoy/v3/v3cidrrange.py @@ -2,6 +2,7 @@ from ipaddress import ip_address, IPv4Address, IPv6Address + class CIDRRange: """ A CIDRRange is an IP address (either v4 or v6) plus a prefix length. It @@ -33,9 +34,9 @@ def __init__(self, spec: str) -> None: pfx_len: Optional[int] = None addr: Optional[Union[IPv4Address, IPv6Address]] = None - if '/' in spec: + if "/" in spec: # CIDR range! Try to separate the address and its length. - address, lenstr = spec.split('/', 1) + address, lenstr = spec.split("/", 1) try: pfx_len = int(lenstr) @@ -71,9 +72,7 @@ def __bool__(self) -> bool: is not None, and the prefix_len is not None. """ - return ((not self.error) and - (self.address is not None) and - (self.prefix_len is not None)) + return (not self.error) and (self.address is not None) and (self.prefix_len is not None) def __str__(self) -> str: if self: @@ -87,7 +86,4 @@ def as_dict(self) -> dict: an Envoy config as an envoy.api.v3.core.CidrRange. """ - return { - "address_prefix": self.address, - "prefix_len": self.prefix_len - } + return {"address_prefix": self.address, "prefix_len": self.prefix_len} diff --git a/python/ambassador/envoy/v3/v3cluster.py b/python/ambassador/envoy/v3/v3cluster.py index d847e48ddc..4e225fb911 100644 --- a/python/ambassador/envoy/v3/v3cluster.py +++ b/python/ambassador/envoy/v3/v3cluster.py @@ -23,20 +23,20 @@ from .v3tls import V3TLSContext if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3Cluster(Cacheable): - def __init__(self, config: 'V3Config', cluster: IRCluster) -> None: + def __init__(self, config: "V3Config", cluster: IRCluster) -> None: super().__init__() - dns_lookup_family = 'V4_ONLY' + dns_lookup_family = "V4_ONLY" if cluster.enable_ipv6: if cluster.enable_ipv4: - dns_lookup_family = 'AUTO' + dns_lookup_family = "AUTO" else: - dns_lookup_family = 'V6_ONLY' + dns_lookup_family = "V6_ONLY" # We must not use cluster.name in the envoy config, since it may be too long # to pass envoy's cluster name length constraint, currently 60 characters. @@ -45,85 +45,96 @@ def __init__(self, config: 'V3Config', cluster: IRCluster) -> None: # In practice, the envoy_name is a short-form of cluster.name with the first # 40 characters followed by `-n` where `n` is an incremental value, one for # every cluster whose name contains the same first 40 characters. - assert(cluster.envoy_name) - assert(len(cluster.envoy_name) <= 60) + assert cluster.envoy_name + assert len(cluster.envoy_name) <= 60 cmap_entry = cluster.clustermap_entry() - if cmap_entry['kind'] == 'KubernetesServiceResolver': + if cmap_entry["kind"] == "KubernetesServiceResolver": ctype = cluster.type.upper() # For now we are only allowing Logical_dns for the cluster since it is similar enough to strict_dns that we dont need any other config changes # It should be easy to add the other dns_types here in the future if we decide to support them - if ctype not in [ "STRICT_DNS", "LOGICAL_DNS" ]: - cluster.ir.logger.warning("dns_type %s, is an invalid type. Options are STRICT_DNS or LOGICAL_DNS. Using default of STRICT_DNS" % (ctype)) + if ctype not in ["STRICT_DNS", "LOGICAL_DNS"]: + cluster.ir.logger.warning( + "dns_type %s, is an invalid type. Options are STRICT_DNS or LOGICAL_DNS. Using default of STRICT_DNS" + % (ctype) + ) ctype = "STRICT_DNS" else: - ctype = 'EDS' + ctype = "EDS" fields = { - 'name': cluster.envoy_name, - 'type': ctype, - 'lb_policy': cluster.lb_type.upper(), - 'connect_timeout':"%0.3fs" % (float(cluster.connect_timeout_ms) / 1000.0), - 'dns_lookup_family': dns_lookup_family + "name": cluster.envoy_name, + "type": ctype, + "lb_policy": cluster.lb_type.upper(), + "connect_timeout": "%0.3fs" % (float(cluster.connect_timeout_ms) / 1000.0), + "dns_lookup_family": dns_lookup_family, } - if cluster.get('stats_name', ''): - fields['alt_stat_name'] = cluster.stats_name + if cluster.get("stats_name", ""): + fields["alt_stat_name"] = cluster.stats_name if cluster.respect_dns_ttl: - fields['respect_dns_ttl'] = cluster.respect_dns_ttl + fields["respect_dns_ttl"] = cluster.respect_dns_ttl - if ctype == 'EDS': - fields['eds_cluster_config'] = { - 'eds_config': { - 'ads': {}, + if ctype == "EDS": + fields["eds_cluster_config"] = { + "eds_config": { + "ads": {}, # Envoy may default to an older API version if we are not explicit about V3 here. - 'resource_api_version': 'V3' + "resource_api_version": "V3", }, - 'service_name': cmap_entry['endpoint_path'] + "service_name": cmap_entry["endpoint_path"], } else: - fields['load_assignment'] = { - 'cluster_name': cluster.envoy_name, - 'endpoints': [ - { - 'lb_endpoints': self.get_endpoints(cluster) - } - ] + fields["load_assignment"] = { + "cluster_name": cluster.envoy_name, + "endpoints": [{"lb_endpoints": self.get_endpoints(cluster)}], } if cluster.cluster_idle_timeout_ms: cluster_idle_timeout_ms = cluster.cluster_idle_timeout_ms else: - cluster_idle_timeout_ms = cluster.ir.ambassador_module.get('cluster_idle_timeout_ms', None) + cluster_idle_timeout_ms = cluster.ir.ambassador_module.get( + "cluster_idle_timeout_ms", None + ) if cluster_idle_timeout_ms: common_http_options = self.setdefault("common_http_protocol_options", {}) - common_http_options['idle_timeout'] = "%0.3fs" % (float(cluster_idle_timeout_ms) / 1000.0) + common_http_options["idle_timeout"] = "%0.3fs" % ( + float(cluster_idle_timeout_ms) / 1000.0 + ) if cluster.cluster_max_connection_lifetime_ms: cluster_max_connection_lifetime_ms = cluster.cluster_max_connection_lifetime_ms else: - cluster_max_connection_lifetime_ms = cluster.ir.ambassador_module.get('cluster_max_connection_lifetime_ms', None) + cluster_max_connection_lifetime_ms = cluster.ir.ambassador_module.get( + "cluster_max_connection_lifetime_ms", None + ) if cluster_max_connection_lifetime_ms: common_http_options = self.setdefault("common_http_protocol_options", {}) - common_http_options['max_connection_duration'] = "%0.3fs" % (float(cluster_max_connection_lifetime_ms) / 1000.0) + common_http_options["max_connection_duration"] = "%0.3fs" % ( + float(cluster_max_connection_lifetime_ms) / 1000.0 + ) circuit_breakers = self.get_circuit_breakers(cluster) if circuit_breakers is not None: - fields['circuit_breakers'] = circuit_breakers + fields["circuit_breakers"] = circuit_breakers # If this cluster is using http2 for grpc, set http2_protocol_options # Otherwise, check for http1-specific configuration. - if cluster.get('grpc', False): + if cluster.get("grpc", False): self["http2_protocol_options"] = {} else: - proper_case: bool = cluster.ir.ambassador_module['proper_case'] + proper_case: bool = cluster.ir.ambassador_module["proper_case"] # Get the list of upstream headers whose casing should be overriden # from the Ambassador module. We configure the downstream side of this # in v3listener.py - header_case_overrides = cluster.ir.ambassador_module.get('header_case_overrides', None) - if header_case_overrides and not proper_case and isinstance(header_case_overrides, list): + header_case_overrides = cluster.ir.ambassador_module.get("header_case_overrides", None) + if ( + header_case_overrides + and not proper_case + and isinstance(header_case_overrides, list) + ): # We have this config validation here because the Ambassador module is # still an untyped config. That is, we aren't yet using a CRD or a # python schema to constrain the configuration that can be present. @@ -134,16 +145,12 @@ def __init__(self, config: 'V3Config', cluster: IRCluster) -> None: rules.append(hdr) if len(rules) > 0: custom_header_rules: Dict[str, Dict[str, dict]] = { - 'custom': { - 'rules': { - header.lower() : header for header in rules - } - } + "custom": {"rules": {header.lower(): header for header in rules}} } http_options = self.setdefault("http_protocol_options", {}) http_options["header_key_format"] = custom_header_rules - ctx = cluster.get('tls_context', None) + ctx = cluster.get("tls_context", None) if ctx is not None: # If this is a null TLS Context (_ambassador_enabled is True), then we at need to specify a @@ -154,100 +161,97 @@ def __init__(self, config: 'V3Config', cluster: IRCluster) -> None: # XXX That's a silly reason to not do proper typing. envoy_ctx: dict - if ctx.get('_ambassador_enabled', False): - envoy_ctx = { - 'common_tls_context': {} - } + if ctx.get("_ambassador_enabled", False): + envoy_ctx = {"common_tls_context": {}} else: - envoy_ctx = V3TLSContext(ctx=ctx, host_rewrite=cluster.get('host_rewrite', None)) + envoy_ctx = V3TLSContext(ctx=ctx, host_rewrite=cluster.get("host_rewrite", None)) if envoy_ctx: - fields['transport_socket'] = { - 'name': 'envoy.transport_sockets.tls', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext', - **envoy_ctx - } + fields["transport_socket"] = { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + **envoy_ctx, + }, } - - keepalive = cluster.get('keepalive', None) - #in case of empty keepalive for service, we can try to fallback to default + keepalive = cluster.get("keepalive", None) + # in case of empty keepalive for service, we can try to fallback to default if keepalive is None: - if cluster.ir.ambassador_module and cluster.ir.ambassador_module.get('keepalive', None): - keepalive = cluster.ir.ambassador_module['keepalive'] + if cluster.ir.ambassador_module and cluster.ir.ambassador_module.get("keepalive", None): + keepalive = cluster.ir.ambassador_module["keepalive"] if keepalive is not None: keepalive_options = {} - keepalive_time = keepalive.get('time', None) - keepalive_interval = keepalive.get('interval', None) - keepalive_probes = keepalive.get('probes', None) + keepalive_time = keepalive.get("time", None) + keepalive_interval = keepalive.get("interval", None) + keepalive_probes = keepalive.get("probes", None) if keepalive_time is not None: - keepalive_options['keepalive_time'] = keepalive_time + keepalive_options["keepalive_time"] = keepalive_time if keepalive_interval is not None: - keepalive_options['keepalive_interval'] = keepalive_interval + keepalive_options["keepalive_interval"] = keepalive_interval if keepalive_probes is not None: - keepalive_options['keepalive_probes'] = keepalive_probes + keepalive_options["keepalive_probes"] = keepalive_probes - fields['upstream_connection_options'] = {'tcp_keepalive' : keepalive_options } + fields["upstream_connection_options"] = {"tcp_keepalive": keepalive_options} self.update(fields) def get_endpoints(self, cluster: IRCluster): result = [] - targetlist = cluster.get('targets', []) + targetlist = cluster.get("targets", []) if len(targetlist) > 0: for target in targetlist: address = { - 'address': target['ip'], - 'port_value': target['port'], - 'protocol': 'TCP' # Yes, really. Envoy uses the TLS context to determine whether to originate TLS. + "address": target["ip"], + "port_value": target["port"], + "protocol": "TCP", # Yes, really. Envoy uses the TLS context to determine whether to originate TLS. } - result.append({'endpoint': {'address': {'socket_address': address}}}) + result.append({"endpoint": {"address": {"socket_address": address}}}) else: for u in cluster.urls: p = urllib.parse.urlparse(u) - address = { - 'address': p.hostname, - 'port_value': int(p.port) - } + address = {"address": p.hostname, "port_value": int(p.port)} if p.scheme: - address['protocol'] = p.scheme.upper() - result.append({'endpoint': {'address': {'socket_address': address}}}) + address["protocol"] = p.scheme.upper() + result.append({"endpoint": {"address": {"socket_address": address}}}) return result def get_circuit_breakers(self, cluster: IRCluster): - cluster_circuit_breakers = cluster.get('circuit_breakers', None) + cluster_circuit_breakers = cluster.get("circuit_breakers", None) if cluster_circuit_breakers is None: return None - circuit_breakers: Dict[str, List[Dict[str, Union[str, int]]]] = { - 'thresholds': [] - } + circuit_breakers: Dict[str, List[Dict[str, Union[str, int]]]] = {"thresholds": []} for circuit_breaker in cluster_circuit_breakers: threshold = {} - if 'priority' in circuit_breaker: - threshold['priority'] = circuit_breaker.get('priority').upper() + if "priority" in circuit_breaker: + threshold["priority"] = circuit_breaker.get("priority").upper() else: - threshold['priority'] = 'DEFAULT' - - digit_fields = ['max_connections', 'max_pending_requests', 'max_requests', 'max_retries'] + threshold["priority"] = "DEFAULT" + + digit_fields = [ + "max_connections", + "max_pending_requests", + "max_requests", + "max_retries", + ] for field in digit_fields: if field in circuit_breaker: threshold[field] = int(circuit_breaker.get(field)) if len(threshold) > 0: - circuit_breakers['thresholds'].append(threshold) + circuit_breakers["thresholds"].append(threshold) return circuit_breakers @classmethod - def generate(self, config: 'V3Config') -> None: - cluster: 'V3Cluster' + def generate(self, config: "V3Config") -> None: + cluster: "V3Cluster" config.clusters = [] config.clustermap = {} @@ -260,7 +264,7 @@ def generate(self, config: 'V3Config') -> None: if cached_cluster is None: # Cache miss. - cluster = config.save_element('cluster', ircluster, V3Cluster(config, ircluster)) + cluster = config.save_element("cluster", ircluster, V3Cluster(config, ircluster)) # Cheat a bit and force the route's cache key. cluster.cache_key = cache_key @@ -277,7 +281,7 @@ def generate(self, config: 'V3Config') -> None: else: # Cache hit. We know a priori that it's a V3Cluster, but let's assert # that rather than casting. - assert(isinstance(cached_cluster, V3Cluster)) + assert isinstance(cached_cluster, V3Cluster) cluster = cached_cluster config.clusters.append(cluster) diff --git a/python/ambassador/envoy/v3/v3config.py b/python/ambassador/envoy/v3/v3config.py index d40be9897c..55e4a5d3e9 100644 --- a/python/ambassador/envoy/v3/v3config.py +++ b/python/ambassador/envoy/v3/v3config.py @@ -29,15 +29,15 @@ from .v3ratelimit import V3RateLimit if TYPE_CHECKING: - from ...ir import IR # pragma: no cover - from ...ir.irserviceresolver import ClustermapEntry # pragma: no cover + from ...ir import IR # pragma: no cover + from ...ir.irserviceresolver import ClustermapEntry # pragma: no cover # ############################################################################# # ## v3config.py -- the Envoy V3 configuration engine # # -class V3Config (EnvoyConfig): +class V3Config(EnvoyConfig): admin: V3Admin tracing: Optional[V3Tracing] ratelimit: Optional[V3RateLimit] @@ -49,7 +49,7 @@ class V3Config (EnvoyConfig): static_resources: V3StaticResources clustermap: Dict[str, Any] - def __init__(self, ir: 'IR', cache: Optional[Cache]=None) -> None: + def __init__(self, ir: "IR", cache: Optional[Cache] = None) -> None: ir.logger.info("EnvoyConfig: Generating V3") # Init our superclass... @@ -74,33 +74,29 @@ def has_listeners(self) -> bool: def as_dict(self) -> Dict[str, Any]: bootstrap_config, ads_config, clustermap = self.split_config() - d = { - 'bootstrap': bootstrap_config, - 'clustermap': clustermap, - **ads_config - } + d = {"bootstrap": bootstrap_config, "clustermap": clustermap, **ads_config} return d - def split_config(self) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, 'ClustermapEntry']]: + def split_config(self) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, "ClustermapEntry"]]: ads_config = { - '@type': '/envoy.config.bootstrap.v3.Bootstrap', - 'static_resources': self.static_resources, - 'layered_runtime': { - 'layers': [ + "@type": "/envoy.config.bootstrap.v3.Bootstrap", + "static_resources": self.static_resources, + "layered_runtime": { + "layers": [ { - 'name': 'static_layer', - 'static_layer': { - 're2.max_program_size.error_level': 200, + "name": "static_layer", + "static_layer": { + "re2.max_program_size.error_level": 200, # the new default is that all filters are looked up using the @type which currently we exclude on a lot of # our filters. This will ensure we do not break current config. We can migrate over # in a minor release. see here: https://www.envoyproxy.io/docs/envoy/v1.22.0/version_history/current#minor-behavior-changes # The biggest impact of this is ensuring that ambex imports all the types because we will need to import many more - "envoy.reloadable_features.no_extension_lookup_by_name": False - } + "envoy.reloadable_features.no_extension_lookup_by_name": False, + }, } ] - } + }, } bootstrap_config = dict(self.bootstrap) diff --git a/python/ambassador/envoy/v3/v3httpfilter.py b/python/ambassador/envoy/v3/v3httpfilter.py index 63acff77ac..b2c5c2aefc 100644 --- a/python/ambassador/envoy/v3/v3httpfilter.py +++ b/python/ambassador/envoy/v3/v3httpfilter.py @@ -33,57 +33,55 @@ from ...utils import ParsedService as Service # Static header keys normally used in the context of an authorization request. -AllowedRequestHeaders = frozenset([ - 'authorization', - 'cookie', - 'from', - 'proxy-authorization', - 'user-agent', - 'x-forwarded-for', - 'x-forwarded-host', - 'x-forwarded-proto' -]) +AllowedRequestHeaders = frozenset( + [ + "authorization", + "cookie", + "from", + "proxy-authorization", + "user-agent", + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + ] +) # Static header keys normally used in the context of an authorization response. -AllowedAuthorizationHeaders = frozenset([ - 'location', - 'authorization', - 'proxy-authenticate', - 'set-cookie', - 'www-authenticate' -]) +AllowedAuthorizationHeaders = frozenset( + ["location", "authorization", "proxy-authenticate", "set-cookie", "www-authenticate"] +) # This mapping is only used for ambassador/v0. ExtAuthRequestHeaders = { - 'Authorization': True, - 'Cookie': True, - 'Forwarded': True, - 'From': True, - 'Host': True, - 'Proxy-Authenticate': True, - 'Proxy-Authorization': True, - 'Set-Cookie': True, - 'User-Agent': True, - 'x-b3-flags': True, - 'x-b3-parentspanid': True, - 'x-b3-traceid': True, - 'x-b3-sampled': True, - 'x-b3-spanid': True, - 'X-Forwarded-For': True, - 'X-Forwarded-Host': True, - 'X-Forwarded-Proto': True, - 'X-Gateway-Proto': True, - 'x-ot-span-context': True, - 'WWW-Authenticate': True, + "Authorization": True, + "Cookie": True, + "Forwarded": True, + "From": True, + "Host": True, + "Proxy-Authenticate": True, + "Proxy-Authorization": True, + "Set-Cookie": True, + "User-Agent": True, + "x-b3-flags": True, + "x-b3-parentspanid": True, + "x-b3-traceid": True, + "x-b3-sampled": True, + "x-b3-spanid": True, + "X-Forwarded-For": True, + "X-Forwarded-Host": True, + "X-Forwarded-Proto": True, + "X-Gateway-Proto": True, + "x-ot-span-context": True, + "WWW-Authenticate": True, } def header_pattern_key(x: Dict[str, str]) -> List[Tuple[str, str]]: - return sorted([ (k, v) for k, v in x.items() ]) + return sorted([(k, v) for k, v in x.items()]) @singledispatch -def V3HTTPFilter(irfilter: IRFilter, v3config: 'V3Config'): +def V3HTTPFilter(irfilter: IRFilter, v3config: "V3Config"): # Fallback for the filters that don't have their own IR* type and therefor can't participate in # @singledispatch. fn = { @@ -97,79 +95,81 @@ def V3HTTPFilter(irfilter: IRFilter, v3config: 'V3Config'): return fn(irfilter, v3config) + @V3HTTPFilter.register -def V3HTTPFilter_buffer(buffer: IRBuffer, v3config: 'V3Config'): +def V3HTTPFilter_buffer(buffer: IRBuffer, v3config: "V3Config"): del v3config # silence unused-variable warning return { - 'name': 'envoy.filters.http.buffer', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer', - "max_request_bytes": buffer.max_request_bytes - } + "name": "envoy.filters.http.buffer", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer", + "max_request_bytes": buffer.max_request_bytes, + }, } + @V3HTTPFilter.register -def V3HTTPFilter_gzip(gzip: IRGzip, v3config: 'V3Config'): +def V3HTTPFilter_gzip(gzip: IRGzip, v3config: "V3Config"): del v3config # silence unused-variable warning common_config = { - 'min_content_length': gzip.content_length, - 'content_type': gzip.content_type, + "min_content_length": gzip.content_length, + "content_type": gzip.content_type, } return { - 'name': 'envoy.filters.http.gzip', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor', - 'compressor_library': { + "name": "envoy.filters.http.gzip", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor", + "compressor_library": { "name": "envoy.compression.gzip.compressor", "typed_config": { "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip", - 'memory_level': gzip.memory_level, - 'compression_level': gzip.compression_level, - 'compression_strategy': gzip.compression_strategy, - 'window_bits': gzip.window_bits, - } + "memory_level": gzip.memory_level, + "compression_level": gzip.compression_level, + "compression_strategy": gzip.compression_strategy, + "window_bits": gzip.window_bits, + }, }, - 'response_direction_config': { - 'disable_on_etag_header': gzip.disable_on_etag_header, - 'remove_accept_encoding_header': gzip.remove_accept_encoding_header, - 'common_config': common_config, - } - } + "response_direction_config": { + "disable_on_etag_header": gzip.disable_on_etag_header, + "remove_accept_encoding_header": gzip.remove_accept_encoding_header, + "common_config": common_config, + }, + }, } -def V3HTTPFilter_grpc_http1_bridge(irfilter: IRFilter, v3config: 'V3Config'): + +def V3HTTPFilter_grpc_http1_bridge(irfilter: IRFilter, v3config: "V3Config"): del irfilter # silence unused-variable warning del v3config # silence unused-variable warning - return { - 'name': 'envoy.filters.http.grpc_http1_bridge' - } + return {"name": "envoy.filters.http.grpc_http1_bridge"} -def V3HTTPFilter_grpc_web(irfilter: IRFilter, v3config: 'V3Config'): + +def V3HTTPFilter_grpc_web(irfilter: IRFilter, v3config: "V3Config"): del irfilter # silence unused-variable warning del v3config # silence unused-variable warning - return { - 'name': 'envoy.filters.http.grpc_web' - } + return {"name": "envoy.filters.http.grpc_web"} + -def V3HTTPFilter_grpc_stats(irfilter: IRFilter, v3config: 'V3Config'): +def V3HTTPFilter_grpc_stats(irfilter: IRFilter, v3config: "V3Config"): del v3config # silence unused-variable warning config = typecast(Dict[str, Any], irfilter.config_dict()) return { - 'name': 'envoy.filters.http.grpc_stats', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig', + "name": "envoy.filters.http.grpc_stats", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig", **config, - } + }, } + def auth_cluster_uri(auth: IRAuth, cluster: IRCluster) -> str: - cluster_context = cluster.get('tls_context') - scheme = 'https' if cluster_context else 'http' + cluster_context = cluster.get("tls_context") + scheme = "https" if cluster_context else "http" prefix = auth.get("path_prefix") or "" @@ -183,8 +183,9 @@ def auth_cluster_uri(auth: IRAuth, cluster: IRCluster) -> str: return server_uri + @V3HTTPFilter.register -def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): +def V3HTTPFilter_authv1(auth: IRAuth, v3config: "V3Config"): del v3config # silence unused-variable warning assert auth.cluster @@ -192,24 +193,21 @@ def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): assert auth.proto - raw_body_info: Optional[Dict[str, int]] = auth.get('include_body') + raw_body_info: Optional[Dict[str, int]] = auth.get("include_body") - if not raw_body_info and auth.get('allow_request_body', False): - raw_body_info = { - 'max_bytes': 4096, - 'allow_partial': True - } + if not raw_body_info and auth.get("allow_request_body", False): + raw_body_info = {"max_bytes": 4096, "allow_partial": True} body_info: Optional[Dict[str, int]] = None if raw_body_info: body_info = {} - if 'max_bytes' in raw_body_info: - body_info['max_request_bytes'] = raw_body_info['max_bytes'] + if "max_bytes" in raw_body_info: + body_info["max_request_bytes"] = raw_body_info["max_bytes"] - if 'allow_partial' in raw_body_info: - body_info['allow_partial_message'] = raw_body_info['allow_partial'] + if "allow_partial" in raw_body_info: + body_info["allow_partial_message"] = raw_body_info["allow_partial"] auth_info: Dict[str, Any] = {} @@ -221,10 +219,10 @@ def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): # 'errors'. But that's kinda tricky because while we have "json_escape()" in the Python # stdlib, we don't have a "lua_escape()"; and I'm on a tight deadline. auth_info = { - 'name': 'envoy.filters.http.lua', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua', - 'inline_code': """ + "name": "envoy.filters.http.lua", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua", + "inline_code": """ function envoy_on_request(request_handle) local path = request_handle:headers():get(':path') if path == '/ambassador/v0/check_alive' or path == '/ambassador/v0/check_ready' @@ -238,7 +236,9 @@ def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): {[":status"] = "500", ["content-type"] = "application/json"}, '{\\n'.. - ' "message": "the """+auth.rkey+""" AuthService is misconfigured; see the logs for more information",\\n'.. + ' "message": "the """ + + auth.rkey + + """ AuthService is misconfigured; see the logs for more information",\\n'.. ' "request_id": "'..request_handle:headers():get('x-request-id')..'",\\n'.. ' "status_code": 500\\n'.. '}\\n') @@ -251,11 +251,13 @@ def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): allowed_authorization_headers = [] headers_to_add = [] - for k, v in auth.get('add_auth_headers', {}).items(): - headers_to_add.append({ - 'key': k, - 'value': v, - }) + for k, v in auth.get("add_auth_headers", {}).items(): + headers_to_add.append( + { + "key": k, + "value": v, + } + ) for key in list(set(auth.allowed_authorization_headers).union(AllowedAuthorizationHeaders)): allowed_authorization_headers.append({"exact": key, "ignore_case": True}) @@ -265,74 +267,72 @@ def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): for key in list(set(auth.allowed_request_headers).union(AllowedRequestHeaders)): allowed_request_headers.append({"exact": key, "ignore_case": True}) - if auth.get('add_linkerd_headers', False): + if auth.get("add_linkerd_headers", False): svc = Service(auth.ir.logger, auth_cluster_uri(auth, cluster)) - headers_to_add.append({ - 'key' : 'l5d-dst-override', - 'value': svc.hostname_port - }) + headers_to_add.append({"key": "l5d-dst-override", "value": svc.hostname_port}) auth_info = { - 'name': 'envoy.filters.http.ext_authz', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz', - 'http_service': { - 'server_uri': { - 'uri': auth_cluster_uri(auth, cluster), - 'cluster': cluster.envoy_name, - 'timeout': "%0.3fs" % (float(auth.timeout_ms) / 1000.0) + "name": "envoy.filters.http.ext_authz", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz", + "http_service": { + "server_uri": { + "uri": auth_cluster_uri(auth, cluster), + "cluster": cluster.envoy_name, + "timeout": "%0.3fs" % (float(auth.timeout_ms) / 1000.0), }, - 'path_prefix': auth.path_prefix, - 'authorization_request': { - 'allowed_headers': { - 'patterns': sorted(allowed_request_headers, key=header_pattern_key) + "path_prefix": auth.path_prefix, + "authorization_request": { + "allowed_headers": { + "patterns": sorted(allowed_request_headers, key=header_pattern_key) }, - 'headers_to_add' : headers_to_add + "headers_to_add": headers_to_add, }, - 'authorization_response' : { - 'allowed_upstream_headers': { - 'patterns': sorted(allowed_authorization_headers, key=header_pattern_key) + "authorization_response": { + "allowed_upstream_headers": { + "patterns": sorted( + allowed_authorization_headers, key=header_pattern_key + ) }, - 'allowed_client_headers': { - 'patterns': sorted(allowed_authorization_headers, key=header_pattern_key) - } - } + "allowed_client_headers": { + "patterns": sorted( + allowed_authorization_headers, key=header_pattern_key + ) + }, + }, }, - } + }, } elif auth.proto == "grpc": auth_info = { - 'name': 'envoy.filters.http.ext_authz', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz', - 'grpc_service': { - 'envoy_grpc': { - 'cluster_name': cluster.envoy_name - }, - 'timeout': "%0.3fs" % (float(auth.timeout_ms) / 1000.0) + "name": "envoy.filters.http.ext_authz", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz", + "grpc_service": { + "envoy_grpc": {"cluster_name": cluster.envoy_name}, + "timeout": "%0.3fs" % (float(auth.timeout_ms) / 1000.0), }, - 'transport_api_version': auth.protocol_version.upper(), - } + "transport_api_version": auth.protocol_version.upper(), + }, } - if auth_info['name'] == 'envoy.filters.http.ext_authz': - auth_info['typed_config']['clear_route_cache'] = True + if auth_info["name"] == "envoy.filters.http.ext_authz": + auth_info["typed_config"]["clear_route_cache"] = True if body_info: - auth_info['typed_config']['with_request_body'] = body_info + auth_info["typed_config"]["with_request_body"] = body_info - if 'failure_mode_allow' in auth: - auth_info['typed_config']["failure_mode_allow"] = auth.failure_mode_allow + if "failure_mode_allow" in auth: + auth_info["typed_config"]["failure_mode_allow"] = auth.failure_mode_allow - if 'status_on_error' in auth: - status_on_error: Optional[Dict[str, int]] = auth.get('status_on_error') - auth_info['typed_config']["status_on_error"] = status_on_error + if "status_on_error" in auth: + status_on_error: Optional[Dict[str, int]] = auth.get("status_on_error") + auth_info["typed_config"]["status_on_error"] = status_on_error return auth_info - # Careful: this function returns None to indicate that no Envoy response_map # filter needs to be instantiated, because either no Module nor Mapping # has error_response_overrides, or the ones that exist are not valid. @@ -340,7 +340,7 @@ def V3HTTPFilter_authv1(auth: IRAuth, v3config: 'V3Config'): # By not instantiating the filter in those cases, we prevent adding a useless # filter onto the chain. @V3HTTPFilter.register -def V3HTTPFilter_error_response(error_response: IRErrorResponse, v3config: 'V3Config'): +def V3HTTPFilter_error_response(error_response: IRErrorResponse, v3config: "V3Config"): # Error response configuration can come from the Ambassador module, on a # a Mapping, or both. We need to use the response_map filter if either one # of these sources defines error responses. First, check if any route @@ -348,35 +348,37 @@ def V3HTTPFilter_error_response(error_response: IRErrorResponse, v3config: 'V3Co # defined error responses. route_has_error_responses = False for route in v3config.routes: - typed_per_filter_config = route.get('typed_per_filter_config', {}) - if 'envoy.filters.http.response_map' in typed_per_filter_config: + typed_per_filter_config = route.get("typed_per_filter_config", {}) + if "envoy.filters.http.response_map" in typed_per_filter_config: route_has_error_responses = True break filter_config: Dict[str, Any] = { # The IRErrorResponse filter builds on the 'envoy.filters.http.response_map' filter. - 'name': 'envoy.filters.http.response_map' + "name": "envoy.filters.http.response_map" } module_config = error_response.config() if module_config: # Mappers are required, otherwise this the response map has nothing to do. We really # shouldn't have a config with nothing in it, but we defend against this case anyway. - if 'mappers' not in module_config or len(module_config['mappers']) == 0: - error_response.post_error('ErrorResponse Module config has no mappers, cannot configure.') + if "mappers" not in module_config or len(module_config["mappers"]) == 0: + error_response.post_error( + "ErrorResponse Module config has no mappers, cannot configure." + ) return None # If there's module config for error responses, create config for that here. # If not, there must be some Mapping config for it, so we'll just return # a filter with no global config and let the Mapping's per-route config # take action instead. - filter_config['typed_config'] = { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.response_map.v3.ResponseMap', + filter_config["typed_config"] = { + "@type": "type.googleapis.com/envoy.extensions.filters.http.response_map.v3.ResponseMap", # The response map filter supports an array of mappers for matching as well # as default actions to take if there are no overrides on a mapper. We do # not take advantage of any default actions, and instead ensure that all of # the mappers we generate contain some action (eg: body_format_override). - 'mappers': module_config['mappers'] + "mappers": module_config["mappers"], } return filter_config elif route_has_error_responses: @@ -392,27 +394,27 @@ def V3HTTPFilter_error_response(error_response: IRErrorResponse, v3config: 'V3Co @V3HTTPFilter.register -def V3HTTPFilter_ratelimit(ratelimit: IRRateLimit, v3config: 'V3Config'): +def V3HTTPFilter_ratelimit(ratelimit: IRRateLimit, v3config: "V3Config"): config = dict(ratelimit.config) - if 'timeout_ms' in config: - tm_ms = config.pop('timeout_ms') + if "timeout_ms" in config: + tm_ms = config.pop("timeout_ms") - config['timeout'] = "%0.3fs" % (float(tm_ms) / 1000.0) + config["timeout"] = "%0.3fs" % (float(tm_ms) / 1000.0) # If here, we must have a ratelimit service configured. assert v3config.ratelimit - config['rate_limit_service'] = dict(v3config.ratelimit) - config['@type'] = 'type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit' + config["rate_limit_service"] = dict(v3config.ratelimit) + config["@type"] = "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit" return { - 'name': 'envoy.filters.http.ratelimit', - 'typed_config': config, + "name": "envoy.filters.http.ratelimit", + "typed_config": config, } @V3HTTPFilter.register -def V3HTTPFilter_ipallowdeny(irfilter: IRIPAllowDeny, v3config: 'V3Config'): +def V3HTTPFilter_ipallowdeny(irfilter: IRIPAllowDeny, v3config: "V3Config"): del v3config # silence unused-variable warning # Go ahead and convert the irfilter to its dictionary form; it's @@ -436,11 +438,7 @@ def V3HTTPFilter_ipallowdeny(irfilter: IRIPAllowDeny, v3config: 'V3Config'): principals = fdict["principals"][0] else: # Multiple principals, so we have to set up an or_ids set. - principals = { - "or_ids": { - "ids": fdict["principals"] - } - } + principals = {"or_ids": {"ids": fdict["principals"]}} return { "name": "envoy.filters.http.rbac", @@ -450,60 +448,56 @@ def V3HTTPFilter_ipallowdeny(irfilter: IRIPAllowDeny, v3config: 'V3Config'): "action": irfilter.action.upper(), "policies": { f"ambassador-ip-{irfilter.action.lower()}": { - "permissions": [ - { - "any": True - } - ], - "principals": [ principals ] + "permissions": [{"any": True}], + "principals": [principals], } - } - } - } + }, + }, + }, } -def V3HTTPFilter_cors(cors: IRFilter, v3config: 'V3Config'): - del cors # silence unused-variable warning +def V3HTTPFilter_cors(cors: IRFilter, v3config: "V3Config"): + del cors # silence unused-variable warning del v3config # silence unused-variable warning - return { 'name': 'envoy.filters.http.cors' } + return {"name": "envoy.filters.http.cors"} -def V3HTTPFilter_router(router: IRFilter, v3config: 'V3Config'): +def V3HTTPFilter_router(router: IRFilter, v3config: "V3Config"): del v3config # silence unused-variable warning - od: Dict[str, Any] = { 'name': 'envoy.filters.http.router' } + od: Dict[str, Any] = {"name": "envoy.filters.http.router"} # Use this config base if we actually need to set config fields below. We don't set # this on `od` by default because it would be an error to end up returning a typed # config that has no real config fields, only a type. typed_config_base = { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.router.v3.Router', + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router", } if router.ir.tracing: - typed_config = od.setdefault('typed_config', typed_config_base) - typed_config['start_child_span'] = True + typed_config = od.setdefault("typed_config", typed_config_base) + typed_config["start_child_span"] = True - if parse_bool(router.ir.ambassador_module.get('suppress_envoy_headers', 'false')): - typed_config = od.setdefault('typed_config', typed_config_base) - typed_config['suppress_envoy_headers'] = True + if parse_bool(router.ir.ambassador_module.get("suppress_envoy_headers", "false")): + typed_config = od.setdefault("typed_config", typed_config_base) + typed_config["suppress_envoy_headers"] = True return od -def V3HTTPFilter_lua(irfilter: IRFilter, v3config: 'V3Config'): +def V3HTTPFilter_lua(irfilter: IRFilter, v3config: "V3Config"): del v3config # silence unused-variable warning config_dict = irfilter.config_dict() config: Dict[str, Any] - config = { - 'name': 'envoy.filters.http.lua' - } + config = {"name": "envoy.filters.http.lua"} if config_dict: - config['typed_config'] = config_dict - config['typed_config']['@type'] = 'type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua' + config["typed_config"] = config_dict + config["typed_config"][ + "@type" + ] = "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" return config diff --git a/python/ambassador/envoy/v3/v3listener.py b/python/ambassador/envoy/v3/v3listener.py index 34586de3e1..2382c21dfa 100644 --- a/python/ambassador/envoy/v3/v3listener.py +++ b/python/ambassador/envoy/v3/v3listener.py @@ -29,9 +29,9 @@ from .v3tls import V3TLSContext if TYPE_CHECKING: - from ...ir.irhost import IRHost # pragma: no cover - from ...ir.irtlscontext import IRTLSContext # pragma: no cover - from . import V3Config # pragma: no cover + from ...ir.irhost import IRHost # pragma: no cover + from ...ir.irtlscontext import IRTLSContext # pragma: no cover + from . import V3Config # pragma: no cover # Model an Envoy filter chain. @@ -52,8 +52,9 @@ # be used. (And yes, that implies that at the moment, you can't mix HTTP Mappings and TCP Mappings # on the same port. Possible near-future feature.) + class V3Chain(dict): - def __init__(self, config: 'V3Config', type: str, host: Optional[IRHost]) -> None: + def __init__(self, config: "V3Config", type: str, host: Optional[IRHost]) -> None: self._config = config self._logger = self._config.ir.logger self._log_debug = self._logger.isEnabledFor(logging.DEBUG) @@ -63,7 +64,7 @@ def __init__(self, config: 'V3Config', type: str, host: Optional[IRHost]) -> Non # We can have multiple hosts here, primarily so that HTTP chains can DTRT -- # but it would be fine to have multiple HTTPS hosts too, as long as they all # share a TLSContext. - self.context: Optional[IRTLSContext]= None + self.context: Optional[IRTLSContext] = None self.hosts: Dict[str, IRHost] = {} # It's OK if an HTTP chain has no Host. @@ -85,8 +86,10 @@ def add_host(self, host: IRHost) -> None: if not self.context: self.context = host.context elif self.context != host.context: - self._config.ir.post_error("Chain context mismatch: Host %s cannot combine with %s" % - (host.name, ", ".join(sorted(self.hosts.keys())))) + self._config.ir.post_error( + "Chain context mismatch: Host %s cannot combine with %s" + % (host.name, ", ".join(sorted(self.hosts.keys()))) + ) def hostglobs(self) -> List[str]: # Get a list of host globs currently set up for this chain. @@ -94,7 +97,9 @@ def hostglobs(self) -> List[str]: def matching_hosts(self, route: V3Route) -> List[IRHost]: # Get a list of _IRHosts_ that the given route should be matched with. - rv: List[IRHost] = [ host for host in self.hosts.values() if host.matches_httpgroup(route._group) ] + rv: List[IRHost] = [ + host for host in self.hosts.values() if host.matches_httpgroup(route._group) + ] return rv @@ -107,8 +112,11 @@ def add_tcpmapping(self, tcpmapping: IRTCPMappingGroup) -> None: def __str__(self) -> str: ctxstr = f" ctx {self.context.name}" if self.context else "" - return "CHAIN: %s%s [ %s ]" % \ - (self.type.upper(), ctxstr, ", ".join(sorted(self.hostglobs()))) + return "CHAIN: %s%s [ %s ]" % ( + self.type.upper(), + ctxstr, + ", ".join(sorted(self.hostglobs())), + ) # Model an Envoy listener. @@ -119,8 +127,9 @@ def __str__(self) -> str: # There is a one-to-one correspondence between an IRListener and an Envoy listener: the logic # here is all about constructing the Envoy configuration implied by the IRListener. + class V3Listener(dict): - def __init__(self, config: 'V3Config', irlistener: IRListener) -> None: + def __init__(self, config: "V3Config", irlistener: IRListener) -> None: super().__init__() self.config = config @@ -130,17 +139,21 @@ def __init__(self, config: 'V3Config', irlistener: IRListener) -> None: self.port = irlistener.port self.bind_to = irlistener.bind_to() - bindstr = f"-{irlistener.socket_protocol.lower()}-{self.bind_address}" if (self.bind_address != "0.0.0.0") else "" + bindstr = ( + f"-{irlistener.socket_protocol.lower()}-{self.bind_address}" + if (self.bind_address != "0.0.0.0") + else "" + ) self.name = irlistener.name or f"ambassador-listener{bindstr}-{self.port}" self.use_proxy_proto = False self.listener_filters: List[dict] = [] self.traffic_direction: str = "UNSPECIFIED" self.per_connection_buffer_limit_bytes: Optional[int] = None - self._irlistener = irlistener # We cache the IRListener to use its match method later + self._irlistener = irlistener # We cache the IRListener to use its match method later self._stats_prefix = irlistener.statsPrefix self._security_model: str = irlistener.securityModel - self._l7_depth: int = irlistener.get('l7Depth', 0) + self._l7_depth: int = irlistener.get("l7Depth", 0) self._insecure_only: bool = False self._filter_chains: List[dict] = [] self._base_http_config: Optional[Dict[str, Any]] = None @@ -152,12 +165,14 @@ def __init__(self, config: 'V3Config', irlistener: IRListener) -> None: # representations) that won't get logged anyway. self._log_debug = self.config.ir.logger.isEnabledFor(logging.DEBUG) if self._log_debug: - self.config.ir.logger.debug(f"V3Listener {self.name} created -- {self._security_model}, l7Depth {self._l7_depth}") + self.config.ir.logger.debug( + f"V3Listener {self.name} created -- {self._security_model}, l7Depth {self._l7_depth}" + ) # If the IRListener is marked insecure-only, so are we. self._insecure_only = irlistener.insecure_only - buffer_limit_bytes = self.config.ir.ambassador_module.get('buffer_limit_bytes', None) + buffer_limit_bytes = self.config.ir.ambassador_module.get("buffer_limit_bytes", None) if buffer_limit_bytes: self.per_connection_buffer_limit_bytes = buffer_limit_bytes @@ -170,9 +185,7 @@ def __init__(self, config: 'V3Config', irlistener: IRListener) -> None: if proto == "PROXY": # The PROXY protocol needs a listener filter. - self.listener_filters.append({ - 'name': 'envoy.filters.listener.proxy_protocol' - }) + self.listener_filters.append({"name": "envoy.filters.listener.proxy_protocol"}) if proto == "TLS": # TLS needs a listener filter _and_ we need to remember that this @@ -183,9 +196,7 @@ def __init__(self, config: 'V3Config', irlistener: IRListener) -> None: ## When UDP we assume it is http/3 listener and configured for quic which has TLS built into the protocol ## therefore, we only need to add this when socket_protocol is TCP if self.isProtocolTCP(): - self.listener_filters.append({ - 'name': 'envoy.filters.listener.tls_inspector' - }) + self.listener_filters.append({"name": "envoy.filters.listener.tls_inspector"}) if proto == "TCP": # TCP doesn't require any specific listener filters, but it @@ -217,8 +228,8 @@ def add_chain(self, chain_type: str, host: Optional[IRHost]) -> V3Chain: # is also why there's no vhost data structure). chain_key = chain_type - hoststr = host.hostname if host else '(no host)' - hostname = (host.hostname if host else None) or '*' + hoststr = host.hostname if host else "(no host)" + hostname = (host.hostname if host else None) or "*" if host: chain_key = "%s-%s" % (chain_type, hostname) @@ -229,15 +240,21 @@ def add_chain(self, chain_type: str, host: Optional[IRHost]) -> V3Chain: if host: chain.add_host(host) if self._log_debug: - self.config.ir.logger.debug(" CHAIN ADD: host %s chain_key %s -- %s", hoststr, chain_key, chain) + self.config.ir.logger.debug( + " CHAIN ADD: host %s chain_key %s -- %s", hoststr, chain_key, chain + ) else: if self._log_debug: - self.config.ir.logger.debug(" CHAIN NOOP: host %s chain_key %s -- %s", hoststr, chain_key, chain) + self.config.ir.logger.debug( + " CHAIN NOOP: host %s chain_key %s -- %s", hoststr, chain_key, chain + ) else: chain = V3Chain(self.config, chain_type, host) self._chains[chain_key] = chain if self._log_debug: - self.config.ir.logger.debug(" CHAIN CREATE: host %s chain_key %s -- %s", hoststr, chain_key, chain) + self.config.ir.logger.debug( + " CHAIN CREATE: host %s chain_key %s -- %s", hoststr, chain_key, chain + ) return chain @@ -246,11 +263,16 @@ def add_tcp_group(self, irgroup: IRTCPMappingGroup) -> None: # mapping group rather than a Host. Same deal applies with TLS: you can't do # host-based matching without it. - group_host = irgroup.get('host', None) + group_host = irgroup.get("host", None) if self._log_debug: - self.config.ir.logger.debug("V3Listener %s on %s: take TCPMappingGroup on %s (%s)", - self.name, self.bind_to, irgroup.bind_to(), group_host or "i'*'") + self.config.ir.logger.debug( + "V3Listener %s on %s: take TCPMappingGroup on %s (%s)", + self.name, + self.bind_to, + irgroup.bind_to(), + group_host or "i'*'", + ) if not group_host: # Special case. No Host in a TCPMapping means an unconditional forward, @@ -265,13 +287,23 @@ def add_tcp_group(self, irgroup: IRTCPMappingGroup) -> None: if not host.context: if self._log_debug: - self.config.ir.logger.debug("V3Listener %s @ %s TCP %s: skip %s", - self.name, self.bind_to, group_host, host) + self.config.ir.logger.debug( + "V3Listener %s @ %s TCP %s: skip %s", + self.name, + self.bind_to, + group_host, + host, + ) continue if self._log_debug: - self.config.ir.logger.debug("V3Listener %s @ %s TCP %s: consider %s", - self.name, self.bind_to, group_host, host) + self.config.ir.logger.debug( + "V3Listener %s @ %s TCP %s: consider %s", + self.name, + self.bind_to, + group_host, + host, + ) if hostglob_matches(host.hostname, group_host): chain = self.add_chain("tcp", host) @@ -282,100 +314,102 @@ def access_log(self) -> List[dict]: access_log: List[dict] = [] for al in self.config.ir.log_services.values(): - access_log_obj: Dict[str, Any] = { "common_config": al.get_common_config() } + access_log_obj: Dict[str, Any] = {"common_config": al.get_common_config()} req_headers = [] resp_headers = [] trailer_headers = [] for additional_header in al.get_additional_headers(): - if additional_header.get('during_request', True): - req_headers.append(additional_header.get('header_name')) - if additional_header.get('during_response', True): - resp_headers.append(additional_header.get('header_name')) - if additional_header.get('during_trailer', True): - trailer_headers.append(additional_header.get('header_name')) - - if al.driver == 'http': - access_log_obj['additional_request_headers_to_log'] = req_headers - access_log_obj['additional_response_headers_to_log'] = resp_headers - access_log_obj['additional_response_trailers_to_log'] = trailer_headers - access_log_obj['@type'] = 'type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig' - access_log.append({ - "name": "envoy.access_loggers.http_grpc", - "typed_config": access_log_obj - }) + if additional_header.get("during_request", True): + req_headers.append(additional_header.get("header_name")) + if additional_header.get("during_response", True): + resp_headers.append(additional_header.get("header_name")) + if additional_header.get("during_trailer", True): + trailer_headers.append(additional_header.get("header_name")) + + if al.driver == "http": + access_log_obj["additional_request_headers_to_log"] = req_headers + access_log_obj["additional_response_headers_to_log"] = resp_headers + access_log_obj["additional_response_trailers_to_log"] = trailer_headers + access_log_obj[ + "@type" + ] = "type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig" + access_log.append( + {"name": "envoy.access_loggers.http_grpc", "typed_config": access_log_obj} + ) else: # inherently TCP right now # tcp loggers do not support additional headers - access_log_obj['@type'] = 'type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.TcpGrpcAccessLogConfig' - access_log.append({ - "name": "envoy.access_loggers.tcp_grpc", - "typed_config": access_log_obj - }) + access_log_obj[ + "@type" + ] = "type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.TcpGrpcAccessLogConfig" + access_log.append( + {"name": "envoy.access_loggers.tcp_grpc", "typed_config": access_log_obj} + ) # Use sane access log spec in JSON if self.config.ir.ambassador_module.envoy_log_type.lower() == "json": - log_format = self.config.ir.ambassador_module.get('envoy_log_format', None) + log_format = self.config.ir.ambassador_module.get("envoy_log_format", None) if log_format is None: log_format = { - 'start_time': '%START_TIME%', - 'method': '%REQ(:METHOD)%', - 'path': '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%', - 'protocol': '%PROTOCOL%', - 'response_code': '%RESPONSE_CODE%', - 'response_flags': '%RESPONSE_FLAGS%', - 'bytes_received': '%BYTES_RECEIVED%', - 'bytes_sent': '%BYTES_SENT%', - 'duration': '%DURATION%', - 'upstream_service_time': '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%', - 'x_forwarded_for': '%REQ(X-FORWARDED-FOR)%', - 'user_agent': '%REQ(USER-AGENT)%', - 'request_id': '%REQ(X-REQUEST-ID)%', - 'authority': '%REQ(:AUTHORITY)%', - 'upstream_host': '%UPSTREAM_HOST%', - 'upstream_cluster': '%UPSTREAM_CLUSTER%', - 'upstream_local_address': '%UPSTREAM_LOCAL_ADDRESS%', - 'downstream_local_address': '%DOWNSTREAM_LOCAL_ADDRESS%', - 'downstream_remote_address': '%DOWNSTREAM_REMOTE_ADDRESS%', - 'requested_server_name': '%REQUESTED_SERVER_NAME%', - 'istio_policy_status': '%DYNAMIC_METADATA(istio.mixer:status)%', - 'upstream_transport_failure_reason': '%UPSTREAM_TRANSPORT_FAILURE_REASON%' + "start_time": "%START_TIME%", + "method": "%REQ(:METHOD)%", + "path": "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%", + "protocol": "%PROTOCOL%", + "response_code": "%RESPONSE_CODE%", + "response_flags": "%RESPONSE_FLAGS%", + "bytes_received": "%BYTES_RECEIVED%", + "bytes_sent": "%BYTES_SENT%", + "duration": "%DURATION%", + "upstream_service_time": "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%", + "x_forwarded_for": "%REQ(X-FORWARDED-FOR)%", + "user_agent": "%REQ(USER-AGENT)%", + "request_id": "%REQ(X-REQUEST-ID)%", + "authority": "%REQ(:AUTHORITY)%", + "upstream_host": "%UPSTREAM_HOST%", + "upstream_cluster": "%UPSTREAM_CLUSTER%", + "upstream_local_address": "%UPSTREAM_LOCAL_ADDRESS%", + "downstream_local_address": "%DOWNSTREAM_LOCAL_ADDRESS%", + "downstream_remote_address": "%DOWNSTREAM_REMOTE_ADDRESS%", + "requested_server_name": "%REQUESTED_SERVER_NAME%", + "istio_policy_status": "%DYNAMIC_METADATA(istio.mixer:status)%", + "upstream_transport_failure_reason": "%UPSTREAM_TRANSPORT_FAILURE_REASON%", } tracing_config = self.config.ir.tracing - if tracing_config and tracing_config.driver == 'envoy.tracers.datadog': - log_format['dd.trace_id'] = '%REQ(X-DATADOG-TRACE-ID)%' - log_format['dd.span_id'] = '%REQ(X-DATADOG-PARENT-ID)%' - - access_log.append({ - 'name': 'envoy.access_loggers.file', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog', - 'path': self.config.ir.ambassador_module.envoy_log_path, - 'json_format': log_format + if tracing_config and tracing_config.driver == "envoy.tracers.datadog": + log_format["dd.trace_id"] = "%REQ(X-DATADOG-TRACE-ID)%" + log_format["dd.span_id"] = "%REQ(X-DATADOG-PARENT-ID)%" + + access_log.append( + { + "name": "envoy.access_loggers.file", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog", + "path": self.config.ir.ambassador_module.envoy_log_path, + "json_format": log_format, + }, } - }) + ) else: # Use a sane access log spec - log_format = self.config.ir.ambassador_module.get('envoy_log_format', None) + log_format = self.config.ir.ambassador_module.get("envoy_log_format", None) if not log_format: - log_format = 'ACCESS [%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"' + log_format = 'ACCESS [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"' if self._log_debug: self.config.ir.logger.debug("V3Listener: Using log_format '%s'" % log_format) - access_log.append({ - 'name': 'envoy.access_loggers.file', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog', - 'path': self.config.ir.ambassador_module.envoy_log_path, - 'log_format': { - 'text_format_source': { - 'inline_string': log_format + '\n' - } - } + access_log.append( + { + "name": "envoy.access_loggers.file", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog", + "path": self.config.ir.ambassador_module.envoy_log_path, + "log_format": {"text_format_source": {"inline_string": log_format + "\n"}}, + }, } - }) + ) return access_log @@ -383,20 +417,21 @@ def access_log(self) -> List[dict]: # V3Listener's http_connection_manager filter. def base_http_config(self) -> Dict[str, Any]: base_http_config: Dict[str, Any] = { - 'stat_prefix': self._stats_prefix, - 'access_log': self.access_log(), - 'http_filters': [], - 'normalize_path': True + "stat_prefix": self._stats_prefix, + "access_log": self.access_log(), + "http_filters": [], + "normalize_path": True, } # Instructs the HTTP Connection Mananger to support http/3. This is required for both TCP and UDP Listeners if self.http3_enabled: - base_http_config['http3_protocol_options'] = {} + base_http_config["http3_protocol_options"] = {} if self.isProtocolUDP(): - base_http_config['codec_type'] = "HTTP3" + base_http_config["codec_type"] = "HTTP3" # Assemble base HTTP filters from .v3httpfilter import V3HTTPFilter + for f in self.config.ir.filters: v3hf: dict = V3HTTPFilter(f, self.config) @@ -406,51 +441,73 @@ def base_http_config(self) -> Dict[str, Any]: # v3config is generated before deciding if it needs to be # instantiated. See IRErrorResponse for an example. if v3hf: - base_http_config['http_filters'].append(v3hf) + base_http_config["http_filters"].append(v3hf) - if 'use_remote_address' in self.config.ir.ambassador_module: - base_http_config["use_remote_address"] = self.config.ir.ambassador_module.use_remote_address + if "use_remote_address" in self.config.ir.ambassador_module: + base_http_config[ + "use_remote_address" + ] = self.config.ir.ambassador_module.use_remote_address if self._l7_depth > 0: base_http_config["xff_num_trusted_hops"] = self._l7_depth - if 'server_name' in self.config.ir.ambassador_module: + if "server_name" in self.config.ir.ambassador_module: base_http_config["server_name"] = self.config.ir.ambassador_module.server_name - listener_idle_timeout_ms = self.config.ir.ambassador_module.get('listener_idle_timeout_ms', None) + listener_idle_timeout_ms = self.config.ir.ambassador_module.get( + "listener_idle_timeout_ms", None + ) if listener_idle_timeout_ms: - if 'common_http_protocol_options' in base_http_config: - base_http_config["common_http_protocol_options"]["idle_timeout"] = "%0.3fs" % (float(listener_idle_timeout_ms) / 1000.0) + if "common_http_protocol_options" in base_http_config: + base_http_config["common_http_protocol_options"]["idle_timeout"] = "%0.3fs" % ( + float(listener_idle_timeout_ms) / 1000.0 + ) else: - base_http_config["common_http_protocol_options"] = { 'idle_timeout': "%0.3fs" % (float(listener_idle_timeout_ms) / 1000.0) } + base_http_config["common_http_protocol_options"] = { + "idle_timeout": "%0.3fs" % (float(listener_idle_timeout_ms) / 1000.0) + } - if 'headers_with_underscores_action' in self.config.ir.ambassador_module: - if 'common_http_protocol_options' in base_http_config: - base_http_config["common_http_protocol_options"]["headers_with_underscores_action"] = self.config.ir.ambassador_module.headers_with_underscores_action + if "headers_with_underscores_action" in self.config.ir.ambassador_module: + if "common_http_protocol_options" in base_http_config: + base_http_config["common_http_protocol_options"][ + "headers_with_underscores_action" + ] = self.config.ir.ambassador_module.headers_with_underscores_action else: - base_http_config["common_http_protocol_options"] = { 'headers_with_underscores_action': self.config.ir.ambassador_module.headers_with_underscores_action } + base_http_config["common_http_protocol_options"] = { + "headers_with_underscores_action": self.config.ir.ambassador_module.headers_with_underscores_action + } - max_request_headers_kb = self.config.ir.ambassador_module.get('max_request_headers_kb', None) + max_request_headers_kb = self.config.ir.ambassador_module.get( + "max_request_headers_kb", None + ) if max_request_headers_kb: base_http_config["max_request_headers_kb"] = max_request_headers_kb - if 'enable_http10' in self.config.ir.ambassador_module: + if "enable_http10" in self.config.ir.ambassador_module: http_options = base_http_config.setdefault("http_protocol_options", {}) - http_options['accept_http_10'] = self.config.ir.ambassador_module.enable_http10 + http_options["accept_http_10"] = self.config.ir.ambassador_module.enable_http10 - if 'allow_chunked_length' in self.config.ir.ambassador_module: + if "allow_chunked_length" in self.config.ir.ambassador_module: if self.config.ir.ambassador_module.allow_chunked_length != None: http_options = base_http_config.setdefault("http_protocol_options", {}) - http_options['allow_chunked_length'] = self.config.ir.ambassador_module.allow_chunked_length + http_options[ + "allow_chunked_length" + ] = self.config.ir.ambassador_module.allow_chunked_length - if 'preserve_external_request_id' in self.config.ir.ambassador_module: - base_http_config["preserve_external_request_id"] = self.config.ir.ambassador_module.preserve_external_request_id + if "preserve_external_request_id" in self.config.ir.ambassador_module: + base_http_config[ + "preserve_external_request_id" + ] = self.config.ir.ambassador_module.preserve_external_request_id - if 'forward_client_cert_details' in self.config.ir.ambassador_module: - base_http_config["forward_client_cert_details"] = self.config.ir.ambassador_module.forward_client_cert_details + if "forward_client_cert_details" in self.config.ir.ambassador_module: + base_http_config[ + "forward_client_cert_details" + ] = self.config.ir.ambassador_module.forward_client_cert_details - if 'set_current_client_cert_details' in self.config.ir.ambassador_module: - base_http_config["set_current_client_cert_details"] = self.config.ir.ambassador_module.set_current_client_cert_details + if "set_current_client_cert_details" in self.config.ir.ambassador_module: + base_http_config[ + "set_current_client_cert_details" + ] = self.config.ir.ambassador_module.set_current_client_cert_details if self.config.ir.tracing: base_http_config["generate_request_id"] = True @@ -458,7 +515,7 @@ def base_http_config(self) -> Dict[str, Any]: base_http_config["tracing"] = {} self.traffic_direction = "OUTBOUND" - req_hdrs = self.config.ir.tracing.get('tag_headers', []) + req_hdrs = self.config.ir.tracing.get("tag_headers", []) if req_hdrs: base_http_config["tracing"]["custom_tags"] = [] @@ -466,48 +523,44 @@ def base_http_config(self) -> Dict[str, Any]: custom_tag = { "request_header": { "name": hdr, - }, + }, "tag": hdr, } base_http_config["tracing"]["custom_tags"].append(custom_tag) - - sampling = self.config.ir.tracing.get('sampling', {}) + sampling = self.config.ir.tracing.get("sampling", {}) if sampling: - client_sampling = sampling.get('client', None) + client_sampling = sampling.get("client", None) if client_sampling is not None: - base_http_config["tracing"]["client_sampling"] = { - "value": client_sampling - } + base_http_config["tracing"]["client_sampling"] = {"value": client_sampling} - random_sampling = sampling.get('random', None) + random_sampling = sampling.get("random", None) if random_sampling is not None: - base_http_config["tracing"]["random_sampling"] = { - "value": random_sampling - } + base_http_config["tracing"]["random_sampling"] = {"value": random_sampling} - overall_sampling = sampling.get('overall', None) + overall_sampling = sampling.get("overall", None) if overall_sampling is not None: - base_http_config["tracing"]["overall_sampling"] = { - "value": overall_sampling - } + base_http_config["tracing"]["overall_sampling"] = {"value": overall_sampling} - proper_case: bool = self.config.ir.ambassador_module['proper_case'] + proper_case: bool = self.config.ir.ambassador_module["proper_case"] # Get the list of downstream headers whose casing should be overriden # from the Ambassador module. We configure the upstream side of this # in v3cluster.py - header_case_overrides = self.config.ir.ambassador_module.get('header_case_overrides', None) + header_case_overrides = self.config.ir.ambassador_module.get("header_case_overrides", None) if header_case_overrides: if proper_case: self.config.ir.post_error( - "Only one of 'proper_case' or 'header_case_overrides' fields may be set on " +\ - "the Ambassador module. Honoring proper_case and ignoring " +\ - "header_case_overrides.") + "Only one of 'proper_case' or 'header_case_overrides' fields may be set on " + + "the Ambassador module. Honoring proper_case and ignoring " + + "header_case_overrides." + ) header_case_overrides = None if not isinstance(header_case_overrides, list): # The header_case_overrides field must be an array. - self.config.ir.post_error("Ambassador module config 'header_case_overrides' must be an array") + self.config.ir.post_error( + "Ambassador module config 'header_case_overrides' must be an array" + ) header_case_overrides = None elif len(header_case_overrides) == 0: # Allow an empty list to mean "do nothing". @@ -520,12 +573,16 @@ def base_http_config(self) -> Dict[str, Any]: rules = [] for hdr in header_case_overrides: if not isinstance(hdr, str): - self.config.ir.post_error("Skipping non-string header in 'header_case_overrides': {hdr}") + self.config.ir.post_error( + "Skipping non-string header in 'header_case_overrides': {hdr}" + ) continue rules.append(hdr) if len(rules) == 0: - self.config.ir.post_error(f"Could not parse any valid string headers in 'header_case_overrides': {header_case_overrides}") + self.config.ir.post_error( + f"Could not parse any valid string headers in 'header_case_overrides': {header_case_overrides}" + ) else: # Create custom header rules that map the lowercase version of every element in # `header_case_overrides` to the the respective original casing. @@ -535,18 +592,16 @@ def base_http_config(self) -> Dict[str, Any]: # overrides the response header case by remapping the lowercased version (the default # casing in envoy) back to the casing provided in the config. custom_header_rules: Dict[str, Dict[str, dict]] = { - 'custom': { - 'rules': { - header.lower() : header for header in rules - } - } + "custom": {"rules": {header.lower(): header for header in rules}} } http_options = base_http_config.setdefault("http_protocol_options", {}) http_options["header_key_format"] = custom_header_rules if proper_case: - proper_case_header: Dict[str, Dict[str, dict]] = {'header_key_format': {'proper_case_words': {}}} - if 'http_protocol_options' in base_http_config: + proper_case_header: Dict[str, Dict[str, dict]] = { + "header_key_format": {"proper_case_words": {}} + } + if "http_protocol_options" in base_http_config: base_http_config["http_protocol_options"].update(proper_case_header) else: base_http_config["http_protocol_options"] = proper_case_header @@ -562,7 +617,7 @@ def finalize(self) -> None: "socket_address": { "address": self.bind_address, "port_value": self.port, - "protocol": self.socket_protocol ## "TCP" or "UDP" + "protocol": self.socket_protocol, ## "TCP" or "UDP" } } @@ -589,29 +644,23 @@ def finalize_tcp(self) -> None: for irgroup in chain.tcpmappings: # First up, which clusters do we need to talk to? - clusters = [{ - 'name': mapping.cluster.envoy_name, - 'weight': mapping._weight - } for mapping in irgroup.mappings] + clusters = [ + {"name": mapping.cluster.envoy_name, "weight": mapping._weight} + for mapping in irgroup.mappings + ] # From that, we can sort out a basic tcp_proxy filter config. tcp_filter = { - 'name': 'envoy.filters.network.tcp_proxy', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy', - 'stat_prefix': self._stats_prefix, - 'weighted_clusters': { - 'clusters': clusters - } - } + "name": "envoy.filters.network.tcp_proxy", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "stat_prefix": self._stats_prefix, + "weighted_clusters": {"clusters": clusters}, + }, } # OK. Basic filter chain entry next. - filter_chain: Dict[str, Any] = { - 'filters': [ - tcp_filter - ] - } + filter_chain: Dict[str, Any] = {"filters": [tcp_filter]} # The chain as a whole has a single matcher. filter_chain_match: Dict[str, Any] = {} @@ -627,12 +676,12 @@ def finalize_tcp(self) -> None: # filter_chain_match. envoy_ctx = V3TLSContext(chain.context) - filter_chain['transport_socket'] = { - 'name': 'envoy.transport_sockets.tls', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext', - **envoy_ctx - } + filter_chain["transport_socket"] = { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + **envoy_ctx, + }, } # We do server-name matching whether or not we have TLS, just to help @@ -640,10 +689,10 @@ def finalize_tcp(self) -> None: # criterion (since Envoy will reject such a configuration). if len(chain_hosts) > 0: - filter_chain_match['server_names'] = chain_hosts + filter_chain_match["server_names"] = chain_hosts # Once all of that is done, hook in the match... - filter_chain['filter_chain_match'] = filter_chain_match + filter_chain["filter_chain_match"] = filter_chain_match # ...and stick this chain into our filter. self._filter_chains.append(filter_chain) @@ -676,7 +725,9 @@ def compute_chains(self) -> None: if self._insecure_only and (self.port != host.insecure_addl_port): if self._log_debug: - self.config.ir.logger.debug(" drop %s, insecure-only port mismatch", host.name) + self.config.ir.logger.debug( + " drop %s, insecure-only port mismatch", host.name + ) continue @@ -691,7 +742,7 @@ def compute_chains(self) -> None: # Listener can ever produce will be rejected. In any other case, we'll set up an # HTTPS chain for this Host, as long as we think TLS is OK. - will_reject_secure = ((not secure_action) or (secure_action == "Reject")) + will_reject_secure = (not secure_action) or (secure_action == "Reject") if self._tls_ok and (not ((security_model == "SECURE") and will_reject_secure)): if self._log_debug: self.config.ir.logger.debug(" take SECURE %s", host) @@ -735,11 +786,15 @@ def compute_routes(self) -> None: matching_hosts = chain.matching_hosts(rv.route) if self._log_debug: - logger.debug(" = matching_hosts %s", ", ".join([ h.hostname for h in matching_hosts ])) + logger.debug( + " = matching_hosts %s", ", ".join([h.hostname for h in matching_hosts]) + ) if not matching_hosts: if self._log_debug: - logger.debug(f" drop outright: no hosts match {sorted(rv.route['_host_constraints'])}") + logger.debug( + f" drop outright: no hosts match {sorted(rv.route['_host_constraints'])}" + ) continue for host in matching_hosts: @@ -754,20 +809,20 @@ def compute_routes(self) -> None: if (host.secure_action is not None) and (self._security_model != "INSECURE"): # We have a secure action, and we're willing to believe that at least some of # our requests will be secure. - matcher = 'always' if (self._security_model == 'SECURE') else 'xfp-https' + matcher = "always" if (self._security_model == "SECURE") else "xfp-https" - candidates.append(( host, matcher, 'Route', rv )) + candidates.append((host, matcher, "Route", rv)) if (host.insecure_action is not None) and (self._security_model != "SECURE"): # We have an insecure action, and we're willing to believe that at least some of # our requests will be insecure. - matcher = 'always' if (self._security_model == 'INSECURE') else 'xfp-http' + matcher = "always" if (self._security_model == "INSECURE") else "xfp-http" action = host.insecure_action - candidates.append(( host, matcher, action, rv )) + candidates.append((host, matcher, action, rv)) for host, matcher, action, rv in candidates: - route_precedence = rv.route.get('_precedence', None) + route_precedence = rv.route.get("_precedence", None) extra_info = "" if rv.route["match"].get("prefix", None) == "/.well-known/acme-challenge/": @@ -776,28 +831,44 @@ def compute_routes(self) -> None: extra_info = " (force Route for ACME challenge)" action = "Route" found_acme = True - elif (self.config.ir.edge_stack_allowed and - (route_precedence == -1000000) and - (rv.route["match"].get("safe_regex", {}).get("regex", None) == "^/$")): + elif ( + self.config.ir.edge_stack_allowed + and (route_precedence == -1000000) + and ( + rv.route["match"].get("safe_regex", {}).get("regex", None) == "^/$" + ) + ): extra_info = " (force Route for fallback Mapping)" action = "Route" - if action != 'Reject': + if action != "Reject": # Worth noting here that "Route" really means "do what the V3Route really # says", which might be a host redirect. When we talk about "Redirect", we # really mean "redirect to HTTPS" specifically. if self._log_debug: - logger.debug(" %s - %s: accept on %s %s%s", - matcher, action, self.name, hostname, extra_info) + logger.debug( + " %s - %s: accept on %s %s%s", + matcher, + action, + self.name, + hostname, + extra_info, + ) variant = dict(rv.get_variant(matcher, action.lower())) - variant["_host_constraints"] = set([ hostname ]) + variant["_host_constraints"] = set([hostname]) chain.add_route(variant) else: if self._log_debug: - logger.debug(" %s - %s: drop from %s %s%s", - matcher, action, self.name, hostname, extra_info) + logger.debug( + " %s - %s: drop from %s %s%s", + matcher, + action, + self.name, + hostname, + extra_info, + ) # If we're on Edge Stack and we don't already have an ACME route, add one. if self.config.ir.edge_stack_allowed and not found_acme: @@ -807,7 +878,9 @@ def compute_routes(self) -> None: if not self.config.ir.sidecar_cluster_name: # Uh whut? how is Edge Stack running exactly? - raise Exception("Edge Stack claims to be running, but we have no sidecar cluster??") + raise Exception( + "Edge Stack claims to be running, but we have no sidecar cluster??" + ) if self._log_debug: logger.debug(" punching a hole for ACME") @@ -816,18 +889,18 @@ def compute_routes(self) -> None: # # XXX This is needed only because we're dictifying the V3Route too early. - chain.routes.insert(0, { - "_host_constraints": set(), - "match": { - "case_sensitive": True, - "prefix": "/.well-known/acme-challenge/" + chain.routes.insert( + 0, + { + "_host_constraints": set(), + "match": {"case_sensitive": True, "prefix": "/.well-known/acme-challenge/"}, + "route": { + "cluster": self.config.ir.sidecar_cluster_name, + "prefix_rewrite": "/.well-known/acme-challenge/", + "timeout": "3.000s", + }, }, - "route": { - "cluster": self.config.ir.sidecar_cluster_name, - "prefix_rewrite": "/.well-known/acme-challenge/", - "timeout": "3.000s" - } - }) + ) if self._log_debug: for route in chain.routes: @@ -864,23 +937,24 @@ def finalize_http(self) -> None: if not filter_chain: if self._log_debug: - self._irlistener.logger.debug("FHTTP create filter_chain %s / empty match", chain_key) - filter_chain = { - "filter_chain_match": {}, - "_vhosts": {} - } + self._irlistener.logger.debug( + "FHTTP create filter_chain %s / empty match", chain_key + ) + filter_chain = {"filter_chain_match": {}, "_vhosts": {}} filter_chains[chain_key] = filter_chain else: if self._log_debug: - self._irlistener.logger.debug("FHTTP use filter_chain %s: vhosts %d", chain_key, len(filter_chain["_vhosts"])) + self._irlistener.logger.debug( + "FHTTP use filter_chain %s: vhosts %d", + chain_key, + len(filter_chain["_vhosts"]), + ) elif chain.type == "https": # Since chain_key is a dictionary key in its own right, we can't already # have a matching chain for this. - filter_chain = { - "_vhosts": {} - } + filter_chain = {"_vhosts": {}} filter_chain_match: Dict[str, Any] = {} chain_hosts = chain.hostglobs() @@ -900,10 +974,12 @@ def finalize_http(self) -> None: # at all. if (len(chain_hosts) > 0) and ("*" not in chain_hosts): - filter_chain_match['server_names'] = chain_hosts + filter_chain_match["server_names"] = chain_hosts # Likewise, an HTTPS chain will ask for TLS or QUIC (when udp) - filter_chain_match["transport_protocol"] = "quic" if self.isProtocolUDP() and self.http3_enabled else "tls" + filter_chain_match["transport_protocol"] = ( + "quic" if self.isProtocolUDP() and self.http3_enabled else "tls" + ) if chain.context: # ...uh. How could we not have a context if we're doing TLS? @@ -912,31 +988,31 @@ def finalize_http(self) -> None: envoy_ctx = V3TLSContext(chain.context) envoy_tls_config = { - 'name': 'envoy.transport_sockets.tls', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext', - **envoy_ctx - } + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + **envoy_ctx, + }, } if self.isProtocolUDP(): envoy_tls_config = { - 'name': 'envoy.transport_sockets.quic', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicDownstreamTransport', - 'downstream_tls_context':{ - **envoy_ctx - } - } + "name": "envoy.transport_sockets.quic", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicDownstreamTransport", + "downstream_tls_context": {**envoy_ctx}, + }, } - filter_chain['transport_socket'] = envoy_tls_config + filter_chain["transport_socket"] = envoy_tls_config else: # Envoy doesn't like having a UDP Listener with QUIC filter chain that doesn't have a "transport_socket" set with a TLS # certificate. If the fall-back cert is removed or no certificate is provided then we should not add the quic filter chain if self.isProtocolUDP() and self.http3_enabled: - self._irlistener.logger.warn(f"Listener: quic network protocol requires a TLSContext to be provided, no tls context found for Listener: {self._irlistener.bind_to()}") + self._irlistener.logger.warn( + f"Listener: quic network protocol requires a TLSContext to be provided, no tls context found for Listener: {self._irlistener.bind_to()}" + ) continue # Finally, stash the match in the chain... @@ -956,7 +1032,7 @@ def finalize_http(self) -> None: routes = [] for r in chain.routes: - routes.append({ k: v for k, v in r.items() if k[0] != '_' }) + routes.append({k: v for k, v in r.items() if k[0] != "_"}) # Do we - somehow - already have a vhost for this hostname? (This should # be "impossible".) @@ -967,8 +1043,8 @@ def finalize_http(self) -> None: vhost = { "name": f"{self.name}-{host.hostname}", "response_headers_to_add": [], - "domains": [ host.hostname ], - "routes": [] + "domains": [host.hostname], + "routes": [], } if self.http3_enabled and (self.socket_protocol == "TCP"): @@ -978,11 +1054,14 @@ def finalize_http(self) -> None: # Additional reading on alt-svc header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Alt-Svc # # The default sets the max-age in seconds to be 1 day and supports clients that speak h3 & h3-29 specifications - alt_svc_hdr = { "key": "alt-svc", "value": f'h3=":443"; ma=86400, h3-29=":443"; ma=86400'} + alt_svc_hdr = { + "key": "alt-svc", + "value": f'h3=":443"; ma=86400, h3-29=":443"; ma=86400', + } - vhost['response_headers_to_add'].append({ "header": alt_svc_hdr}) + vhost["response_headers_to_add"].append({"header": alt_svc_hdr}) else: - del(vhost['response_headers_to_add']) + del vhost["response_headers_to_add"] filter_chain["_vhosts"][host.hostname] = vhost @@ -994,21 +1073,25 @@ def finalize_http(self) -> None: http_config = dict(typecast(dict, self._base_http_config)) # ...and unfold our vhosts dict into a list for Envoy. - http_config["route_config"] = { - "virtual_hosts": list(filter_chain["_vhosts"].values()) - } + http_config["route_config"] = {"virtual_hosts": list(filter_chain["_vhosts"].values())} # Now that we've saved our vhosts as a list, drop the dict version. - del(filter_chain["_vhosts"]) + del filter_chain["_vhosts"] # Finish up config for this filter chain... - if parse_bool(self.config.ir.ambassador_module.get("strip_matching_host_port", "false")): + if parse_bool( + self.config.ir.ambassador_module.get("strip_matching_host_port", "false") + ): http_config["strip_matching_host_port"] = True if parse_bool(self.config.ir.ambassador_module.get("merge_slashes", "false")): http_config["merge_slashes"] = True - if parse_bool(self.config.ir.ambassador_module.get("reject_requests_with_escaped_slashes", "false")): + if parse_bool( + self.config.ir.ambassador_module.get( + "reject_requests_with_escaped_slashes", "false" + ) + ): http_config["path_with_escaped_slashes_action"] = "REJECT_REQUEST" filter_chain["filters"] = [ @@ -1016,8 +1099,8 @@ def finalize_http(self) -> None: "name": "envoy.filters.network.http_connection_manager", "typed_config": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - **http_config - } + **http_config, + }, } ] @@ -1029,19 +1112,19 @@ def as_dict(self) -> dict: "name": self.name, "address": self.address, "filter_chains": self._filter_chains, - "traffic_direction": self.traffic_direction + "traffic_direction": self.traffic_direction, } if self.isProtocolUDP(): - listener['udp_listener_config'] = { - 'quic_options': {}, - 'downstream_socket_config': { 'prefer_gro': True } + listener["udp_listener_config"] = { + "quic_options": {}, + "downstream_socket_config": {"prefer_gro": True}, } # We only want to add the buffer limit setting to the listener if specified in the module. # Otherwise, we want to leave it unset and allow Envoys Default 1MiB setting. if self.per_connection_buffer_limit_bytes: - listener['per_connection_buffer_limit_bytes'] = self.per_connection_buffer_limit_bytes + listener["per_connection_buffer_limit_bytes"] = self.per_connection_buffer_limit_bytes if self.listener_filters: listener["listener_filters"] = self.listener_filters @@ -1059,19 +1142,22 @@ def pretty(self) -> dict: def __str__(self) -> str: return "" % ( "HTTP" if self._base_http_config else "TCP", - self.name, self.bind_address, self.port, self._security_model + self.name, + self.bind_address, + self.port, + self._security_model, ) def isProtocolTCP(self) -> bool: """Whether the listener is configured to use the TCP protocol or not?""" - return (self.socket_protocol == "TCP") + return self.socket_protocol == "TCP" def isProtocolUDP(self) -> bool: """Whether the listener is configured to use the UDP protocol or not?""" - return (self.socket_protocol == "UDP") + return self.socket_protocol == "UDP" @classmethod - def generate(cls, config: 'V3Config') -> None: + def generate(cls, config: "V3Config") -> None: config.listeners = [] logger = config.ir.logger diff --git a/python/ambassador/envoy/v3/v3ratelimit.py b/python/ambassador/envoy/v3/v3ratelimit.py index 25826f01ec..939bf3c098 100644 --- a/python/ambassador/envoy/v3/v3ratelimit.py +++ b/python/ambassador/envoy/v3/v3ratelimit.py @@ -18,11 +18,11 @@ from ...ir.irratelimit import IRRateLimit if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3RateLimit(dict): - def __init__(self, config: 'V3Config') -> None: + def __init__(self, config: "V3Config") -> None: # We should never be instantiated unless there is, in fact, defined ratelimit stuff. assert config.ir.ratelimit @@ -30,18 +30,16 @@ def __init__(self, config: 'V3Config') -> None: ratelimit = typecast(IRRateLimit, config.ir.ratelimit) - assert(ratelimit.cluster.envoy_name) + assert ratelimit.cluster.envoy_name - self['transport_api_version'] = ratelimit.protocol_version.upper() - self['grpc_service'] = { - 'envoy_grpc': { - 'cluster_name': ratelimit.cluster.envoy_name - } - } + self["transport_api_version"] = ratelimit.protocol_version.upper() + self["grpc_service"] = {"envoy_grpc": {"cluster_name": ratelimit.cluster.envoy_name}} @classmethod - def generate(cls, config: 'V3Config') -> None: + def generate(cls, config: "V3Config") -> None: config.ratelimit = None if config.ir.ratelimit: - config.ratelimit = config.save_element('ratelimit', config.ir.ratelimit, V3RateLimit(config)) + config.ratelimit = config.save_element( + "ratelimit", config.ir.ratelimit, V3RateLimit(config) + ) diff --git a/python/ambassador/envoy/v3/v3ratelimitaction.py b/python/ambassador/envoy/v3/v3ratelimitaction.py index 3f71da3fe7..30c30b4f4b 100644 --- a/python/ambassador/envoy/v3/v3ratelimitaction.py +++ b/python/ambassador/envoy/v3/v3ratelimitaction.py @@ -17,7 +17,7 @@ # from ...utils import RichStatus if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3RateLimitAction(dict): @@ -30,7 +30,7 @@ class V3RateLimitAction(dict): already_errored: ClassVar[bool] = False - def __init__(self, config: 'V3Config', rate_limit: Dict[str, Any]) -> None: + def __init__(self, config: "V3Config", rate_limit: Dict[str, Any]) -> None: super().__init__() self.valid = False @@ -46,7 +46,9 @@ def __init__(self, config: 'V3Config', rate_limit: Dict[str, Any]) -> None: lkeys = rate_limit.keys() if len(lkeys) > 1: # "Impossible". This should've been caught earlier. - config.ir.post_error("Label for RateLimit has multiple entries instead of just one: %s" % rate_limit) + config.ir.post_error( + "Label for RateLimit has multiple entries instead of just one: %s" % rate_limit + ) return lkey = list(lkeys)[0] @@ -58,43 +60,54 @@ def __init__(self, config: 'V3Config', rate_limit: Dict[str, Any]) -> None: # This should be a dict with a single key. keylist = list(action.keys()) if len(keylist) != 1: - config.ir.post_error("Label for RateLimit has invalid custom header '%s' (%s)" % - (action, rate_limit)) + config.ir.post_error( + "Label for RateLimit has invalid custom header '%s' (%s)" % (action, rate_limit) + ) continue dkey = keylist[0] - if dkey == 'source_cluster': - self.save_action({ - 'source_cluster': {}, - }) - elif dkey == 'destination_cluster': - self.save_action({ - 'destination_cluster': {}, - }) - elif dkey == 'remote_address': - self.save_action({ - 'remote_address': {}, - }) - elif dkey == 'generic_key': - self.save_action({ - 'generic_key': { - 'descriptor_key': action[dkey].get('key', 'generic_key'), - 'descriptor_value': action[dkey]['value'], - }, - }) - elif dkey == 'request_headers': - self.save_action({ - 'request_headers': { - 'descriptor_key': action[dkey]['key'], - 'header_name': action[dkey]['header_name'], - # This line was written and commented out with the comment "Need to upgrade - # to Envoy API v3 to set `skip_if_absent`." Well, we're on Envoy API v3, - # but I'm leaving it commented out because it seems that `skip_if_absent` - # doesn't quite work the way that this line of code implies it does. - # - #'skip_if_absent': action[dkey].get('omit_if_not_present', False), - }, - }) + if dkey == "source_cluster": + self.save_action( + { + "source_cluster": {}, + } + ) + elif dkey == "destination_cluster": + self.save_action( + { + "destination_cluster": {}, + } + ) + elif dkey == "remote_address": + self.save_action( + { + "remote_address": {}, + } + ) + elif dkey == "generic_key": + self.save_action( + { + "generic_key": { + "descriptor_key": action[dkey].get("key", "generic_key"), + "descriptor_value": action[dkey]["value"], + }, + } + ) + elif dkey == "request_headers": + self.save_action( + { + "request_headers": { + "descriptor_key": action[dkey]["key"], + "header_name": action[dkey]["header_name"], + # This line was written and commented out with the comment "Need to upgrade + # to Envoy API v3 to set `skip_if_absent`." Well, we're on Envoy API v3, + # but I'm leaving it commented out because it seems that `skip_if_absent` + # doesn't quite work the way that this line of code implies it does. + # + #'skip_if_absent': action[dkey].get('omit_if_not_present', False), + }, + } + ) ### This whole bit doesn't work with the existing RateLimit filter. We're ### going to have to tweak it to allow request_headers with a default value. ### @@ -144,7 +157,4 @@ def save_action(self, action): self.valid = True def to_dict(self): - return { - 'stage': self.stage, - 'actions': self.actions - } + return {"stage": self.stage, "actions": self.actions} diff --git a/python/ambassador/envoy/v3/v3route.py b/python/ambassador/envoy/v3/v3route.py index beb69b0a05..c0a18340fd 100644 --- a/python/ambassador/envoy/v3/v3route.py +++ b/python/ambassador/envoy/v3/v3route.py @@ -24,7 +24,7 @@ from .v3ratelimitaction import V3RateLimitAction if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover # This is the root of a certain amount of ugliness in this file -- it's a V3Route @@ -58,16 +58,16 @@ def v3prettyroute(route: DictifiedV3Route) -> str: name = header.get("name", None).lower() exact = header.get("exact_match", None) - if header == ':authority': + if header == ":authority": if exact: host = exact - elif 'prefix_match' in header: - host = header['prefix_match'] + '*' - elif 'suffix_match' in header: - host = '*' + header['suffix_match'] - elif 'safe_regex_match' in header: - host = header['safe_regex_match']['regex'] - elif name == 'x-forwarded-proto': + elif "prefix_match" in header: + host = header["prefix_match"] + "*" + elif "suffix_match" in header: + host = "*" + header["suffix_match"] + elif "safe_regex_match" in header: + host = header["safe_regex_match"]["regex"] + elif name == "x-forwarded-proto": xfp = exact if xfp: @@ -93,20 +93,13 @@ def v3prettyroute(route: DictifiedV3Route) -> str: # regex_matcher generates Envoy configuration to do a regex match in a Route. It's complex # here because, even though we don't have to deal with safe and unsafe regexes, it's simpler # to keep the weird baroqueness of this stuff wrapped in a function. -def regex_matcher(config: 'V3Config', regex: str, key="regex", safe_key=None) -> Dict[str, Any]: - max_size = int(config.ir.ambassador_module.get('regex_max_size', 200)) +def regex_matcher(config: "V3Config", regex: str, key="regex", safe_key=None) -> Dict[str, Any]: + max_size = int(config.ir.ambassador_module.get("regex_max_size", 200)) if not safe_key: safe_key = "safe_" + key - return { - safe_key: { - "google_re2": { - "max_program_size": max_size - }, - "regex": regex - } - } + return {safe_key: {"google_re2": {"max_program_size": max_size}, "regex": regex}} class V3RouteVariants: @@ -152,10 +145,10 @@ class V3RouteVariants: such a lazy collection of route variants for a given V3Route. """ - route: 'V3Route' + route: "V3Route" variants: Dict[str, DictifiedV3Route] - def __init__(self, route: 'V3Route') -> None: + def __init__(self, route: "V3Route") -> None: self.route = route self.variants = {} @@ -224,17 +217,13 @@ def matcher_xfp(self, variant: DictifiedV3Route, value: Optional[str]) -> None: # ...then make a copy of match["headers"], but don't include # any existing XFP header match. headers = match_copy.get("headers") or [] - headers_copy = [ h for h in headers - if h.get("name", "").lower() != "x-forwarded-proto" ] + headers_copy = [h for h in headers if h.get("name", "").lower() != "x-forwarded-proto"] # OK, if the new XFP value is anything, write a match for it. If not, # we'll just match any XFP. if value: - headers_copy.append({ - "name": "x-forwarded-proto", - "exact_match": value - }) + headers_copy.append({"name": "x-forwarded-proto", "exact_match": value}) # Don't bother writing headers_copy back if it's empty. if headers_copy: @@ -249,9 +238,7 @@ def action_route(self, variant) -> None: # instead. def action_redirect(self, variant) -> None: variant.pop("route", None) - variant["redirect"] = { - "https_redirect": True - } + variant["redirect"] = {"https_redirect": True} # Model an Envoy route. @@ -264,8 +251,11 @@ def action_redirect(self, variant) -> None: # context-matching madness happens up at the chain level, so we only need to # mess with the one host glob at this point. + class V3Route(Cacheable): - def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBaseMapping) -> None: + def __init__( + self, config: "V3Config", group: IRHTTPMappingGroup, mapping: IRBaseMapping + ) -> None: super().__init__() # Save the logger and the group. @@ -274,74 +264,81 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas # Passing a list to set is _very important_ here, lest you get a set of # the individual characters in group.host! - self['_host_constraints'] = set( [ group.get("host") or "*" ] ) + self["_host_constraints"] = set([group.get("host") or "*"]) - if group.get('precedence'): - self['_precedence'] = group['precedence'] + if group.get("precedence"): + self["_precedence"] = group["precedence"] envoy_route = EnvoyRoute(group).envoy_route - mapping_prefix = mapping.get('prefix', None) - route_prefix = mapping_prefix if mapping_prefix is not None else group.get('prefix') + mapping_prefix = mapping.get("prefix", None) + route_prefix = mapping_prefix if mapping_prefix is not None else group.get("prefix") - mapping_case_sensitive = mapping.get('case_sensitive', None) - case_sensitive = mapping_case_sensitive if mapping_case_sensitive is not None else group.get('case_sensitive', True) + mapping_case_sensitive = mapping.get("case_sensitive", None) + case_sensitive = ( + mapping_case_sensitive + if mapping_case_sensitive is not None + else group.get("case_sensitive", True) + ) runtime_fraction: Dict[str, Union[dict, str]] = { - 'default_value': { - 'numerator': mapping.get('_weight', 100), - 'denominator': 'HUNDRED' - } + "default_value": {"numerator": mapping.get("_weight", 100), "denominator": "HUNDRED"} } if len(mapping) > 0: - if not 'cluster' in mapping: - config.ir.logger.error("%s: Mapping %s has no cluster? %s", mapping.rkey, route_prefix, mapping.as_json()) - self['_failed'] = True + if not "cluster" in mapping: + config.ir.logger.error( + "%s: Mapping %s has no cluster? %s", + mapping.rkey, + route_prefix, + mapping.as_json(), + ) + self["_failed"] = True else: - runtime_fraction['runtime_key'] = f'routing.traffic_shift.{mapping.cluster.envoy_name}' + runtime_fraction[ + "runtime_key" + ] = f"routing.traffic_shift.{mapping.cluster.envoy_name}" - match = { - 'case_sensitive': case_sensitive, - 'runtime_fraction': runtime_fraction - } + match = {"case_sensitive": case_sensitive, "runtime_fraction": runtime_fraction} - if envoy_route == 'prefix': - match['prefix'] = route_prefix - elif envoy_route == 'path': - match['path'] = route_prefix + if envoy_route == "prefix": + match["prefix"] = route_prefix + elif envoy_route == "path": + match["path"] = route_prefix else: # Cheat. - if config.ir.edge_stack_allowed and (self.get('_precedence', 0) == -1000000): + if config.ir.edge_stack_allowed and (self.get("_precedence", 0) == -1000000): # Force the safe_regex engine. - match.update({ - "safe_regex": { - "google_re2": { - "max_program_size": 200, - }, - "regex": route_prefix + match.update( + { + "safe_regex": { + "google_re2": { + "max_program_size": 200, + }, + "regex": route_prefix, + } } - }) + ) else: match.update(regex_matcher(config, route_prefix)) headers = self.generate_headers(config, group) if len(headers) > 0: - match['headers'] = headers + match["headers"] = headers query_parameters = self.generate_query_parameters(config, group) if len(query_parameters) > 0: - match['query_parameters'] = query_parameters + match["query_parameters"] = query_parameters - self['match'] = match + self["match"] = match # `typed_per_filter_config` is used to pass typed configuration to Envoy filters typed_per_filter_config = {} - if mapping.get('bypass_error_response_overrides', False): - typed_per_filter_config['envoy.filters.http.response_map'] = { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.response_map.v3.ResponseMapPerRoute', - 'disabled': True, + if mapping.get("bypass_error_response_overrides", False): + typed_per_filter_config["envoy.filters.http.response_map"] = { + "@type": "type.googleapis.com/envoy.extensions.filters.http.response_map.v3.ResponseMapPerRoute", + "disabled": True, } else: # The error_response_overrides field is set on the Mapping as input config @@ -351,7 +348,7 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas # # Therefore, if the field is present at this point, it means it's a valid # IRErrorResponse with a 'config' field, since setup must have succeded. - error_response_overrides = mapping.get('error_response_overrides', None) + error_response_overrides = mapping.get("error_response_overrides", None) if error_response_overrides: # The error reponse IR only has optional response map config to use. # On this particular code path, we're protected by both Mapping schema @@ -363,82 +360,75 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas if filter_config: # The error response IR itself guarantees that any resulting config() has # at least one mapper in 'mappers', so assert on that here. - assert 'mappers' in filter_config - assert len(filter_config['mappers']) > 0 - typed_per_filter_config['envoy.filters.http.response_map'] = { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.response_map.v3.ResponseMapPerRoute', + assert "mappers" in filter_config + assert len(filter_config["mappers"]) > 0 + typed_per_filter_config["envoy.filters.http.response_map"] = { + "@type": "type.googleapis.com/envoy.extensions.filters.http.response_map.v3.ResponseMapPerRoute", # The ResponseMapPerRoute Envoy config is similar to the ResponseMap filter # config, except that it is wrapped in another object with key 'response_map'. - 'response_map': { - 'mappers': filter_config['mappers'] - } + "response_map": {"mappers": filter_config["mappers"]}, } - if mapping.get('bypass_auth', False): - typed_per_filter_config['envoy.filters.http.ext_authz'] = { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute', - 'disabled': True, + if mapping.get("bypass_auth", False): + typed_per_filter_config["envoy.filters.http.ext_authz"] = { + "@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute", + "disabled": True, } else: # Additional ext_auth configuration only makes sense when not bypassing auth. - auth_context_extensions = mapping.get('auth_context_extensions', False) + auth_context_extensions = mapping.get("auth_context_extensions", False) if auth_context_extensions: - typed_per_filter_config['envoy.filters.http.ext_authz'] = { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute', - 'check_settings': {'context_extensions': auth_context_extensions} + typed_per_filter_config["envoy.filters.http.ext_authz"] = { + "@type": "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute", + "check_settings": {"context_extensions": auth_context_extensions}, } if len(typed_per_filter_config) > 0: - self['typed_per_filter_config'] = typed_per_filter_config + self["typed_per_filter_config"] = typed_per_filter_config - request_headers_to_add = group.get('add_request_headers', None) + request_headers_to_add = group.get("add_request_headers", None) if request_headers_to_add: - self['request_headers_to_add'] = self.generate_headers_to_add(request_headers_to_add) + self["request_headers_to_add"] = self.generate_headers_to_add(request_headers_to_add) - response_headers_to_add = group.get('add_response_headers', None) + response_headers_to_add = group.get("add_response_headers", None) if response_headers_to_add: - self['response_headers_to_add'] = self.generate_headers_to_add(response_headers_to_add) + self["response_headers_to_add"] = self.generate_headers_to_add(response_headers_to_add) - request_headers_to_remove = group.get('remove_request_headers', None) + request_headers_to_remove = group.get("remove_request_headers", None) if request_headers_to_remove: if type(request_headers_to_remove) != list: - request_headers_to_remove = [ request_headers_to_remove ] - self['request_headers_to_remove'] = request_headers_to_remove + request_headers_to_remove = [request_headers_to_remove] + self["request_headers_to_remove"] = request_headers_to_remove - response_headers_to_remove = group.get('remove_response_headers', None) + response_headers_to_remove = group.get("remove_response_headers", None) if response_headers_to_remove: if type(response_headers_to_remove) != list: - response_headers_to_remove = [ response_headers_to_remove ] - self['response_headers_to_remove'] = response_headers_to_remove + response_headers_to_remove = [response_headers_to_remove] + self["response_headers_to_remove"] = response_headers_to_remove - host_redirect = group.get('host_redirect', None) + host_redirect = group.get("host_redirect", None) if host_redirect: # We have a host_redirect. Deal with it. - self['redirect'] = { - 'host_redirect': host_redirect.service - } + self["redirect"] = {"host_redirect": host_redirect.service} - path_redirect = host_redirect.get('path_redirect', None) - prefix_redirect = host_redirect.get('prefix_redirect', None) - regex_redirect = host_redirect.get('regex_redirect', None) - response_code = host_redirect.get('redirect_response_code', None) + path_redirect = host_redirect.get("path_redirect", None) + prefix_redirect = host_redirect.get("prefix_redirect", None) + regex_redirect = host_redirect.get("regex_redirect", None) + response_code = host_redirect.get("redirect_response_code", None) # We enforce that only one of path_redirect or prefix_redirect is set in the IR. # But here, we just prefer path_redirect if that's set. if path_redirect: - self['redirect']['path_redirect'] = path_redirect + self["redirect"]["path_redirect"] = path_redirect elif prefix_redirect: # In Envoy, it's called prefix_rewrite. - self['redirect']['prefix_rewrite'] = prefix_redirect + self["redirect"]["prefix_rewrite"] = prefix_redirect elif regex_redirect: # In Envoy, it's called regex_rewrite. - self['redirect']['regex_rewrite'] = { - 'pattern': { - 'google_re2': {}, - 'regex': regex_redirect.get('pattern', '') - }, - 'substitution': regex_redirect.get('substitution', '') + self["redirect"]["regex_rewrite"] = { + "pattern": {"google_re2": {}, "regex": regex_redirect.get("pattern", "")}, + "substitution": regex_redirect.get("substitution", ""), } # In Ambassador, we express the redirect_reponse_code as the actual @@ -457,41 +447,42 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas enum_code = 4 else: config.ir.post_error( - f"Unknown redirect_response_code={response_code}, must be one of [301, 302, 303,307, 308]. Using default redirect_response_code=301") + f"Unknown redirect_response_code={response_code}, must be one of [301, 302, 303,307, 308]. Using default redirect_response_code=301" + ) enum_code = 0 - self['redirect']['response_code'] = enum_code + self["redirect"]["response_code"] = enum_code return # Take the default `timeout_ms` value from the Ambassador module using `cluster_request_timeout_ms`. # If that isn't set, use 3000ms. The mapping below will override this if its own `timeout_ms` is set. - default_timeout_ms = config.ir.ambassador_module.get('cluster_request_timeout_ms', 3000) + default_timeout_ms = config.ir.ambassador_module.get("cluster_request_timeout_ms", 3000) route = { - 'priority': group.get('priority'), - 'timeout': "%0.3fs" % (mapping.get('timeout_ms', default_timeout_ms) / 1000.0), - 'cluster': mapping.cluster.envoy_name + "priority": group.get("priority"), + "timeout": "%0.3fs" % (mapping.get("timeout_ms", default_timeout_ms) / 1000.0), + "cluster": mapping.cluster.envoy_name, } - idle_timeout_ms = mapping.get('idle_timeout_ms', None) + idle_timeout_ms = mapping.get("idle_timeout_ms", None) if idle_timeout_ms is not None: - route['idle_timeout'] = "%0.3fs" % (idle_timeout_ms / 1000.0) + route["idle_timeout"] = "%0.3fs" % (idle_timeout_ms / 1000.0) regex_rewrite = self.generate_regex_rewrite(config, group) if len(regex_rewrite) > 0: - route['regex_rewrite'] = regex_rewrite - elif mapping.get('rewrite', None): - route['prefix_rewrite'] = mapping['rewrite'] + route["regex_rewrite"] = regex_rewrite + elif mapping.get("rewrite", None): + route["prefix_rewrite"] = mapping["rewrite"] - if 'host_rewrite' in mapping: - route['host_rewrite_literal'] = mapping['host_rewrite'] + if "host_rewrite" in mapping: + route["host_rewrite_literal"] = mapping["host_rewrite"] - if 'auto_host_rewrite' in mapping: - route['auto_host_rewrite'] = mapping['auto_host_rewrite'] + if "auto_host_rewrite" in mapping: + route["auto_host_rewrite"] = mapping["auto_host_rewrite"] hash_policy = self.generate_hash_policy(group) if len(hash_policy) > 0: - route['hash_policy'] = [ hash_policy ] + route["hash_policy"] = [hash_policy] cors = None @@ -505,7 +496,7 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas cors = cors.dup() cors.set_id(group.group_id) - route['cors'] = cors.as_dict() + route["cors"] = cors.as_dict() retry_policy = None @@ -515,7 +506,7 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas retry_policy = config.ir.ambassador_module.retry_policy.as_dict() if retry_policy: - route['retry_policy'] = retry_policy + route["retry_policy"] = retry_policy # Is shadowing enabled? shadow = group.get("shadows", None) @@ -523,19 +514,16 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas if shadow: shadow = shadow[0] - weight = shadow.get('weight', 100) + weight = shadow.get("weight", 100) - route['request_mirror_policies'] = [ + route["request_mirror_policies"] = [ { - 'cluster': shadow.cluster.envoy_name, - 'runtime_fraction': { - 'default_value': { - 'numerator': weight, - 'denominator': 'HUNDRED' - } - } + "cluster": shadow.cluster.envoy_name, + "runtime_fraction": { + "default_value": {"numerator": weight, "denominator": "HUNDRED"} + }, } - ] + ] # Is RateLimit a thing? rlsvc = config.ir.ratelimit @@ -561,10 +549,12 @@ def __init__(self, config: 'V3Config', group: IRHTTPMappingGroup, mapping: IRBas route["rate_limits"] = rate_limits # Save upgrade configs. - if group.get('allow_upgrade'): - route["upgrade_configs"] = [ { 'upgrade_type': proto } for proto in group.get('allow_upgrade', []) ] + if group.get("allow_upgrade"): + route["upgrade_configs"] = [ + {"upgrade_type": proto} for proto in group.get("allow_upgrade", []) + ] - self['route'] = route + self["route"] = route # matches_domain and matches_domains are both still written assuming a _host_constraints # with more than element. Not changing that yet. @@ -585,7 +575,7 @@ def matches_domains(self, domains: List[str]) -> bool: self.logger.debug(f" - matches_domains: nonspecific domains") return True - if any([ self.matches_domain(domain) for domain in domains ]): + if any([self.matches_domain(domain) for domain in domains]): self.logger.debug(f" - matches_domains: domain match") return True @@ -593,9 +583,10 @@ def matches_domains(self, domains: List[str]) -> bool: return False @classmethod - def get_route(cls, config: 'V3Config', cache_key: str, - irgroup: IRHTTPMappingGroup, mapping: IRBaseMapping) -> 'V3Route': - route: 'V3Route' + def get_route( + cls, config: "V3Config", cache_key: str, irgroup: IRHTTPMappingGroup, mapping: IRBaseMapping + ) -> "V3Route": + route: "V3Route" cached_route = config.cache[cache_key] @@ -616,7 +607,7 @@ def get_route(cls, config: 'V3Config', cache_key: str, else: # Cache hit. We know a priori that it's a V3Route, but let's assert that # before casting. - assert(isinstance(cached_route, V3Route)) + assert isinstance(cached_route, V3Route) route = cached_route # config.ir.logger.info(f"V3Route: cache hit for {cache_key}") @@ -625,7 +616,7 @@ def get_route(cls, config: 'V3Config', cache_key: str, return route @classmethod - def generate(cls, config: 'V3Config') -> None: + def generate(cls, config: "V3Config") -> None: config.routes = [] for irgroup in config.ir.ordered_groups(): @@ -633,7 +624,7 @@ def generate(cls, config: 'V3Config') -> None: # We only want HTTP mapping groups here. continue - if irgroup.get('host_redirect') is not None and len(irgroup.get('mappings', [])) == 0: + if irgroup.get("host_redirect") is not None and len(irgroup.get("mappings", [])) == 0: # This is a host-redirect-only group, which is weird, but can happen. Do we # have a cached route for it? key = f"Route-{irgroup.group_id}-hostredirect" @@ -644,7 +635,11 @@ def generate(cls, config: 'V3Config') -> None: # # (We could also have written V3Route to allow the mapping to be Optional, but that # makes a lot of its constructor much uglier.) - route = config.save_element('route', irgroup, cls.get_route(config, key, irgroup, typecast(IRBaseMapping, {}))) + route = config.save_element( + "route", + irgroup, + cls.get_route(config, key, irgroup, typecast(IRBaseMapping, {})), + ) config.routes.append(route) # Repeat for our real mappings. @@ -653,8 +648,8 @@ def generate(cls, config: 'V3Config') -> None: route = cls.get_route(config, key, irgroup, mapping) - if not route.get('_failed', False): - config.routes.append(config.save_element('route', irgroup, route)) + if not route.get("_failed", False): + config.routes.append(config.save_element("route", irgroup, route)) # Once that's done, go build the variants on each route. config.route_variants = [] @@ -664,22 +659,22 @@ def generate(cls, config: 'V3Config') -> None: config.route_variants.append(V3RouteVariants(route)) @staticmethod - def generate_headers(config: 'V3Config', mapping_group: IRHTTPMappingGroup) -> List[dict]: + def generate_headers(config: "V3Config", mapping_group: IRHTTPMappingGroup) -> List[dict]: headers = [] - group_headers = mapping_group.get('headers', []) + group_headers = mapping_group.get("headers", []) for group_header in group_headers: - header_name = group_header.get('name') - header_value = group_header.get('value') + header_name = group_header.get("name") + header_value = group_header.get("value") - header = { 'name': header_name } + header = {"name": header_name} # Is this a regex? - if group_header.get('regex'): - header.update(regex_matcher(config, header_value, key='regex_match')) + if group_header.get("regex"): + header.update(regex_matcher(config, header_value, key="regex_match")) else: - if header_name == ':authority': + if header_name == ":authority": # The authority header is special, because its value is a glob. # (This works without the user marking it as such because '*' isn't # valid in DNS names, so we know that treating a name with a '*' as @@ -687,51 +682,49 @@ def generate_headers(config: 'V3Config', mapping_group: IRHTTPMappingGroup) -> L if header_value == "*": # This is actually a noop, so just don't include this header. continue - elif header_value.startswith('*'): - header['suffix_match'] = header_value[1:] - elif header_value.endswith('*'): - header['prefix_match'] = header_value[:-1] + elif header_value.startswith("*"): + header["suffix_match"] = header_value[1:] + elif header_value.endswith("*"): + header["prefix_match"] = header_value[:-1] else: # But wait! What about 'foo.*.com'?? Turns out Envoy doesn't # support that in the places it actually does host globbing, # so we won't either for the moment. - header['exact_match'] = header_value + header["exact_match"] = header_value else: - header['exact_match'] = header_value + header["exact_match"] = header_value headers.append(header) return headers @staticmethod - def generate_query_parameters(config: 'V3Config', mapping_group: IRHTTPMappingGroup) -> List[dict]: + def generate_query_parameters( + config: "V3Config", mapping_group: IRHTTPMappingGroup + ) -> List[dict]: query_parameters = [] - group_query_parameters = mapping_group.get('query_parameters', []) + group_query_parameters = mapping_group.get("query_parameters", []) for group_query_parameter in group_query_parameters: - query_parameter = { 'name': group_query_parameter.get('name') } - - if group_query_parameter.get('regex'): - query_parameter.update({ - 'string_match': regex_matcher( - config, - group_query_parameter.get('value'), - key='regex' - ) - }) + query_parameter = {"name": group_query_parameter.get("name")} + + if group_query_parameter.get("regex"): + query_parameter.update( + { + "string_match": regex_matcher( + config, group_query_parameter.get("value"), key="regex" + ) + } + ) else: - value = group_query_parameter.get('value', None) + value = group_query_parameter.get("value", None) if value is not None: - query_parameter.update({ - 'string_match': { - 'exact': group_query_parameter.get('value') - } - }) + query_parameter.update( + {"string_match": {"exact": group_query_parameter.get("value")}} + ) else: - query_parameter.update({ - 'present_match': True - }) + query_parameter.update({"present_match": True}) query_parameters.append(query_parameter) @@ -740,30 +733,24 @@ def generate_query_parameters(config: 'V3Config', mapping_group: IRHTTPMappingGr @staticmethod def generate_hash_policy(mapping_group: IRHTTPMappingGroup) -> dict: hash_policy = {} - load_balancer = mapping_group.get('load_balancer', None) + load_balancer = mapping_group.get("load_balancer", None) if load_balancer is not None: - lb_policy = load_balancer.get('policy') - if lb_policy in ['ring_hash', 'maglev']: - cookie = load_balancer.get('cookie') - header = load_balancer.get('header') - source_ip = load_balancer.get('source_ip') + lb_policy = load_balancer.get("policy") + if lb_policy in ["ring_hash", "maglev"]: + cookie = load_balancer.get("cookie") + header = load_balancer.get("header") + source_ip = load_balancer.get("source_ip") if cookie is not None: - hash_policy['cookie'] = { - 'name': cookie.get('name') - } - if 'path' in cookie: - hash_policy['cookie']['path'] = cookie['path'] - if 'ttl' in cookie: - hash_policy['cookie']['ttl'] = cookie['ttl'] + hash_policy["cookie"] = {"name": cookie.get("name")} + if "path" in cookie: + hash_policy["cookie"]["path"] = cookie["path"] + if "ttl" in cookie: + hash_policy["cookie"]["ttl"] = cookie["ttl"] elif header is not None: - hash_policy['header'] = { - 'header_name': header - } + hash_policy["header"] = {"header_name": header} elif source_ip is not None: - hash_policy['connection_properties'] = { - 'source_ip': source_ip - } + hash_policy["connection_properties"] = {"source_ip": source_ip} return hash_policy @@ -771,36 +758,31 @@ def generate_hash_policy(mapping_group: IRHTTPMappingGroup) -> dict: def generate_headers_to_add(header_dict: dict) -> List[dict]: headers = [] for k, v in header_dict.items(): - append = True - if isinstance(v,dict): - if 'append' in v: - append = bool(v['append']) - headers.append({ - 'header': { - 'key': k, - 'value': v['value'] - }, - 'append': append - }) - else: - headers.append({ - 'header': { - 'key': k, - 'value': v - }, - 'append': append # Default append True, for backward compatability - }) + append = True + if isinstance(v, dict): + if "append" in v: + append = bool(v["append"]) + headers.append({"header": {"key": k, "value": v["value"]}, "append": append}) + else: + headers.append( + { + "header": {"key": k, "value": v}, + "append": append, # Default append True, for backward compatability + } + ) return headers @staticmethod - def generate_regex_rewrite(config: 'V3Config', mapping_group: IRHTTPMappingGroup) -> dict: + def generate_regex_rewrite(config: "V3Config", mapping_group: IRHTTPMappingGroup) -> dict: regex_rewrite = {} - group_regex_rewrite = mapping_group.get('regex_rewrite', None) + group_regex_rewrite = mapping_group.get("regex_rewrite", None) if group_regex_rewrite is not None: - pattern = group_regex_rewrite.get('pattern', None) - if (pattern is not None): - regex_rewrite.update(regex_matcher(config, pattern, key='regex',safe_key='pattern')) # regex_rewrite should never ever be unsafe - substitution = group_regex_rewrite.get('substitution', None) - if (substitution is not None): + pattern = group_regex_rewrite.get("pattern", None) + if pattern is not None: + regex_rewrite.update( + regex_matcher(config, pattern, key="regex", safe_key="pattern") + ) # regex_rewrite should never ever be unsafe + substitution = group_regex_rewrite.get("substitution", None) + if substitution is not None: regex_rewrite["substitution"] = substitution return regex_rewrite diff --git a/python/ambassador/envoy/v3/v3tls.py b/python/ambassador/envoy/v3/v3tls.py index e9cdf22021..a7e8ffc18e 100644 --- a/python/ambassador/envoy/v3/v3tls.py +++ b/python/ambassador/envoy/v3/v3tls.py @@ -48,8 +48,10 @@ class V3TLSContext(Dict): "v1.3": "TLSv1_3", } - def __init__(self, ctx: Optional[IRTLSContext]=None, host_rewrite: Optional[str]=None) -> None: - del host_rewrite # quiesce warning + def __init__( + self, ctx: Optional[IRTLSContext] = None, host_rewrite: Optional[str] = None + ) -> None: + del host_rewrite # quiesce warning super().__init__() @@ -59,14 +61,14 @@ def __init__(self, ctx: Optional[IRTLSContext]=None, host_rewrite: Optional[str] self.add_context(ctx) def get_common(self) -> EnvoyCommonTLSContext: - return self.setdefault('common_tls_context', {}) + return self.setdefault("common_tls_context", {}) def get_params(self) -> EnvoyTLSParams: common = self.get_common() # This boils down to "params = common.setdefault('tls_params', {})" with typing. empty_params = typecast(EnvoyTLSParams, {}) - params = typecast(EnvoyTLSParams, common.setdefault('tls_params', empty_params)) + params = typecast(EnvoyTLSParams, common.setdefault("tls_params", empty_params)) return params @@ -75,7 +77,7 @@ def get_certs(self) -> ListOfCerts: # We have to explicitly cast this empty list to a list of strings. empty_cert_list: List[str] = [] - cert_list = common.setdefault('tls_certificates', empty_cert_list) + cert_list = common.setdefault("tls_certificates", empty_cert_list) # cert_list is of type EnvoyCommonTLSElements right now, so we need to cast it. return typecast(ListOfCerts, cert_list) @@ -86,12 +88,12 @@ def update_cert_zero(self, key: str, value: str) -> None: if not certs: certs.append({}) - src: EnvoyCoreSource = { 'filename': value } + src: EnvoyCoreSource = {"filename": value} certs[0][key] = src def update_alpn(self, key: str, value: str) -> None: common = self.get_common() - common[key] = [ value ] + common[key] = [value] def update_tls_version(self, key: str, value: str) -> None: params = self.get_params() @@ -110,34 +112,37 @@ def update_validation(self, key: str, value: str) -> None: # This looks weirder than you might expect, because self.get_common().setdefault() is a truly # crazy Union type, so we need to cast it to an EnvoyValidationContext to be able to work # with it. - validation = typecast(EnvoyValidationContext, self.get_common().setdefault('validation_context', empty_context)) + validation = typecast( + EnvoyValidationContext, + self.get_common().setdefault("validation_context", empty_context), + ) - src: EnvoyCoreSource = { 'filename': value } + src: EnvoyCoreSource = {"filename": value} validation[key] = src def add_context(self, ctx: IRTLSContext) -> None: if TYPE_CHECKING: # This is needed because otherwise self.__setitem__ confuses things. - handler: Callable[[str, str], None] # pragma: no cover + handler: Callable[[str, str], None] # pragma: no cover if ctx.is_fallback: self.is_fallback = True for secretinfokey, handler, hkey in [ - ( 'cert_chain_file', self.update_cert_zero, 'certificate_chain' ), - ( 'private_key_file', self.update_cert_zero, 'private_key' ), - ( 'cacert_chain_file', self.update_validation, 'trusted_ca' ), - ( 'crl_file', self.update_validation, 'crl' ), + ("cert_chain_file", self.update_cert_zero, "certificate_chain"), + ("private_key_file", self.update_cert_zero, "private_key"), + ("cacert_chain_file", self.update_validation, "trusted_ca"), + ("crl_file", self.update_validation, "crl"), ]: - if secretinfokey in ctx['secret_info']: - handler(hkey, ctx['secret_info'][secretinfokey]) + if secretinfokey in ctx["secret_info"]: + handler(hkey, ctx["secret_info"][secretinfokey]) for ctxkey, handler, hkey in [ - ( 'alpn_protocols', self.update_alpn, 'alpn_protocols' ), - ( 'cert_required', self.__setitem__, 'require_client_certificate' ), - ( 'min_tls_version', self.update_tls_version, 'tls_minimum_protocol_version' ), - ( 'max_tls_version', self.update_tls_version, 'tls_maximum_protocol_version' ), - ( 'sni', self.__setitem__, 'sni' ), + ("alpn_protocols", self.update_alpn, "alpn_protocols"), + ("cert_required", self.__setitem__, "require_client_certificate"), + ("min_tls_version", self.update_tls_version, "tls_minimum_protocol_version"), + ("max_tls_version", self.update_tls_version, "tls_maximum_protocol_version"), + ("sni", self.__setitem__, "sni"), ]: value = ctx.get(ctxkey, None) @@ -149,8 +154,8 @@ def add_context(self, ctx: IRTLSContext) -> None: # string. Getting mypy to be happy with that is _annoying_. for ctxkey, list_handler, hkey in [ - ( 'cipher_suites', self.update_tls_cipher, 'cipher_suites' ), - ( 'ecdh_curves', self.update_tls_cipher, 'ecdh_curves' ), + ("cipher_suites", self.update_tls_cipher, "cipher_suites"), + ("ecdh_curves", self.update_tls_cipher, "ecdh_curves"), ]: value = ctx.get(ctxkey, None) @@ -169,5 +174,7 @@ def pretty(self) -> str: dirname = os.path.basename(os.path.dirname(filename)) filename = f".../{dirname}/{basename}" - return "" % \ - (" (fallback)" if self.is_fallback else "", filename) + return "" % ( + " (fallback)" if self.is_fallback else "", + filename, + ) diff --git a/python/ambassador/envoy/v3/v3tracing.py b/python/ambassador/envoy/v3/v3tracing.py index 4ae39e5aee..1cf42dc3ff 100644 --- a/python/ambassador/envoy/v3/v3tracing.py +++ b/python/ambassador/envoy/v3/v3tracing.py @@ -18,11 +18,11 @@ from ...ir.irtracing import IRTracing if TYPE_CHECKING: - from . import V3Config # pragma: no cover + from . import V3Config # pragma: no cover class V3Tracing(dict): - def __init__(self, config: 'V3Config') -> None: + def __init__(self, config: "V3Config") -> None: # We should never be instantiated unless there is, in fact, defined tracing stuff. assert config.ir.tracing @@ -30,43 +30,40 @@ def __init__(self, config: 'V3Config') -> None: tracing = typecast(IRTracing, config.ir.tracing) - name = tracing['driver'] + name = tracing["driver"] - if not name.startswith('envoy.'): - name = 'envoy.%s' % (name.lower()) + if not name.startswith("envoy."): + name = "envoy.%s" % (name.lower()) - driver_config = tracing['driver_config'] + driver_config = tracing["driver_config"] # We check for the full 'envoy.tracers.datadog' below because that's how it's set in the # IR code. The other tracers are configured by their short name and then 'envoy.' is # appended above. - if name.lower() == 'envoy.zipkin': - driver_config['@type'] = 'type.googleapis.com/envoy.config.trace.v3.ZipkinConfig' + if name.lower() == "envoy.zipkin": + driver_config["@type"] = "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig" # In xDS v3 the old Zipkin-v1 API can only be specified as the implicit default; it # cannot be specified explicitly. # https://www.envoyproxy.io/docs/envoy/latest/version_history/v1.12.0.html?highlight=http_json_v1 # https://github.com/envoyproxy/envoy/blob/ae1ed1fa74f096dabe8dd5b19fc70333621b0309/api/envoy/config/trace/v3/zipkin.proto#L27 - if driver_config['collector_endpoint_version'] == 'HTTP_JSON_V1': - del driver_config['collector_endpoint_version'] - elif name.lower() == 'envoy.tracers.datadog': - driver_config['@type'] = 'type.googleapis.com/envoy.config.trace.v3.DatadogConfig' - if not driver_config.get('service_name'): - driver_config['service_name'] = 'ambassador' - elif name.lower() == 'envoy.lightstep': - driver_config['@type'] = 'type.googleapis.com/envoy.config.trace.v3.LightstepConfig' + if driver_config["collector_endpoint_version"] == "HTTP_JSON_V1": + del driver_config["collector_endpoint_version"] + elif name.lower() == "envoy.tracers.datadog": + driver_config["@type"] = "type.googleapis.com/envoy.config.trace.v3.DatadogConfig" + if not driver_config.get("service_name"): + driver_config["service_name"] = "ambassador" + elif name.lower() == "envoy.lightstep": + driver_config["@type"] = "type.googleapis.com/envoy.config.trace.v3.LightstepConfig" else: # This should be impossible, because we ought to have validated the input driver # in ambassador/pkg/api/getambassador.io/v2/tracingservice_types.go:47 - raise Exception("Unsupported tracing driver \"%s\"" % name.lower()) + raise Exception('Unsupported tracing driver "%s"' % name.lower()) - self['http'] = { - "name": name, - "typed_config": driver_config - } + self["http"] = {"name": name, "typed_config": driver_config} @classmethod - def generate(cls, config: 'V3Config') -> None: + def generate(cls, config: "V3Config") -> None: config.tracing = None if config.ir.tracing: - config.tracing = config.save_element('tracing', config.ir.tracing, V3Tracing(config)) + config.tracing = config.save_element("tracing", config.ir.tracing, V3Tracing(config)) diff --git a/python/ambassador/fetch/ambassador.py b/python/ambassador/fetch/ambassador.py index 3c7fdfd3c1..c860705c02 100644 --- a/python/ambassador/fetch/ambassador.py +++ b/python/ambassador/fetch/ambassador.py @@ -7,32 +7,35 @@ from .resource import NormalizedResource -class AmbassadorProcessor (ManagedKubernetesProcessor): +class AmbassadorProcessor(ManagedKubernetesProcessor): """ A Kubernetes object processor that emits direct IR from an Ambassador CRD. """ def kinds(self) -> FrozenSet[KubernetesGVK]: kinds = [ - 'AuthService', - 'ConsulResolver', - 'Host', - 'KubernetesEndpointResolver', - 'KubernetesServiceResolver', - 'Listener', - 'LogService', - 'Mapping', - 'Module', - 'RateLimitService', - 'DevPortal', - 'TCPMapping', - 'TLSContext', - 'TracingService', + "AuthService", + "ConsulResolver", + "Host", + "KubernetesEndpointResolver", + "KubernetesServiceResolver", + "Listener", + "LogService", + "Mapping", + "Module", + "RateLimitService", + "DevPortal", + "TCPMapping", + "TLSContext", + "TracingService", ] - return frozenset([ - KubernetesGVK.for_ambassador(kind, version=version) for (kind, version) in itertools.product(kinds, ['v1', 'v2', 'v3alpha1']) - ]) + return frozenset( + [ + KubernetesGVK.for_ambassador(kind, version=version) + for (kind, version) in itertools.product(kinds, ["v1", "v2", "v3alpha1"]) + ] + ) def _process(self, obj: KubernetesObject) -> None: self.manager.emit(NormalizedResource.from_kubernetes_object(obj)) diff --git a/python/ambassador/fetch/dependency.py b/python/ambassador/fetch/dependency.py index 33ede40a71..5502e6a2dc 100644 --- a/python/ambassador/fetch/dependency.py +++ b/python/ambassador/fetch/dependency.py @@ -1,4 +1,15 @@ -from typing import Any, Collection, Iterator, Mapping, MutableSet, Optional, Protocol, Sequence, Type, TypeVar +from typing import ( + Any, + Collection, + Iterator, + Mapping, + MutableSet, + Optional, + Protocol, + Sequence, + Type, + TypeVar, +) from collections import defaultdict import dataclasses @@ -6,7 +17,7 @@ from .k8sobject import KubernetesObject -class Dependency (Protocol): +class Dependency(Protocol): """ Dependencies link information provided by processors of a given Watt invocation to other processors that need the processed result. This results @@ -14,10 +25,11 @@ class Dependency (Protocol): without direct knowledge of where data is coming from. """ - def watt_key(self) -> str: ... + def watt_key(self) -> str: + ... -class ServiceDependency (Dependency): +class ServiceDependency(Dependency): """ A dependency that exposes information about the Kubernetes service for Ambassador itself. @@ -29,20 +41,20 @@ def __init__(self) -> None: self.ambassador_service = None def watt_key(self) -> str: - return 'service' + return "service" -class SecretDependency (Dependency): +class SecretDependency(Dependency): """ A dependency that is satisfied once secret information has been mapped and emitted. """ def watt_key(self) -> str: - return 'secret' + return "secret" -class IngressClassesDependency (Dependency): +class IngressClassesDependency(Dependency): """ A dependency that provides the list of ingress classes that are valid (i.e., have the proper controller) for this cluster. @@ -54,16 +66,18 @@ def __init__(self): self.ingress_classes = set() def watt_key(self) -> str: - return 'ingressclasses' + return "ingressclasses" -D = TypeVar('D', bound=Dependency) +D = TypeVar("D", bound=Dependency) -class DependencyMapping (Protocol): +class DependencyMapping(Protocol): + def __contains__(self, key: Type[D]) -> bool: + ... - def __contains__(self, key: Type[D]) -> bool: ... - def __getitem__(self, key: Type[D]) -> D: ... + def __getitem__(self, key: Type[D]) -> D: + ... class DependencyInjector: @@ -140,7 +154,7 @@ def traverse(self) -> Iterator[Any]: # No roots of a graph with at least one vertex indicates a cycle. if len(queue) == 0: - raise ValueError('cyclic') + raise ValueError("cyclic") while len(queue) > 0: cur = queue.pop(0) @@ -151,7 +165,7 @@ def traverse(self) -> Iterator[Any]: if in_counts[obj] == 0: queue.append(obj) - assert sum(in_counts.values()) == 0, 'Traversal did not reach every vertex exactly once' + assert sum(in_counts.values()) == 0, "Traversal did not reach every vertex exactly once" class DependencyManager: diff --git a/python/ambassador/fetch/fetcher.py b/python/ambassador/fetch/fetcher.py index 495c33ed26..397ba33e48 100644 --- a/python/ambassador/fetch/fetcher.py +++ b/python/ambassador/fetch/fetcher.py @@ -9,7 +9,12 @@ from ..config import ACResource, Config from ..utils import parse_yaml, parse_json, dump_json, parse_bool -from .dependency import DependencyManager, IngressClassesDependency, SecretDependency, ServiceDependency +from .dependency import ( + DependencyManager, + IngressClassesDependency, + SecretDependency, + ServiceDependency, +) from .resource import NormalizedResource, ResourceManager from .k8sobject import KubernetesGVK, KubernetesObject from .k8sprocessor import ( @@ -51,7 +56,7 @@ # - Endpoint resources probably have just a name, a service name, and an endpoint # address. -k8sLabelMatcher = re.compile(r'([\w\-_./]+)=\"(.+)\"') +k8sLabelMatcher = re.compile(r"([\w\-_./]+)=\"(.+)\"") class ResourceFetcher: @@ -59,25 +64,40 @@ class ResourceFetcher: k8s_processor: KubernetesProcessor invalid: List[Dict] - def __init__(self, logger: logging.Logger, aconf: 'Config', - skip_init_dir: bool=False, watch_only=False) -> None: + def __init__( + self, logger: logging.Logger, aconf: "Config", skip_init_dir: bool = False, watch_only=False + ) -> None: self.aconf = aconf self.logger = logger - self.manager = ResourceManager(self.logger, self.aconf, DependencyManager([ - ServiceDependency(), - SecretDependency(), - IngressClassesDependency(), - ])) - - self.k8s_processor = DeduplicatingKubernetesProcessor(AggregateKubernetesProcessor([ - CountingKubernetesProcessor(self.aconf, KubernetesGVK.for_knative_networking('Ingress'), 'knative_ingress'), - AmbassadorProcessor(self.manager), - SecretProcessor(self.manager), - IngressClassProcessor(self.manager), - IngressProcessor(self.manager), - ServiceProcessor(self.manager, watch_only=watch_only), - KnativeIngressProcessor(self.manager), - ])) + self.manager = ResourceManager( + self.logger, + self.aconf, + DependencyManager( + [ + ServiceDependency(), + SecretDependency(), + IngressClassesDependency(), + ] + ), + ) + + self.k8s_processor = DeduplicatingKubernetesProcessor( + AggregateKubernetesProcessor( + [ + CountingKubernetesProcessor( + self.aconf, + KubernetesGVK.for_knative_networking("Ingress"), + "knative_ingress", + ), + AmbassadorProcessor(self.manager), + SecretProcessor(self.manager), + IngressClassProcessor(self.manager), + IngressProcessor(self.manager), + ServiceProcessor(self.manager, watch_only=watch_only), + KnativeIngressProcessor(self.manager), + ] + ) + ) self.alerted_about_labels = False @@ -105,17 +125,20 @@ def __init__(self, logger: logging.Logger, aconf: 'Config', # Check /ambassador/init-config for initialization resources -- note NOT # $AMBASSADOR_CONFIG_BASE_DIR/init-config! This is compile-time stuff that # doesn't move around if you change the configuration base. - init_dir = '/ambassador/init-config' + init_dir = "/ambassador/init-config" automatic_manifests = [] edge_stack_mappings_path = os.path.join(init_dir, "edge-stack-mappings.yaml") - if parse_bool(os.environ.get('EDGE_STACK', 'false')) and not os.path.exists(edge_stack_mappings_path): + if parse_bool(os.environ.get("EDGE_STACK", "false")) and not os.path.exists( + edge_stack_mappings_path + ): # HACK # If we're running in Edge Stack via environment variable and the magic "edge-stack-mappings.yaml" file doesn't # exist in its well known location, then go ahead and add it. This should _not_ be necessary under # normal circumstances where Edge Stack is running in its container. We do this so that tests can # run outside of a container with this environment variable set. - automatic_manifests.append(''' + automatic_manifests.append( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -132,10 +155,17 @@ def __init__(self, logger: logging.Logger, aconf: 'Config', rewrite: "" service: "127.0.0.1:8500" precedence: 1000000 -''') +""" + ) if os.path.isdir(init_dir) or len(automatic_manifests) > 0: - self.load_from_filesystem(init_dir, k8s=True, recurse=True, finalize=False, automatic_manifests=automatic_manifests) + self.load_from_filesystem( + init_dir, + k8s=True, + recurse=True, + finalize=False, + automatic_manifests=automatic_manifests, + ) @property def elements(self) -> List[ACResource]: @@ -145,13 +175,18 @@ def elements(self) -> List[ACResource]: def location(self) -> str: return str(self.manager.locations.current) - def load_from_filesystem(self, config_dir_path, recurse: bool=False, - k8s: bool=False, finalize: bool=True, - automatic_manifests: List[str]=[]): + def load_from_filesystem( + self, + config_dir_path, + recurse: bool = False, + k8s: bool = False, + finalize: bool = True, + automatic_manifests: List[str] = [], + ): inputs: List[Tuple[str, str]] = [] if os.path.isdir(config_dir_path): - dirs = [ config_dir_path ] + dirs = [config_dir_path] while dirs: dirpath = dirs.pop(0) @@ -168,7 +203,7 @@ def load_from_filesystem(self, config_dir_path, recurse: bool=False, # self.logger.debug("%s: SKIP non-file" % filepath) continue - if not filename.lower().endswith('.yaml'): + if not filename.lower().endswith(".yaml"): # self.logger.debug("%s: SKIP non-YAML" % filepath) continue @@ -182,7 +217,10 @@ def load_from_filesystem(self, config_dir_path, recurse: bool=False, elif len(automatic_manifests) == 0: # The config_dir_path wasn't a directory nor a file, and there are # no automatic manifests. Nothing to do. - self.logger.debug("no init directory/file at path %s and no automatic manifests, doing nothing" % config_dir_path) + self.logger.debug( + "no init directory/file at path %s and no automatic manifests, doing nothing" + % config_dir_path + ) for filepath, filename in inputs: self.logger.debug("reading %s (%s)" % (filename, filepath)) @@ -203,8 +241,14 @@ def load_from_filesystem(self, config_dir_path, recurse: bool=False, if finalize: self.finalize() - def parse_yaml(self, serialization: str, k8s=False, rkey: Optional[str] = None, - filename: Optional[str] = None, finalize: bool = True) -> None: + def parse_yaml( + self, + serialization: str, + k8s=False, + rkey: Optional[str] = None, + filename: Optional[str] = None, + finalize: bool = True, + ) -> None: # self.logger.info(f"RF YAML: {serialization}") # Expand environment variables allowing interpolation in manifests. @@ -228,32 +272,44 @@ def parse_yaml(self, serialization: str, k8s=False, rkey: Optional[str] = None, if finalize: self.finalize() - def parse_watt(self, serialization: str, finalize: bool=True) -> None: - basedir = os.environ.get('AMBASSADOR_CONFIG_BASE_DIR', '/ambassador') + def parse_watt(self, serialization: str, finalize: bool = True) -> None: + basedir = os.environ.get("AMBASSADOR_CONFIG_BASE_DIR", "/ambassador") - if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_crds')): - self.aconf.post_error("Ambassador could not find core CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored...") + if os.path.isfile(os.path.join(basedir, ".ambassador_ignore_crds")): + self.aconf.post_error( + "Ambassador could not find core CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored..." + ) - if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_crds_2')): - self.aconf.post_error("Ambassador could not find Resolver type CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but ConsulResolver, KubernetesEndpointResolver, and KubernetesServiceResolver resources will be ignored...") + if os.path.isfile(os.path.join(basedir, ".ambassador_ignore_crds_2")): + self.aconf.post_error( + "Ambassador could not find Resolver type CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but ConsulResolver, KubernetesEndpointResolver, and KubernetesServiceResolver resources will be ignored..." + ) - if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_crds_3')): - self.aconf.post_error("Ambassador could not find the Host CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but Host resources will be ignored...") + if os.path.isfile(os.path.join(basedir, ".ambassador_ignore_crds_3")): + self.aconf.post_error( + "Ambassador could not find the Host CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but Host resources will be ignored..." + ) - if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_crds_4')): - self.aconf.post_error("Ambassador could not find the LogService CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but LogService resources will be ignored...") + if os.path.isfile(os.path.join(basedir, ".ambassador_ignore_crds_4")): + self.aconf.post_error( + "Ambassador could not find the LogService CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but LogService resources will be ignored..." + ) - if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_crds_5')): - self.aconf.post_error("Ambassador could not find the DevPortal CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but DevPortal resources will be ignored...") + if os.path.isfile(os.path.join(basedir, ".ambassador_ignore_crds_5")): + self.aconf.post_error( + "Ambassador could not find the DevPortal CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador, but DevPortal resources will be ignored..." + ) # We could be posting errors about the missing IngressClass resource, but given it's new in K8s 1.18 # and we assume most users would be worried about it when running on older clusters, we'll rely on # Ambassador logs "Ambassador does not have permission to read IngressClass resources" for the moment. - #if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_ingress_class')): + # if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_ingress_class')): # self.aconf.post_error("Ambassador is not permitted to read IngressClass resources. Please visit https://www.getambassador.io/user-guide/ingress-controller/ for more information. You can continue using Ambassador, but IngressClass resources will be ignored...") - if os.path.isfile(os.path.join(basedir, '.ambassador_ignore_ingress')): - self.aconf.post_error("Ambassador is not permitted to read Ingress resources. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/running/ingress-controller/#ambassador-as-an-ingress-controller for more information. You can continue using Ambassador, but Ingress resources will be ignored...") + if os.path.isfile(os.path.join(basedir, ".ambassador_ignore_ingress")): + self.aconf.post_error( + "Ambassador is not permitted to read Ingress resources. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/running/ingress-controller/#ambassador-as-an-ingress-controller for more information. You can continue using Ambassador, but Ingress resources will be ignored..." + ) # Expand environment variables allowing interpolation in manifests. serialization = os.path.expandvars(serialization) @@ -264,10 +320,10 @@ def parse_watt(self, serialization: str, finalize: bool=True) -> None: watt_dict = parse_json(serialization) # Grab deltas if they're present... - self.deltas = watt_dict.get('Deltas', []) + self.deltas = watt_dict.get("Deltas", []) # ...then it's off to deal with Kubernetes. - watt_k8s = watt_dict.get('Kubernetes', {}) + watt_k8s = watt_dict.get("Kubernetes", {}) # First, though, let's fold any invalid objects into the main watt_k8s # tree. They're in the "Invalid" dict simply because we don't fully trust @@ -278,10 +334,10 @@ def parse_watt(self, serialization: str, finalize: bool=True) -> None: # processed??? It's because they have error information that we need to # propagate to the user, and this is the simplest way to do that. - self.invalid: List[Dict] = watt_dict.get('Invalid') or [] + self.invalid: List[Dict] = watt_dict.get("Invalid") or [] for obj in self.invalid: - kind = obj.get('kind', None) + kind = obj.get("kind", None) if not kind: # Can't work with this at _all_. @@ -300,7 +356,7 @@ def parse_watt(self, serialization: str, finalize: bool=True) -> None: watt_list.append(obj) # Remove annotations from the snapshot; we'll process them separately. - annotations = watt_k8s.pop('annotations', {}) + annotations = watt_k8s.pop("annotations", {}) # These objects have to be processed first, in order, as they depend # on each other. @@ -317,13 +373,13 @@ def parse_watt(self, serialization: str, finalize: bool=True) -> None: # self.logger.debug(f"Handling Kubernetes {key}...") with self.manager.locations.push_reset(): self.handle_k8s(obj) - if 'errors' not in obj: + if "errors" not in obj: ann_parent_key = f"{obj['kind']}/{obj['metadata']['name']}.{obj['metadata'].get('namespace')}" - for ann_obj in (annotations.get(ann_parent_key) or []): + for ann_obj in annotations.get(ann_parent_key) or []: self.handle_annotation(ann_parent_key, ann_obj) - watt_consul = watt_dict.get('Consul', {}) - consul_endpoints = watt_consul.get('Endpoints', {}) + watt_consul = watt_dict.get("Consul", {}) + consul_endpoints = watt_consul.get("Endpoints", {}) for consul_rkey, consul_object in consul_endpoints.items(): self.handle_consul_service(consul_rkey, consul_object) @@ -334,10 +390,12 @@ def parse_watt(self, serialization: str, finalize: bool=True) -> None: self.finalize() def load_pod_labels(self): - pod_labels_path = '/tmp/ambassador-pod-info/labels' + pod_labels_path = "/tmp/ambassador-pod-info/labels" if not os.path.isfile(pod_labels_path): if not self.alerted_about_labels: - self.aconf.post_error(f"Pod labels are not mounted in the Ambassador container; Kubernetes Ingress support is likely to be limited") + self.aconf.post_error( + f"Pod labels are not mounted in the Ambassador container; Kubernetes Ingress support is likely to be limited" + ) self.alerted_about_labels = True return False @@ -379,16 +437,15 @@ def handle_annotation(self, parent_key: str, raw_obj: dict) -> None: return with self.manager.locations.mark_annotated(): - rkey = parent_key.split('/', 1)[1] + rkey = parent_key.split("/", 1)[1] self.manager.emit(NormalizedResource.from_kubernetes_object(obj, rkey=rkey)) # Handler for Consul services - def handle_consul_service(self, - consul_rkey: str, consul_object: AnyDict) -> None: + def handle_consul_service(self, consul_rkey: str, consul_object: AnyDict) -> None: # resource_identifier = f'consul-{consul_rkey}' - endpoints = consul_object.get('Endpoints', []) - name = consul_object.get('Service', consul_rkey) + endpoints = consul_object.get("Endpoints", []) + name = consul_object.get("Service", consul_rkey) if len(endpoints) < 1: # Bzzt. @@ -404,34 +461,34 @@ def handle_consul_service(self, normalized_endpoints: Dict[str, List[Dict[str, Any]]] = {} for ep in endpoints: - ep_addr = ep.get('Address') - ep_port = ep.get('Port') + ep_addr = ep.get("Address") + ep_port = ep.get("Port") if not ep_addr or not ep_port: - self.logger.debug(f"ignoring Consul service {name} endpoint {ep['ID']} missing address info") + self.logger.debug( + f"ignoring Consul service {name} endpoint {ep['ID']} missing address info" + ) continue # Consul services don't have the weird indirections that Kube services do, so just # lump all the endpoints together under the same source port of '*'. - svc_eps = normalized_endpoints.setdefault('*', []) - svc_eps.append({ - 'ip': ep_addr, - 'port': ep_port, - 'target_kind': 'Consul' - }) + svc_eps = normalized_endpoints.setdefault("*", []) + svc_eps.append({"ip": ep_addr, "port": ep_port, "target_kind": "Consul"}) spec = { - 'ambassador_id': Config.ambassador_id, - 'datacenter': consul_object.get('Id') or 'dc1', - 'endpoints': normalized_endpoints, + "ambassador_id": Config.ambassador_id, + "datacenter": consul_object.get("Id") or "dc1", + "endpoints": normalized_endpoints, } - self.manager.emit(NormalizedResource.from_data( - 'Service', - name, - spec=spec, - rkey=f"consul-{name}-{spec['datacenter']}", - )) + self.manager.emit( + NormalizedResource.from_data( + "Service", + name, + spec=spec, + rkey=f"consul-{name}-{spec['datacenter']}", + ) + ) def finalize(self) -> None: self.k8s_processor.finalize() diff --git a/python/ambassador/fetch/ingress.py b/python/ambassador/fetch/ingress.py index 83e00f944a..d31a548fdf 100644 --- a/python/ambassador/fetch/ingress.py +++ b/python/ambassador/fetch/ingress.py @@ -8,9 +8,9 @@ from .resource import NormalizedResource, ResourceManager -class IngressClassProcessor (ManagedKubernetesProcessor): +class IngressClassProcessor(ManagedKubernetesProcessor): - CONTROLLER: ClassVar[str] = 'getambassador.io/ingress-controller' + CONTROLLER: ClassVar[str] = "getambassador.io/ingress-controller" ingress_classes_dep: IngressClassesDependency @@ -20,19 +20,25 @@ def __init__(self, manager: ResourceManager) -> None: self.ingress_classes_dep = self.deps.provide(IngressClassesDependency) def kinds(self) -> FrozenSet[KubernetesGVK]: - return frozenset([ - KubernetesGVK('networking.k8s.io/v1beta1', 'IngressClass'), - KubernetesGVK('networking.k8s.io/v1', 'IngressClass'), - ]) + return frozenset( + [ + KubernetesGVK("networking.k8s.io/v1beta1", "IngressClass"), + KubernetesGVK("networking.k8s.io/v1", "IngressClass"), + ] + ) def _process(self, obj: KubernetesObject) -> None: # We only want to deal with IngressClasses that belong to "spec.controller: getambassador.io/ingress-controller" - if obj.spec.get('controller', '').lower() != self.CONTROLLER: - self.logger.debug(f'ignoring IngressClass {obj.name} without controller - getambassador.io/ingress-controller') + if obj.spec.get("controller", "").lower() != self.CONTROLLER: + self.logger.debug( + f"ignoring IngressClass {obj.name} without controller - getambassador.io/ingress-controller" + ) return if obj.ambassador_id != Config.ambassador_id: - self.logger.debug(f'IngressClass {obj.name} does not have Ambassador ID {Config.ambassador_id}, ignoring...') + self.logger.debug( + f"IngressClass {obj.name} does not have Ambassador ID {Config.ambassador_id}, ignoring..." + ) return # TODO: Do we intend to use this parameter in any way? @@ -45,10 +51,12 @@ def _process(self, obj: KubernetesObject) -> None: # # It was designed to reference a CRD for this specific ingress-controller # implementation... although usage is optional and not prescribed. - ingress_parameters = obj.spec.get('parameters', {}) + ingress_parameters = obj.spec.get("parameters", {}) - self.logger.debug(f'Handling IngressClass {obj.name} with parameters {ingress_parameters}...') - self.aconf.incr_count('k8s_ingress_class') + self.logger.debug( + f"Handling IngressClass {obj.name} with parameters {ingress_parameters}..." + ) + self.aconf.incr_count("k8s_ingress_class") # Don't emit this directly. We use it when we handle ingresses below. If # we want to use the parameters, we should add them to this dependency @@ -56,7 +64,7 @@ def _process(self, obj: KubernetesObject) -> None: self.ingress_classes_dep.ingress_classes.add(obj.name) -class IngressProcessor (ManagedKubernetesProcessor): +class IngressProcessor(ManagedKubernetesProcessor): service_dep: ServiceDependency ingress_classes_dep: IngressClassesDependency @@ -69,17 +77,21 @@ def __init__(self, manager: ResourceManager) -> None: self.ingress_classes_dep = self.deps.want(IngressClassesDependency) def kinds(self) -> FrozenSet[KubernetesGVK]: - return frozenset([ - KubernetesGVK('extensions/v1beta1', 'Ingress'), - KubernetesGVK('networking.k8s.io/v1beta1', 'Ingress'), - KubernetesGVK('networking.k8s.io/v1', 'Ingress'), - ]) + return frozenset( + [ + KubernetesGVK("extensions/v1beta1", "Ingress"), + KubernetesGVK("networking.k8s.io/v1beta1", "Ingress"), + KubernetesGVK("networking.k8s.io/v1", "Ingress"), + ] + ) def _update_status(self, obj: KubernetesObject) -> None: service_status = None if not self.service_dep.ambassador_service or not self.service_dep.ambassador_service.name: - self.logger.error(f"Unable to set Ingress {obj.name}'s load balancer, could not find Ambassador service") + self.logger.error( + f"Unable to set Ingress {obj.name}'s load balancer, could not find Ambassador service" + ) else: service_status = self.service_dep.ambassador_service.status @@ -87,15 +99,19 @@ def _update_status(self, obj: KubernetesObject) -> None: if service_status: status_update = (obj.gvk.kind, obj.namespace, service_status) self.logger.debug(f"Updating Ingress {obj.name} status to {status_update}") - self.aconf.k8s_status_updates[f'{obj.name}.{obj.namespace}'] = status_update + self.aconf.k8s_status_updates[f"{obj.name}.{obj.namespace}"] = status_update else: - self.logger.debug(f"Not reconciling Ingress {obj.name}: observed and current statuses are in sync") + self.logger.debug( + f"Not reconciling Ingress {obj.name}: observed and current statuses are in sync" + ) def _process(self, obj: KubernetesObject) -> None: - ingress_class_name = obj.spec.get('ingressClassName', '') + ingress_class_name = obj.spec.get("ingressClassName", "") has_ingress_class = ingress_class_name in self.ingress_classes_dep.ingress_classes - has_ambassador_ingress_class_annotation = obj.annotations.get('kubernetes.io/ingress.class', '').lower() == 'ambassador' + has_ambassador_ingress_class_annotation = ( + obj.annotations.get("kubernetes.io/ingress.class", "").lower() == "ambassador" + ) # check the Ingress resource has either: # - a `kubernetes.io/ingress.class: "ambassador"` annotation @@ -107,57 +123,49 @@ def _process(self, obj: KubernetesObject) -> None: # annotations: # ingressclass.kubernetes.io/is-default-class: "true" if not (has_ingress_class or has_ambassador_ingress_class_annotation): - self.logger.debug(f'ignoring Ingress {obj.name} without annotation (kubernetes.io/ingress.class: "ambassador") or IngressClass controller (getambassador.io/ingress-controller)') + self.logger.debug( + f'ignoring Ingress {obj.name} without annotation (kubernetes.io/ingress.class: "ambassador") or IngressClass controller (getambassador.io/ingress-controller)' + ) return # We don't want to deal with non-matching Ambassador IDs if obj.ambassador_id != Config.ambassador_id: - self.logger.debug(f"Ingress {obj.name} does not have Ambassador ID {Config.ambassador_id}, ignoring...") + self.logger.debug( + f"Ingress {obj.name} does not have Ambassador ID {Config.ambassador_id}, ignoring..." + ) return self.logger.debug(f"Handling Ingress {obj.name}...") - self.aconf.incr_count('k8s_ingress') + self.aconf.incr_count("k8s_ingress") # We'll generate an ingress_id to match up this Ingress with its Mappings, but # only if this Ingress defines a Host. If no Host is defined, ingress_id will stay # None. ingress_id: Optional[str] = None - ingress_tls = obj.spec.get('tls', []) + ingress_tls = obj.spec.get("tls", []) for tls_count, tls in enumerate(ingress_tls): # Use the name and namespace to make a unique ID for this Ingress. We'll use # this for matching up this Ingress with its Mappings. ingress_id = f"a10r-ingress-{obj.name}-{obj.namespace}" - tls_secret = tls.get('secretName', None) + tls_secret = tls.get("secretName", None) if tls_secret is not None: - for host_count, host in enumerate(tls.get('hosts', ['*'])): + for host_count, host in enumerate(tls.get("hosts", ["*"])): tls_unique_identifier = f"{obj.name}-{tls_count}-{host_count}" spec = { - 'ambassador_id': [obj.ambassador_id], - 'hostname': host, - 'acmeProvider': { - 'authority': 'none' - }, - 'tlsSecret': { - 'name': tls_secret - }, - 'selector': { - 'matchLabels': { - 'a10r-k8s-ingress': ingress_id - } - }, - 'requestPolicy': { - 'insecure': { - 'action': 'Route' - } - } + "ambassador_id": [obj.ambassador_id], + "hostname": host, + "acmeProvider": {"authority": "none"}, + "tlsSecret": {"name": tls_secret}, + "selector": {"matchLabels": {"a10r-k8s-ingress": ingress_id}}, + "requestPolicy": {"insecure": {"action": "Route"}}, } ingress_host = NormalizedResource.from_data( - 'Host', + "Host", tls_unique_identifier, namespace=obj.namespace, labels=obj.labels, @@ -169,9 +177,9 @@ def _process(self, obj: KubernetesObject) -> None: # parse ingress.spec.defaultBackend # using ingress.spec.backend as a fallback, for older versions of the Ingress resource. - default_backend = obj.spec.get('defaultBackend', obj.spec.get('backend', {})) - db_service_name = default_backend.get('serviceName', None) - db_service_port = default_backend.get('servicePort', None) + default_backend = obj.spec.get("defaultBackend", obj.spec.get("backend", {})) + db_service_name = default_backend.get("serviceName", None) + db_service_port = default_backend.get("servicePort", None) if db_service_name is not None and db_service_port is not None: db_mapping_identifier = f"{obj.name}-default-backend" @@ -181,36 +189,38 @@ def _process(self, obj: KubernetesObject) -> None: mapping_labels["a10r-k8s-ingress"] = ingress_id default_backend_mapping = NormalizedResource.from_data( - 'Mapping', + "Mapping", db_mapping_identifier, namespace=obj.namespace, labels=mapping_labels, spec={ - 'ambassador_id': obj.ambassador_id, - 'hostname': '*', - 'prefix': '/', - 'service': f'{db_service_name}.{obj.namespace}:{db_service_port}' + "ambassador_id": obj.ambassador_id, + "hostname": "*", + "prefix": "/", + "service": f"{db_service_name}.{obj.namespace}:{db_service_port}", }, ) - self.logger.debug(f"Generated Mapping from Ingress {obj.name}: {default_backend_mapping}") + self.logger.debug( + f"Generated Mapping from Ingress {obj.name}: {default_backend_mapping}" + ) self.manager.emit(default_backend_mapping) # parse ingress.spec.rules - ingress_rules = obj.spec.get('rules', []) + ingress_rules = obj.spec.get("rules", []) for rule_count, rule in enumerate(ingress_rules): - rule_http = rule.get('http', {}) + rule_http = rule.get("http", {}) - rule_host = rule.get('host', None) + rule_host = rule.get("host", None) - http_paths = rule_http.get('paths', []) + http_paths = rule_http.get("paths", []) for path_count, path in enumerate(http_paths): - path_backend = path.get('backend', {}) - path_type = path.get('pathType', 'ImplementationSpecific') + path_backend = path.get("backend", {}) + path_type = path.get("pathType", "ImplementationSpecific") - service_name = path_backend.get('serviceName', None) - service_port = path_backend.get('servicePort', None) - path_location = path.get('path', '/') + service_name = path_backend.get("serviceName", None) + service_port = path_backend.get("servicePort", None) + path_location = path.get("path", "/") if not service_name or not service_port or not path_location: continue @@ -220,33 +230,38 @@ def _process(self, obj: KubernetesObject) -> None: # For cases where `pathType: Exact`, # otherwise `Prefix` and `ImplementationSpecific` are handled as regular Mapping prefixes - is_exact_prefix = True if path_type == 'Exact' else False + is_exact_prefix = True if path_type == "Exact" else False spec = { - 'ambassador_id': obj.ambassador_id, - 'prefix': path_location, - 'prefix_exact': is_exact_prefix, - 'precedence': 1 if is_exact_prefix else 0, # Make sure exact paths are evaluated before prefix - 'service': f'{service_name}.{obj.namespace}:{service_port}' + "ambassador_id": obj.ambassador_id, + "prefix": path_location, + "prefix_exact": is_exact_prefix, + "precedence": 1 + if is_exact_prefix + else 0, # Make sure exact paths are evaluated before prefix + "service": f"{service_name}.{obj.namespace}:{service_port}", } if rule_host is not None: - if rule_host.startswith('*.'): + if rule_host.startswith("*."): # Ingress allow specifying hosts with a single wildcard as the first label in the hostname. # Transform the rule_host into a host_regex: # *.star.com becomes ^[a-z0-9]([-a-z0-9]*[a-z0-9])?\.star\.com$ - spec['host'] = rule_host\ - .replace('.', '\\.')\ - .replace('*', '^[a-z0-9]([-a-z0-9]*[a-z0-9])?', 1) + '$' - spec['host_regex'] = True + spec["host"] = ( + rule_host.replace(".", "\\.").replace( + "*", "^[a-z0-9]([-a-z0-9]*[a-z0-9])?", 1 + ) + + "$" + ) + spec["host_regex"] = True else: # Use hostname since this can be a hostname of "*" too. - spec['hostname'] = rule_host + spec["hostname"] = rule_host else: # If there's no rule_host, and we don't have an ingress_id, force a hostname # of "*" so that the Mapping we generate doesn't get dropped. if not ingress_id: - spec['hostname'] = "*" + spec["hostname"] = "*" mapping_labels = dict(obj.labels) @@ -254,7 +269,7 @@ def _process(self, obj: KubernetesObject) -> None: mapping_labels["a10r-k8s-ingress"] = ingress_id path_mapping = NormalizedResource.from_data( - 'Mapping', + "Mapping", mapping_identifier, namespace=obj.namespace, labels=mapping_labels, diff --git a/python/ambassador/fetch/k8sobject.py b/python/ambassador/fetch/k8sobject.py index 7ad848a3cf..53d6a4965f 100644 --- a/python/ambassador/fetch/k8sobject.py +++ b/python/ambassador/fetch/k8sobject.py @@ -22,35 +22,35 @@ def api_group(self) -> Optional[str]: # These are backward-indexed to support apiVersion: v1, which has a # version but no group. try: - return self.api_version.split('/', 1)[-2] + return self.api_version.split("/", 1)[-2] except IndexError: return None @property def version(self) -> str: - return self.api_version.split('/', 1)[-1] + return self.api_version.split("/", 1)[-1] @property def domain(self) -> str: if self.api_group: - return f'{self.kind.lower()}.{self.api_group}' + return f"{self.kind.lower()}.{self.api_group}" else: return self.kind.lower() @classmethod - def for_ambassador(cls, kind: str, version: str = 'v2') -> KubernetesGVK: - if 'alpha' in version: - return cls(f'getambassador.io/{version}', kind) + def for_ambassador(cls, kind: str, version: str = "v2") -> KubernetesGVK: + if "alpha" in version: + return cls(f"getambassador.io/{version}", kind) else: - return cls(f'getambassador.io/{version}', kind) + return cls(f"getambassador.io/{version}", kind) @classmethod def for_knative_networking(cls, kind: str) -> KubernetesGVK: - return cls('networking.internal.knative.dev/v1alpha1', kind) + return cls("networking.internal.knative.dev/v1alpha1", kind) @enum.unique -class KubernetesObjectScope (enum.Enum): +class KubernetesObjectScope(enum.Enum): CLUSTER = enum.auto() NAMESPACE = enum.auto() @@ -71,14 +71,18 @@ def kind(self) -> str: @property def scope(self) -> KubernetesObjectScope: - return KubernetesObjectScope.CLUSTER if self.namespace is None else KubernetesObjectScope.NAMESPACE + return ( + KubernetesObjectScope.CLUSTER + if self.namespace is None + else KubernetesObjectScope.NAMESPACE + ) @classmethod def from_object_reference(cls, ref: Dict[str, Any]) -> KubernetesObjectKey: - return cls(KubernetesGVK('v1', ref['kind']), ref.get('namespace'), ref['name']) + return cls(KubernetesGVK("v1", ref["kind"]), ref.get("namespace"), ref["name"]) -class KubernetesObject (collections.abc.Mapping): +class KubernetesObject(collections.abc.Mapping): """ Represents a raw object from Kubernetes. """ @@ -90,7 +94,7 @@ def __init__(self, delegate: Dict[str, Any]) -> None: self.gvk self.name except KeyError: - raise ValueError('delegate is not a valid Kubernetes object') + raise ValueError("delegate is not a valid Kubernetes object") def __getitem__(self, key: str) -> Any: return self.delegate[key] @@ -103,7 +107,7 @@ def __len__(self) -> int: @property def gvk(self) -> KubernetesGVK: - return KubernetesGVK(self['apiVersion'], self['kind']) + return KubernetesGVK(self["apiVersion"], self["kind"]) @property def kind(self) -> str: @@ -111,21 +115,23 @@ def kind(self) -> str: @property def metadata(self) -> Dict[str, Any]: - return self['metadata'] + return self["metadata"] @property def namespace(self) -> str: - val = self.metadata.get('namespace') - if val == '_automatic_': + val = self.metadata.get("namespace") + if val == "_automatic_": val = Config.ambassador_namespace elif val is None: - raise AttributeError(f'{self.__class__.__name__} {self.gvk.domain} {self.name} has no namespace (it is cluster-scoped)') + raise AttributeError( + f"{self.__class__.__name__} {self.gvk.domain} {self.name} has no namespace (it is cluster-scoped)" + ) return val @property def name(self) -> str: - return self.metadata['name'] + return self.metadata["name"] @property def key(self) -> KubernetesObjectKey: @@ -142,24 +148,24 @@ def scope(self) -> KubernetesObjectScope: @property def generation(self) -> int: - return self.metadata.get('generation', 1) + return self.metadata.get("generation", 1) @property def annotations(self) -> Dict[str, str]: - return self.metadata.get('annotations', {}) + return self.metadata.get("annotations", {}) @property def ambassador_id(self) -> str: - return self.annotations.get('getambassador.io/ambassador-id', 'default') + return self.annotations.get("getambassador.io/ambassador-id", "default") @property def labels(self) -> Dict[str, str]: - return self.metadata.get('labels', {}) + return self.metadata.get("labels", {}) @property def spec(self) -> Dict[str, Any]: - return self.get('spec', {}) + return self.get("spec", {}) @property def status(self) -> Dict[str, Any]: - return self.get('status', {}) + return self.get("status", {}) diff --git a/python/ambassador/fetch/k8sprocessor.py b/python/ambassador/fetch/k8sprocessor.py index 013945c68b..6af9b94805 100644 --- a/python/ambassador/fetch/k8sprocessor.py +++ b/python/ambassador/fetch/k8sprocessor.py @@ -54,7 +54,7 @@ def finalize(self) -> None: pass -class ManagedKubernetesProcessor (KubernetesProcessor): +class ManagedKubernetesProcessor(KubernetesProcessor): """ An abstract processor that provides access to a resource manager. """ @@ -77,7 +77,7 @@ def deps(self) -> DependencyInjector: return self.manager.deps.for_instance(self) -class AggregateKubernetesProcessor (KubernetesProcessor): +class AggregateKubernetesProcessor(KubernetesProcessor): """ This processor aggregates many other processors into a single convenient processor. @@ -107,7 +107,7 @@ def finalize(self) -> None: proc.finalize() -class DeduplicatingKubernetesProcessor (KubernetesProcessor): +class DeduplicatingKubernetesProcessor(KubernetesProcessor): """ This processor delegates work to another processor but prevents the same Kubernetes object from being processed multiple times. @@ -134,7 +134,7 @@ def finalize(self) -> None: self.delegate.finalize() -class CountingKubernetesProcessor (KubernetesProcessor): +class CountingKubernetesProcessor(KubernetesProcessor): """ This processor increments a given configuration counter when it receives an object. diff --git a/python/ambassador/fetch/knative.py b/python/ambassador/fetch/knative.py index cac9db8384..e5231cdf44 100644 --- a/python/ambassador/fetch/knative.py +++ b/python/ambassador/fetch/knative.py @@ -14,12 +14,12 @@ from .resource import NormalizedResource, ResourceManager -class KnativeIngressProcessor (ManagedKubernetesProcessor): +class KnativeIngressProcessor(ManagedKubernetesProcessor): """ A Kubernetes object processor that emits mappings from Knative Ingresses. """ - INGRESS_CLASS: ClassVar[str] = 'ambassador.ingress.networking.knative.dev' + INGRESS_CLASS: ClassVar[str] = "ambassador.ingress.networking.knative.dev" service_dep: ServiceDependency @@ -29,7 +29,7 @@ def __init__(self, manager: ResourceManager): self.service_dep = self.deps.want(ServiceDependency) def kinds(self) -> FrozenSet[KubernetesGVK]: - return frozenset([KubernetesGVK.for_knative_networking('Ingress')]) + return frozenset([KubernetesGVK.for_knative_networking("Ingress")]) def _has_required_annotations(self, obj: KubernetesObject) -> bool: annotations = obj.annotations @@ -38,59 +38,69 @@ def _has_required_annotations(self, obj: KubernetesObject) -> bool: # to ignore KnativeIngress iff networking.knative.dev/ingress.class is # present in annotation. If it's not there, then we accept all ingress # classes. - ingress_class = annotations.get('networking.knative.dev/ingress.class', self.INGRESS_CLASS) + ingress_class = annotations.get("networking.knative.dev/ingress.class", self.INGRESS_CLASS) if ingress_class.lower() != self.INGRESS_CLASS: - self.logger.debug(f'Ignoring Knative {obj.kind} {obj.name}; set networking.knative.dev/ingress.class ' - f'annotation to {self.INGRESS_CLASS} for ambassador to parse it.') + self.logger.debug( + f"Ignoring Knative {obj.kind} {obj.name}; set networking.knative.dev/ingress.class " + f"annotation to {self.INGRESS_CLASS} for ambassador to parse it." + ) return False # We don't want to deal with non-matching Ambassador IDs if obj.ambassador_id != Config.ambassador_id: - self.logger.info(f"Knative {obj.kind} {obj.name} does not have Ambassador ID {Config.ambassador_id}, ignoring...") + self.logger.info( + f"Knative {obj.kind} {obj.name} does not have Ambassador ID {Config.ambassador_id}, ignoring..." + ) return False return True def _emit_mapping(self, obj: KubernetesObject, rule_count: int, rule: Dict[str, Any]) -> None: - hosts = rule.get('hosts', []) + hosts = rule.get("hosts", []) split_mapping_specs: List[Dict[str, Any]] = [] - paths = rule.get('http', {}).get('paths', []) + paths = rule.get("http", {}).get("paths", []) for path in paths: - global_headers = path.get('appendHeaders', {}) + global_headers = path.get("appendHeaders", {}) - splits = path.get('splits', []) + splits = path.get("splits", []) for split in splits: - service_name = split.get('serviceName') + service_name = split.get("serviceName") if not service_name: continue - service_namespace = split.get('serviceNamespace', obj.namespace) - service_port = split.get('servicePort', 80) + service_namespace = split.get("serviceNamespace", obj.namespace) + service_port = split.get("servicePort", 80) - headers = split.get('appendHeaders', {}) + headers = split.get("appendHeaders", {}) headers = {**global_headers, **headers} - split_mapping_specs.append({ - 'service': f"{service_name}.{service_namespace}:{service_port}", - 'add_request_headers': headers, - 'weight': split.get('percent', 100), - 'prefix': path.get('path', '/'), - 'timeout_ms': int(durationpy.from_str(path.get('timeout', '15s')).total_seconds() * 1000), - }) + split_mapping_specs.append( + { + "service": f"{service_name}.{service_namespace}:{service_port}", + "add_request_headers": headers, + "weight": split.get("percent", 100), + "prefix": path.get("path", "/"), + "timeout_ms": int( + durationpy.from_str(path.get("timeout", "15s")).total_seconds() * 1000 + ), + } + ) - for split_count, (host, split_mapping_spec) in enumerate(itertools.product(hosts, split_mapping_specs)): + for split_count, (host, split_mapping_spec) in enumerate( + itertools.product(hosts, split_mapping_specs) + ): mapping_identifier = f"{obj.name}-{rule_count}-{split_count}" spec = { - 'ambassador_id': obj.ambassador_id, - 'host': host, + "ambassador_id": obj.ambassador_id, + "host": host, } spec.update(split_mapping_spec) mapping = NormalizedResource.from_data( - 'Mapping', + "Mapping", mapping_identifier, namespace=obj.namespace, generation=obj.generation, @@ -106,22 +116,10 @@ def _make_status(self, generation: int = 1, lb_domain: Optional[str] = None) -> status = { "observedGeneration": generation, "conditions": [ - { - "lastTransitionTime": utcnow, - "status": "True", - "type": "LoadBalancerReady" - }, - { - "lastTransitionTime": utcnow, - "status": "True", - "type": "NetworkConfigured" - }, - { - "lastTransitionTime": utcnow, - "status": "True", - "type": "Ready" - } - ] + {"lastTransitionTime": utcnow, "status": "True", "type": "LoadBalancerReady"}, + {"lastTransitionTime": utcnow, "status": "True", "type": "NetworkConfigured"}, + {"lastTransitionTime": utcnow, "status": "True", "type": "Ready"}, + ], } if lb_domain: @@ -133,13 +131,13 @@ def _make_status(self, generation: int = 1, lb_domain: Optional[str] = None) -> ] } - status['loadBalancer'] = load_balancer - status['privateLoadBalancer'] = load_balancer + status["loadBalancer"] = load_balancer + status["privateLoadBalancer"] = load_balancer return status def _update_status(self, obj: KubernetesObject) -> None: - has_new_generation = obj.generation > obj.status.get('observedGeneration', 0) + has_new_generation = obj.generation > obj.status.get("observedGeneration", 0) # Knative expects the load balancer information on the ingress, which it # then propagates to an ExternalName service for intra-cluster use. We @@ -149,7 +147,9 @@ def _update_status(self, obj: KubernetesObject) -> None: current_lb_domain = None if not self.service_dep.ambassador_service or not self.service_dep.ambassador_service.name: - self.logger.warning(f"Unable to set Knative {obj.kind} {obj.name}'s load balancer, could not find Ambassador service") + self.logger.warning( + f"Unable to set Knative {obj.kind} {obj.name}'s load balancer, could not find Ambassador service" + ) else: # TODO: It is technically possible to use a domain other than # cluster.local (common-ish on bare metal clusters). We can resolve @@ -158,8 +158,10 @@ def _update_status(self, obj: KubernetesObject) -> None: # code as well and probably should just be fixed all at once. current_lb_domain = f"{self.service_dep.ambassador_service.name}.{self.service_dep.ambassador_service.namespace}.svc.cluster.local" - observed_ingress: Dict[str, Any] = next(iter(obj.status.get('privateLoadBalancer', {}).get('ingress', [])), {}) - observed_lb_domain = observed_ingress.get('domainInternal') + observed_ingress: Dict[str, Any] = next( + iter(obj.status.get("privateLoadBalancer", {}).get("ingress", [])), {} + ) + observed_lb_domain = observed_ingress.get("domainInternal") has_new_lb_domain = current_lb_domain != observed_lb_domain @@ -168,16 +170,20 @@ def _update_status(self, obj: KubernetesObject) -> None: if status: status_update = (obj.gvk.domain, obj.namespace, status) - self.logger.info(f"Updating Knative {obj.kind} {obj.name} status to {status_update}") + self.logger.info( + f"Updating Knative {obj.kind} {obj.name} status to {status_update}" + ) self.aconf.k8s_status_updates[f"{obj.name}.{obj.namespace}"] = status_update else: - self.logger.debug(f"Not reconciling Knative {obj.kind} {obj.name}: observed and current generations are in sync") + self.logger.debug( + f"Not reconciling Knative {obj.kind} {obj.name}: observed and current generations are in sync" + ) def _process(self, obj: KubernetesObject) -> None: if not self._has_required_annotations(obj): return - rules = obj.spec.get('rules', []) + rules = obj.spec.get("rules", []) for rule_count, rule in enumerate(rules): self._emit_mapping(obj, rule_count, rule) diff --git a/python/ambassador/fetch/location.py b/python/ambassador/fetch/location.py index e36d6787f8..9490b28641 100644 --- a/python/ambassador/fetch/location.py +++ b/python/ambassador/fetch/location.py @@ -13,7 +13,7 @@ class Location: filename: Optional[str] = None ocount: int = 1 - def filename_default(self, default: str = 'anonymous YAML') -> str: + def filename_default(self, default: str = "anonymous YAML") -> str: return self.filename or default def __str__(self) -> str: @@ -65,8 +65,8 @@ def mark_annotated(self) -> ContextManager[Location]: filename. """ previous_filename = self.current.filename - if self.current.filename and not self.current.filename.endswith(':annotation'): - self.current.filename += ':annotation' + if self.current.filename and not self.current.filename.endswith(":annotation"): + self.current.filename += ":annotation" @contextlib.contextmanager def cleaner(): diff --git a/python/ambassador/fetch/resource.py b/python/ambassador/fetch/resource.py index 636da6d7cb..88697c2170 100644 --- a/python/ambassador/fetch/resource.py +++ b/python/ambassador/fetch/resource.py @@ -21,43 +21,54 @@ class NormalizedResource: object: dict rkey: Optional[str] = None - log_resources: ClassVar[bool] = parse_bool(os.environ.get('AMBASSADOR_LOG_RESOURCES')) + log_resources: ClassVar[bool] = parse_bool(os.environ.get("AMBASSADOR_LOG_RESOURCES")) @classmethod - def from_data(cls, kind: str, name: str, namespace: Optional[str] = None, - generation: Optional[int] = None, version: str = 'v3alpha1', - api_group = 'getambassador.io', - labels: Optional[Dict[str, Any]] = None, - spec: Dict[str, Any] = None, errors: Optional[str] = None, - rkey: Optional[str] = None) -> NormalizedResource: + def from_data( + cls, + kind: str, + name: str, + namespace: Optional[str] = None, + generation: Optional[int] = None, + version: str = "v3alpha1", + api_group="getambassador.io", + labels: Optional[Dict[str, Any]] = None, + spec: Dict[str, Any] = None, + errors: Optional[str] = None, + rkey: Optional[str] = None, + ) -> NormalizedResource: if rkey is None: - rkey = f'{name}.{namespace}' + rkey = f"{name}.{namespace}" ir_obj = {} if spec: ir_obj.update(spec) - ir_obj['apiVersion'] = f'{api_group}/{version}' - ir_obj['kind'] = kind - ir_obj['name'] = name + ir_obj["apiVersion"] = f"{api_group}/{version}" + ir_obj["kind"] = kind + ir_obj["name"] = name if namespace is not None: - ir_obj['namespace'] = namespace + ir_obj["namespace"] = namespace if generation is not None: - ir_obj['generation'] = generation + ir_obj["generation"] = generation - ir_obj['metadata_labels'] = labels or {} + ir_obj["metadata_labels"] = labels or {} if errors: - ir_obj['errors'] = errors + ir_obj["errors"] = errors return cls(ir_obj, rkey) @classmethod - def from_kubernetes_object(cls, obj: KubernetesObject, rkey: Optional[str] = None) -> NormalizedResource: + def from_kubernetes_object( + cls, obj: KubernetesObject, rkey: Optional[str] = None + ) -> NormalizedResource: if obj.namespace is None: - raise ValueError(f'Cannot construct resource from Kubernetes object {obj.key} without namespace') + raise ValueError( + f"Cannot construct resource from Kubernetes object {obj.key} without namespace" + ) labels = dict(obj.labels) @@ -67,10 +78,10 @@ def from_kubernetes_object(cls, obj: KubernetesObject, rkey: Optional[str] = Non # Some other code uses the 'ambassador_crd' label to know which resource to update # .status for with the apiserver. Which is (IMO) a horrible hack, but I'm not up for # changing it at the moment. - labels['ambassador_crd'] = rkey + labels["ambassador_crd"] = rkey else: # Don't let it think that an annotation can have its status updated. - labels.pop('ambassador_crd', None) + labels.pop("ambassador_crd", None) # When creating an Ambassador object from a Kubernetes object, we have to make # sure that we pay attention to 'errors', which will be set IFF watt's validation @@ -79,7 +90,7 @@ def from_kubernetes_object(cls, obj: KubernetesObject, rkey: Optional[str] = Non return cls.from_data( obj.kind, obj.name, - errors=obj.get('errors'), + errors=obj.get("errors"), namespace=obj.namespace, generation=obj.generation, version=obj.gvk.version, @@ -121,24 +132,26 @@ def _emit(self, resource: NormalizedResource) -> bool: if not obj: self.aconf.post_error("%s is empty" % self.location) else: - self.aconf.post_error("%s is not a dictionary? %s" % - (self.location, dump_json(obj, pretty=True))) + self.aconf.post_error( + "%s is not a dictionary? %s" % (self.location, dump_json(obj, pretty=True)) + ) return True if not self.aconf.good_ambassador_id(obj): self.logger.debug("%s ignoring object with mismatched ambassador_id" % self.location) return True - if 'kind' not in obj: + if "kind" not in obj: # Bug!! - self.aconf.post_error("%s is missing 'kind'?? %s" % - (self.location, dump_json(obj, pretty=True))) + self.aconf.post_error( + "%s is missing 'kind'?? %s" % (self.location, dump_json(obj, pretty=True)) + ) return True # Is this a pragma object? - if obj['kind'] == 'Pragma': + if obj["kind"] == "Pragma": # Why did I think this was a good idea? [ :) ] - new_source = obj.get('source', None) + new_source = obj.get("source", None) if new_source: # We don't save the old self.filename here, so this change will last until @@ -149,9 +162,9 @@ def _emit(self, resource: NormalizedResource) -> bool: return False if not rkey: - rkey = self.locations.current.filename_default('unknown') + rkey = self.locations.current.filename_default("unknown") - if obj['kind'] != 'Service': + if obj["kind"] != "Service": # Services are unique and don't get an object count appended to # them. rkey = "%s.%d" % (rkey, self.locations.current.ocount) @@ -165,7 +178,9 @@ def _emit(self, resource: NormalizedResource) -> bool: self.aconf.post_error(e.args[0]) if NormalizedResource.log_resources: - self.logger.debug("%s PROCESS %s save %s: %s", self.location, obj['kind'], rkey, serialization) + self.logger.debug( + "%s PROCESS %s save %s: %s", self.location, obj["kind"], rkey, serialization + ) return True diff --git a/python/ambassador/fetch/secret.py b/python/ambassador/fetch/secret.py index 2cb00b8a12..00331c1657 100644 --- a/python/ambassador/fetch/secret.py +++ b/python/ambassador/fetch/secret.py @@ -9,25 +9,26 @@ from ..utils import dump_json -class SecretProcessor (ManagedKubernetesProcessor): + +class SecretProcessor(ManagedKubernetesProcessor): """ A Kubernetes object processor that emits Ambassador secrets from Kubernetes secrets. """ KNOWN_TYPES = [ - 'Opaque', - 'kubernetes.io/tls', - 'istio.io/key-and-cert', + "Opaque", + "kubernetes.io/tls", + "istio.io/key-and-cert", ] KNOWN_DATA_KEYS = [ - 'tls.crt', # type="kubernetes.io/tls" - 'tls.key', # type="kubernetes.io/tls" - 'user.key', # type="Opaque", used for AES ACME - 'cert-chain.pem', # type="istio.io/key-and-cert" - 'key.pem', # type="istio.io/key-and-cert" - 'root-cert.pem', # type="istio.io/key-and-cert" - 'crl.pem', # type="Opaque", used for TLS CRL + "tls.crt", # type="kubernetes.io/tls" + "tls.key", # type="kubernetes.io/tls" + "user.key", # type="Opaque", used for AES ACME + "cert-chain.pem", # type="istio.io/key-and-cert" + "key.pem", # type="istio.io/key-and-cert" + "root-cert.pem", # type="istio.io/key-and-cert" + "crl.pem", # type="Opaque", used for TLS CRL ] def __init__(self, manager: ResourceManager) -> None: @@ -36,7 +37,7 @@ def __init__(self, manager: ResourceManager) -> None: self.deps.provide(SecretDependency) def kinds(self) -> FrozenSet[KubernetesGVK]: - return frozenset([KubernetesGVK('v1', 'Secret')]) + return frozenset([KubernetesGVK("v1", "Secret")]) def _admit(self, obj: KubernetesObject) -> bool: if not Config.certs_single_namespace: @@ -47,34 +48,36 @@ def _admit(self, obj: KubernetesObject) -> bool: def _process(self, obj: KubernetesObject) -> None: # self.logger.debug("processing K8s Secret %s", dump_json(dict(obj), pretty=True)) - secret_type = obj.get('type') + secret_type = obj.get("type") if secret_type not in self.KNOWN_TYPES: self.logger.debug("ignoring K8s Secret with unknown type %s" % secret_type) return - data = obj.get('data') + data = obj.get("data") if not data: self.logger.debug("ignoring K8s Secret with no data") return if not any(key in data for key in self.KNOWN_DATA_KEYS): # Uh. WTFO? - self.logger.debug(f'ignoring K8s Secret {obj.name}.{obj.namespace} with no keys') + self.logger.debug(f"ignoring K8s Secret {obj.name}.{obj.namespace} with no keys") return spec = { - 'ambassador_id': Config.ambassador_id, - 'secret_type': secret_type, + "ambassador_id": Config.ambassador_id, + "secret_type": secret_type, } for key, value in data.items(): - spec[key.replace('.', '_')] = value - - self.manager.emit(NormalizedResource.from_data( - 'Secret', - obj.name, - namespace=obj.namespace, - labels=obj.labels, - spec=spec, - errors=obj.get('errors'), # Make sure we preserve errors here! - )) + spec[key.replace(".", "_")] = value + + self.manager.emit( + NormalizedResource.from_data( + "Secret", + obj.name, + namespace=obj.namespace, + labels=obj.labels, + spec=spec, + errors=obj.get("errors"), # Make sure we preserve errors here! + ) + ) diff --git a/python/ambassador/fetch/service.py b/python/ambassador/fetch/service.py index f232e516ea..3c93c43539 100644 --- a/python/ambassador/fetch/service.py +++ b/python/ambassador/fetch/service.py @@ -36,7 +36,7 @@ class Endpoints: labels: Dict[str, str] -class InternalServiceProcessor (ManagedKubernetesProcessor): +class InternalServiceProcessor(ManagedKubernetesProcessor): """ An internal Kubernetes object processor for services. Used by the ServiceProcessor aggregate class. @@ -54,14 +54,14 @@ def __init__(self, manager: ResourceManager) -> None: self.discovered_services = {} def kinds(self) -> FrozenSet[KubernetesGVK]: - return frozenset([KubernetesGVK('v1', 'Service')]) + return frozenset([KubernetesGVK("v1", "Service")]) def _is_ambassador_service(self, obj: KubernetesObject) -> bool: - selector = obj.spec.get('selector', {}) + selector = obj.spec.get("selector", {}) # self.logger.info(f"is_ambassador_service checking {obj.labels} - {selector}") # Every Ambassador service must have the label 'app.kubernetes.io/component: ambassador-service' - if obj.labels.get('app.kubernetes.io/component', "").lower() != 'ambassador-service': + if obj.labels.get("app.kubernetes.io/component", "").lower() != "ambassador-service": return False # This service must be in the same namespace as the Ambassador deployment. @@ -88,12 +88,14 @@ def _process(self, obj: KubernetesObject) -> None: # # Again, we're trusting that the input isn't overly bloated on that latter bit. - chart_version = obj.labels.get('helm.sh/chart') + chart_version = obj.labels.get("helm.sh/chart") if chart_version and not self.helm_chart: self.helm_chart = chart_version - if not obj.spec.get('ports'): - self.logger.debug(f"not saving Kubernetes Service {obj.name}.{obj.namespace} with no ports") + if not obj.spec.get("ports"): + self.logger.debug( + f"not saving Kubernetes Service {obj.name}.{obj.namespace} with no ports" + ) else: self.discovered_services[obj.key] = obj @@ -102,7 +104,7 @@ def _process(self, obj: KubernetesObject) -> None: self.service_dep.ambassador_service = obj -class InternalEndpointsProcessor (ManagedKubernetesProcessor): +class InternalEndpointsProcessor(ManagedKubernetesProcessor): """ This processor discovers endpoints, extracts information we care about, and stores the data for referencing by another processor. @@ -116,12 +118,14 @@ def __init__(self, manager: ResourceManager) -> None: self.discovered_endpoints = {} def kinds(self) -> FrozenSet[KubernetesGVK]: - return frozenset([KubernetesGVK('v1', 'Endpoints')]) + return frozenset([KubernetesGVK("v1", "Endpoints")]) def _process(self, obj: KubernetesObject) -> None: - resource_subsets = obj.get('subsets') + resource_subsets = obj.get("subsets") if not resource_subsets: - self.logger.debug(f"ignoring Kubernetes Endpoints {obj.name}.{obj.namespace} with no subsets") + self.logger.debug( + f"ignoring Kubernetes Endpoints {obj.name}.{obj.namespace} with no subsets" + ) return # K8s Endpoints resources are _stupid_ in that they give you a vector of @@ -146,34 +150,38 @@ def _process(self, obj: KubernetesObject) -> None: addresses: List[EndpointAddress] = [] - for address in subset.get('addresses', []): - ip = address.get('ip') + for address in subset.get("addresses", []): + ip = address.get("ip") if not ip: continue target_ref: Optional[KubernetesObjectKey] = None try: - target_ref = KubernetesObjectKey.from_object_reference(address.get('targetRef', {})) + target_ref = KubernetesObjectKey.from_object_reference( + address.get("targetRef", {}) + ) except KeyError: pass - addresses.append(EndpointAddress(ip, node=address.get('nodeName'), target=target_ref)) + addresses.append( + EndpointAddress(ip, node=address.get("nodeName"), target=target_ref) + ) # If we got no addresses, there's no point in messing with ports. if len(addresses) == 0: continue - ports = subset.get('ports', []) + ports = subset.get("ports", []) # A service can reference a port either by name or by port number. port_dict: Dict[str, int] = {} for port in ports: - port_name = port.get('name', None) - port_number = port.get('port', None) - port_proto = port.get('protocol', 'TCP').upper() + port_name = port.get("name", None) + port_number = port.get("port", None) + port_proto = port.get("protocol", "TCP").upper() - if port_proto != 'TCP': + if port_proto != "TCP": continue if port_number is None: @@ -186,13 +194,15 @@ def _process(self, obj: KubernetesObject) -> None: port_dict[port_name] = port_number if not port_dict: - self.logger.debug(f"ignoring K8s Endpoints {obj.name}.{obj.namespace} with no routable ports") + self.logger.debug( + f"ignoring K8s Endpoints {obj.name}.{obj.namespace} with no routable ports" + ) continue self.discovered_endpoints[obj.key] = Endpoints(addresses, port_dict, obj.labels) -class ServiceProcessor (ManagedKubernetesProcessor): +class ServiceProcessor(ManagedKubernetesProcessor): """ This processor handles Service and Endpoints objects and creates relevant Ambassador service resources. @@ -251,7 +261,7 @@ def finalize(self) -> None: # self.logger.debug("==== FINALIZE START\n%s" % dump_json(od, pretty=True)) for k8s_svc in self.services.discovered_services.values(): - key = f'{k8s_svc.name}.{k8s_svc.namespace}' + key = f"{k8s_svc.name}.{k8s_svc.namespace}" target_ports = {} target_addrs = [] @@ -259,7 +269,9 @@ def finalize(self) -> None: if not self.watch_only: # If we're not in watch mode, try to find endpoints for this service. - k8s_ep_key = KubernetesObjectKey(KubernetesGVK('v1', 'Endpoints'), k8s_svc.namespace, k8s_svc.name) + k8s_ep_key = KubernetesObjectKey( + KubernetesGVK("v1", "Endpoints"), k8s_svc.namespace, k8s_svc.name + ) k8s_ep = self.endpoints.discovered_endpoints.get(k8s_ep_key) # OK, Kube is weird. The way all this works goes like this: @@ -294,20 +306,22 @@ def finalize(self) -> None: if not k8s_ep: # No endpoints at all, so we're done with this service. - self.logger.debug(f'{key}: no endpoints at all') + self.logger.debug(f"{key}: no endpoints at all") else: idx = -1 - for port in k8s_svc.spec.get('ports', []): + for port in k8s_svc.spec.get("ports", []): idx += 1 k8s_target: Optional[int] = None - src_port = port.get('port', None) + src_port = port.get("port", None) if not src_port: # WTFO. This is impossible. - self.logger.error(f"Kubernetes service {key} has no port number at index {idx}?") + self.logger.error( + f"Kubernetes service {key} has no port number at index {idx}?" + ) continue if len(k8s_ep.ports) == 1: @@ -315,7 +329,9 @@ def finalize(self) -> None: k8s_target = next(iter(k8s_ep.ports.values())) target_ports[src_port] = k8s_target - self.logger.debug(f'{key} port {src_port}: single endpoint port {k8s_target}') + self.logger.debug( + f"{key} port {src_port}: single endpoint port {k8s_target}" + ) continue # Hmmm, we need to try to actually map whatever ports are listed for @@ -324,13 +340,19 @@ def finalize(self) -> None: found_key = False fallback: Optional[int] = None - for attr in ['targetPort', 'name', 'port']: - port_key = port.get(attr) # This could be a name or a number, in general. + for attr in ["targetPort", "name", "port"]: + port_key = port.get( + attr + ) # This could be a name or a number, in general. if port_key: found_key = True - if not fallback and (port_key != 'name') and str(port_key).isdigit(): + if ( + not fallback + and (port_key != "name") + and str(port_key).isdigit() + ): # fallback can only be digits. fallback = port_key @@ -338,14 +360,20 @@ def finalize(self) -> None: k8s_target = k8s_ep.ports.get(str(port_key), None) if k8s_target: - self.logger.debug(f'{key} port {src_port} #{idx}: {attr} {port_key} -> {k8s_target}') + self.logger.debug( + f"{key} port {src_port} #{idx}: {attr} {port_key} -> {k8s_target}" + ) break else: - self.logger.debug(f'{key} port {src_port} #{idx}: {attr} {port_key} -> miss') + self.logger.debug( + f"{key} port {src_port} #{idx}: {attr} {port_key} -> miss" + ) if not found_key: # WTFO. This is impossible. - self.logger.error(f"Kubernetes service {key} port {src_port} has an empty port spec at index {idx}?") + self.logger.error( + f"Kubernetes service {key} port {src_port} has an empty port spec at index {idx}?" + ) continue if not k8s_target: @@ -355,7 +383,9 @@ def finalize(self) -> None: # It's actually impossible for fallback to be unset, but WTF. k8s_target = fallback or src_port - self.logger.debug(f'{key} port {src_port} #{idx}: falling back to {k8s_target}') + self.logger.debug( + f"{key} port {src_port} #{idx}: falling back to {k8s_target}" + ) target_ports[src_port] = k8s_target @@ -372,30 +402,31 @@ def finalize(self) -> None: # OK! If we have no target addresses, just use service routing. if not target_addrs: if not self.watch_only: - self.logger.debug(f'{key} falling back to service routing') + self.logger.debug(f"{key} falling back to service routing") target_addrs = [key] for src_port, target_port in target_ports.items(): - svc_endpoints[src_port] = [{ - 'ip': target_addr, - 'port': target_port - } for target_addr in target_addrs] + svc_endpoints[src_port] = [ + {"ip": target_addr, "port": target_port} for target_addr in target_addrs + ] spec = { - 'ambassador_id': Config.ambassador_id, - 'endpoints': svc_endpoints, + "ambassador_id": Config.ambassador_id, + "endpoints": svc_endpoints, } if self.services.helm_chart: - spec['helm_chart'] = self.services.helm_chart - - self.manager.emit(NormalizedResource.from_data( - 'Service', - k8s_svc.name, - namespace=k8s_svc.namespace, - labels=k8s_svc.labels, - spec=spec, - rkey=f'k8s-{k8s_svc.name}-{k8s_svc.namespace}', - )) + spec["helm_chart"] = self.services.helm_chart + + self.manager.emit( + NormalizedResource.from_data( + "Service", + k8s_svc.name, + namespace=k8s_svc.namespace, + labels=k8s_svc.labels, + spec=spec, + rkey=f"k8s-{k8s_svc.name}-{k8s_svc.namespace}", + ) + ) # self.logger.debug("==== FINALIZE END\n%s" % dump_json(od, pretty=True)) diff --git a/python/ambassador/ir/__init__.py b/python/ambassador/ir/__init__.py index 880d9fb740..ccc272554b 100644 --- a/python/ambassador/ir/__init__.py +++ b/python/ambassador/ir/__init__.py @@ -14,4 +14,3 @@ from .irresource import IRResource from .ir import IR - diff --git a/python/ambassador/ir/ir.py b/python/ambassador/ir/ir.py index dd218be898..d2f0cdf896 100644 --- a/python/ambassador/ir/ir.py +++ b/python/ambassador/ir/ir.py @@ -72,6 +72,7 @@ IRFileChecker = Callable[[str], bool] + class IR: ambassador_module: IRAmbassador ambassador_id: str @@ -108,7 +109,9 @@ class IR: tracing: Optional[IRTracing] @classmethod - def check_deltas(cls, logger: logging.Logger, fetcher: 'ResourceFetcher', cache: Optional[Cache]=None) -> Tuple[str, bool, List[str]]: + def check_deltas( + cls, logger: logging.Logger, fetcher: "ResourceFetcher", cache: Optional[Cache] = None + ) -> Tuple[str, bool, List[str]]: # Assume that this should be marked as a complete reconfigure, and that we'll be # resetting the cache. config_type = "complete" @@ -156,11 +159,11 @@ def check_deltas(cls, logger: logging.Logger, fetcher: 'ResourceFetcher', cache: # The "kind" of a Delta must be a string; assert that to make # mypy happy. - delta_kind = delta['kind'] - assert(isinstance(delta_kind, str)) + delta_kind = delta["kind"] + assert isinstance(delta_kind, str) # Only worry about Mappings and TCPMappings right now. - if (delta_kind == 'Mapping') or (delta_kind == 'TCPMapping'): + if (delta_kind == "Mapping") or (delta_kind == "TCPMapping"): # XXX C'mon, mypy, is this cast really necessary? metadata = typecast(Dict[str, str], delta.get("metadata", {})) name = metadata.get("name", "") @@ -201,13 +204,16 @@ def check_deltas(cls, logger: logging.Logger, fetcher: 'ResourceFetcher', cache: return (config_type, reset_cache, invalidate_groups_for) - def __init__(self, aconf: Config, - secret_handler: SecretHandler, - file_checker: Optional[IRFileChecker]=None, - logger: Optional[logging.Logger]=None, - invalidate_groups_for: Optional[List[str]]=None, - cache: Optional[Cache]=None, - watch_only=False) -> None: + def __init__( + self, + aconf: Config, + secret_handler: SecretHandler, + file_checker: Optional[IRFileChecker] = None, + logger: Optional[logging.Logger] = None, + invalidate_groups_for: Optional[List[str]] = None, + cache: Optional[Cache] = None, + watch_only=False, + ) -> None: # Initialize the basics... self.ambassador_id = Config.ambassador_id self.ambassador_namespace = Config.ambassador_namespace @@ -227,11 +233,11 @@ def __init__(self, aconf: Config, self.cache.dump("Fetcher") # We're using setattr since since mypy complains about assigning directly to a method. - secret_root = os.environ.get('AMBASSADOR_CONFIG_BASE_DIR', "/ambassador") + secret_root = os.environ.get("AMBASSADOR_CONFIG_BASE_DIR", "/ambassador") # This setattr business is because mypy seems to think that, since self.file_checker is # callable, any mention of self.file_checker must be a function call. Sigh. - setattr(self, 'file_checker', file_checker if file_checker is not None else os.path.isfile) + setattr(self, "file_checker", file_checker if file_checker is not None else os.path.isfile) # The secret_handler is _required_. self.secret_handler = secret_handler @@ -243,9 +249,11 @@ def __init__(self, aconf: Config, self.logger.debug("IR: AMBASSADOR_ID %s" % self.ambassador_id) self.logger.debug("IR: Namespace %s" % self.ambassador_namespace) self.logger.debug("IR: Nodename %s" % self.ambassador_nodename) - self.logger.debug("IR: Endpoints %s" % "enabled" if Config.enable_endpoints else "disabled") + self.logger.debug( + "IR: Endpoints %s" % "enabled" if Config.enable_endpoints else "disabled" + ) - self.logger.debug("IR: file checker: %s" % getattr(self, 'file_checker').__name__) + self.logger.debug("IR: file checker: %s" % getattr(self, "file_checker").__name__) self.logger.debug("IR: secret handler: %s" % type(self.secret_handler).__name__) # First up: save the Config object. Its source map may be necessary later. @@ -293,11 +301,13 @@ def __init__(self, aconf: Config, # Check on the intercept agent and edge stack. Note that the Edge Stack touchfile is _not_ # within $AMBASSADOR_CONFIG_BASE_DIR: it stays in /ambassador no matter what. - self.agent_active = (os.environ.get("AGENT_SERVICE", None) != None) + self.agent_active = os.environ.get("AGENT_SERVICE", None) != None # Allow an environment variable to state whether we're in Edge Stack. But keep the # existing condition as sufficient, so that there is less of a chance of breaking # things running in a container with this file present. - self.edge_stack_allowed = parse_bool(os.environ.get('EDGE_STACK', 'false')) or os.path.exists('/ambassador/.edge_stack') + self.edge_stack_allowed = parse_bool( + os.environ.get("EDGE_STACK", "false") + ) or os.path.exists("/ambassador/.edge_stack") self.agent_origination_ctx = None # OK, time to get this show on the road. First things first: set up the @@ -310,7 +320,9 @@ def __init__(self, aconf: Config, # stuff fully set up. # # So. First, create the module. - self.ambassador_module = typecast(IRAmbassador, self.save_resource(IRAmbassador(self, aconf))) + self.ambassador_module = typecast( + IRAmbassador, self.save_resource(IRAmbassador(self, aconf)) + ) # Next, grab whatever information our aconf has about secrets... self.save_secret_info(aconf) @@ -339,7 +351,7 @@ def __init__(self, aconf: Config, # we wait until all the listeners are loaded so that we can check for the existance of a # "companion" TCP Listener. If a UDP listener was the first to be parsed then # we wouldn't know at that time. Thus we need to wait until after all of them have been loaded. - udp_listeners = (l for l in self.listeners.values() if l.socket_protocol == "UDP") + udp_listeners = (l for l in self.listeners.values() if l.socket_protocol == "UDP") for udp_listener in udp_listeners: ## this matches the `listener.bind_to` for the tcp listener tcp_listener_key = f"tcp-{udp_listener.bind_address}-{udp_listener.port}" @@ -364,15 +376,15 @@ def __init__(self, aconf: Config, # here so that secrets and TLS contexts are available. if not self.ambassador_module.finalize(self, aconf): # Uhoh. - self.ambassador_module.set_active(False) # This can't be good. + self.ambassador_module.set_active(False) # This can't be good. - _activity_str = 'watching' if watch_only else 'starting' - _mode_str = 'OSS' + _activity_str = "watching" if watch_only else "starting" + _mode_str = "OSS" if self.agent_active: - _mode_str = 'Intercept Agent' + _mode_str = "Intercept Agent" elif self.edge_stack_allowed: - _mode_str = 'Edge Stack' + _mode_str = "Edge Stack" self.logger.debug(f"IR: {_activity_str} {_mode_str}") @@ -403,9 +415,9 @@ def __init__(self, aconf: Config, # filter chains. Note that order of the filters matters. Start with CORS, # so that preflights will work even for things behind auth. - self.save_filter(IRFilter(ir=self, aconf=aconf, - rkey="ir.cors", kind="ir.cors", name="cors", - config={})) + self.save_filter( + IRFilter(ir=self, aconf=aconf, rkey="ir.cors", kind="ir.cors", name="cors", config={}) + ) # Next is auth... self.save_filter(IRAuth(self, aconf)) @@ -415,26 +427,39 @@ def __init__(self, aconf: Config, self.save_filter(self.ratelimit, already_saved=True) # ...and the error response filter... - self.save_filter(IRErrorResponse(self, aconf, - self.ambassador_module.get('error_response_overrides', None), - referenced_by_obj=self.ambassador_module)) + self.save_filter( + IRErrorResponse( + self, + aconf, + self.ambassador_module.get("error_response_overrides", None), + referenced_by_obj=self.ambassador_module, + ) + ) # ...and, finally, the barely-configurable router filter. router_config = {} if self.tracing: - router_config['start_child_span'] = True - - self.save_filter(IRFilter(ir=self, aconf=aconf, - rkey="ir.router", kind="ir.router", name="router", type="decoder", - config=router_config)) + router_config["start_child_span"] = True + + self.save_filter( + IRFilter( + ir=self, + aconf=aconf, + rkey="ir.router", + kind="ir.router", + name="router", + type="decoder", + config=router_config, + ) + ) # We would handle other modules here -- but guess what? There aren't any. # At this point ambassador, tls, and the deprecated auth module are all there # are, and they're handled above. So. At this point go sort out all the Mappings. MappingFactory.load_all(self, aconf) - self.walk_saved_resources(aconf, 'add_mappings') + self.walk_saved_resources(aconf, "add_mappings") TLSModuleFactory.finalize(self, aconf) MappingFactory.finalize(self, aconf) @@ -469,7 +494,7 @@ def __init__(self, aconf: Config, collision_list.append(name) else: # Short enough, set the envoy name to the cluster name. - self.clusters[name]['envoy_name'] = name + self.clusters[name]["envoy_name"] = name for short_name in sorted(collisions.keys()): name_list = collisions[short_name] @@ -506,11 +531,13 @@ def __init__(self, aconf: Config, # v2cluster.py and v3cluster.py. self.cache.invalidate(f"V2-{cluster.cache_key}") self.cache.invalidate(f"V3-{cluster.cache_key}") - self.cache.dump("Invalidate clusters V2-%s, V3-%s", cluster.cache_key, cluster.cache_key) + self.cache.dump( + "Invalidate clusters V2-%s, V3-%s", cluster.cache_key, cluster.cache_key + ) # OK. Finally, we can update the envoy_name. - cluster['envoy_name'] = mangled_name - self.logger.debug("COLLISION: envoy_name %s" % cluster['envoy_name']) + cluster["envoy_name"] = mangled_name + self.logger.debug("COLLISION: envoy_name %s" % cluster["envoy_name"]) # After we have the cluster names fixed up, go finalize filters. if self.tracing: @@ -524,7 +551,13 @@ def __init__(self, aconf: Config, # XXX Brutal hackery here! Probably this is a clue that Config and IR and such should have # a common container that can hold errors. - def post_error(self, rc: Union[str, RichStatus], resource: Optional[IRResource]=None, rkey: Optional[str]=None, log_level=logging.INFO): + def post_error( + self, + rc: Union[str, RichStatus], + resource: Optional[IRResource] = None, + rkey: Optional[str] = None, + log_level=logging.INFO, + ): self.aconf.post_error(rc, resource=resource, rkey=rkey, log_level=log_level) def agent_init(self, aconf: Config) -> None: @@ -557,14 +590,8 @@ def agent_init(self, aconf: Config) -> None: # case will we do ACME. Set additionalPort to -1 so we don't grab 8080 in the TLS case. host_args: Dict[str, Any] = { "hostname": "*", - "selector": { - "matchLabels": { - "intercept": self.agent_service - } - }, - "acmeProvider": { - "authority": "none" - }, + "selector": {"matchLabels": {"intercept": self.agent_service}}, + "acmeProvider": {"authority": "none"}, "requestPolicy": { "insecure": { "additionalPort": -1, @@ -577,14 +604,19 @@ def agent_init(self, aconf: Config) -> None: if agent_termination_secret: # Yup. - host_args["tlsSecret"] = { "name": agent_termination_secret } + host_args["tlsSecret"] = {"name": agent_termination_secret} else: # No termination secret, so do cleartext. host_args["requestPolicy"]["insecure"]["action"] = "Route" - host = IRHost(self, aconf, rkey=self.ambassador_module.rkey, location=self.ambassador_module.location, - name="agent-host", - **host_args) + host = IRHost( + self, + aconf, + rkey=self.ambassador_module.rkey, + location=self.ambassador_module.location, + name="agent-host", + **host_args, + ) if host.is_active(): host.referenced_by(self.ambassador_module) @@ -603,9 +635,14 @@ def agent_init(self, aconf: Config) -> None: # Uhhhh. Synthesize a TLSContext for this, I guess. # # XXX What if they already have a context with this name? - ctx = IRTLSContext(self, aconf, rkey=self.ambassador_module.rkey, location=self.ambassador_module.location, - name="agent-origination-context", - secret=agent_origination_secret) + ctx = IRTLSContext( + self, + aconf, + rkey=self.ambassador_module.rkey, + location=self.ambassador_module.location, + name="agent-origination-context", + secret=agent_origination_secret, + ) ctx.referenced_by(self.ambassador_module) self.save_tls_context(ctx) @@ -667,18 +704,23 @@ def agent_finalize(self, aconf) -> None: if self.agent_origination_ctx: ctx_name = self.agent_origination_ctx.name - mapping = IRHTTPMapping(self, aconf, rkey=self.ambassador_module.rkey, location=self.ambassador_module.location, - name="agent-fallback-mapping", - metadata_labels={"ambassador_diag_class": "private"}, - prefix="/", - rewrite="/", - service=f"127.0.0.1:{agent_port}", - grpc=agent_grpc, - # Making sure we don't have shorter timeouts on intercepts than the original Mapping - timeout_ms=60000, - idle_timeout_ms=60000, - tls=ctx_name, - precedence=-999999) # No, really. See comment above. + mapping = IRHTTPMapping( + self, + aconf, + rkey=self.ambassador_module.rkey, + location=self.ambassador_module.location, + name="agent-fallback-mapping", + metadata_labels={"ambassador_diag_class": "private"}, + prefix="/", + rewrite="/", + service=f"127.0.0.1:{agent_port}", + grpc=agent_grpc, + # Making sure we don't have shorter timeouts on intercepts than the original Mapping + timeout_ms=60000, + idle_timeout_ms=60000, + tls=ctx_name, + precedence=-999999, + ) # No, really. See comment above. mapping.referenced_by(self.ambassador_module) self.add_mapping(aconf, mapping) @@ -700,7 +742,7 @@ def cache_fetch(self, key: str) -> Optional[IRResource]: if rsrc is not None: # By definition, anything the IR layer pulls from the cache must be # an IRResource. - assert(isinstance(rsrc, IRResource)) + assert isinstance(rsrc, IRResource) # Since it's an IRResource, it has a pointer to the IR. Reset that. rsrc.ir = self @@ -732,7 +774,9 @@ def save_host(self, host: IRHost) -> None: is_valid = True if extant_host: - self.post_error("Duplicate Host %s; keeping definition from %s" % (host.name, extant_host.location)) + self.post_error( + "Duplicate Host %s; keeping definition from %s" % (host.name, extant_host.location) + ) is_valid = False if is_valid: @@ -754,30 +798,48 @@ def save_secret_info(self, aconf): # should not generate errors.) # (We include 'crl_pem' here because CRL secrets use that, and they # should not generate errors.) - if aconf_secret.get('tls_crt') or aconf_secret.get('cert-chain_pem') or aconf_secret.get('user_key') or aconf_secret.get('crl_pem'): + if ( + aconf_secret.get("tls_crt") + or aconf_secret.get("cert-chain_pem") + or aconf_secret.get("user_key") + or aconf_secret.get("crl_pem") + ): secret_info = SecretInfo.from_aconf_secret(aconf_secret) secret_name = secret_info.name secret_namespace = secret_info.namespace - self.logger.debug('saving "%s.%s" (from %s) in secret_info', secret_name, secret_namespace, secret_key) - self.secret_info[f'{secret_name}.{secret_namespace}'] = secret_info + self.logger.debug( + 'saving "%s.%s" (from %s) in secret_info', + secret_name, + secret_namespace, + secret_key, + ) + self.secret_info[f"{secret_name}.{secret_namespace}"] = secret_info else: - self.logger.debug('not saving secret_info from %s because there is no public half', secret_key) + self.logger.debug( + "not saving secret_info from %s because there is no public half", secret_key + ) def save_tls_context(self, ctx: IRTLSContext) -> None: extant_ctx = self.tls_contexts.get(ctx.name, None) is_valid = True if extant_ctx: - self.post_error("Duplicate TLSContext %s; keeping definition from %s" % (ctx.name, extant_ctx.location)) + self.post_error( + "Duplicate TLSContext %s; keeping definition from %s" + % (ctx.name, extant_ctx.location) + ) is_valid = False - if ctx.get('redirect_cleartext_from', None) is not None: + if ctx.get("redirect_cleartext_from", None) is not None: if self.redirect_cleartext_from is None: self.redirect_cleartext_from = ctx.redirect_cleartext_from else: if self.redirect_cleartext_from != ctx.redirect_cleartext_from: - self.post_error("TLSContext: %s; configured conflicting redirect_from port: %s" % (ctx.name, ctx.redirect_cleartext_from)) + self.post_error( + "TLSContext: %s; configured conflicting redirect_from port: %s" + % (ctx.name, ctx.redirect_cleartext_from) + ) is_valid = False if is_valid: @@ -800,7 +862,7 @@ def get_tls_contexts(self) -> ValuesView[IRTLSContext]: def resolve_secret(self, resource: IRResource, secret_name: str, namespace: str): # OK. Do we already have a SavedSecret for this? - ss_key = f'{secret_name}.{namespace}' + ss_key = f"{secret_name}.{namespace}" ss = self.saved_secrets.get(ss_key, None) @@ -837,10 +899,12 @@ def resolve_secret(self, resource: IRResource, secret_name: str, namespace: str) self.saved_secrets[secret_name] = ss return ss - def resolve_resolver(self, cluster: IRCluster, resolver_name: Optional[str]) -> IRServiceResolver: + def resolve_resolver( + self, cluster: IRCluster, resolver_name: Optional[str] + ) -> IRServiceResolver: # Which resolver should we use? if not resolver_name: - resolver_name = self.ambassador_module.get('resolver', 'kubernetes-service') + resolver_name = self.ambassador_module.get("resolver", "kubernetes-service") # Casting to str is OK because the Ambassador module's resolver must be a string, # so all the paths for resolver_name land with it being a string. @@ -848,9 +912,14 @@ def resolve_resolver(self, cluster: IRCluster, resolver_name: Optional[str]) -> assert resolver is not None return resolver - - def resolve_targets(self, cluster: IRCluster, resolver_name: Optional[str], - hostname: str, namespace: str, port: int) -> Optional[SvcEndpointSet]: + def resolve_targets( + self, + cluster: IRCluster, + resolver_name: Optional[str], + hostname: str, + namespace: str, + port: int, + ) -> Optional[SvcEndpointSet]: # Is the host already an IP address? is_ip_address = False @@ -862,21 +931,17 @@ def resolve_targets(self, cluster: IRCluster, resolver_name: Optional[str], if is_ip_address: # Already an IP address, great. - self.logger.debug(f'cluster {cluster.name}: {hostname} is already an IP address') + self.logger.debug(f"cluster {cluster.name}: {hostname} is already an IP address") - return [ - { - 'ip': hostname, - 'port': port, - 'target_kind': 'IPaddr' - } - ] + return [{"ip": hostname, "port": port, "target_kind": "IPaddr"}] resolver = self.resolve_resolver(cluster, resolver_name) # It should not be possible for resolver to be unset here. if not resolver: - self.post_error(f"cluster {cluster.name} has invalid resolver {resolver_name}?", rkey=cluster.rkey) + self.post_error( + f"cluster {cluster.name} has invalid resolver {resolver_name}?", rkey=cluster.rkey + ) return None # OK, ask the resolver for the target list. Understanding the mechanics of resolution @@ -900,8 +965,10 @@ def save_listener(self, listener: IRListener) -> None: extant_listener = self.listeners.get(listener_key, None) is_valid = True if extant_listener: - err_msg = f"Duplicate listener {listener.name} on {listener.socket_protocol.lower()}://{listener.bind_address}:{listener.port};" \ - f" keeping definition from {extant_listener.location}" + err_msg = ( + f"Duplicate listener {listener.name} on {listener.socket_protocol.lower()}://{listener.bind_address}:{listener.port};" + f" keeping definition from {extant_listener.location}" + ) self.post_error(err_msg) is_valid = False @@ -923,14 +990,17 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> Optional[IRBaseM self.logger.debug(f"IR: synthesizing group for {mapping.name}") group_name = "GROUP: %s" % mapping.name group_class = mapping.group_class() - group = group_class(ir=self, aconf=aconf, - location=mapping.location, - name=group_name, - mapping=mapping) + group = group_class( + ir=self, + aconf=aconf, + location=mapping.location, + name=group_name, + mapping=mapping, + ) # There's no way group can be anything but a non-None IRBaseMappingGroup # here. assert() that so that mypy understands it. - assert(isinstance(group, IRBaseMappingGroup)) # for mypy + assert isinstance(group, IRBaseMappingGroup) # for mypy self.groups[group.group_id] = group else: self.logger.debug(f"IR: already have group for {mapping.name}") @@ -946,7 +1016,7 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> Optional[IRBaseM return None def ordered_groups(self) -> Iterable[IRBaseMappingGroup]: - return reversed(sorted(self.groups.values(), key=lambda x: x['group_weight'])) + return reversed(sorted(self.groups.values(), key=lambda x: x["group_weight"])) def has_cluster(self, name: str) -> bool: return name in self.clusters @@ -963,7 +1033,10 @@ def add_cluster(self, cluster: IRCluster) -> IRCluster: # self.logger.debug(f"IR: cluster {cluster.name} is the sidecar") self.sidecar_cluster_name = cluster.name else: - self.logger.debug("IR: add_cluster: extant cluster %s (%s)" % (cluster.name, cluster.get("envoy_name", "-"))) + self.logger.debug( + "IR: add_cluster: extant cluster %s (%s)" + % (cluster.name, cluster.get("envoy_name", "-")) + ) return self.clusters[cluster.name] @@ -990,33 +1063,35 @@ def add_grpc_service(self, name: str, cluster: IRCluster) -> IRCluster: def as_dict(self) -> Dict[str, Any]: od = { - 'identity': { - 'ambassador_id': self.ambassador_id, - 'ambassador_namespace': self.ambassador_namespace, - 'ambassador_nodename': self.ambassador_nodename, + "identity": { + "ambassador_id": self.ambassador_id, + "ambassador_namespace": self.ambassador_namespace, + "ambassador_nodename": self.ambassador_nodename, }, - 'ambassador': self.ambassador_module.as_dict(), - 'clusters': { cluster_name: cluster.as_dict() - for cluster_name, cluster in self.clusters.items() }, - 'grpc_services': { svc_name: cluster.as_dict() - for svc_name, cluster in self.grpc_services.items() }, - 'hosts': [ host.as_dict() for host in self.hosts.values() ], - 'listeners': [ self.listeners[x].as_dict() for x in sorted(self.listeners.keys()) ], - 'filters': [ filt.as_dict() for filt in self.filters ], - 'groups': [ group.as_dict() for group in self.ordered_groups() ], - 'tls_contexts': [ context.as_dict() for context in self.tls_contexts.values() ], - 'services': self.services, - 'k8s_status_updates': self.k8s_status_updates + "ambassador": self.ambassador_module.as_dict(), + "clusters": { + cluster_name: cluster.as_dict() for cluster_name, cluster in self.clusters.items() + }, + "grpc_services": { + svc_name: cluster.as_dict() for svc_name, cluster in self.grpc_services.items() + }, + "hosts": [host.as_dict() for host in self.hosts.values()], + "listeners": [self.listeners[x].as_dict() for x in sorted(self.listeners.keys())], + "filters": [filt.as_dict() for filt in self.filters], + "groups": [group.as_dict() for group in self.ordered_groups()], + "tls_contexts": [context.as_dict() for context in self.tls_contexts.values()], + "services": self.services, + "k8s_status_updates": self.k8s_status_updates, } if self.log_services: - od['log_services'] = [ srv.as_dict() for srv in self.log_services.values() ] + od["log_services"] = [srv.as_dict() for srv in self.log_services.values()] if self.tracing: - od['tracing'] = self.tracing.as_dict() + od["tracing"] = self.tracing.as_dict() if self.ratelimit: - od['ratelimit'] = self.ratelimit.as_dict() + od["ratelimit"] = self.ratelimit.as_dict() return od @@ -1027,86 +1102,100 @@ def features(self) -> Dict[str, Any]: od: Dict[str, Union[bool, int, Optional[str], Dict]] = {} if self.aconf.helm_chart: - od['helm_chart'] = self.aconf.helm_chart - od['managed_by'] = self.aconf.pod_labels.get('app.kubernetes.io/managed-by', '') + od["helm_chart"] = self.aconf.helm_chart + od["managed_by"] = self.aconf.pod_labels.get("app.kubernetes.io/managed-by", "") - tls_termination_count = 0 # TLS termination contexts - tls_origination_count = 0 # TLS origination contexts - tls_crl_file_count = 0 # CRL files used + tls_termination_count = 0 # TLS termination contexts + tls_origination_count = 0 # TLS origination contexts + tls_crl_file_count = 0 # CRL files used using_tls_module = False using_tls_contexts = False for ctx in self.get_tls_contexts(): if ctx: - secret_info = ctx.get('secret_info', {}) + secret_info = ctx.get("secret_info", {}) if secret_info: using_tls_contexts = True - if secret_info.get('certificate_chain_file', None): + if secret_info.get("certificate_chain_file", None): tls_termination_count += 1 - if secret_info.get('cacert_chain_file', None): + if secret_info.get("cacert_chain_file", None): tls_origination_count += 1 - if secret_info.get('crl_file', None): + if secret_info.get("crl_file", None): tls_crl_file_count += 1 - if ctx.get('_legacy', False): + if ctx.get("_legacy", False): using_tls_module = True - od['tls_using_module'] = using_tls_module - od['tls_using_contexts'] = using_tls_contexts - od['tls_termination_count'] = tls_termination_count - od['tls_origination_count'] = tls_origination_count - od['tls_crl_file_count'] = tls_crl_file_count - - for key in [ 'diagnostics', 'liveness_probe', 'readiness_probe', 'statsd' ]: - od[key] = self.ambassador_module.get(key, {}).get('enabled', False) - - for key in [ 'use_proxy_proto', 'use_remote_address', 'x_forwarded_proto_redirect', 'enable_http10', - 'add_linkerd_headers', 'use_ambassador_namespace_for_service_resolution', 'proper_case', 'preserve_external_request_id' ]: + od["tls_using_module"] = using_tls_module + od["tls_using_contexts"] = using_tls_contexts + od["tls_termination_count"] = tls_termination_count + od["tls_origination_count"] = tls_origination_count + od["tls_crl_file_count"] = tls_crl_file_count + + for key in ["diagnostics", "liveness_probe", "readiness_probe", "statsd"]: + od[key] = self.ambassador_module.get(key, {}).get("enabled", False) + + for key in [ + "use_proxy_proto", + "use_remote_address", + "x_forwarded_proto_redirect", + "enable_http10", + "add_linkerd_headers", + "use_ambassador_namespace_for_service_resolution", + "proper_case", + "preserve_external_request_id", + ]: od[key] = self.ambassador_module.get(key, False) - od['service_resource_total'] = len(list(self.services.keys())) + od["service_resource_total"] = len(list(self.services.keys())) - od['listener_idle_timeout_ms'] = self.ambassador_module.get('listener_idle_timeout_ms', None) - od['headers_with_underscores_action'] = self.ambassador_module.get('headers_with_underscores_action', None) - od['max_request_headers_kb'] = self.ambassador_module.get('max_request_headers_kb', None) + od["listener_idle_timeout_ms"] = self.ambassador_module.get( + "listener_idle_timeout_ms", None + ) + od["headers_with_underscores_action"] = self.ambassador_module.get( + "headers_with_underscores_action", None + ) + od["max_request_headers_kb"] = self.ambassador_module.get("max_request_headers_kb", None) - od['server_name'] = bool(self.ambassador_module.server_name != 'envoy') + od["server_name"] = bool(self.ambassador_module.server_name != "envoy") - od['custom_ambassador_id'] = bool(self.ambassador_id != 'default') + od["custom_ambassador_id"] = bool(self.ambassador_id != "default") - od['buffer_limit_bytes'] = self.ambassador_module.get('buffer_limit_bytes', None) + od["buffer_limit_bytes"] = self.ambassador_module.get("buffer_limit_bytes", None) - default_port = Constants.SERVICE_PORT_HTTPS if tls_termination_count else Constants.SERVICE_PORT_HTTP + default_port = ( + Constants.SERVICE_PORT_HTTPS if tls_termination_count else Constants.SERVICE_PORT_HTTP + ) - od['custom_listener_port'] = bool(self.ambassador_module.service_port != default_port) + od["custom_listener_port"] = bool(self.ambassador_module.service_port != default_port) - od['allow_chunked_length'] = self.ambassador_module.get('allow_chunked_length', None) + od["allow_chunked_length"] = self.ambassador_module.get("allow_chunked_length", None) cluster_count = 0 - cluster_grpc_count = 0 # clusters using GRPC upstream - cluster_http_count = 0 # clusters using HTTP or HTTPS upstream - cluster_tls_count = 0 # clusters using TLS origination + cluster_grpc_count = 0 # clusters using GRPC upstream + cluster_http_count = 0 # clusters using HTTP or HTTPS upstream + cluster_tls_count = 0 # clusters using TLS origination - cluster_routing_kube_count = 0 # clusters routing using kube - cluster_routing_envoy_rr_count = 0 # clusters routing using envoy round robin - cluster_routing_envoy_rh_count = 0 # clusters routing using envoy ring hash + cluster_routing_kube_count = 0 # clusters routing using kube + cluster_routing_envoy_rr_count = 0 # clusters routing using envoy round robin + cluster_routing_envoy_rh_count = 0 # clusters routing using envoy ring hash cluster_routing_envoy_maglev_count = 0 # clusters routing using envoy maglev - cluster_routing_envoy_lr_count = 0 # clusters routing using envoy least request + cluster_routing_envoy_lr_count = 0 # clusters routing using envoy least request - endpoint_grpc_count = 0 # endpoints using GRPC upstream - endpoint_http_count = 0 # endpoints using HTTP/HTTPS upstream - endpoint_tls_count = 0 # endpoints using TLS origination + endpoint_grpc_count = 0 # endpoints using GRPC upstream + endpoint_http_count = 0 # endpoints using HTTP/HTTPS upstream + endpoint_tls_count = 0 # endpoints using TLS origination - endpoint_routing_kube_count = 0 # endpoints Kube is routing to - endpoint_routing_envoy_rr_count = 0 # endpoints Envoy round robin is routing to - endpoint_routing_envoy_rh_count = 0 # endpoints Envoy ring hash is routing to + endpoint_routing_kube_count = 0 # endpoints Kube is routing to + endpoint_routing_envoy_rr_count = 0 # endpoints Envoy round robin is routing to + endpoint_routing_envoy_rh_count = 0 # endpoints Envoy ring hash is routing to endpoint_routing_envoy_maglev_count = 0 # endpoints Envoy maglev is routing to - endpoint_routing_envoy_lr_count = 0 # endpoints Envoy least request is routing to + endpoint_routing_envoy_lr_count = 0 # endpoints Envoy least request is routing to for cluster in self.clusters.values(): cluster_count += 1 @@ -1114,34 +1203,34 @@ def features(self) -> Dict[str, Any]: using_http = False using_grpc = False - lb_type = 'kube' + lb_type = "kube" - if cluster.get('enable_endpoints', False): - lb_type = cluster.get('lb_type', 'round_robin') + if cluster.get("enable_endpoints", False): + lb_type = cluster.get("lb_type", "round_robin") - if lb_type == 'kube': + if lb_type == "kube": cluster_routing_kube_count += 1 - elif lb_type == 'ring_hash': + elif lb_type == "ring_hash": cluster_routing_envoy_rh_count += 1 - elif lb_type == 'maglev': + elif lb_type == "maglev": cluster_routing_envoy_maglev_count += 1 - elif lb_type == 'least_request': + elif lb_type == "least_request": cluster_routing_envoy_lr_count += 1 else: cluster_routing_envoy_rr_count += 1 - if cluster.get('tls_context', None): + if cluster.get("tls_context", None): using_tls = True cluster_tls_count += 1 - if cluster.get('grpc', False): + if cluster.get("grpc", False): using_grpc = True cluster_grpc_count += 1 else: using_http = True cluster_http_count += 1 - cluster_endpoints = cluster.urls if (lb_type == 'kube') else cluster.get('targets', []) + cluster_endpoints = cluster.urls if (lb_type == "kube") else cluster.get("targets", []) # Paranoia, really. if not cluster_endpoints: @@ -1160,43 +1249,43 @@ def features(self) -> Dict[str, Any]: if using_grpc: endpoint_grpc_count += num_endpoints - if lb_type == 'kube': + if lb_type == "kube": endpoint_routing_kube_count += num_endpoints - elif lb_type == 'ring_hash': + elif lb_type == "ring_hash": endpoint_routing_envoy_rh_count += num_endpoints - elif lb_type == 'maglev': + elif lb_type == "maglev": endpoint_routing_envoy_maglev_count += num_endpoints - elif lb_type == 'least_request': + elif lb_type == "least_request": endpoint_routing_envoy_lr_count += num_endpoints else: endpoint_routing_envoy_rr_count += num_endpoints - od['cluster_count'] = cluster_count - od['cluster_grpc_count'] = cluster_grpc_count - od['cluster_http_count'] = cluster_http_count - od['cluster_tls_count'] = cluster_tls_count - od['cluster_routing_kube_count'] = cluster_routing_kube_count - od['cluster_routing_envoy_rr_count'] = cluster_routing_envoy_rr_count - od['cluster_routing_envoy_rh_count'] = cluster_routing_envoy_rh_count - od['cluster_routing_envoy_maglev_count'] = cluster_routing_envoy_maglev_count - od['cluster_routing_envoy_lr_count'] = cluster_routing_envoy_lr_count - - od['endpoint_routing'] = Config.enable_endpoints - - od['endpoint_grpc_count'] = endpoint_grpc_count - od['endpoint_http_count'] = endpoint_http_count - od['endpoint_tls_count'] = endpoint_tls_count - od['endpoint_routing_kube_count'] = endpoint_routing_kube_count - od['endpoint_routing_envoy_rr_count'] = endpoint_routing_envoy_rr_count - od['endpoint_routing_envoy_rh_count'] = endpoint_routing_envoy_rh_count - od['endpoint_routing_envoy_maglev_count'] = endpoint_routing_envoy_maglev_count - od['endpoint_routing_envoy_lr_count'] = endpoint_routing_envoy_lr_count - - od['cluster_ingress_count'] = 0 # Provided for backward compatibility only. - od['knative_ingress_count'] = self.aconf.get_count('knative_ingress') - - od['k8s_ingress_count'] = self.aconf.get_count('k8s_ingress') - od['k8s_ingress_class_count'] = self.aconf.get_count('k8s_ingress_class') + od["cluster_count"] = cluster_count + od["cluster_grpc_count"] = cluster_grpc_count + od["cluster_http_count"] = cluster_http_count + od["cluster_tls_count"] = cluster_tls_count + od["cluster_routing_kube_count"] = cluster_routing_kube_count + od["cluster_routing_envoy_rr_count"] = cluster_routing_envoy_rr_count + od["cluster_routing_envoy_rh_count"] = cluster_routing_envoy_rh_count + od["cluster_routing_envoy_maglev_count"] = cluster_routing_envoy_maglev_count + od["cluster_routing_envoy_lr_count"] = cluster_routing_envoy_lr_count + + od["endpoint_routing"] = Config.enable_endpoints + + od["endpoint_grpc_count"] = endpoint_grpc_count + od["endpoint_http_count"] = endpoint_http_count + od["endpoint_tls_count"] = endpoint_tls_count + od["endpoint_routing_kube_count"] = endpoint_routing_kube_count + od["endpoint_routing_envoy_rr_count"] = endpoint_routing_envoy_rr_count + od["endpoint_routing_envoy_rh_count"] = endpoint_routing_envoy_rh_count + od["endpoint_routing_envoy_maglev_count"] = endpoint_routing_envoy_maglev_count + od["endpoint_routing_envoy_lr_count"] = endpoint_routing_envoy_lr_count + + od["cluster_ingress_count"] = 0 # Provided for backward compatibility only. + od["knative_ingress_count"] = self.aconf.get_count("knative_ingress") + + od["k8s_ingress_count"] = self.aconf.get_count("k8s_ingress") + od["k8s_ingress_class_count"] = self.aconf.get_count("k8s_ingress_class") extauth = False extauth_proto: Optional[str] = None @@ -1211,66 +1300,66 @@ def features(self) -> Dict[str, Any]: tracing_driver: Optional[str] = None for filter in self.filters: - if filter.kind == 'IRAuth': + if filter.kind == "IRAuth": extauth = True - extauth_proto = filter.get('proto', 'http') - extauth_allow_body = filter.get('allow_request_body', False) + extauth_proto = filter.get("proto", "http") + extauth_allow_body = filter.get("allow_request_body", False) extauth_host_count = len(filter.hosts.keys()) if self.ratelimit: ratelimit = True - ratelimit_data_plane_proto = self.ratelimit.get('data_plane_proto', False) - ratelimit_custom_domain = bool(self.ratelimit.domain != 'ambassador') + ratelimit_data_plane_proto = self.ratelimit.get("data_plane_proto", False) + ratelimit_custom_domain = bool(self.ratelimit.domain != "ambassador") if self.tracing: tracing = True tracing_driver = self.tracing.driver - od['extauth'] = extauth - od['extauth_proto'] = extauth_proto - od['extauth_allow_body'] = extauth_allow_body - od['extauth_host_count'] = extauth_host_count - od['ratelimit'] = ratelimit - od['ratelimit_data_plane_proto'] = ratelimit_data_plane_proto - od['ratelimit_custom_domain'] = ratelimit_custom_domain - od['tracing'] = tracing - od['tracing_driver'] = tracing_driver + od["extauth"] = extauth + od["extauth_proto"] = extauth_proto + od["extauth_allow_body"] = extauth_allow_body + od["extauth_host_count"] = extauth_host_count + od["ratelimit"] = ratelimit + od["ratelimit_data_plane_proto"] = ratelimit_data_plane_proto + od["ratelimit_custom_domain"] = ratelimit_custom_domain + od["tracing"] = tracing + od["tracing_driver"] = tracing_driver group_count = 0 - group_http_count = 0 # HTTPMappingGroups - group_tcp_count = 0 # TCPMappingGroups - group_precedence_count = 0 # groups using explicit precedence - group_header_match_count = 0 # groups using header matches - group_regex_header_count = 0 # groups using regex header matches - group_regex_prefix_count = 0 # groups using regex prefix matches - group_shadow_count = 0 # groups using shadows - group_shadow_weighted_count = 0 # groups using shadows with non-100% weights - group_host_redirect_count = 0 # groups using host_redirect - group_host_rewrite_count = 0 # groups using host_rewrite - group_canary_count = 0 # groups coalescing multiple mappings - group_resolver_kube_service = 0 # groups using the KubernetesServiceResolver + group_http_count = 0 # HTTPMappingGroups + group_tcp_count = 0 # TCPMappingGroups + group_precedence_count = 0 # groups using explicit precedence + group_header_match_count = 0 # groups using header matches + group_regex_header_count = 0 # groups using regex header matches + group_regex_prefix_count = 0 # groups using regex prefix matches + group_shadow_count = 0 # groups using shadows + group_shadow_weighted_count = 0 # groups using shadows with non-100% weights + group_host_redirect_count = 0 # groups using host_redirect + group_host_rewrite_count = 0 # groups using host_rewrite + group_canary_count = 0 # groups coalescing multiple mappings + group_resolver_kube_service = 0 # groups using the KubernetesServiceResolver group_resolver_kube_endpoint = 0 # groups using the KubernetesServiceResolver - group_resolver_consul = 0 # groups using the ConsulResolver - mapping_count = 0 # total mappings + group_resolver_consul = 0 # groups using the ConsulResolver + mapping_count = 0 # total mappings for group in self.ordered_groups(): group_count += 1 - if group.get('kind', "IRHTTPMappingGroup") == 'IRTCPMappingGroup': + if group.get("kind", "IRHTTPMappingGroup") == "IRTCPMappingGroup": group_tcp_count += 1 else: group_http_count += 1 - if group.get('precedence', 0) != 0: + if group.get("precedence", 0) != 0: group_precedence_count += 1 using_headers = False using_regex_headers = False - for header in group.get('headers', []): + for header in group.get("headers", []): using_headers = True - if header['regex']: + if header["regex"]: using_regex_headers = True break @@ -1285,48 +1374,50 @@ def features(self) -> Dict[str, Any]: mapping_count += len(group.mappings) - if group.get('shadows', []): + if group.get("shadows", []): group_shadow_count += 1 - if group.get('weight', 100) != 100: + if group.get("weight", 100) != 100: group_shadow_weighted_count += 1 - if group.get('host_redirect', {}): + if group.get("host_redirect", {}): group_host_redirect_count += 1 - if group.get('host_rewrite', None): + if group.get("host_rewrite", None): group_host_rewrite_count += 1 - res_name = group.get('resolver', self.ambassador_module.get('resolver', 'kubernetes-service')) + res_name = group.get( + "resolver", self.ambassador_module.get("resolver", "kubernetes-service") + ) resolver = self.get_resolver(res_name) if resolver: - if resolver.kind == 'KubernetesServiceResolver': + if resolver.kind == "KubernetesServiceResolver": group_resolver_kube_service += 1 - elif resolver.kind == 'KubernetesEndpoinhResolver': + elif resolver.kind == "KubernetesEndpoinhResolver": group_resolver_kube_endpoint += 1 - elif resolver.kind == 'ConsulResolver': + elif resolver.kind == "ConsulResolver": group_resolver_consul += 1 - od['group_count'] = group_count - od['group_http_count'] = group_http_count - od['group_tcp_count'] = group_tcp_count - od['group_precedence_count'] = group_precedence_count - od['group_header_match_count'] = group_header_match_count - od['group_regex_header_count'] = group_regex_header_count - od['group_regex_prefix_count'] = group_regex_prefix_count - od['group_shadow_count'] = group_shadow_count - od['group_shadow_weighted_count'] = group_shadow_weighted_count - od['group_host_redirect_count'] = group_host_redirect_count - od['group_host_rewrite_count'] = group_host_rewrite_count - od['group_canary_count'] = group_canary_count - od['group_resolver_kube_service'] = group_resolver_kube_service - od['group_resolver_kube_endpoint'] = group_resolver_kube_endpoint - od['group_resolver_consul'] = group_resolver_consul - od['mapping_count'] = mapping_count - - od['listener_count'] = len(self.listeners) - od['host_count'] = len(self.hosts) + od["group_count"] = group_count + od["group_http_count"] = group_http_count + od["group_tcp_count"] = group_tcp_count + od["group_precedence_count"] = group_precedence_count + od["group_header_match_count"] = group_header_match_count + od["group_regex_header_count"] = group_regex_header_count + od["group_regex_prefix_count"] = group_regex_prefix_count + od["group_shadow_count"] = group_shadow_count + od["group_shadow_weighted_count"] = group_shadow_weighted_count + od["group_host_redirect_count"] = group_host_redirect_count + od["group_host_rewrite_count"] = group_host_rewrite_count + od["group_canary_count"] = group_canary_count + od["group_resolver_kube_service"] = group_resolver_kube_service + od["group_resolver_kube_endpoint"] = group_resolver_kube_endpoint + od["group_resolver_consul"] = group_resolver_consul + od["mapping_count"] = mapping_count + + od["listener_count"] = len(self.listeners) + od["host_count"] = len(self.hosts) invalid_counts: Dict[str, int] = {} @@ -1336,7 +1427,7 @@ def features(self) -> Dict[str, Any]: invalid_counts[kind] = invalid_counts.get(kind, 0) + 1 - od['invalid_counts'] = invalid_counts + od["invalid_counts"] = invalid_counts # Fast reconfiguration information is supplied in check_scout in diagd.py. diff --git a/python/ambassador/ir/irambassador.py b/python/ambassador/ir/irambassador.py index 70a35598a8..63600fbb91 100644 --- a/python/ambassador/ir/irambassador.py +++ b/python/ambassador/ir/irambassador.py @@ -17,10 +17,10 @@ from .irfilter import IRFilter if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRAmbassador (IRResource): +class IRAmbassador(IRResource): # All the AModTransparentKeys are copied from the incoming Ambassador resource # into the IRAmbassador object partway through IRAmbassador.finalize(). @@ -28,57 +28,57 @@ class IRAmbassador (IRResource): # PLEASE KEEP THIS LIST SORTED. AModTransparentKeys: ClassVar = [ - 'add_linkerd_headers', - 'admin_port', - 'auth_enabled', - 'allow_chunked_length', - 'buffer_limit_bytes', - 'circuit_breakers', - 'cluster_idle_timeout_ms', - 'cluster_max_connection_lifetime_ms', - 'cluster_request_timeout_ms', - 'debug_mode', + "add_linkerd_headers", + "admin_port", + "auth_enabled", + "allow_chunked_length", + "buffer_limit_bytes", + "circuit_breakers", + "cluster_idle_timeout_ms", + "cluster_max_connection_lifetime_ms", + "cluster_request_timeout_ms", + "debug_mode", # Do not include defaults, that's handled manually in setup. - 'default_label_domain', - 'default_labels', - 'diagnostics', - 'enable_http10', - 'enable_ipv4', - 'enable_ipv6', - 'envoy_log_format', - 'envoy_log_path', - 'envoy_log_type', - 'forward_client_cert_details', + "default_label_domain", + "default_labels", + "diagnostics", + "enable_http10", + "enable_ipv4", + "enable_ipv6", + "envoy_log_format", + "envoy_log_path", + "envoy_log_type", + "forward_client_cert_details", # Do not include envoy_validation_timeout; we let finalize() type-check it. # Do not include ip_allow or ip_deny; we let finalize() type-check them. - 'headers_with_underscores_action', - 'keepalive', - 'listener_idle_timeout_ms', - 'liveness_probe', - 'load_balancer', - 'max_request_headers_kb', - 'merge_slashes', - 'reject_requests_with_escaped_slashes', - 'preserve_external_request_id', - 'proper_case', - 'prune_unreachable_routes', - 'readiness_probe', - 'regex_max_size', - 'regex_type', - 'resolver', - 'error_response_overrides', - 'header_case_overrides', - 'server_name', - 'service_port', - 'set_current_client_cert_details', - 'statsd', - 'strip_matching_host_port', - 'suppress_envoy_headers', - 'use_ambassador_namespace_for_service_resolution', - 'use_proxy_proto', - 'use_remote_address', - 'x_forwarded_proto_redirect', - 'xff_num_trusted_hops', + "headers_with_underscores_action", + "keepalive", + "listener_idle_timeout_ms", + "liveness_probe", + "load_balancer", + "max_request_headers_kb", + "merge_slashes", + "reject_requests_with_escaped_slashes", + "preserve_external_request_id", + "proper_case", + "prune_unreachable_routes", + "readiness_probe", + "regex_max_size", + "regex_type", + "resolver", + "error_response_overrides", + "header_case_overrides", + "server_name", + "service_port", + "set_current_client_cert_details", + "statsd", + "strip_matching_host_port", + "suppress_envoy_headers", + "use_ambassador_namespace_for_service_resolution", + "use_proxy_proto", + "use_remote_address", + "x_forwarded_proto_redirect", + "xff_num_trusted_hops", ] service_port: int @@ -108,16 +108,24 @@ class IRAmbassador (IRResource): # large enough to exceed this threshold. default_validation_timeout: ClassVar[int] = 60 - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.ambassador", - kind: str="IRAmbassador", - name: str="ir.ambassador", - use_remote_address: bool=True, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.ambassador", + kind: str = "IRAmbassador", + name: str = "ir.ambassador", + use_remote_address: bool = True, + **kwargs, + ) -> None: # print("IRAmbassador __init__ (%s %s %s)" % (kind, name, kwargs)) super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, + ir=ir, + aconf=aconf, + rkey=rkey, + kind=kind, + name=name, service_port=Constants.SERVICE_PORT_HTTP, admin_port=Constants.ADMIN_PORT, auth_enabled=None, @@ -131,12 +139,12 @@ def __init__(self, ir: 'IR', aconf: Config, liveness_probe={"enabled": True}, readiness_probe={"enabled": True}, diagnostics={"enabled": True}, # TODO(lukeshu): In getambassador.io/v3alpha2, change - # the default to {"enabled": False}. See the related - # comment in crd_module.go. + # the default to {"enabled": False}. See the related + # comment in crd_module.go. use_proxy_proto=False, enable_http10=False, proper_case=False, - prune_unreachable_routes=True, # default True; can be updated in finalize() + prune_unreachable_routes=True, # default True; can be updated in finalize() use_remote_address=use_remote_address, x_forwarded_proto_redirect=False, load_balancer=None, @@ -147,13 +155,13 @@ def __init__(self, ir: 'IR', aconf: Config, debug_mode=False, preserve_external_request_id=False, max_request_headers_kb=None, - **kwargs + **kwargs, ) self.ip_allow_deny: Optional[IRIPAllowDeny] = None self._finalized = False - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # The heavy lifting here is mostly in the finalize() method, so that when we do fallback # lookups for TLS configuration stuff, the defaults are present in the Ambassador module. # @@ -162,12 +170,12 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # We're interested in the 'ambassador' module from the Config, if any... amod = aconf.get_module("ambassador") - if amod and 'defaults' in amod: - self['defaults'] = amod['defaults'] + if amod and "defaults" in amod: + self["defaults"] = amod["defaults"] return True - def finalize(self, ir: 'IR', aconf: Config) -> bool: + def finalize(self, ir: "IR", aconf: Config) -> bool: self._finalized = True # Check TLSContext resources to see if we should enable TLS termination. @@ -178,17 +186,19 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: # Welllll this ain't good. ctx.set_active(False) to_delete.append(ctx_name) - elif ctx.get('hosts', None): + elif ctx.get("hosts", None): # This is a termination context - self.logger.debug("TLSContext %s is a termination context, enabling TLS termination" % ctx.name) + self.logger.debug( + "TLSContext %s is a termination context, enabling TLS termination" % ctx.name + ) self.service_port = Constants.SERVICE_PORT_HTTPS - if ctx.get('ca_cert', None): + if ctx.get("ca_cert", None): # Client-side TLS is enabled. self.logger.debug("TLSContext %s enables client certs!" % ctx.name) for ctx_name in to_delete: - del(ir.tls_contexts[ctx_name]) + del ir.tls_contexts[ctx_name] # After that, walk the AModTransparentKeys and copy all those things from the # input into our IRAmbassador. @@ -204,87 +214,92 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: self[key] = amod[key] # If we have an envoy_validation_timeout... - if 'envoy_validation_timeout' in amod: + if "envoy_validation_timeout" in amod: # ...then set our timeout from it. try: - self.envoy_validation_timeout = int(amod['envoy_validation_timeout']) + self.envoy_validation_timeout = int(amod["envoy_validation_timeout"]) except ValueError: self.post_error("envoy_validation_timeout must be an integer number of seconds") # If we don't have a default label domain, force it to 'ambassador'. - if not self.get('default_label_domain'): - self.default_label_domain = 'ambassador' + if not self.get("default_label_domain"): + self.default_label_domain = "ambassador" # Likewise, if we have no default labels, force an empty dict (it makes life easier # on other modules). - if not self.get('default_labels'): + if not self.get("default_labels"): self.default_labels: Dict[str, Any] = {} # Next up: diag port & services. diag_service = "127.0.0.1:%d" % Constants.DIAG_PORT for name, cur, dflt in [ - ("liveness", self.liveness_probe, IRAmbassador.default_liveness_probe), - ("readiness", self.readiness_probe, IRAmbassador.default_readiness_probe), - ("diagnostics", self.diagnostics, IRAmbassador.default_diagnostics) + ("liveness", self.liveness_probe, IRAmbassador.default_liveness_probe), + ("readiness", self.readiness_probe, IRAmbassador.default_readiness_probe), + ("diagnostics", self.diagnostics, IRAmbassador.default_diagnostics), ]: if cur and cur.get("enabled", False): - if not cur.get('prefix', None): - cur['prefix'] = dflt['prefix'] - - if not cur.get('rewrite', None): - cur['rewrite'] = dflt['rewrite'] - - if not cur.get('service', None): - cur['service'] = diag_service - - if amod and ('enable_grpc_http11_bridge' in amod): - self.grpc_http11_bridge = IRFilter(ir=ir, aconf=aconf, - kind='ir.grpc_http1_bridge', - name='grpc_http1_bridge', - config=dict()) + if not cur.get("prefix", None): + cur["prefix"] = dflt["prefix"] + + if not cur.get("rewrite", None): + cur["rewrite"] = dflt["rewrite"] + + if not cur.get("service", None): + cur["service"] = diag_service + + if amod and ("enable_grpc_http11_bridge" in amod): + self.grpc_http11_bridge = IRFilter( + ir=ir, + aconf=aconf, + kind="ir.grpc_http1_bridge", + name="grpc_http1_bridge", + config=dict(), + ) self.grpc_http11_bridge.sourced_by(amod) ir.save_filter(self.grpc_http11_bridge) - if amod and ('enable_grpc_web' in amod): - self.grpc_web = IRFilter(ir=ir, aconf=aconf, kind='ir.grpc_web', name='grpc_web', config=dict()) + if amod and ("enable_grpc_web" in amod): + self.grpc_web = IRFilter( + ir=ir, aconf=aconf, kind="ir.grpc_web", name="grpc_web", config=dict() + ) self.grpc_web.sourced_by(amod) ir.save_filter(self.grpc_web) - if amod and (grpc_stats := amod.get('grpc_stats')) is not None: + if amod and (grpc_stats := amod.get("grpc_stats")) is not None: # grpc_stats = { 'all_methods': False} if amod.grpc_stats is None else amod.grpc_stats # default config with safe values - config: Dict[str, Any] = { - 'enable_upstream_stats': False - } + config: Dict[str, Any] = {"enable_upstream_stats": False} # Only one of config['individual_method_stats_allowlist'] or # config['stats_for_all_methods'] can be set. - if 'services' in grpc_stats: - config['individual_method_stats_allowlist'] = { - 'services': grpc_stats['services'] - } + if "services" in grpc_stats: + config["individual_method_stats_allowlist"] = {"services": grpc_stats["services"]} else: - config['stats_for_all_methods'] = bool(grpc_stats.get('all_methods', False)) + config["stats_for_all_methods"] = bool(grpc_stats.get("all_methods", False)) - if ('upstream_stats' in grpc_stats): - config['enable_upstream_stats'] = bool(grpc_stats['upstream_stats']) + if "upstream_stats" in grpc_stats: + config["enable_upstream_stats"] = bool(grpc_stats["upstream_stats"]) - self.grpc_stats = IRFilter(ir=ir, aconf=aconf, - kind='ir.grpc_stats', - name='grpc_stats', - config=config) + self.grpc_stats = IRFilter( + ir=ir, aconf=aconf, kind="ir.grpc_stats", name="grpc_stats", config=config + ) self.grpc_stats.sourced_by(amod) ir.save_filter(self.grpc_stats) - if amod and ('lua_scripts' in amod): - self.lua_scripts = IRFilter(ir=ir, aconf=aconf, kind='ir.lua_scripts', name='lua_scripts', - config={'inline_code': amod.lua_scripts}) + if amod and ("lua_scripts" in amod): + self.lua_scripts = IRFilter( + ir=ir, + aconf=aconf, + kind="ir.lua_scripts", + name="lua_scripts", + config={"inline_code": amod.lua_scripts}, + ) self.lua_scripts.sourced_by(amod) ir.save_filter(self.lua_scripts) # Gzip. - if amod and ('gzip' in amod): + if amod and ("gzip" in amod): self.gzip = IRGzip(ir=ir, aconf=aconf, location=self.location, **amod.gzip) if self.gzip: @@ -292,8 +307,8 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: else: return False - # Buffer. - if amod and ('buffer' in amod): + # Buffer. + if amod and ("buffer" in amod): self.buffer = IRBuffer(ir=ir, aconf=aconf, location=self.location, **amod.buffer) if self.buffer: @@ -301,11 +316,11 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: else: return False - if amod and ('keepalive' in amod): - self.keepalive = amod['keepalive'] + if amod and ("keepalive" in amod): + self.keepalive = amod["keepalive"] # Finally, default CORS stuff. - if amod and ('cors' in amod): + if amod and ("cors" in amod): self.cors = IRCORS(ir=ir, aconf=aconf, location=self.location, **amod.cors) if self.cors: @@ -313,8 +328,10 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: else: return False - if amod and ('retry_policy' in amod): - self.retry_policy = IRRetryPolicy(ir=ir, aconf=aconf, location=self.location, **amod.retry_policy) + if amod and ("retry_policy" in amod): + self.retry_policy = IRRetryPolicy( + ir=ir, aconf=aconf, location=self.location, **amod.retry_policy + ) if self.retry_policy: self.retry_policy.referenced_by(self) @@ -322,10 +339,10 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: return False if amod: - if 'ip_allow' in amod: + if "ip_allow" in amod: self.handle_ip_allow_deny(allow=True, principals=amod.ip_allow) - if 'ip_deny' in amod: + if "ip_deny" in amod: self.handle_ip_allow_deny(allow=False, principals=amod.ip_deny) if self.ip_allow_deny is not None: @@ -335,69 +352,96 @@ def finalize(self, ir: 'IR', aconf: Config) -> bool: # Ambassador module. self.ip_allow_deny = None - if self.get('load_balancer', None) is not None: - if not IRHTTPMapping.validate_load_balancer(self['load_balancer']): - self.post_error("Invalid load_balancer specified: {}".format(self['load_balancer'])) + if self.get("load_balancer", None) is not None: + if not IRHTTPMapping.validate_load_balancer(self["load_balancer"]): + self.post_error("Invalid load_balancer specified: {}".format(self["load_balancer"])) return False - if self.get('circuit_breakers', None) is not None: - if not IRBaseMapping.validate_circuit_breakers(self.ir, self['circuit_breakers']): - self.post_error("Invalid circuit_breakers specified: {}".format(self['circuit_breakers'])) + if self.get("circuit_breakers", None) is not None: + if not IRBaseMapping.validate_circuit_breakers(self.ir, self["circuit_breakers"]): + self.post_error( + "Invalid circuit_breakers specified: {}".format(self["circuit_breakers"]) + ) return False - if self.get('envoy_log_type') == 'text': - if self.get('envoy_log_format', None) is not None and not isinstance(self.get('envoy_log_format'), str): + if self.get("envoy_log_type") == "text": + if self.get("envoy_log_format", None) is not None and not isinstance( + self.get("envoy_log_format"), str + ): self.post_error( "envoy_log_type 'text' requires a string in envoy_log_format: {}, invalidating...".format( - self.get('envoy_log_format'))) - self['envoy_log_format'] = "" + self.get("envoy_log_format") + ) + ) + self["envoy_log_format"] = "" return False - elif self.get('envoy_log_type') == 'json': - if self.get('envoy_log_format', None) is not None and not isinstance(self.get('envoy_log_format'), dict): + elif self.get("envoy_log_type") == "json": + if self.get("envoy_log_format", None) is not None and not isinstance( + self.get("envoy_log_format"), dict + ): self.post_error( "envoy_log_type 'json' requires a dictionary in envoy_log_format: {}, invalidating...".format( - self.get('envoy_log_format'))) - self['envoy_log_format'] = {} + self.get("envoy_log_format") + ) + ) + self["envoy_log_format"] = {} return False else: - self.post_error("Invalid log_type specified: {}. Supported: json, text".format(self.get('envoy_log_type'))) + self.post_error( + "Invalid log_type specified: {}. Supported: json, text".format( + self.get("envoy_log_type") + ) + ) return False - if self.get('forward_client_cert_details') is not None: + if self.get("forward_client_cert_details") is not None: # https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-enum-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-forwardclientcertdetails - valid_values = ('SANITIZE', 'FORWARD_ONLY', 'APPEND_FORWARD', 'SANITIZE_SET', 'ALWAYS_FORWARD_ONLY') - - value = self.get('forward_client_cert_details') + valid_values = ( + "SANITIZE", + "FORWARD_ONLY", + "APPEND_FORWARD", + "SANITIZE_SET", + "ALWAYS_FORWARD_ONLY", + ) + + value = self.get("forward_client_cert_details") if value not in valid_values: self.post_error( "'forward_client_cert_details' may not be set to '{}'; it may only be set to one of: {}".format( - value, ', '.join(valid_values))) + value, ", ".join(valid_values) + ) + ) return False - cert_details = self.get('set_current_client_cert_details') + cert_details = self.get("set_current_client_cert_details") if cert_details: # https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-msg-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-setcurrentclientcertdetails - valid_keys = ('subject', 'cert', 'chain', 'dns', 'uri') + valid_keys = ("subject", "cert", "chain", "dns", "uri") for k, v in cert_details.items(): if k not in valid_keys: self.post_error( "'set_current_client_cert_details' may not contain key '{}'; it may only contain keys: {}".format( - k, ', '.join(valid_keys))) + k, ", ".join(valid_keys) + ) + ) return False if v not in (True, False): self.post_error( - "'set_current_client_cert_details' value for key '{}' may only be 'true' or 'false', not '{}'".format(k, v)) + "'set_current_client_cert_details' value for key '{}' may only be 'true' or 'false', not '{}'".format( + k, v + ) + ) return False return True - def add_mappings(self, ir: 'IR', aconf: Config): + def add_mappings(self, ir: "IR", aconf: Config): for name, cur in [ - ( "liveness", self.liveness_probe ), - ( "readiness", self.readiness_probe ), - ( "diagnostics", self.diagnostics ) + ("liveness", self.liveness_probe), + ("readiness", self.readiness_probe), + ("diagnostics", self.diagnostics), ]: if cur and cur.get("enabled", False): name = "internal_%s_probe_mapping" % name @@ -408,11 +452,19 @@ def add_mappings(self, ir: 'IR', aconf: Config): if mapping is not None: # Cache hit. We know a priori that anything in the cache under a Mapping # key must be an IRBaseMapping, but let's assert that rather than casting. - assert(isinstance(mapping, IRBaseMapping)) + assert isinstance(mapping, IRBaseMapping) else: - mapping = IRHTTPMapping(ir, aconf, kind="InternalMapping", - rkey=self.rkey, name=name, location=self.location, - timeout_ms=10000, hostname="*", **cur) + mapping = IRHTTPMapping( + ir, + aconf, + kind="InternalMapping", + rkey=self.rkey, + name=name, + location=self.location, + timeout_ms=10000, + hostname="*", + **cur, + ) mapping.referenced_by(self) ir.add_mapping(aconf, mapping) @@ -452,7 +504,7 @@ def add_mappings(self, ir: 'IR', aconf: Config): def get_default_label_domain(self) -> str: return self.default_label_domain - def get_default_labels(self, domain: Optional[str]=None) -> Optional[List]: + def get_default_labels(self, domain: Optional[str] = None) -> Optional[List]: if not domain: domain = self.get_default_label_domain() @@ -460,7 +512,7 @@ def get_default_labels(self, domain: Optional[str]=None) -> Optional[List]: self.logger.debug("default_labels info for %s: %s" % (domain, domain_info)) - return domain_info.get('defaults') + return domain_info.get("defaults") def handle_ip_allow_deny(self, allow: bool, principals: List[str]) -> None: """ @@ -475,14 +527,18 @@ def handle_ip_allow_deny(self, allow: bool, principals: List[str]) -> None: :param principals: list of IP addresses or CIDR ranges to match """ - if self.get('ip_allow_deny') is not None: + if self.get("ip_allow_deny") is not None: self.post_error("ip_allow and ip_deny may not both be set") return - ipa = IRIPAllowDeny(self.ir, self.ir.aconf, rkey=self.rkey, - parent=self, - action="ALLOW" if allow else "DENY", - principals=principals) + ipa = IRIPAllowDeny( + self.ir, + self.ir.aconf, + rkey=self.rkey, + parent=self, + action="ALLOW" if allow else "DENY", + principals=principals, + ) if ipa: - self['ip_allow_deny'] = ipa + self["ip_allow_deny"] = ipa diff --git a/python/ambassador/ir/irauth.py b/python/ambassador/ir/irauth.py index 4ad2fa1627..c7ec9cb904 100644 --- a/python/ambassador/ir/irauth.py +++ b/python/ambassador/ir/irauth.py @@ -10,24 +10,32 @@ from .irretrypolicy import IRRetryPolicy if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRAuth (IRFilter): +class IRAuth(IRFilter): cluster: Optional[IRCluster] - protocol_version: Literal['v2', 'v3'] - - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.auth", - kind: str="IRAuth", - name: str="extauth", - namespace: Optional[str] = None, - type: Optional[str] = "decoder", - **kwargs) -> None: - + protocol_version: Literal["v2", "v3"] + + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.auth", + kind: str = "IRAuth", + name: str = "extauth", + namespace: Optional[str] = None, + type: Optional[str] = "decoder", + **kwargs, + ) -> None: super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, namespace=namespace, + ir=ir, + aconf=aconf, + rkey=rkey, + kind=kind, + name=name, + namespace=namespace, cluster=None, timeout_ms=None, connect_timeout_ms=3000, @@ -38,9 +46,10 @@ def __init__(self, ir: 'IR', aconf: Config, allowed_authorization_headers=[], hosts={}, type=type, - **kwargs) + **kwargs, + ) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: module_info = aconf.get_module("authentication") if module_info: @@ -60,8 +69,8 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return True - def add_mappings(self, ir: 'IR', aconf: Config): - cluster_hosts = self.get('hosts', { '127.0.0.1:5000': ( 100, None, '-internal-' ) }) + def add_mappings(self, ir: "IR", aconf: Config): + cluster_hosts = self.get("hosts", {"127.0.0.1:5000": (100, None, "-internal-")}) self.cluster = None cluster_good = False @@ -69,16 +78,21 @@ def add_mappings(self, ir: 'IR', aconf: Config): for service, params in cluster_hosts.items(): weight, grpc, ctx_name, location = params - self.logger.debug("IRAuth: svc %s, weight %s, grpc %s, ctx_name %s, location %s" % - (service, weight, grpc, ctx_name, location)) + self.logger.debug( + "IRAuth: svc %s, weight %s, grpc %s, ctx_name %s, location %s" + % (service, weight, grpc, ctx_name, location) + ) cluster = IRCluster( - ir=ir, aconf=aconf, parent_ir_resource=self, location=location, + ir=ir, + aconf=aconf, + parent_ir_resource=self, + location=location, service=service, - host_rewrite=self.get('host_rewrite', False), + host_rewrite=self.get("host_rewrite", False), ctx_name=ctx_name, grpc=grpc, - marker='extauth', + marker="extauth", stats_name=self.get("stats_name", None), circuit_breakers=self.get("circuit_breakers", None), ) @@ -89,7 +103,11 @@ def add_mappings(self, ir: 'IR', aconf: Config): if self.cluster: if not self.cluster.merge(cluster): - self.post_error(RichStatus.fromError("auth canary %s can only change service!" % cluster.name)) + self.post_error( + RichStatus.fromError( + "auth canary %s can only change service!" % cluster.name + ) + ) cluster_good = False else: self.cluster = cluster @@ -98,12 +116,12 @@ def add_mappings(self, ir: 'IR', aconf: Config): ir.add_cluster(typecast(IRCluster, self.cluster)) self.referenced_by(typecast(IRCluster, self.cluster)) - def _load_auth(self, module: Resource, ir: 'IR'): + def _load_auth(self, module: Resource, ir: "IR"): self.namespace = module.get("namespace", self.namespace) - if self.location == '--internal--': + if self.location == "--internal--": self.sourced_by(module) - for key in [ 'path_prefix', 'timeout_ms', 'cluster', 'allow_request_body', 'proto' ]: + for key in ["path_prefix", "timeout_ms", "cluster", "allow_request_body", "proto"]: value = module.get(key, None) if value: @@ -114,7 +132,7 @@ def _load_auth(self, module: Resource, ir: 'IR'): # resource. And don't use self.ir.post_error, since our module isn't an IRResource. self.ir.aconf.post_error( "AuthService cannot support multiple %s values; using %s" % (key, previous), - resource=module + resource=module, ) else: self[key] = value @@ -124,17 +142,17 @@ def _load_auth(self, module: Resource, ir: 'IR'): if module.get("add_linkerd_headers"): self["add_linkerd_headers"] = module.get("add_linkerd_headers") else: - add_linkerd_headers = module.get('add_linkerd_headers', None) + add_linkerd_headers = module.get("add_linkerd_headers", None) if add_linkerd_headers is None: - self["add_linkerd_headers"] = ir.ambassador_module.get('add_linkerd_headers', False) + self["add_linkerd_headers"] = ir.ambassador_module.get("add_linkerd_headers", False) - if module.get('circuit_breakers', None): - self['circuit_breakers'] = module.get('circuit_breakers') + if module.get("circuit_breakers", None): + self["circuit_breakers"] = module.get("circuit_breakers") else: - cb = ir.ambassador_module.get('circuit_breakers') + cb = ir.ambassador_module.get("circuit_breakers") if cb: - self['circuit_breakers'] = cb + self["circuit_breakers"] = cb self["allow_request_body"] = module.get("allow_request_body", False) self["include_body"] = module.get("include_body", None) @@ -143,26 +161,32 @@ def _load_auth(self, module: Resource, ir: 'IR'): self["timeout_ms"] = module.get("timeout_ms", 5000) self["connect_timeout_ms"] = module.get("connect_timeout_ms", 3000) self["cluster_idle_timeout_ms"] = module.get("cluster_idle_timeout_ms", None) - self["cluster_max_connection_lifetime_ms"] = module.get("cluster_max_connection_lifetime_ms", None) + self["cluster_max_connection_lifetime_ms"] = module.get( + "cluster_max_connection_lifetime_ms", None + ) self["add_auth_headers"] = module.get("add_auth_headers", {}) - self.__to_header_list('allowed_headers', module) - self.__to_header_list('allowed_request_headers', module) - self.__to_header_list('allowed_authorization_headers', module) + self.__to_header_list("allowed_headers", module) + self.__to_header_list("allowed_request_headers", module) + self.__to_header_list("allowed_authorization_headers", module) if self["proto"] not in ["grpc", "http"]: - self.post_error(f'AuthService: proto_version {self["proto"]} is unsupported, proto must be "grpc" or "http"') + self.post_error( + f'AuthService: proto_version {self["proto"]} is unsupported, proto must be "grpc" or "http"' + ) self.protocol_version = module.get("protocol_version", "v2") if self["proto"] == "grpc" and self.protocol_version not in ["v3"]: - self.post_error(f'AuthService: protocol_version {self.protocol_version} is unsupported, protocol_version must be "v3"') + self.post_error( + f'AuthService: protocol_version {self.protocol_version} is unsupported, protocol_version must be "v3"' + ) - status_on_error = module.get('status_on_error', None) + status_on_error = module.get("status_on_error", None) if status_on_error: - self['status_on_error'] = status_on_error + self["status_on_error"] = status_on_error - failure_mode_allow = module.get('failure_mode_allow', None) + failure_mode_allow = module.get("failure_mode_allow", None) if failure_mode_allow: - self['failure_mode_allow'] = failure_mode_allow + self["failure_mode_allow"] = failure_mode_allow # Required fields check. if self["api_version"] == None: @@ -172,15 +196,15 @@ def _load_auth(self, module: Resource, ir: 'IR'): self.post_error(RichStatus.fromError("AuthService requires proto field.")) if self.get("include_body") and self.get("allow_request_body"): - self.post_error('AuthService ignoring allow_request_body since include_body is present') - del(self['allow_request_body']) + self.post_error("AuthService ignoring allow_request_body since include_body is present") + del self["allow_request_body"] auth_service = module.get("auth_service", None) - weight = 100 # Can't support arbitrary weights right now. + weight = 100 # Can't support arbitrary weights right now. if auth_service: is_grpc = True if self["proto"] == "grpc" else False - self.hosts[auth_service] = ( weight, is_grpc, module.get('tls', None), module.location) + self.hosts[auth_service] = (weight, is_grpc, module.get("tls", None), module.location) def __to_header_list(self, list_name, module): headers = module.get(list_name, None) diff --git a/python/ambassador/ir/irbasemapping.py b/python/ambassador/ir/irbasemapping.py index 7540685e5b..dbbb058f86 100644 --- a/python/ambassador/ir/irbasemapping.py +++ b/python/ambassador/ir/irbasemapping.py @@ -11,7 +11,8 @@ from .irresource import IRResource if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover + def would_confuse_urlparse(url: str) -> bool: """Returns whether an URL-ish string would be interpretted by urlparse() @@ -22,7 +23,7 @@ def would_confuse_urlparse(url: str) -> bool: Note: This has a Go equivalent in github.com/emissary-ingress/emissary/v3/pkg/emissaryutil. Please keep them in-sync. """ - if url.find(':') > 0 and url.lstrip(scheme_chars).startswith("://"): + if url.find(":") > 0 and url.lstrip(scheme_chars).startswith("://"): # has a scheme return False if url.startswith("//"): @@ -30,7 +31,14 @@ def would_confuse_urlparse(url: str) -> bool: return False return True -def normalize_service_name(ir: 'IR', in_service: str, mapping_namespace: Optional[str], resolver_kind: str, rkey: Optional[str]=None) -> str: + +def normalize_service_name( + ir: "IR", + in_service: str, + mapping_namespace: Optional[str], + resolver_kind: str, + rkey: Optional[str] = None, +) -> str: """ Note: This has a Go equivalent in github.com/emissary-ingress/emissary/v3/pkg/emissaryutil. Please keep them in-sync. @@ -41,7 +49,11 @@ def normalize_service_name(ir: 'IR', in_service: str, mapping_namespace: Optiona if not parsed.hostname: raise ValueError("No hostname") # urlib.parse.unquote is permissive, but we want to be strict - bad_seqs = [seq for seq in re.findall(r'%.{,2}', parsed.hostname) if not re.fullmatch(r'%[0-9a-fA-F]{2}', seq)] + bad_seqs = [ + seq + for seq in re.findall(r"%.{,2}", parsed.hostname) + if not re.fullmatch(r"%[0-9a-fA-F]{2}", seq) + ] if bad_seqs: raise ValueError(f"Invalid percent-escape in hostname: {bad_seqs[0]}") hostname = urlunquote(parsed.hostname) @@ -63,31 +75,45 @@ def normalize_service_name(ir: 'IR', in_service: str, mapping_namespace: Optiona # Consul Resolvers don't allow service names to include subdomains, but # Kubernetes Resolvers _require_ subdomains to correctly handle namespaces. - want_qualified = not ir.ambassador_module.use_ambassador_namespace_for_service_resolution and resolver_kind.startswith('Kubernetes') + want_qualified = ( + not ir.ambassador_module.use_ambassador_namespace_for_service_resolution + and resolver_kind.startswith("Kubernetes") + ) is_qualified = "." in hostname or ":" in hostname or "localhost" == hostname - if mapping_namespace and mapping_namespace != ir.ambassador_namespace and want_qualified and not is_qualified: - hostname += "."+mapping_namespace - - out_service = urlquote(hostname, safe="!$&'()*+,;=:[]<>\"") # match 'encodeHost' behavior of Go stdlib net/url/url.go - if ':' in out_service: + if ( + mapping_namespace + and mapping_namespace != ir.ambassador_namespace + and want_qualified + and not is_qualified + ): + hostname += "." + mapping_namespace + + out_service = urlquote( + hostname, safe="!$&'()*+,;=:[]<>\"" + ) # match 'encodeHost' behavior of Go stdlib net/url/url.go + if ":" in out_service: out_service = f"[{out_service}]" if scheme: out_service = f"{scheme}://{out_service}" if port: out_service += f":{port}" - ir.logger.debug("%s use_ambassador_namespace_for_service_resolution %s, fully qualified %s, upstream hostname %s" % ( - resolver_kind, - ir.ambassador_module.use_ambassador_namespace_for_service_resolution, - is_qualified, - out_service - )) + ir.logger.debug( + "%s use_ambassador_namespace_for_service_resolution %s, fully qualified %s, upstream hostname %s" + % ( + resolver_kind, + ir.ambassador_module.use_ambassador_namespace_for_service_resolution, + is_qualified, + out_service, + ) + ) return out_service -class IRBaseMapping (IRResource): + +class IRBaseMapping(IRResource): group_id: str host: Optional[str] route_weight: List[Union[str, int]] @@ -96,17 +122,21 @@ class IRBaseMapping (IRResource): cluster_key: Optional[str] _weight: int - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, # REQUIRED - name: str, # REQUIRED - location: str, # REQUIRED - kind: str, # REQUIRED - namespace: Optional[str] = None, - metadata_labels: Optional[Dict[str, str]] = None, - apiVersion: str="getambassador.io/v3alpha1", - precedence: int=0, - cluster_tag: Optional[str]=None, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, # REQUIRED + name: str, # REQUIRED + location: str, # REQUIRED + kind: str, # REQUIRED + namespace: Optional[str] = None, + metadata_labels: Optional[Dict[str, str]] = None, + apiVersion: str = "getambassador.io/v3alpha1", + precedence: int = 0, + cluster_tag: Optional[str] = None, + **kwargs, + ) -> None: # Default status... self.cached_status = None self.status_update = None @@ -119,14 +149,22 @@ def __init__(self, ir: 'IR', aconf: Config, # Init the superclass... super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, - kind=kind, name=name, namespace=namespace, metadata_labels=metadata_labels, - apiVersion=apiVersion, precedence=precedence, cluster_tag=cluster_tag, - **kwargs + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + kind=kind, + name=name, + namespace=namespace, + metadata_labels=metadata_labels, + apiVersion=apiVersion, + precedence=precedence, + cluster_tag=cluster_tag, + **kwargs, ) @classmethod - def make_cache_key(cls, kind: str, name: str, namespace: str, version: str="v2") -> str: + def make_cache_key(cls, kind: str, name: str, namespace: str, version: str = "v2") -> str: # Why is this split on the name necessary? # the name of a Mapping when we fetch it from the aconf will match the metadata.name of # the Mapping that the config comes from _only if_ it is the only Mapping with that exact name. @@ -143,7 +181,7 @@ def make_cache_key(cls, kind: str, name: str, namespace: str, version: str="v2") name = name.split(".")[0] return f"{kind}-{version}-{name}-{namespace}" - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # Set up our cache key. We're using this format so that it'll be easy # to generate it just from the Mapping's K8s metadata. self._cache_key = IRBaseMapping.make_cache_key(self.kind, self.name, self.namespace) @@ -159,77 +197,85 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # We can also default the resolver, and scream if it doesn't match a resolver we # know about. - if not self.get('resolver'): - self.resolver = self.ir.ambassador_module.get('resolver', 'kubernetes-service') + if not self.get("resolver"): + self.resolver = self.ir.ambassador_module.get("resolver", "kubernetes-service") resolver = self.ir.get_resolver(self.resolver) if not resolver: - self.post_error(f'resolver {self.resolver} is unknown!') + self.post_error(f"resolver {self.resolver} is unknown!") return False - self.ir.logger.debug("%s: GID %s route_weight %s, resolver %s" % - (self, self.group_id, self.route_weight, resolver)) + self.ir.logger.debug( + "%s: GID %s route_weight %s, resolver %s" + % (self, self.group_id, self.route_weight, resolver) + ) # And, of course, we can make sure that the resolver thinks that this Mapping is OK. if not resolver.valid_mapping(ir, self): # If there's trouble, the resolver should've already posted about it. return False - if self.get('circuit_breakers', None) is None: - self['circuit_breakers'] = ir.ambassador_module.circuit_breakers + if self.get("circuit_breakers", None) is None: + self["circuit_breakers"] = ir.ambassador_module.circuit_breakers - if self.get('circuit_breakers', None) is not None: - if not self.validate_circuit_breakers(ir, self['circuit_breakers']): - self.post_error("Invalid circuit_breakers specified: {}, invalidating mapping".format(self['circuit_breakers'])) + if self.get("circuit_breakers", None) is not None: + if not self.validate_circuit_breakers(ir, self["circuit_breakers"]): + self.post_error( + "Invalid circuit_breakers specified: {}, invalidating mapping".format( + self["circuit_breakers"] + ) + ) return False return True @staticmethod - def validate_circuit_breakers(ir: 'IR', circuit_breakers) -> bool: + def validate_circuit_breakers(ir: "IR", circuit_breakers) -> bool: if not isinstance(circuit_breakers, (list, tuple)): return False for circuit_breaker in circuit_breakers: - if '_name' in circuit_breaker: + if "_name" in circuit_breaker: # Already reconciled. ir.logger.debug(f'Breaker validation: good breaker {circuit_breaker["_name"]}') continue - ir.logger.debug(f'Breaker validation: {dump_json(circuit_breakers, pretty=True)}') + ir.logger.debug(f"Breaker validation: {dump_json(circuit_breakers, pretty=True)}") - name_fields = [ 'cb' ] + name_fields = ["cb"] - if 'priority' in circuit_breaker: - prio = circuit_breaker.get('priority').lower() - if prio not in ['default', 'high']: + if "priority" in circuit_breaker: + prio = circuit_breaker.get("priority").lower() + if prio not in ["default", "high"]: return False name_fields.append(prio[0]) else: - name_fields.append('n') + name_fields.append("n") - digit_fields = [ ( 'max_connections', 'c' ), - ( 'max_pending_requests', 'p' ), - ( 'max_requests', 'r' ), - ( 'max_retries', 't' ) ] + digit_fields = [ + ("max_connections", "c"), + ("max_pending_requests", "p"), + ("max_requests", "r"), + ("max_retries", "t"), + ] for field, abbrev in digit_fields: if field in circuit_breaker: try: value = int(circuit_breaker[field]) - name_fields.append(f'{abbrev}{value}') + name_fields.append(f"{abbrev}{value}") except ValueError: return False - circuit_breaker['_name'] = ''.join(name_fields) + circuit_breaker["_name"] = "".join(name_fields) ir.logger.debug(f'Breaker valid: {circuit_breaker["_name"]}') return True def get_label(self, key: str) -> Optional[str]: - labels = self.get('metadata_labels') or {} + labels = self.get("metadata_labels") or {} return labels.get(key) or None def status(self) -> Optional[Dict[str, Any]]: @@ -242,7 +288,7 @@ def status(self) -> Optional[Dict[str, Any]]: return None def check_status(self) -> None: - crd_name = self.get_label('ambassador_crd') + crd_name = self.get_label("ambassador_crd") if not crd_name: return @@ -253,12 +299,12 @@ def check_status(self) -> None: wanted = self.status() if wanted != self.cached_status: - self.ir.k8s_status_updates[crd_name] = ('Mapping', self.namespace, wanted) + self.ir.k8s_status_updates[crd_name] = ("Mapping", self.namespace, wanted) def _group_id(self) -> str: - """ Compute the group ID for this Mapping. Must be defined by subclasses. """ - raise NotImplementedError("%s._group_id is not implemented?" % self.__class__.__name__) + """Compute the group ID for this Mapping. Must be defined by subclasses.""" + raise NotImplementedError("%s._group_id is not implemented?" % self.__class__.__name__) def _route_weight(self) -> List[Union[str, int]]: - """ Compute the route weight for this Mapping. Must be defined by subclasses. """ - raise NotImplementedError("%s._route_weight is not implemented?" % self.__class__.__name__) + """Compute the route weight for this Mapping. Must be defined by subclasses.""" + raise NotImplementedError("%s._route_weight is not implemented?" % self.__class__.__name__) diff --git a/python/ambassador/ir/irbasemappinggroup.py b/python/ambassador/ir/irbasemappinggroup.py index e4e3a3b2c5..c2a2e85157 100644 --- a/python/ambassador/ir/irbasemappinggroup.py +++ b/python/ambassador/ir/irbasemappinggroup.py @@ -6,22 +6,26 @@ from .irbasemapping import IRBaseMapping if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRBaseMappingGroup (IRResource): +class IRBaseMappingGroup(IRResource): mappings: List[IRBaseMapping] group_id: str group_weight: List[Union[str, int]] labels: Dict[str, Any] _cache_key: Optional[str] - def __init__(self, ir: 'IR', aconf: Config, - location: str, - rkey: str="ir.mappinggroup", - kind: str="IRBaseMappingGroup", - name: str="ir.mappinggroup", - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + location: str, + rkey: str = "ir.mappinggroup", + kind: str = "IRBaseMappingGroup", + name: str = "ir.mappinggroup", + **kwargs, + ) -> None: # Default to no cache key... self._cache_key = None @@ -30,8 +34,7 @@ def __init__(self, ir: 'IR', aconf: Config, # ...before we init the superclass, which will call self.setup(). super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, - kind=kind, name=name, **kwargs + ir=ir, aconf=aconf, rkey=rkey, location=location, kind=kind, name=name, **kwargs ) @classmethod @@ -41,7 +44,7 @@ def key_for_id(cls, group_id: str) -> str: # XXX WTFO, I hear you cry. Why is this "type: ignore here?" So here's the deal: # mypy doesn't like it if you override just the getter of a property that has a # setter, too, and I cannot figure out how else to shut it up. - @property # type: ignore + @property # type: ignore def cache_key(self) -> str: # XXX WTFO, I hear you cry again! Can this possibly be thread-safe??! # Well, no, not really. But as long as you're not trying to use the @@ -57,7 +60,9 @@ def normalize_weights_in_mappings(self) -> bool: # If there's only one mapping in the group, it's automatically weighted # at 100%. if len(self.mappings) == 1: - self.logger.debug("Assigning weight 100 to single mapping %s in group", self.mappings[0].name) + self.logger.debug( + "Assigning weight 100 to single mapping %s in group", self.mappings[0].name + ) self.mappings[0]._weight = 100 return True @@ -69,7 +74,7 @@ def normalize_weights_in_mappings(self) -> bool: current_weight = 0 for mapping in self.mappings: - if 'weight' in mapping: + if "weight" in mapping: if mapping.weight > 100: self.post_error(f"Mapping {mapping.name} has invalid weight {mapping.weight}") return False @@ -78,7 +83,9 @@ def normalize_weights_in_mappings(self) -> bool: current_weight += round(mapping.weight) # set mapping's calculated weight to current weight - self.logger.debug(f"Assigning calculated weight {current_weight} to mapping {mapping.name}") + self.logger.debug( + f"Assigning calculated weight {current_weight} to mapping {mapping.name}" + ) mapping._weight = current_weight # add this mapping to normalized mappings @@ -89,7 +96,9 @@ def normalize_weights_in_mappings(self) -> bool: # Did we go over 100%? if current_weight > 100: - self.post_error(f"Total weight of mappings exceeds 100, please reconfigure for correct behavior...") + self.post_error( + f"Total weight of mappings exceeds 100, please reconfigure for correct behavior..." + ) return False if num_weightless_mappings > 0: @@ -99,7 +108,7 @@ def normalize_weights_in_mappings(self) -> bool: # (much like we do for Argo rollouts). # # Likewise, you might expect that we'd generate errors if we're at less than 100% and - # have no weightless mappings. We don't do that because it's not entirely clear what + # have no weightless mappings. We don't do that because it's not entirely clear what # to do -- a straightforward answer is to simply scale the weights we do have to hit # 100%, and we may well do that for the next major version. # @@ -109,9 +118,11 @@ def normalize_weights_in_mappings(self) -> bool: # what we want in the "scale the canary to 100% and then delete the original" case # described above. (Not coincidentally, our CanaryDiffMapping tests exercise this.) remaining_weight = 100 - current_weight - weight_per_weightless_mapping = round(remaining_weight/num_weightless_mappings) + weight_per_weightless_mapping = round(remaining_weight / num_weightless_mappings) - self.logger.debug(f"Assigning calculated weight {weight_per_weightless_mapping} of remaining weight {remaining_weight} to each of {num_weightless_mappings} weightless mappings") + self.logger.debug( + f"Assigning calculated weight {weight_per_weightless_mapping} of remaining weight {remaining_weight} to each of {num_weightless_mappings} weightless mappings" + ) # Now, let's add weight to every weightless mapping and push to normalized_mappings for i, weightless_mapping in enumerate(weightless_mappings): @@ -122,7 +133,9 @@ def normalize_weights_in_mappings(self) -> bool: else: current_weight += weight_per_weightless_mapping - self.logger.debug(f"Assigning weight {current_weight} to weightless mapping {weightless_mapping.name}") + self.logger.debug( + f"Assigning weight {current_weight} to weightless mapping {weightless_mapping.name}" + ) weightless_mapping._weight = current_weight normalized_mappings.append(weightless_mapping) diff --git a/python/ambassador/ir/irbuffer.py b/python/ambassador/ir/irbuffer.py index 0adcb611ef..f8407e87cc 100644 --- a/python/ambassador/ir/irbuffer.py +++ b/python/ambassador/ir/irbuffer.py @@ -9,29 +9,32 @@ from .ircluster import IRCluster if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRBuffer (IRFilter): - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.buffer", - name: str="ir.buffer", - kind: str="IRBuffer", - **kwargs) -> None: +class IRBuffer(IRFilter): + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.buffer", + name: str = "ir.buffer", + kind: str = "IRBuffer", + **kwargs + ) -> None: - super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) + super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: - max_request_bytes = self.pop('max_request_bytes', None) + max_request_bytes = self.pop("max_request_bytes", None) if max_request_bytes is not None: self["max_request_bytes"] = max_request_bytes else: self.post_error(RichStatus.fromError("missing required field: max_request_bytes")) return False - if self.pop('max_request_time', None): + if self.pop("max_request_time", None): self.ir.aconf.post_notice("'max_request_time' is no longer supported, ignoring", self) return True diff --git a/python/ambassador/ir/ircluster.py b/python/ambassador/ir/ircluster.py index 88cffda05f..e023a32957 100644 --- a/python/ambassador/ir/ircluster.py +++ b/python/ambassador/ir/ircluster.py @@ -26,8 +26,8 @@ from .irtlscontext import IRTLSContext if TYPE_CHECKING: - from .ir import IR # pragma: no cover - from .ir.irserviceresolver import IRServiceResolver # pragma: no cover + from .ir import IR # pragma: no cover + from .ir.irserviceresolver import IRServiceResolver # pragma: no cover ############################################################################# ## ircluster.py -- the ircluster configuration object for Ambassador @@ -38,36 +38,37 @@ ## entity. -class IRCluster (IRResource): - def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', - location: str, # REQUIRED - - service: str, # REQUIRED - resolver: Optional[str] = None, - connect_timeout_ms: Optional[int] = 3000, - cluster_idle_timeout_ms: Optional[int] = None, - cluster_max_connection_lifetime_ms: Optional[int] = None, - marker: Optional[str] = None, # extra marker for this context name - stats_name: Optional[str] = None, # Override the stats name for this cluster - - ctx_name: Optional[Union[str, bool]]=None, - host_rewrite: Optional[str]=None, - - dns_type: Optional[str]="strict_dns", - enable_ipv4: Optional[bool]=None, - enable_ipv6: Optional[bool]=None, - lb_type: str="round_robin", - grpc: Optional[bool] = False, - allow_scheme: Optional[bool] = True, - load_balancer: Optional[dict] = None, - keepalive: Optional[dict] = None, - circuit_breakers: Optional[list] = None, - respect_dns_ttl: Optional[bool] = False, - - rkey: str="-override-", - kind: str="IRCluster", - apiVersion: str="getambassador.io/v0", # Not a typo! See below. - **kwargs) -> None: +class IRCluster(IRResource): + def __init__( + self, + ir: "IR", + aconf: Config, + parent_ir_resource: "IRResource", + location: str, # REQUIRED + service: str, # REQUIRED + resolver: Optional[str] = None, + connect_timeout_ms: Optional[int] = 3000, + cluster_idle_timeout_ms: Optional[int] = None, + cluster_max_connection_lifetime_ms: Optional[int] = None, + marker: Optional[str] = None, # extra marker for this context name + stats_name: Optional[str] = None, # Override the stats name for this cluster + ctx_name: Optional[Union[str, bool]] = None, + host_rewrite: Optional[str] = None, + dns_type: Optional[str] = "strict_dns", + enable_ipv4: Optional[bool] = None, + enable_ipv6: Optional[bool] = None, + lb_type: str = "round_robin", + grpc: Optional[bool] = False, + allow_scheme: Optional[bool] = True, + load_balancer: Optional[dict] = None, + keepalive: Optional[dict] = None, + circuit_breakers: Optional[list] = None, + respect_dns_ttl: Optional[bool] = False, + rkey: str = "-override-", + kind: str = "IRCluster", + apiVersion: str = "getambassador.io/v0", # Not a typo! See below. + **kwargs, + ) -> None: # Step one: look at the service and such and figure out a cluster name # and TLS origination info. @@ -87,7 +88,7 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # if we're originating TLS, 80 if not. originate_tls: bool = False - name_fields: List[str] = [ 'cluster' ] + name_fields: List[str] = ["cluster"] ctx: Optional[IRTLSContext] = None errors: List[str] = [] unknown_breakers = 0 @@ -122,41 +123,47 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # TODO: lots of duplication of here, need to replace with broken down functions if allow_scheme and service.lower().startswith("https://"): - service = service[len("https://"):] + service = service[len("https://") :] originate_tls = True - name_fields.append('otls') + name_fields.append("otls") elif allow_scheme and service.lower().startswith("http://"): - service = service[ len("http://"): ] + service = service[len("http://") :] if ctx: - errors.append("Originate-TLS context %s being used even though service %s lists HTTP" % - (ctx_name, service)) + errors.append( + "Originate-TLS context %s being used even though service %s lists HTTP" + % (ctx_name, service) + ) originate_tls = True - name_fields.append('otls') + name_fields.append("otls") else: originate_tls = False elif ctx: # No scheme (or schemes are ignored), but we have a context. originate_tls = True - name_fields.append('otls') + name_fields.append("otls") name_fields.append(ctx.name) - if '://' in service: + if "://" in service: # WTF is this? - idx = service.index('://') + idx = service.index("://") scheme = service[0:idx] if allow_scheme: - errors.append("service %s has unknown scheme %s, assuming %s" % - (service, scheme, "HTTPS" if originate_tls else "HTTP")) + errors.append( + "service %s has unknown scheme %s, assuming %s" + % (service, scheme, "HTTPS" if originate_tls else "HTTP") + ) else: - errors.append("ignoring scheme %s for service %s, since it is being used for a non-HTTP mapping" % - (scheme, service)) + errors.append( + "ignoring scheme %s for service %s, since it is being used for a non-HTTP mapping" + % (scheme, service) + ) - service = service[idx + 3:] + service = service[idx + 3 :] # XXX Should this be checking originate_tls? Why does it do that? if originate_tls and host_rewrite: @@ -166,12 +173,15 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # parser, because it's kind of stupid. ir.logger.debug("cluster setup: service %s otls %s ctx %s" % (service, originate_tls, ctx)) - p = urllib.parse.urlparse('random://' + service) + p = urllib.parse.urlparse("random://" + service) # Is there any junk after the host? if p.path or p.params or p.query or p.fragment: - errors.append("service %s has extra URL components; ignoring everything but the host and port" % service) + errors.append( + "service %s has extra URL components; ignoring everything but the host and port" + % service + ) # p is read-only, so break stuff out. @@ -183,14 +193,20 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # Do we actually have a hostname? if not hostname: # We don't. That ain't good. - errors.append("service %s has no hostname and will be ignored; please re-configure" % service) + errors.append( + "service %s has no hostname and will be ignored; please re-configure" % service + ) self.ignore_cluster = True hostname = "unknown" try: port = p.port except ValueError as e: - errors.append("found invalid port for service {}. Please specify a valid port between 0 and 65535 - {}. Service {} cluster will be ignored, please re-configure".format(service, e, service)) + errors.append( + "found invalid port for service {}. Please specify a valid port between 0 and 65535 - {}. Service {} cluster will be ignored, please re-configure".format( + service, e, service + ) + ) self.ignore_cluster = True port = 0 @@ -206,14 +222,14 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # Is there a circuit breaker involved here? if circuit_breakers: for breaker in circuit_breakers: - name = breaker.get('_name', None) + name = breaker.get("_name", None) if name: name_fields.append(name) else: # This is "impossible", but... let it go I guess? errors.append(f"{service}: unvalidated circuit breaker {breaker}!") - name_fields.append(f'cbu{unknown_breakers}') + name_fields.append(f"cbu{unknown_breakers}") unknown_breakers += 1 # The Ambassador module will always have a load_balancer (which may be None). @@ -229,7 +245,9 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', if self.endpoints_required(load_balancer): if not Config.enable_endpoints: # Bzzt. - errors.append(f"{service}: endpoint routing is not enabled, falling back to {global_load_balancer}") + errors.append( + f"{service}: endpoint routing is not enabled, falling back to {global_load_balancer}" + ) load_balancer = global_load_balancer else: enable_endpoints = True @@ -238,27 +256,27 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # This is used only for cluster naming; it doesn't need to be a real # load balancer policy. - lb_type = load_balancer.get('policy', 'default') + lb_type = load_balancer.get("policy", "default") - key_fields = ['er', lb_type.lower()] + key_fields = ["er", lb_type.lower()] # XXX Should we really include these things? - if 'header' in load_balancer: - key_fields.append('hdr') - key_fields.append(load_balancer['header']) + if "header" in load_balancer: + key_fields.append("hdr") + key_fields.append(load_balancer["header"]) - if 'cookie' in load_balancer: - key_fields.append('cookie') - key_fields.append(load_balancer['cookie']['name']) + if "cookie" in load_balancer: + key_fields.append("cookie") + key_fields.append(load_balancer["cookie"]["name"]) - if 'source_ip' in load_balancer: - key_fields.append('srcip') + if "source_ip" in load_balancer: + key_fields.append("srcip") name_fields.append("-".join(key_fields)) # Finally we can construct the cluster name. name = "_".join(name_fields) - name = re.sub(r'[^0-9A-Za-z_]', '_', name) + name = re.sub(r"[^0-9A-Za-z_]", "_", name) # OK. Build our default args. # @@ -266,27 +284,31 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', if enable_ipv4 is None: enable_ipv4 = ir.ambassador_module.enable_ipv4 - ir.logger.debug("%s: copying enable_ipv4 %s from Ambassador Module" % (name, enable_ipv4)) + ir.logger.debug( + "%s: copying enable_ipv4 %s from Ambassador Module" % (name, enable_ipv4) + ) if enable_ipv6 is None: enable_ipv6 = ir.ambassador_module.enable_ipv6 - ir.logger.debug("%s: copying enable_ipv6 %s from Ambassador Module" % (name, enable_ipv6)) + ir.logger.debug( + "%s: copying enable_ipv6 %s from Ambassador Module" % (name, enable_ipv6) + ) new_args: Dict[str, Any] = { "type": dns_type, "lb_type": lb_type, - "urls": [ url ], # TODO: Should we completely eliminate `urls` in favor of `targets`? + "urls": [url], # TODO: Should we completely eliminate `urls` in favor of `targets`? "load_balancer": load_balancer, "keepalive": keepalive, "circuit_breakers": circuit_breakers, "service": service, - 'enable_ipv4': enable_ipv4, - 'enable_ipv6': enable_ipv6, - 'enable_endpoints': enable_endpoints, - 'connect_timeout_ms': connect_timeout_ms, - 'cluster_idle_timeout_ms': cluster_idle_timeout_ms, - 'cluster_max_connection_lifetime_ms': cluster_max_connection_lifetime_ms, - 'respect_dns_ttl': respect_dns_ttl, + "enable_ipv4": enable_ipv4, + "enable_ipv6": enable_ipv6, + "enable_endpoints": enable_endpoints, + "connect_timeout_ms": connect_timeout_ms, + "cluster_idle_timeout_ms": cluster_idle_timeout_ms, + "cluster_max_connection_lifetime_ms": cluster_max_connection_lifetime_ms, + "respect_dns_ttl": respect_dns_ttl, } # If we have a stats_name, use it. If not, default it to the service to make life @@ -294,23 +316,23 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', # to underscores, just in case. if stats_name: - new_args['stats_name'] = stats_name + new_args["stats_name"] = stats_name else: - new_args['stats_name'] = re.sub(r'[^0-9A-Za-z_]', '_', service) + new_args["stats_name"] = re.sub(r"[^0-9A-Za-z_]", "_", service) if grpc: - new_args['grpc'] = True + new_args["grpc"] = True if host_rewrite: - new_args['host_rewrite'] = host_rewrite + new_args["host_rewrite"] = host_rewrite if originate_tls: if ctx: - new_args['tls_context'] = typecast(IRTLSContext, ctx) + new_args["tls_context"] = typecast(IRTLSContext, ctx) else: - new_args['tls_context'] = IRTLSContext.null_context(ir=ir) + new_args["tls_context"] = IRTLSContext.null_context(ir=ir) - if rkey == '-override-': + if rkey == "-override-": rkey = name # Stash the resolver, hostname, and port for setup. @@ -320,13 +342,18 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', self._port = port self._is_sidecar = False - if self._hostname == '127.0.0.1' and self._port == 8500: + if self._hostname == "127.0.0.1" and self._port == 8500: self._is_sidecar = True super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, - kind=kind, name=name, apiVersion=apiVersion, - **new_args + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + kind=kind, + name=name, + apiVersion=apiVersion, + **new_args, ) if ctx: @@ -336,14 +363,16 @@ def __init__(self, ir: 'IR', aconf: Config, parent_ir_resource: 'IRResource', for error in errors: ir.post_error(error, resource=self) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: self._cache_key = f"Cluster-{self.name}" if self.ignore_cluster: return False # Resolve our actual targets. - targets = ir.resolve_targets(self, self._resolver, self._hostname, self._namespace, self._port) + targets = ir.resolve_targets( + self, self._resolver, self._hostname, self._namespace, self._port + ) self.targets = targets @@ -359,10 +388,12 @@ def endpoints_required(self, load_balancer) -> bool: required = False if load_balancer: - lb_policy = load_balancer.get('policy') + lb_policy = load_balancer.get("policy") - if lb_policy in ['round_robin', 'least_request', 'ring_hash', 'maglev']: - self.logger.debug("Endpoints are required for load balancing policy {}".format(lb_policy)) + if lb_policy in ["round_robin", "least_request", "ring_hash", "maglev"]: + self.logger.debug( + "Endpoints are required for load balancing policy {}".format(lb_policy) + ) required = True return required @@ -372,19 +403,32 @@ def add_url(self, url: str) -> List[str]: return self.urls - def merge(self, other: 'IRCluster') -> bool: + def merge(self, other: "IRCluster") -> bool: # Is this mergeable? mismatches = [] - for key in [ 'type', 'lb_type', 'host_rewrite', - 'tls_context', 'originate_tls', 'grpc', 'connect_timeout_ms', 'cluster_idle_timeout_ms', 'cluster_max_connection_lifetime_ms' ]: + for key in [ + "type", + "lb_type", + "host_rewrite", + "tls_context", + "originate_tls", + "grpc", + "connect_timeout_ms", + "cluster_idle_timeout_ms", + "cluster_max_connection_lifetime_ms", + ]: if self.get(key, None) != other.get(key, None): mismatches.append(key) if mismatches: - self.post_error(RichStatus.fromError("cannot merge cluster %s: mismatched attributes %s" % - (other.name, ", ".join(mismatches)))) + self.post_error( + RichStatus.fromError( + "cannot merge cluster %s: mismatched attributes %s" + % (other.name, ", ".join(mismatches)) + ) + ) return False # All good. @@ -399,12 +443,16 @@ def merge(self, other: 'IRCluster') -> bool: if self.targets == None: self.targets = other.targets else: - self.targets = typecast(List[Dict[str, Union[int, str]]], self.targets) + other.targets + self.targets = ( + typecast(List[Dict[str, Union[int, str]]], self.targets) + other.targets + ) return True - def get_resolver(self) -> 'IRServiceResolver': + def get_resolver(self) -> "IRServiceResolver": return self.ir.resolve_resolver(self, self._resolver) def clustermap_entry(self) -> Dict: - return self.get_resolver().clustermap_entry(self.ir, self, self._hostname, self._namespace, self._port) + return self.get_resolver().clustermap_entry( + self.ir, self, self._hostname, self._namespace, self._port + ) diff --git a/python/ambassador/ir/ircors.py b/python/ambassador/ir/ircors.py index 06ee783951..c83788ccba 100644 --- a/python/ambassador/ir/ircors.py +++ b/python/ambassador/ir/ircors.py @@ -8,16 +8,19 @@ from .irresource import IRResource if TYPE_CHECKING: - from .ir import IR # pragma: no cover - - -class IRCORS (IRResource): - def __init__(self, ir: 'IR', aconf: Config, - - rkey: str="ir.cors", - kind: str="IRCORS", - name: str="ir.cors", - **kwargs) -> None: + from .ir import IR # pragma: no cover + + +class IRCORS(IRResource): + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.cors", + kind: str = "IRCORS", + name: str = "ir.cors", + **kwargs, + ) -> None: # print("IRCORS __init__ (%s %s %s)" % (kind, name, kwargs)) # Convert our incoming kwargs into the things that Envoy actually wants. @@ -26,11 +29,13 @@ def __init__(self, ir: 'IR', aconf: Config, new_kwargs: Dict[str, Any] = {} - for from_key, to_key in [ ( 'max_age', 'max_age' ), - ( 'credentials', 'allow_credentials' ), - ( 'methods', 'allow_methods' ), - ( 'headers', 'allow_headers' ), - ( 'exposed_headers', 'expose_headers' ) ]: + for from_key, to_key in [ + ("max_age", "max_age"), + ("credentials", "allow_credentials"), + ("methods", "allow_methods"), + ("headers", "allow_headers"), + ("exposed_headers", "expose_headers"), + ]: value = kwargs.get(from_key, None) if value: @@ -38,31 +43,25 @@ def __init__(self, ir: 'IR', aconf: Config, # 'origins' cannot be treated like other keys, because we have to transform it; Envoy wants # it in a different shape than it is in the CRD. - origins = kwargs.get('origins', None) + origins = kwargs.get("origins", None) if origins is not None: - new_kwargs['allow_origin_string_match'] = [{'exact': origin} for origin in origins] + new_kwargs["allow_origin_string_match"] = [{"exact": origin} for origin in origins] - super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, - **new_kwargs - ) + super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **new_kwargs) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # This IRCORS has not been finalized with an ID, so leave with an 'unset' ID so far. - self.set_id('unset') + self.set_id("unset") return True def set_id(self, group_id: str): - self['filter_enabled'] = { - "default_value": { - "denominator": "HUNDRED", - "numerator": 100 - }, - "runtime_key": f"routing.cors_enabled.{group_id}" + self["filter_enabled"] = { + "default_value": {"denominator": "HUNDRED", "numerator": 100}, + "runtime_key": f"routing.cors_enabled.{group_id}", } - def dup(self) -> 'IRCORS': + def dup(self) -> "IRCORS": return copy.copy(self) @staticmethod @@ -73,7 +72,7 @@ def _cors_normalize(value: Any) -> Any: """ if type(value) == list: - return ", ".join([ str(x) for x in value ]) + return ", ".join([str(x) for x in value]) else: return value @@ -81,8 +80,17 @@ def as_dict(self) -> dict: raw_dict = super().as_dict() for key in list(raw_dict): - if key in ["_active", "_errored", "_referenced_by", "_rkey", - "kind", "location", "name", "namespace", "metadata_labels"]: + if key in [ + "_active", + "_errored", + "_referenced_by", + "_rkey", + "kind", + "location", + "name", + "namespace", + "metadata_labels", + ]: raw_dict.pop(key, None) return raw_dict diff --git a/python/ambassador/ir/irerrorresponse.py b/python/ambassador/ir/irerrorresponse.py index b5f49eacd4..ec6ae9088d 100644 --- a/python/ambassador/ir/irerrorresponse.py +++ b/python/ambassador/ir/irerrorresponse.py @@ -6,8 +6,8 @@ from .irfilter import IRFilter if TYPE_CHECKING: - from .ir import IR # pragma: no cover - from .ir.irresource import IRResource # pragma: no cover + from .ir import IR # pragma: no cover + from .ir.irresource import IRResource # pragma: no cover import re @@ -15,30 +15,72 @@ # Use a whitelist to validate that any command operators in error response body are supported by envoy # TODO: remove this after support for escaping "%" lands in envoy ALLOWED_ENVOY_FMT_TOKENS = [ - "START_TIME", "REQUEST_HEADERS_BYTES", "BYTES_RECEIVED", - "PROTOCOL", "RESPONSE_CODE", "RESPONSE_CODE_DETAILS", - "CONNECTION_TERMINATION_DETAILS", "RESPONSE_HEADERS_BYTES", - "RESPONSE_TRAILERS_BYTES", "BYTES_SENT", "UPSTREAM_WIRE_BYTES_SENT", - "UPSTREAM_WIRE_BYTES_RECEIVED", "UPSTREAM_HEADER_BYTES_SENT", - "UPSTREAM_HEADER_BYTES_RECEIVED", "DOWNSTREAM_WIRE_BYTES_SENT", - "DOWNSTREAM_WIRE_BYTES_RECEIVED", "DOWNSTREAM_HEADER_BYTES_SENT", - "DOWNSTREAM_HEADER_BYTES_RECEIVED", "DURATION", "REQUEST_DURATION", - "REQUEST_TX_DURATION", "RESPONSE_DURATION", "RESPONSE_TX_DURATION", - "RESPONSE_FLAGS", "ROUTE_NAME", "UPSTREAM_HOST", "UPSTREAM_CLUSTER", - "UPSTREAM_LOCAL_ADDRESS", "UPSTREAM_TRANSPORT_FAILURE_REASON", - "DOWNSTREAM_REMOTE_ADDRESS", "DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT", - "DOWNSTREAM_DIRECT_REMOTE_ADDRESS", "DOWNSTREAM_DIRECT_REMOTE_ADDRESS_WITHOUT_PORT", - "DOWNSTREAM_LOCAL_ADDRESS", "DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", - "CONNECTION_ID", "GRPC_STATUS", "DOWNSTREAM_LOCAL_PORT", "REQ", - "RESP", "TRAILER", "DYNAMIC_METADATA", "CLUSTER_METADATA", - "FILTER_STATE", "REQUESTED_SERVER_NAME", "DOWNSTREAM_LOCAL_URI_SAN", - "DOWNSTREAM_PEER_URI_SAN", "DOWNSTREAM_LOCAL_SUBJECT", "DOWNSTREAM_PEER_SUBJECT", - "DOWNSTREAM_PEER_ISSUER", "DOWNSTREAM_TLS_SESSION_ID", "DOWNSTREAM_TLS_CIPHER", - "DOWNSTREAM_TLS_VERSION", "DOWNSTREAM_PEER_FINGERPRINT_256", "DOWNSTREAM_PEER_FINGERPRINT_1", - "DOWNSTREAM_PEER_SERIAL", "DOWNSTREAM_PEER_CERT", "DOWNSTREAM_PEER_CERT_V_START", - "DOWNSTREAM_PEER_CERT_V_END", "HOSTNAME", "LOCAL_REPLY_BODY", "FILTER_CHAIN_NAME" + "START_TIME", + "REQUEST_HEADERS_BYTES", + "BYTES_RECEIVED", + "PROTOCOL", + "RESPONSE_CODE", + "RESPONSE_CODE_DETAILS", + "CONNECTION_TERMINATION_DETAILS", + "RESPONSE_HEADERS_BYTES", + "RESPONSE_TRAILERS_BYTES", + "BYTES_SENT", + "UPSTREAM_WIRE_BYTES_SENT", + "UPSTREAM_WIRE_BYTES_RECEIVED", + "UPSTREAM_HEADER_BYTES_SENT", + "UPSTREAM_HEADER_BYTES_RECEIVED", + "DOWNSTREAM_WIRE_BYTES_SENT", + "DOWNSTREAM_WIRE_BYTES_RECEIVED", + "DOWNSTREAM_HEADER_BYTES_SENT", + "DOWNSTREAM_HEADER_BYTES_RECEIVED", + "DURATION", + "REQUEST_DURATION", + "REQUEST_TX_DURATION", + "RESPONSE_DURATION", + "RESPONSE_TX_DURATION", + "RESPONSE_FLAGS", + "ROUTE_NAME", + "UPSTREAM_HOST", + "UPSTREAM_CLUSTER", + "UPSTREAM_LOCAL_ADDRESS", + "UPSTREAM_TRANSPORT_FAILURE_REASON", + "DOWNSTREAM_REMOTE_ADDRESS", + "DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT", + "DOWNSTREAM_DIRECT_REMOTE_ADDRESS", + "DOWNSTREAM_DIRECT_REMOTE_ADDRESS_WITHOUT_PORT", + "DOWNSTREAM_LOCAL_ADDRESS", + "DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", + "CONNECTION_ID", + "GRPC_STATUS", + "DOWNSTREAM_LOCAL_PORT", + "REQ", + "RESP", + "TRAILER", + "DYNAMIC_METADATA", + "CLUSTER_METADATA", + "FILTER_STATE", + "REQUESTED_SERVER_NAME", + "DOWNSTREAM_LOCAL_URI_SAN", + "DOWNSTREAM_PEER_URI_SAN", + "DOWNSTREAM_LOCAL_SUBJECT", + "DOWNSTREAM_PEER_SUBJECT", + "DOWNSTREAM_PEER_ISSUER", + "DOWNSTREAM_TLS_SESSION_ID", + "DOWNSTREAM_TLS_CIPHER", + "DOWNSTREAM_TLS_VERSION", + "DOWNSTREAM_PEER_FINGERPRINT_256", + "DOWNSTREAM_PEER_FINGERPRINT_1", + "DOWNSTREAM_PEER_SERIAL", + "DOWNSTREAM_PEER_CERT", + "DOWNSTREAM_PEER_CERT_V_START", + "DOWNSTREAM_PEER_CERT_V_END", + "HOSTNAME", + "LOCAL_REPLY_BODY", + "FILTER_CHAIN_NAME", ] -ENVOY_FMT_TOKEN_REGEX = "\%([A-Za-z0-9_]+?)(\([A-Za-z0-9_.]+?((:|\?)[A-Za-z0-9_.]+?)+\))?(:[A-Za-z0-9_]+?)?\%" +ENVOY_FMT_TOKEN_REGEX = ( + "\%([A-Za-z0-9_]+?)(\([A-Za-z0-9_.]+?((:|\?)[A-Za-z0-9_.]+?)+\))?(:[A-Za-z0-9_]+?)?\%" +) # IRErrorResponse implements custom error response bodies using Envoy's HTTP response_map filter. # @@ -49,7 +91,7 @@ # # The Ambassador module config isn't subject to strict typing at higher layers, so this IR has # to pay special attention to the types and format of the incoming config. -class IRErrorResponse (IRFilter): +class IRErrorResponse(IRFilter): # The list of mappers that will make up the final error response config _mappers: Optional[List[Dict[str, Any]]] @@ -60,20 +102,24 @@ class IRErrorResponse (IRFilter): # The object that references this IRErrorResource. # Use by diagnostics to report the exact source of configuration errors. - _referenced_by_obj: Optional['IRResource'] - - def __init__(self, ir: 'IR', aconf: Config, error_response_config: List[Dict[str, Any]], - referenced_by_obj: Optional['IRResource']=None, - rkey: str="ir.error_response", - kind: str="IRErrorResponse", - name: str="error_response", - type: Optional[str] = "decoder", - **kwargs) -> None: + _referenced_by_obj: Optional["IRResource"] + + def __init__( + self, + ir: "IR", + aconf: Config, + error_response_config: List[Dict[str, Any]], + referenced_by_obj: Optional["IRResource"] = None, + rkey: str = "ir.error_response", + kind: str = "IRErrorResponse", + name: str = "error_response", + type: Optional[str] = "decoder", + **kwargs, + ) -> None: self._ir_config = error_response_config self._referenced_by_obj = referenced_by_obj self._mappers = None - super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) + super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) # Return the final config, or None if there isn't any, either because # there was no input config, or none of the input config was valid. @@ -84,25 +130,25 @@ def __init__(self, ir: 'IR', aconf: Config, error_response_config: List[Dict[str def config(self) -> Optional[Dict[str, Any]]: if not self._mappers: return None - return { - 'mappers': self._mappers - } + return {"mappers": self._mappers} # Runs setup and always returns true to indicate success. This is safe because # _setup is tolerant of missing or invalid config. At the end of setup, the caller # should retain this object and use `config()` get the final, good config, if any. - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: self._setup(ir, aconf) return True - def _setup(self, ir: 'IR', aconf: Config): + def _setup(self, ir: "IR", aconf: Config): # Do nothing (and post no errors) if there's no config. if not self._ir_config: return # The error_response_overrides config must be an array if not isinstance(self._ir_config, list): - self.post_error(f"IRErrorResponse: error_response_overrides: field must be an array, got {type(self._ir_config)}") + self.post_error( + f"IRErrorResponse: error_response_overrides: field must be an array, got {type(self._ir_config)}" + ) return # Do nothing (and post no errors) if there's config, but it's empty. @@ -120,7 +166,6 @@ def _setup(self, ir: 'IR', aconf: Config): if self._referenced_by_obj is not None: self.referenced_by(self._referenced_by_obj) - def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: all_mappers: List[Dict[str, Any]] = [] for error_response in self._ir_config: @@ -139,7 +184,7 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: if code < 400 or code >= 600: raise ValueError("field must be an integer >= 400 and < 600") - status_code_str: str=str(code) + status_code_str: str = str(code) except ValueError as e: self.post_error(f"IRErrorResponse: on_status_code: %s" % e) continue @@ -151,7 +196,8 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: continue if not isinstance(ir_body, dict): self.post_error( - f"IRErrorResponse: body: field must be an object, found %s" % ir_body) + f"IRErrorResponse: body: field must be an object, found %s" % ir_body + ) continue # We currently only support filtering using an equality match on status codes. @@ -168,12 +214,11 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: # has an associated "runtime_key". This is used as a key # in the runtime config system for changing config values # without restarting Envoy. - # We definitely do not want this value to ever change # inside of Envoy at runtime, so the best we can do is name # this key something arbitrary and hopefully unused. - "runtime_key": "_donotsetthiskey" - } + "runtime_key": "_donotsetthiskey", + }, } } } @@ -194,33 +239,38 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: # Only one of text_format, json_format, or text_format_source may be set. # Post an error if we found more than one these fields set. formats_set: int = 0 - for f in [ ir_text_format_source, ir_text_format, ir_json_format ]: + for f in [ir_text_format_source, ir_text_format, ir_json_format]: if f is not None: formats_set += 1 if formats_set > 1: self.post_error( - "IRErrorResponse: only one of \"text_format\", \"json_format\", " - +"or \"text_format_source\" may be set, found %d of these fields set." % - formats_set) + 'IRErrorResponse: only one of "text_format", "json_format", ' + + 'or "text_format_source" may be set, found %d of these fields set.' + % formats_set + ) continue body_format_override: Dict[str, Any] = {} if ir_text_format_source is not None: # Verify that the text_format_source field is an object with a string filename. - if not isinstance(ir_text_format_source, dict) or \ - not isinstance(ir_text_format_source.get('filename', None), str): + if not isinstance(ir_text_format_source, dict) or not isinstance( + ir_text_format_source.get("filename", None), str + ): self.post_error( - f"IRErrorResponse: text_format_source field must be an object with a single filename field, found \"{ir_text_format_source}\"") + f'IRErrorResponse: text_format_source field must be an object with a single filename field, found "{ir_text_format_source}"' + ) continue body_format_override["text_format_source"] = ir_text_format_source try: - fmt_file = open(ir_text_format_source["filename"], mode='r') + fmt_file = open(ir_text_format_source["filename"], mode="r") format_body = fmt_file.read() fmt_file.close() except OSError: - self.post_error("IRErrorResponse: text_format_source field references a file that does not exist") + self.post_error( + "IRErrorResponse: text_format_source field references a file that does not exist" + ) continue elif ir_text_format is not None: @@ -233,7 +283,9 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: elif ir_json_format is not None: # Verify that the json_format field is an object if not isinstance(ir_json_format, dict): - self.post_error(f"IRErrorResponse: json_format field must be an object, found \"{ir_json_format}\"") + self.post_error( + f'IRErrorResponse: json_format field must be an object, found "{ir_json_format}"' + ) continue # Envoy requires string values for json_format. Validate that every field in the @@ -253,7 +305,7 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: sanitized[k] = str(v) format_body += f"{k}: {str(v)}, " else: - error = f"IRErrorResponse: json_format only supports string values, and type \"{type(v)}\" for key \"{k}\" cannot be implicitly converted to string" + error = f'IRErrorResponse: json_format only supports string values, and type "{type(v)}" for key "{k}" cannot be implicitly converted to string' break except ValueError as e: # This really shouldn't be possible, because the string casts we do above @@ -267,7 +319,8 @@ def _generate_mappers(self) -> Optional[List[Dict[str, Any]]]: body_format_override["json_format"] = sanitized else: self.post_error( - f"IRErrorResponse: could not find a valid format field in body \"{ir_body}\"") + f'IRErrorResponse: could not find a valid format field in body "{ir_body}"' + ) continue if ir_content_type is not None: diff --git a/python/ambassador/ir/irfilter.py b/python/ambassador/ir/irfilter.py index 93ae5171c2..11ca1e0a39 100644 --- a/python/ambassador/ir/irfilter.py +++ b/python/ambassador/ir/irfilter.py @@ -5,24 +5,33 @@ from .irresource import IRResource if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover class IRFilter(IRResource): - def __init__(self, ir: 'IR', aconf: Config, - rkey: str = "ir.filter", - kind: str = "IRFilter", - name: str = "ir.filter", - location: str = "--internal--", - type: Optional[str] = None, - config: Optional[Dict[str, Any]] = None, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.filter", + kind: str = "IRFilter", + name: str = "ir.filter", + location: str = "--internal--", + type: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + **kwargs + ) -> None: super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, + ir=ir, + aconf=aconf, + rkey=rkey, + kind=kind, + name=name, location=location, type=type, config=config, - **kwargs) + **kwargs + ) def config_dict(self) -> Optional[Dict[str, Any]]: return self.config diff --git a/python/ambassador/ir/irgzip.py b/python/ambassador/ir/irgzip.py index a83655eccb..3a03d1b547 100644 --- a/python/ambassador/ir/irgzip.py +++ b/python/ambassador/ir/irgzip.py @@ -9,27 +9,30 @@ from .ircluster import IRCluster if TYPE_CHECKING: - from .ir import IR # pragma: no cover - -class IRGzip (IRFilter): - - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.gzip", - name: str="ir.gzip", - kind: str="IRGzip", - **kwargs) -> None: - - super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) - - def setup(self, ir: 'IR', aconf: Config) -> bool: - self["memory_level"] = self.pop('memory_level', None) - self["content_length"] = self.pop('min_content_length', None) - self["compression_level"] = self.pop('compression_level', None) - self["compression_strategy"] = self.pop('compression_strategy', None) - self["window_bits"] = self.pop('window_bits', None) - self["content_type"] = self.pop('content_type', []) - self["disable_on_etag_header"] = self.pop('disable_on_etag_header', None) - self["remove_accept_encoding_header"] = self.pop('remove_accept_encoding_header', None) + from .ir import IR # pragma: no cover + + +class IRGzip(IRFilter): + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.gzip", + name: str = "ir.gzip", + kind: str = "IRGzip", + **kwargs + ) -> None: + + super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) + + def setup(self, ir: "IR", aconf: Config) -> bool: + self["memory_level"] = self.pop("memory_level", None) + self["content_length"] = self.pop("min_content_length", None) + self["compression_level"] = self.pop("compression_level", None) + self["compression_strategy"] = self.pop("compression_strategy", None) + self["window_bits"] = self.pop("window_bits", None) + self["content_type"] = self.pop("content_type", []) + self["disable_on_etag_header"] = self.pop("disable_on_etag_header", None) + self["remove_accept_encoding_header"] = self.pop("remove_accept_encoding_header", None) return True diff --git a/python/ambassador/ir/irhost.py b/python/ambassador/ir/irhost.py index cd61e37796..5157c25b8f 100644 --- a/python/ambassador/ir/irhost.py +++ b/python/ambassador/ir/irhost.py @@ -10,21 +10,21 @@ from .irutils import hostglob_matches, selector_matches if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover from .irhttpmappinggroup import IRHTTPMappingGroup class IRHost(IRResource): AllowedKeys = { - 'acmeProvider', - 'hostname', - 'mappingSelector', - 'metadata_labels', - 'requestPolicy', - 'selector', - 'tlsSecret', - 'tlsContext', - 'tls', + "acmeProvider", + "hostname", + "mappingSelector", + "metadata_labels", + "requestPolicy", + "selector", + "tlsSecret", + "tlsContext", + "tls", } hostname: str @@ -32,60 +32,67 @@ class IRHost(IRResource): insecure_action: str insecure_addl_port: Optional[int] - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, # REQUIRED - name: str, # REQUIRED - location: str, # REQUIRED - namespace: Optional[str]=None, - kind: str="IRHost", - apiVersion: str="getambassador.io/v3alpha1", # Not a typo! See below. - **kwargs) -> None: - - new_args = { - x: kwargs[x] for x in kwargs.keys() - if x in IRHost.AllowedKeys - } + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, # REQUIRED + name: str, # REQUIRED + location: str, # REQUIRED + namespace: Optional[str] = None, + kind: str = "IRHost", + apiVersion: str = "getambassador.io/v3alpha1", # Not a typo! See below. + **kwargs, + ) -> None: + + new_args = {x: kwargs[x] for x in kwargs.keys() if x in IRHost.AllowedKeys} self.context: Optional[IRTLSContext] = None super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, - kind=kind, name=name, namespace=namespace, apiVersion=apiVersion, - **new_args + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + kind=kind, + name=name, + namespace=namespace, + apiVersion=apiVersion, + **new_args, ) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: ir.logger.debug(f"Host {self.name} setting up") - if not self.get('hostname', None): - self.hostname = '*' + if not self.get("hostname", None): + self.hostname = "*" tls_ss: Optional[SavedSecret] = None pkey_ss: Optional[SavedSecret] = None # Go ahead and cache some things to make life easier later. - request_policy = self.get('requestPolicy', {}) + request_policy = self.get("requestPolicy", {}) # XXX This will change later!! - self.secure_action = 'Route' + self.secure_action = "Route" - insecure_policy = request_policy.get('insecure', {}) - self.insecure_action = insecure_policy.get('action', 'Redirect') - self.insecure_addl_port: Optional[int] = insecure_policy.get('additionalPort', None) + insecure_policy = request_policy.get("insecure", {}) + self.insecure_action = insecure_policy.get("action", "Redirect") + self.insecure_addl_port: Optional[int] = insecure_policy.get("additionalPort", None) # If we have no mappingSelector, check for selector. - mapsel = self.get('mappingSelector', None) + mapsel = self.get("mappingSelector", None) if not mapsel: - mapsel = self.get('selector', None) + mapsel = self.get("selector", None) if mapsel: self.mappingSelector = mapsel - del self['selector'] + del self["selector"] - if self.get('tlsSecret', None): + if self.get("tlsSecret", None): tls_secret = self.tlsSecret - tls_name = tls_secret.get('name', None) + tls_name = tls_secret.get("name", None) if tls_name: ir.logger.debug(f"Host {self.name}: resolving spec.tlsSecret.name: {tls_name}") @@ -99,31 +106,39 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: ctx_name = f"{self.name}-context" implicit_tls_exists = ir.has_tls_context(ctx_name) - self.logger.debug(f"Host {self.name}: implicit TLSContext {ctx_name} {'exists' if implicit_tls_exists else 'missing'}") + self.logger.debug( + f"Host {self.name}: implicit TLSContext {ctx_name} {'exists' if implicit_tls_exists else 'missing'}" + ) - host_tls_context_obj = self.get('tlsContext', {}) - host_tls_context_name = host_tls_context_obj.get('name', None) + host_tls_context_obj = self.get("tlsContext", {}) + host_tls_context_name = host_tls_context_obj.get("name", None) self.logger.debug(f"Host {self.name}: spec.tlsContext: {host_tls_context_name}") - host_tls_config = self.get('tls', None) + host_tls_config = self.get("tls", None) self.logger.debug(f"Host {self.name}: spec.tls: {host_tls_config}") # Choose explicit TLS configuration over implicit TLSContext name if implicit_tls_exists and (host_tls_context_name or host_tls_config): - self.logger.info(f"Host {self.name}: even though TLSContext {ctx_name} exists in the cluster," - f"it will be ignored in favor of 'tls'/'tlsConfig' specified in the Host.") + self.logger.info( + f"Host {self.name}: even though TLSContext {ctx_name} exists in the cluster," + f"it will be ignored in favor of 'tls'/'tlsConfig' specified in the Host." + ) # Even though this is unlikely because we have a oneOf is proto definitions, but just in case the # objects have a different source :shrug: if host_tls_context_name and host_tls_config: - self.post_error(f"Host {self.name}: both TLSContext name and TLS config specified, ignoring " - f"Host...") + self.post_error( + f"Host {self.name}: both TLSContext name and TLS config specified, ignoring " + f"Host..." + ) return False if host_tls_context_name: # They named a TLSContext, so try to use that. self.save_context will check the # context to make sure it works for us, and save it if so. - ir.logger.debug(f"Host {self.name}: resolving spec.tlsContext: {host_tls_context_name}") + ir.logger.debug( + f"Host {self.name}: resolving spec.tlsContext: {host_tls_context_name}" + ) if not self.save_context(ir, host_tls_context_name, tls_ss, tls_name): return False @@ -133,19 +148,19 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: ir.logger.debug(f"Host {self.name}: examining spec.tls {host_tls_config}") camel_snake_map = { - 'alpnProtocols': 'alpn_protocols', - 'cipherSuites': 'cipher_suites', - 'ecdhCurves': 'ecdh_curves', - 'redirectCleartextFrom': 'redirect_cleartext_from', - 'certRequired': 'cert_required', - 'minTlsVersion': 'min_tls_version', - 'maxTlsVersion': 'max_tls_version', - 'certChainFile': 'cert_chain_file', - 'privateKeyFile': 'private_key_file', - 'cacertChainFile': 'cacert_chain_file', - 'crlSecret': 'crl_secret', - 'crlFile': 'crl_file', - 'caSecret': 'ca_secret', + "alpnProtocols": "alpn_protocols", + "cipherSuites": "cipher_suites", + "ecdhCurves": "ecdh_curves", + "redirectCleartextFrom": "redirect_cleartext_from", + "certRequired": "cert_required", + "minTlsVersion": "min_tls_version", + "maxTlsVersion": "max_tls_version", + "certChainFile": "cert_chain_file", + "privateKeyFile": "private_key_file", + "cacertChainFile": "cacert_chain_file", + "crlSecret": "crl_secret", + "crlFile": "crl_file", + "caSecret": "ca_secret", # 'sni': 'sni' (this field is not required in snake-camel but adding for completeness) } @@ -155,16 +170,26 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # We use .pop() to actually replace the camelCase name with snake case host_tls_config[snake] = host_tls_config.pop(camel) - if 'min_tls_version' in host_tls_config: - if host_tls_config['min_tls_version'] not in IRTLSContext.AllowedTLSVersions: - self.post_error(f"Host {self.name}: Invalid min_tls_version set in Host.tls: " - f"{host_tls_config['min_tls_version']}") + if "min_tls_version" in host_tls_config: + if ( + host_tls_config["min_tls_version"] + not in IRTLSContext.AllowedTLSVersions + ): + self.post_error( + f"Host {self.name}: Invalid min_tls_version set in Host.tls: " + f"{host_tls_config['min_tls_version']}" + ) return False - if 'max_tls_version' in host_tls_config: - if host_tls_config['max_tls_version'] not in IRTLSContext.AllowedTLSVersions: - self.post_error(f"Host {self.name}: Invalid max_tls_version set in Host.tls: " - f"{host_tls_config['max_tls_version']}") + if "max_tls_version" in host_tls_config: + if ( + host_tls_config["max_tls_version"] + not in IRTLSContext.AllowedTLSVersions + ): + self.post_error( + f"Host {self.name}: Invalid max_tls_version set in Host.tls: " + f"{host_tls_config['max_tls_version']}" + ) return False tls_context_init = dict( @@ -176,14 +201,16 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: secret=tls_name, ) - tls_config_context = IRTLSContext(ir, aconf, **tls_context_init, **host_tls_config) + tls_config_context = IRTLSContext( + ir, aconf, **tls_context_init, **host_tls_config + ) # This code was here because, while 'selector' was controlling things to be watched # for, we figured we should update the labels on this generated TLSContext so that # it would actually match the 'selector'. Nothing was actually using that, though, so # we're not doing that any more. # - #----------------------------------------------------------------------------------- + # ----------------------------------------------------------------------------------- # # XXX This seems kind of pointless -- nothing looks at the context's labels? # match_labels = self.get('selector', {}).get('matchLabels') @@ -197,8 +224,10 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: ir.save_tls_context(tls_config_context) else: - self.post_error(f"Host {self.name}: generated TLSContext {tls_config_context.name} from " - f"Host.tls is not valid") + self.post_error( + f"Host {self.name}: generated TLSContext {tls_config_context.name} from " + f"Host.tls is not valid" + ) return False elif implicit_tls_exists: @@ -216,19 +245,19 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: name=ctx_name, namespace=self.namespace, location=self.location, - hosts=[ self.hostname or self.name ], - secret=tls_name + hosts=[self.hostname or self.name], + secret=tls_name, ) ctx = IRTLSContext(ir, aconf, **new_ctx) - match_labels = self.get('matchLabels') + match_labels = self.get("matchLabels") if not match_labels: - match_labels = self.get('match_labels') + match_labels = self.get("match_labels") if match_labels: - ctx['metadata_labels'] = match_labels + ctx["metadata_labels"] = match_labels if ctx.is_active(): self.context = ctx @@ -237,19 +266,23 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: ir.save_tls_context(ctx) else: - ir.logger.error(f"Host {self.name}: new TLSContext {ctx_name} is not valid") + ir.logger.error( + f"Host {self.name}: new TLSContext {ctx_name} is not valid" + ) else: - ir.logger.error(f"Host {self.name}: invalid TLS secret {tls_name}, marking inactive") + ir.logger.error( + f"Host {self.name}: invalid TLS secret {tls_name}, marking inactive" + ) return False - if self.get('acmeProvider', None): + if self.get("acmeProvider", None): acme = self.acmeProvider # The ACME client is disabled if we're running as an intercept agent. if ir.edge_stack_allowed and not ir.agent_active: - authority = acme.get('authority', None) + authority = acme.get("authority", None) - if authority and (authority.lower() != 'none'): + if authority and (authority.lower() != "none"): # ACME is active, which means that we must have an insecure_addl_port. # Make sure we do -- if no port is set at all, just silently default it, # but if for some reason they tried to force it disabled, be noisy. @@ -261,7 +294,9 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: override_insecure = True elif self.insecure_addl_port < 0: # Override noisily, since they tried to explicitly disable it. - self.post_error("ACME requires insecure.additionalPort to function; forcing to 8080") + self.post_error( + "ACME requires insecure.additionalPort to function; forcing to 8080" + ) override_insecure = True if override_insecure: @@ -269,20 +304,20 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: self.insecure_addl_port = 8080 # ...but also update the actual policy dict, too. - insecure_policy['additionalPort'] = 8080 + insecure_policy["additionalPort"] = 8080 - if 'action' not in insecure_policy: + if "action" not in insecure_policy: # No action when we're overriding the additionalPort already means that we # default the action to Reject (the hole-puncher will do the right thing). - insecure_policy['action'] = 'Reject' + insecure_policy["action"] = "Reject" - request_policy['insecure'] = insecure_policy - self['requestPolicy'] = request_policy + request_policy["insecure"] = insecure_policy + self["requestPolicy"] = request_policy - pkey_secret = acme.get('privateKeySecret', None) + pkey_secret = acme.get("privateKeySecret", None) if pkey_secret: - pkey_name = pkey_secret.get('name', None) + pkey_name = pkey_secret.get("name", None) if pkey_name: ir.logger.debug(f"Host {self.name}: ACME private key name is {pkey_name}") @@ -290,50 +325,60 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: pkey_ss = self.resolve(ir, pkey_name) if not pkey_ss: - ir.logger.error(f"Host {self.name}: continuing with invalid private key secret {pkey_name}; ACME will not be able to renew this certificate") - self.post_error(f"continuing with invalid ACME private key secret {pkey_name}; ACME will not be able to renew this certificate") + ir.logger.error( + f"Host {self.name}: continuing with invalid private key secret {pkey_name}; ACME will not be able to renew this certificate" + ) + self.post_error( + f"continuing with invalid ACME private key secret {pkey_name}; ACME will not be able to renew this certificate" + ) ir.logger.debug(f"Host setup OK: {self}") return True # Check a TLSContext name, and save the linked TLSContext if it'll work for us. - def save_context(self, ir: 'IR', ctx_name: str, tls_ss: SavedSecret, tls_name: str): + def save_context(self, ir: "IR", ctx_name: str, tls_ss: SavedSecret, tls_name: str): # First obvious thing: does a TLSContext with the right name even exist? if not ir.has_tls_context(ctx_name): - self.post_error("Host %s: Specified TLSContext does not exist: %s" % (self.name, ctx_name)) + self.post_error( + "Host %s: Specified TLSContext does not exist: %s" % (self.name, ctx_name) + ) return False ctx = ir.get_tls_context(ctx_name) - assert(ctx) # For mypy -- we checked above to be sure it exists. + assert ctx # For mypy -- we checked above to be sure it exists. # Make sure that the TLSContext is "compatible" i.e. it at least has the same cert related # configuration as the one in this Host AND hosts are same as well. if ctx.has_secret(): secret_name = ctx.secret_name() - assert(secret_name) # For mypy -- if has_secret() is true, secret_name() will be there. + assert secret_name # For mypy -- if has_secret() is true, secret_name() will be there. # This is a little weird. Basically we're going to resolve the secret (which should just # be a cache lookup here) so that we can use SavedSecret.__str__() as a serializer to # compare the configurations. context_ss = self.resolve(ir, secret_name) - self.logger.debug(f"Host {self.name}, ctx {ctx.name}, secret {secret_name}, resolved {context_ss}") + self.logger.debug( + f"Host {self.name}, ctx {ctx.name}, secret {secret_name}, resolved {context_ss}" + ) if str(context_ss) != str(tls_ss): - self.post_error("Secret info mismatch between Host %s (secret: %s) and TLSContext %s: (secret: %s)" % - (self.name, tls_name, ctx_name, secret_name)) + self.post_error( + "Secret info mismatch between Host %s (secret: %s) and TLSContext %s: (secret: %s)" + % (self.name, tls_name, ctx_name, secret_name) + ) return False else: # This will often be a no-op. ctx.set_secret_name(tls_name) # TLS config is good, let's make sure the hosts line up too. - context_hosts = ctx.get('hosts') + context_hosts = ctx.get("hosts") # XXX WTF? self.name is not OK as a hostname! Leaving this for the moment, but it's # almost certainly getting shredded before 2.0 GAs. - host_hosts = [ self.name ] + host_hosts = [self.name] if self.hostname: host_hosts.append(self.hostname) @@ -347,12 +392,14 @@ def save_context(self, ir: 'IR', ctx_name: str, tls_ss: SavedSecret, tls_name: s is_valid_hosts = True if not is_valid_hosts: - self.post_error("Hosts mismatch between Host %s (accepted hosts: %s) and TLSContext %s (hosts: %s)" % - (self.name, host_hosts, ctx_name, context_hosts)) + self.post_error( + "Hosts mismatch between Host %s (accepted hosts: %s) and TLSContext %s (hosts: %s)" + % (self.name, host_hosts, ctx_name, context_hosts) + ) # XXX Shouldn't we return false here? else: # XXX WTF? self.name is not OK as a hostname! - ctx['hosts'] = [self.hostname or self.name] + ctx["hosts"] = [self.hostname or self.name] self.logger.debug(f"Host {self.name}, final ctx {ctx.name}: {ctx.as_json()}") @@ -361,7 +408,7 @@ def save_context(self, ir: 'IR', ctx_name: str, tls_ss: SavedSecret, tls_name: s return True - def matches_httpgroup(self, group: 'IRHTTPMappingGroup') -> bool: + def matches_httpgroup(self, group: "IRHTTPMappingGroup") -> bool: """ Make sure a given IRHTTPMappingGroup is a match for this Host, meaning that at least one of the following is true: @@ -376,7 +423,7 @@ def matches_httpgroup(self, group: 'IRHTTPMappingGroup') -> bool: host_match = False sel_match = False - group_regex = group.get('host_regex') or False + group_regex = group.get("host_regex") or False if group_regex: # It matches. @@ -389,35 +436,45 @@ def matches_httpgroup(self, group: 'IRHTTPMappingGroup') -> bool: # It's possible for group.host_redirect to be None instead of missing, and it's also # conceivably possible for group.host_redirect.host to be "", which we'd rather be # None. Hence we do this two-line dance to massage the various cases. - host_redirect = (group.get('host_redirect') or {}).get('host') - group_glob = group.get('host') or host_redirect # NOT A TYPO: see above. + host_redirect = (group.get("host_redirect") or {}).get("host") + group_glob = group.get("host") or host_redirect # NOT A TYPO: see above. if group_glob: host_match = hostglob_matches(self.hostname, group_glob) - self.logger.debug("-- hostname %s group glob %s => %s", self.hostname, group_glob, host_match) + self.logger.debug( + "-- hostname %s group glob %s => %s", self.hostname, group_glob, host_match + ) - mapsel = self.get('mappingSelector') + mapsel = self.get("mappingSelector") if mapsel: - sel_match = selector_matches(self.logger, mapsel, group.get('metadata_labels', {})) - self.logger.debug("-- host sel %s group labels %s => %s", - dump_json(mapsel), dump_json(group.get('metadata_labels')), sel_match) + sel_match = selector_matches(self.logger, mapsel, group.get("metadata_labels", {})) + self.logger.debug( + "-- host sel %s group labels %s => %s", + dump_json(mapsel), + dump_json(group.get("metadata_labels")), + sel_match, + ) return host_match or sel_match def __str__(self) -> str: - request_policy = self.get('requestPolicy', {}) - insecure_policy = request_policy.get('insecure', {}) - insecure_action = insecure_policy.get('action', 'Redirect') - insecure_addl_port = insecure_policy.get('additionalPort', None) + request_policy = self.get("requestPolicy", {}) + insecure_policy = request_policy.get("insecure", {}) + insecure_action = insecure_policy.get("action", "Redirect") + insecure_addl_port = insecure_policy.get("additionalPort", None) ctx_name = self.context.name if self.context else "-none-" return "" % ( - self.name, self.hostname or '*', self.namespace, ctx_name, - insecure_action, insecure_addl_port + self.name, + self.hostname or "*", + self.namespace, + ctx_name, + insecure_action, + insecure_addl_port, ) - def resolve(self, ir: 'IR', secret_name: str) -> SavedSecret: + def resolve(self, ir: "IR", secret_name: str) -> SavedSecret: # Try to use our namespace for secret resolution. If we somehow have no # namespace, fall back to the Ambassador's namespace. namespace = self.namespace or ir.ambassador_namespace @@ -427,10 +484,10 @@ def resolve(self, ir: 'IR', secret_name: str) -> SavedSecret: class HostFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: + def load_all(cls, ir: "IR", aconf: Config) -> None: assert ir - hosts = aconf.get_config('hosts') + hosts = aconf.get_config("hosts") if hosts: for config in hosts.values(): @@ -448,7 +505,7 @@ def load_all(cls, ir: 'IR', aconf: Config) -> None: ir.logger.debug(f"HostFactory: not saving inactive host {host}") @classmethod - def finalize(cls, ir: 'IR', aconf: Config) -> None: + def finalize(cls, ir: "IR", aconf: Config) -> None: # First up: how many Hosts do we have? host_count = len(ir.get_hosts() or []) @@ -462,51 +519,65 @@ def finalize(cls, ir: 'IR', aconf: Config) -> None: found_termination_context = False for ctx in contexts: - if ctx.get('hosts'): # not None and not the empty list + if ctx.get("hosts"): # not None and not the empty list found_termination_context = True break - ir.logger.debug(f"HostFactory: Host count %d, %s TLS termination contexts" % - (host_count, "with" if found_termination_context else "no")) + ir.logger.debug( + f"HostFactory: Host count %d, %s TLS termination contexts" + % (host_count, "with" if found_termination_context else "no") + ) # OK, do we have any Hosts? if host_count == 0: # Nope. First up, scream if we _do_ have termination contexts... if found_termination_context: - ir.post_error("No Hosts defined, but TLSContexts exist that terminate TLS. The TLSContexts are being ignored.") + ir.post_error( + "No Hosts defined, but TLSContexts exist that terminate TLS. The TLSContexts are being ignored." + ) # If we don't have a fallback secret, don't try to use it. # # We use the Ambassador's namespace here because we'll be creating the # fallback Host in the Ambassador's namespace. - fallback_ss = ir.resolve_secret(ir.ambassador_module, "fallback-self-signed-cert", ir.ambassador_namespace) + fallback_ss = ir.resolve_secret( + ir.ambassador_module, "fallback-self-signed-cert", ir.ambassador_namespace + ) host: IRHost if not fallback_ss: - ir.aconf.post_notice("No TLS termination and no fallback cert -- defaulting to cleartext-only.") + ir.aconf.post_notice( + "No TLS termination and no fallback cert -- defaulting to cleartext-only." + ) ir.logger.debug("HostFactory: creating cleartext-only default host") - host = IRHost(ir, aconf, + host = IRHost( + ir, + aconf, rkey="-internal", name="default-host", location="-internal-", hostname="*", - requestPolicy={ "insecure": { "action": "Route" }}, + requestPolicy={"insecure": {"action": "Route"}}, ) else: ir.logger.debug(f"HostFactory: creating TLS-enabled default Host") - host = IRHost(ir, aconf, + host = IRHost( + ir, + aconf, rkey="-internal", name="default-host", location="-internal-", hostname="*", - tlsSecret={ "name": "fallback-self-signed-cert" } + tlsSecret={"name": "fallback-self-signed-cert"}, ) if not host.is_active(): - ir.post_error("Synthesized default host is inactive? %s" % dump_json(host.as_dict())) + ir.post_error( + "Synthesized default host is inactive? %s" % dump_json(host.as_dict()) + ) else: host.referenced_by(ir.ambassador_module) host.sourced_by(ir.ambassador_module) diff --git a/python/ambassador/ir/irhttpmapping.py b/python/ambassador/ir/irhttpmapping.py index a73dc689c2..ef54296905 100644 --- a/python/ambassador/ir/irhttpmapping.py +++ b/python/ambassador/ir/irhttpmapping.py @@ -15,12 +15,14 @@ import hashlib if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover # Kind of cheating here so that it's easy to json-serialize key-value pairs (including with regex) -class KeyValueDecorator (dict): - def __init__(self, name: str, value: Optional[str]=None, regex: Optional[bool]=False) -> None: +class KeyValueDecorator(dict): + def __init__( + self, name: str, value: Optional[str] = None, regex: Optional[bool] = False + ) -> None: super().__init__() self.name = name self.value = value @@ -33,16 +35,16 @@ def __setattr__(self, key: str, value: Any) -> None: self[key] = value def _get_value(self) -> str: - return self.value or '*' + return self.value or "*" def length(self) -> int: return len(self.name) + len(self._get_value()) + (1 if self.regex else 0) def key(self) -> str: - return self.name + '-' + self._get_value() + return self.name + "-" + self._get_value() -class IRHTTPMapping (IRBaseMapping): +class IRHTTPMapping(IRBaseMapping): prefix: str headers: List[KeyValueDecorator] add_request_headers: Dict[str, str] @@ -55,7 +57,7 @@ class IRHTTPMapping (IRBaseMapping): retry_policy: IRRetryPolicy error_response_overrides: Optional[IRErrorResponse] query_parameters: List[KeyValueDecorator] - regex_rewrite: Dict[str,str] + regex_rewrite: Dict[str, str] # Keys that are present in AllowedKeys are allowed to be set from kwargs. # If the value is True, we'll look for a default in the Ambassador module @@ -99,7 +101,7 @@ class IRHTTPMapping (IRBaseMapping): "host_rewrite": False, "idle_timeout_ms": False, "keepalive": False, - "labels": False, # Not supported in v0; requires v1+; handled in setup + "labels": False, # Not supported in v0; requires v1+; handled in setup "load_balancer": False, "metadata_labels": False, # Do not include method @@ -113,7 +115,7 @@ class IRHTTPMapping (IRBaseMapping): "prefix_exact": False, "prefix_regex": False, "priority": False, - "rate_limits": False, # Only supported in v0; replaced by "labels" in v1; handled in setup + "rate_limits": False, # Only supported in v0; replaced by "labels" in v1; handled in setup # Do not include regex_headers "remove_request_headers": True, "remove_response_headers": True, @@ -121,7 +123,7 @@ class IRHTTPMapping (IRBaseMapping): "respect_dns_ttl": False, "retry_policy": False, # Do not include rewrite - "service": False, # See notes above + "service": False, # See notes above "shadow": False, "stats_name": True, "timeout_ms": False, @@ -129,24 +131,27 @@ class IRHTTPMapping (IRBaseMapping): "use_websocket": False, "allow_upgrade": False, "weight": False, - # Include the serialization, too. "serialization": False, } - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, # REQUIRED - name: str, # REQUIRED - location: str, # REQUIRED - service: str, # REQUIRED - namespace: Optional[str] = None, - metadata_labels: Optional[Dict[str, str]] = None, - kind: str="IRHTTPMapping", - apiVersion: str="getambassador.io/v3alpha1", # Not a typo! See below. - precedence: int=0, - rewrite: str="/", - cluster_tag: Optional[str]=None, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, # REQUIRED + name: str, # REQUIRED + location: str, # REQUIRED + service: str, # REQUIRED + namespace: Optional[str] = None, + metadata_labels: Optional[Dict[str, str]] = None, + kind: str = "IRHTTPMapping", + apiVersion: str = "getambassador.io/v3alpha1", # Not a typo! See below. + precedence: int = 0, + rewrite: str = "/", + cluster_tag: Optional[str] = None, + **kwargs, + ) -> None: # OK, this is a bit of a pain. We want to preserve the name and rkey and # such here, unlike most kinds of IRResource, so we shallow copy the keys # we're going to allow from the incoming kwargs. @@ -180,7 +185,7 @@ def __init__(self, ir: 'IR', aconf: Config, if "add_linkerd_headers" not in new_args: # They didn't set it explicitly, so check for the older way. - add_linkerd_headers = self.ir.ambassador_module.get('add_linkerd_headers', None) + add_linkerd_headers = self.ir.ambassador_module.get("add_linkerd_headers", None) if add_linkerd_headers != None: new_args["add_linkerd_headers"] = add_linkerd_headers @@ -188,7 +193,7 @@ def __init__(self, ir: 'IR', aconf: Config, # OK. On to set up the headers (since we need them to compute our group ID). hdrs = [] query_parameters = [] - regex_rewrite = kwargs.get('regex_rewrite', {}) + regex_rewrite = kwargs.get("regex_rewrite", {}) # Start by assuming that nothing in our arguments mentions hosts (so no host and no host_regex). host = None @@ -198,64 +203,78 @@ def __init__(self, ir: 'IR', aconf: Config, self.host = None # OK. Start by looking for a :authority header match. - if 'headers' in kwargs: - for name, value in kwargs.get('headers', {}).items(): + if "headers" in kwargs: + for name, value in kwargs.get("headers", {}).items(): if value is True: hdrs.append(KeyValueDecorator(name)) else: # An exact match on the :authority header is special -- treat it like # they set the "host" element (but note that we'll allow the actual # "host" element to override it later). - if name.lower() == ':authority': + if name.lower() == ":authority": # This is an _exact_ match, so it mustn't contain a "*" -- that's illegal in the DNS. if "*" in value: # We can't call self.post_error() yet, because we're not initialized yet. So we cheat a bit # and defer the error for later. - new_args["_deferred_error"] = f":authority exact-match '{value}' contains *, which cannot match anything." - ir.logger.debug("IRHTTPMapping %s: self.host contains * (%s, :authority)", name, value) + new_args[ + "_deferred_error" + ] = f":authority exact-match '{value}' contains *, which cannot match anything." + ir.logger.debug( + "IRHTTPMapping %s: self.host contains * (%s, :authority)", + name, + value, + ) else: # No globs, just save it. (We'll end up using it as a glob later, in the Envoy # config part of the world, but that's OK -- a glob with no "*" in it will always # match only itself.) host = value - ir.logger.debug("IRHTTPMapping %s: self.host == %s (:authority)", name, self.host) + ir.logger.debug( + "IRHTTPMapping %s: self.host == %s (:authority)", name, self.host + ) # DO NOT save the ':authority' match here -- we'll pick it up after we've checked # for hostname, too. else: # It's not an :authority match, so we're good. hdrs.append(KeyValueDecorator(name, value)) - if 'regex_headers' in kwargs: + if "regex_headers" in kwargs: # DON'T do anything special with a regex :authority match: we can't # do host-based filtering within the IR for it anyway. - for name, value in kwargs.get('regex_headers', {}).items(): + for name, value in kwargs.get("regex_headers", {}).items(): hdrs.append(KeyValueDecorator(name, value, regex=True)) - if 'host' in kwargs: + if "host" in kwargs: # It's deliberate that we'll allow kwargs['host'] to silently override an exact :authority # header match. - host = kwargs['host'] - host_regex = kwargs.get('host_regex', False) + host = kwargs["host"] + host_regex = kwargs.get("host_regex", False) # If it's not a regex, it's an exact match -- make sure it doesn't contain a '*'. if not host_regex: if "*" in host: # We can't call self.post_error() yet, because we're not initialized yet. So we cheat a bit # and defer the error for later. - new_args["_deferred_error"] = f"host exact-match {host} contains *, which cannot match anything." + new_args[ + "_deferred_error" + ] = f"host exact-match {host} contains *, which cannot match anything." ir.logger.debug("IRHTTPMapping %s: self.host contains * (%s, host)", name, host) else: ir.logger.debug("IRHTTPMapping %s: self.host == %s (host)", name, self.host) # Finally, check for 'hostname'. - if 'hostname' in kwargs: + if "hostname" in kwargs: # It's deliberate that we allow kwargs['hostname'] to override anything else -- even a regex host. # Yell about it, though. if host: - ir.logger.warning("Mapping %s in namespace %s: both host and hostname are set, using hostname and ignoring host", name, namespace) + ir.logger.warning( + "Mapping %s in namespace %s: both host and hostname are set, using hostname and ignoring host", + name, + namespace, + ) # No need to be so careful about "*" here, since hostname is defined to be a glob. - host = kwargs['hostname'] + host = kwargs["hostname"] host_regex = False ir.logger.debug("IRHTTPMapping %s: self.host gl~ %s (hostname)", name, self.host) @@ -269,14 +288,16 @@ def __init__(self, ir: 'IR', aconf: Config, if not host_regex: self.host = host - if 'method' in kwargs: - hdrs.append(KeyValueDecorator(":method", kwargs['method'], kwargs.get('method_regex', False))) + if "method" in kwargs: + hdrs.append( + KeyValueDecorator(":method", kwargs["method"], kwargs.get("method_regex", False)) + ) - if 'use_websocket' in new_args: - allow_upgrade = new_args.setdefault('allow_upgrade', []) - if 'websocket' not in allow_upgrade: - allow_upgrade.append('websocket') - del new_args['use_websocket'] + if "use_websocket" in new_args: + allow_upgrade = new_args.setdefault("allow_upgrade", []) + if "websocket" not in allow_upgrade: + allow_upgrade.append("websocket") + del new_args["use_websocket"] # Next up: figure out what headers we need to add to each request. Again, if the key # is present in kwargs, the kwargs value wins -- this is important to allow explicitly @@ -285,24 +306,26 @@ def __init__(self, ir: 'IR', aconf: Config, add_request_hdrs: dict add_response_hdrs: dict - if 'add_request_headers' in kwargs: - add_request_hdrs = kwargs['add_request_headers'] + if "add_request_headers" in kwargs: + add_request_hdrs = kwargs["add_request_headers"] else: - add_request_hdrs = self.lookup_default('add_request_headers', {}) + add_request_hdrs = self.lookup_default("add_request_headers", {}) - if 'add_response_headers' in kwargs: - add_response_hdrs = kwargs['add_response_headers'] + if "add_response_headers" in kwargs: + add_response_hdrs = kwargs["add_response_headers"] else: - add_response_hdrs = self.lookup_default('add_response_headers', {}) + add_response_hdrs = self.lookup_default("add_response_headers", {}) # Remember that we may need to add the Linkerd headers, too. - add_linkerd_headers = new_args.get('add_linkerd_headers', False) + add_linkerd_headers = new_args.get("add_linkerd_headers", False) # XXX The resolver lookup code is duplicated from IRBaseMapping.setup -- # needs to be fixed after 1.6.1. - resolver_name = kwargs.get('resolver') or self.ir.ambassador_module.get('resolver', 'kubernetes-service') + resolver_name = kwargs.get("resolver") or self.ir.ambassador_module.get( + "resolver", "kubernetes-service" + ) - assert(resolver_name) # for mypy -- resolver_name cannot be None at this point + assert resolver_name # for mypy -- resolver_name cannot be None at this point resolver = self.ir.get_resolver(resolver_name) if resolver: @@ -311,7 +334,7 @@ def __init__(self, ir: 'IR', aconf: Config, # In IRBaseMapping.setup, we post an error if the resolver is unknown. # Here, we just don't bother; we're only using it for service # qualification. - resolver_kind = 'KubernetesBogusResolver' + resolver_kind = "KubernetesBogusResolver" service = normalize_service_name(ir, service, namespace, resolver_kind, rkey=rkey) self.ir.logger.debug(f"Mapping {name} service qualified to {repr(service)}") @@ -319,46 +342,64 @@ def __init__(self, ir: 'IR', aconf: Config, svc = Service(ir.logger, service) if add_linkerd_headers: - add_request_hdrs['l5d-dst-override'] = svc.hostname_port + add_request_hdrs["l5d-dst-override"] = svc.hostname_port # XXX BRUTAL HACK HERE: # If we _don't_ have an origination context, but our IR has an agent_origination_ctx, # force TLS origination because it's the agent. I know, I know. It's a hack. - if ('tls' not in new_args) and ir.agent_origination_ctx: - ir.logger.debug(f"Mapping {name}: Agent forcing origination TLS context to {ir.agent_origination_ctx.name}") - new_args['tls'] = ir.agent_origination_ctx.name - - if 'query_parameters' in kwargs: - for name, value in kwargs.get('query_parameters', {}).items(): + if ("tls" not in new_args) and ir.agent_origination_ctx: + ir.logger.debug( + f"Mapping {name}: Agent forcing origination TLS context to {ir.agent_origination_ctx.name}" + ) + new_args["tls"] = ir.agent_origination_ctx.name + + if "query_parameters" in kwargs: + for name, value in kwargs.get("query_parameters", {}).items(): if value is True: query_parameters.append(KeyValueDecorator(name)) else: query_parameters.append(KeyValueDecorator(name, value)) - if 'regex_query_parameters' in kwargs: - for name, value in kwargs.get('regex_query_parameters', {}).items(): + if "regex_query_parameters" in kwargs: + for name, value in kwargs.get("regex_query_parameters", {}).items(): query_parameters.append(KeyValueDecorator(name, value, regex=True)) - if 'regex_rewrite' in kwargs: + if "regex_rewrite" in kwargs: if rewrite and rewrite != "/": - self.ir.aconf.post_notice("Cannot specify both rewrite and regex_rewrite: using regex_rewrite and ignoring rewrite") + self.ir.aconf.post_notice( + "Cannot specify both rewrite and regex_rewrite: using regex_rewrite and ignoring rewrite" + ) rewrite = "" - rewrite_items = kwargs.get('regex_rewrite', {}) - regex_rewrite = {'pattern' : rewrite_items.get('pattern',''), - 'substitution' : rewrite_items.get('substitution','') } + rewrite_items = kwargs.get("regex_rewrite", {}) + regex_rewrite = { + "pattern": rewrite_items.get("pattern", ""), + "substitution": rewrite_items.get("substitution", ""), + } # ...and then init the superclass. super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, service=service, - kind=kind, name=name, namespace=namespace, metadata_labels=metadata_labels, - apiVersion=apiVersion, headers=hdrs, add_request_headers=add_request_hdrs, add_response_headers = add_response_hdrs, - precedence=precedence, rewrite=rewrite, cluster_tag=cluster_tag, + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + service=service, + kind=kind, + name=name, + namespace=namespace, + metadata_labels=metadata_labels, + apiVersion=apiVersion, + headers=hdrs, + add_request_headers=add_request_hdrs, + add_response_headers=add_response_hdrs, + precedence=precedence, + rewrite=rewrite, + cluster_tag=cluster_tag, query_parameters=query_parameters, regex_rewrite=regex_rewrite, - **new_args + **new_args, ) - if 'outlier_detection' in kwargs: + if "outlier_detection" in kwargs: self.post_error(RichStatus.fromError("outlier_detection is not supported")) @staticmethod @@ -367,10 +408,13 @@ def group_class() -> Type[IRBaseMappingGroup]: def _enforce_mutual_exclusion(self, preferred, other): if preferred in self and other in self: - self.ir.aconf.post_error(f"Cannot specify both {preferred} and {other}. Using {preferred} and ignoring {other}.", resource=self) + self.ir.aconf.post_error( + f"Cannot specify both {preferred} and {other}. Using {preferred} and ignoring {other}.", + resource=self, + ) del self[other] - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # First things first: handle any deferred error. _deferred_error = self.get("_deferred_error") if _deferred_error: @@ -381,7 +425,7 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return False # If we have CORS stuff, normalize it. - if 'cors' in self: + if "cors" in self: self.cors = IRCORS(ir=ir, aconf=aconf, location=self.location, **self.cors) if self.cors: @@ -390,8 +434,10 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return False # If we have RETRY_POLICY stuff, normalize it. - if 'retry_policy' in self: - self.retry_policy = IRRetryPolicy(ir=ir, aconf=aconf, location=self.location, **self.retry_policy) + if "retry_policy" in self: + self.retry_policy = IRRetryPolicy( + ir=ir, aconf=aconf, location=self.location, **self.retry_policy + ) if self.retry_policy: self.retry_policy.referenced_by(self) @@ -399,19 +445,23 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return False # If we have error response overrides, generate an IR for that too. - if 'error_response_overrides' in self: - self.error_response_overrides = IRErrorResponse(self.ir, aconf, - self.get('error_response_overrides', None), - location=self.location) - #if self.error_response_overrides.setup(self.ir, aconf): + if "error_response_overrides" in self: + self.error_response_overrides = IRErrorResponse( + self.ir, aconf, self.get("error_response_overrides", None), location=self.location + ) + # if self.error_response_overrides.setup(self.ir, aconf): if self.error_response_overrides: self.error_response_overrides.referenced_by(self) else: return False - if self.get('load_balancer', None) is not None: - if not self.validate_load_balancer(self['load_balancer']): - self.post_error("Invalid load_balancer specified: {}, invalidating mapping".format(self['load_balancer'])) + if self.get("load_balancer", None) is not None: + if not self.validate_load_balancer(self["load_balancer"]): + self.post_error( + "Invalid load_balancer specified: {}, invalidating mapping".format( + self["load_balancer"] + ) + ) return False # All three redirect fields are mutually exclusive. @@ -419,32 +469,37 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # Prefer path_redirect over the other two. If only prefix_redirect and # regex_redirect are set, prefer prefix_redirect. There's no exact # reason for this, only to arbitrarily prefer "less fancy" features. - self._enforce_mutual_exclusion('path_redirect', 'prefix_redirect') - self._enforce_mutual_exclusion('path_redirect', 'regex_redirect') - self._enforce_mutual_exclusion('prefix_redirect', 'regex_redirect') - - ir.logger.debug("Mapping %s: setup OK: host %s hostname %s regex %s", - self.name, self.get('host'), self.get('hostname'), self.get('host_regex')) + self._enforce_mutual_exclusion("path_redirect", "prefix_redirect") + self._enforce_mutual_exclusion("path_redirect", "regex_redirect") + self._enforce_mutual_exclusion("prefix_redirect", "regex_redirect") + + ir.logger.debug( + "Mapping %s: setup OK: host %s hostname %s regex %s", + self.name, + self.get("host"), + self.get("hostname"), + self.get("host_regex"), + ) return True @staticmethod def validate_load_balancer(load_balancer) -> bool: - lb_policy = load_balancer.get('policy', None) + lb_policy = load_balancer.get("policy", None) is_valid = False - if lb_policy in ['round_robin', 'least_request']: + if lb_policy in ["round_robin", "least_request"]: if len(load_balancer) == 1: is_valid = True - elif lb_policy in ['ring_hash', 'maglev']: + elif lb_policy in ["ring_hash", "maglev"]: if len(load_balancer) == 2: - if 'cookie' in load_balancer: - cookie = load_balancer.get('cookie') - if 'name' in cookie: + if "cookie" in load_balancer: + cookie = load_balancer.get("cookie") + if "name" in cookie: is_valid = True - elif 'header' in load_balancer: + elif "header" in load_balancer: is_valid = True - elif 'source_ip' in load_balancer: + elif "source_ip" in load_balancer: is_valid = True return is_valid @@ -452,32 +507,32 @@ def validate_load_balancer(load_balancer) -> bool: def _group_id(self) -> str: # Yes, we're using a cryptographic hash here. Cope. [ :) ] - h = hashlib.new('sha1') + h = hashlib.new("sha1") # This is an HTTP mapping. - h.update('HTTP-'.encode('utf-8')) + h.update("HTTP-".encode("utf-8")) # method first, but of course method might be None. For calculating the # group_id, 'method' defaults to 'GET' (for historical reasons). - method = self.get('method') or 'GET' - h.update(method.encode('utf-8')) - h.update(self.prefix.encode('utf-8')) + method = self.get("method") or "GET" + h.update(method.encode("utf-8")) + h.update(self.prefix.encode("utf-8")) for hdr in self.headers: - h.update(hdr.name.encode('utf-8')) + h.update(hdr.name.encode("utf-8")) if hdr.value is not None: - h.update(hdr.value.encode('utf-8')) + h.update(hdr.value.encode("utf-8")) for query_parameter in self.query_parameters: - h.update(query_parameter.name.encode('utf-8')) + h.update(query_parameter.name.encode("utf-8")) if query_parameter.value is not None: - h.update(query_parameter.value.encode('utf-8')) + h.update(query_parameter.value.encode("utf-8")) if self.precedence != 0: - h.update(str(self.precedence).encode('utf-8')) + h.update(str(self.precedence).encode("utf-8")) return h.hexdigest() @@ -493,9 +548,16 @@ def _route_weight(self) -> List[Union[str, int]]: # For calculating the route weight, 'method' defaults to '*' (for historical reasons). - weight = [ self.precedence, len(self.prefix), len_headers, len_query_parameters, self.prefix, self.get('method', 'GET') ] - weight += [ hdr.key() for hdr in self.headers ] - weight += [ query_parameter.key() for query_parameter in self.query_parameters] + weight = [ + self.precedence, + len(self.prefix), + len_headers, + len_query_parameters, + self.prefix, + self.get("method", "GET"), + ] + weight += [hdr.key() for hdr in self.headers] + weight += [query_parameter.key() for query_parameter in self.query_parameters] return weight @@ -504,7 +566,7 @@ def summarize_errors(self) -> str: errstr = "(no errors)" if errors: - errstr = errors[0].get('error') or 'unknown error?' + errstr = errors[0].get("error") or "unknown error?" if len(errors) > 1: errstr += " (and more)" @@ -513,6 +575,6 @@ def summarize_errors(self) -> str: def status(self) -> Dict[str, str]: if not self.is_active(): - return { 'state': 'Inactive', 'reason': self.summarize_errors() } + return {"state": "Inactive", "reason": self.summarize_errors()} else: - return { 'state': 'Running' } + return {"state": "Running"} diff --git a/python/ambassador/ir/irhttpmappinggroup.py b/python/ambassador/ir/irhttpmappinggroup.py index 34a3109626..fc1f32dca0 100644 --- a/python/ambassador/ir/irhttpmappinggroup.py +++ b/python/ambassador/ir/irhttpmappinggroup.py @@ -10,14 +10,15 @@ from .irbasemapping import IRBaseMapping if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover ######## ## IRHTTPMappingGroup is a collection of Mappings. We'll use it to build Envoy routes later, ## so the group itself ends up with some of the group-wide attributes of its Mappings. -class IRHTTPMappingGroup (IRBaseMappingGroup): + +class IRHTTPMappingGroup(IRBaseMappingGroup): host_redirect: Optional[IRBaseMapping] shadow: List[IRBaseMapping] rewrite: str @@ -25,28 +26,28 @@ class IRHTTPMappingGroup (IRBaseMappingGroup): add_response_headers: Dict[str, str] CoreMappingKeys: ClassVar[Dict[str, bool]] = { - 'bypass_auth': True, - 'bypass_error_response_overrides': True, - 'circuit_breakers': True, - 'cluster_timeout_ms': True, - 'connect_timeout_ms': True, - 'cluster_idle_timeout_ms': True, - 'cluster_max_connection_lifetime_ms': True, - 'group_id': True, - 'headers': True, + "bypass_auth": True, + "bypass_error_response_overrides": True, + "circuit_breakers": True, + "cluster_timeout_ms": True, + "connect_timeout_ms": True, + "cluster_idle_timeout_ms": True, + "cluster_max_connection_lifetime_ms": True, + "group_id": True, + "headers": True, # 'host_rewrite': True, # 'idle_timeout_ms': True, - 'keepalive': True, + "keepalive": True, # 'labels' doesn't appear in the TransparentKeys list for IRMapping, but it's still # a CoreMappingKey -- if it appears, it can't have multiple values within an IRHTTPMappingGroup. - 'labels': True, - 'load_balancer': True, + "labels": True, + "load_balancer": True, # 'metadata_labels' will get flattened by merging. The group gets all the labels that all its # Mappings have. - 'method': True, - 'prefix': True, - 'prefix_regex': True, - 'prefix_exact': True, + "method": True, + "prefix": True, + "prefix_regex": True, + "prefix_exact": True, # 'rewrite': True, # 'timeout_ms': True } @@ -57,71 +58,87 @@ class IRHTTPMappingGroup (IRBaseMappingGroup): # same stats_name in two unrelated mappings, on your own head be it. DoNotFlattenKeys: ClassVar[Dict[str, bool]] = dict(CoreMappingKeys) - DoNotFlattenKeys.update({ - 'add_request_headers': True, # do this manually. - 'add_response_headers': True, # do this manually. - 'cluster': True, - 'cluster_key': True, # See above about stats. - 'kind': True, - 'location': True, - 'name': True, - 'resolver': True, # can't flatten the resolver... - 'rkey': True, - 'route_weight': True, - 'service': True, - 'stats_name': True, # See above about stats. - 'weight': True, - }) + DoNotFlattenKeys.update( + { + "add_request_headers": True, # do this manually. + "add_response_headers": True, # do this manually. + "cluster": True, + "cluster_key": True, # See above about stats. + "kind": True, + "location": True, + "name": True, + "resolver": True, # can't flatten the resolver... + "rkey": True, + "route_weight": True, + "service": True, + "stats_name": True, # See above about stats. + "weight": True, + } + ) @staticmethod def helper_mappings(res: IRResource, k: str) -> Tuple[str, List[dict]]: - return k, list(reversed(sorted([ x.as_dict() for x in res.mappings ], - key=lambda x: x['route_weight']))) + return k, list( + reversed(sorted([x.as_dict() for x in res.mappings], key=lambda x: x["route_weight"])) + ) @staticmethod def helper_shadows(res: IRResource, k: str) -> Tuple[str, List[dict]]: - return k, list([ x.as_dict() for x in res[k] ]) - - def __init__(self, ir: 'IR', aconf: Config, - location: str, - mapping: IRBaseMapping, - rkey: str="ir.mappinggroup", - kind: str="IRHTTPMappingGroup", - name: str="ir.mappinggroup", - **kwargs) -> None: + return k, list([x.as_dict() for x in res[k]]) + + def __init__( + self, + ir: "IR", + aconf: Config, + location: str, + mapping: IRBaseMapping, + rkey: str = "ir.mappinggroup", + kind: str = "IRHTTPMappingGroup", + name: str = "ir.mappinggroup", + **kwargs, + ) -> None: # print("IRHTTPMappingGroup __init__ (%s %s %s)" % (kind, name, kwargs)) - del rkey # silence unused-variable warning + del rkey # silence unused-variable warning - if 'host_redirect' in kwargs: - raise Exception("IRHTTPMappingGroup cannot accept a host_redirect as a keyword argument") + if "host_redirect" in kwargs: + raise Exception( + "IRHTTPMappingGroup cannot accept a host_redirect as a keyword argument" + ) - if 'path_redirect' in kwargs: - raise Exception("IRHTTPMappingGroup cannot accept a path_redirect as a keyword argument") + if "path_redirect" in kwargs: + raise Exception( + "IRHTTPMappingGroup cannot accept a path_redirect as a keyword argument" + ) - if 'prefix_redirect' in kwargs: - raise Exception("IRHTTPMappingGroup cannot accept a prefix_redirect as a keyword argument") + if "prefix_redirect" in kwargs: + raise Exception( + "IRHTTPMappingGroup cannot accept a prefix_redirect as a keyword argument" + ) - if 'regex_redirect' in kwargs: - raise Exception("IRHTTPMappingGroup cannot accept a regex_redirect as a keyword argument") + if "regex_redirect" in kwargs: + raise Exception( + "IRHTTPMappingGroup cannot accept a regex_redirect as a keyword argument" + ) - if ('shadow' in kwargs) or ('shadows' in kwargs): - raise Exception("IRHTTPMappingGroup cannot accept shadow or shadows as a keyword argument") + if ("shadow" in kwargs) or ("shadows" in kwargs): + raise Exception( + "IRHTTPMappingGroup cannot accept shadow or shadows as a keyword argument" + ) super().__init__( - ir=ir, aconf=aconf, rkey=mapping.rkey, location=location, - kind=kind, name=name, **kwargs + ir=ir, aconf=aconf, rkey=mapping.rkey, location=location, kind=kind, name=name, **kwargs ) self.host_redirect = None self.shadows: List[IRBaseMapping] = [] # XXX This should really be IRHTTPMapping, no? - self.add_dict_helper('mappings', IRHTTPMappingGroup.helper_mappings) - self.add_dict_helper('shadows', IRHTTPMappingGroup.helper_shadows) + self.add_dict_helper("mappings", IRHTTPMappingGroup.helper_mappings) + self.add_dict_helper("shadows", IRHTTPMappingGroup.helper_shadows) # Time to lift a bunch of core stuff from the first mapping up into the # group. - if ('group_weight' not in self) and ('route_weight' in mapping): + if ("group_weight" not in self) and ("route_weight" in mapping): self.group_weight = mapping.route_weight for k in IRHTTPMappingGroup.CoreMappingKeys: @@ -138,25 +155,24 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: mismatches = [] for k in IRHTTPMappingGroup.CoreMappingKeys: - if (k in mapping) and ((k not in self) or - (mapping[k] != self[k])): - mismatches.append((k, mapping[k], self.get(k, '-unset-'))) + if (k in mapping) and ((k not in self) or (mapping[k] != self[k])): + mismatches.append((k, mapping[k], self.get(k, "-unset-"))) if mismatches: - self.post_error("cannot accept new mapping %s with mismatched %s." - "Please verify field is set with the same value in all related mappings." - "Example: When canary is configured, related mappings should have same fields and values" % ( - mapping.name, - ", ".join([ "%s: %s != %s" % (x, y, z) for x, y, z in mismatches ]) - )) + self.post_error( + "cannot accept new mapping %s with mismatched %s." + "Please verify field is set with the same value in all related mappings." + "Example: When canary is configured, related mappings should have same fields and values" + % (mapping.name, ", ".join(["%s: %s != %s" % (x, y, z) for x, y, z in mismatches])) + ) return # self.ir.logger.debug("%s: add mapping %s" % (self, mapping.as_json())) # Per the schema, host_redirect and shadow are Booleans. They won't be _saved_ as # Booleans, though: instead we just save the Mapping that they're a part of. - host_redirect = mapping.get('host_redirect', False) - shadow = mapping.get('shadow', False) + host_redirect = mapping.get("host_redirect", False) + shadow = mapping.get("shadow", False) # First things first: if both shadow and host_redirect are set in this Mapping, # we're going to let shadow win. Kill the host_redirect part. @@ -165,17 +181,19 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: errstr = "At most one of host_redirect and shadow may be set; ignoring host_redirect" aconf.post_error(RichStatus.fromError(errstr), resource=mapping) - mapping.pop('host_redirect', None) - mapping.pop('path_redirect', None) - mapping.pop('prefix_redirect', None) - mapping.pop('regex_redirect', None) + mapping.pop("host_redirect", None) + mapping.pop("path_redirect", None) + mapping.pop("prefix_redirect", None) + mapping.pop("regex_redirect", None) # OK. Is this a shadow Mapping? if shadow: # Yup. Make sure that we don't have multiple shadows. if self.shadows: - errstr = "cannot accept %s as second shadow after %s" % \ - (mapping.name, self.shadows[0].name) + errstr = "cannot accept %s as second shadow after %s" % ( + mapping.name, + self.shadows[0].name, + ) aconf.post_error(RichStatus.fromError(errstr), resource=self) else: # All good. Save it. @@ -185,12 +203,16 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: # those either. if self.host_redirect: - errstr = "cannot accept %s as second host_redirect after %s" % \ - (mapping.name, typecast(IRBaseMapping, self.host_redirect).name) + errstr = "cannot accept %s as second host_redirect after %s" % ( + mapping.name, + typecast(IRBaseMapping, self.host_redirect).name, + ) aconf.post_error(RichStatus.fromError(errstr), resource=self) elif len(self.mappings) > 0: - errstr = "cannot accept %s with host_redirect after mappings without host_redirect (eg %s)" % \ - (mapping.name, self.mappings[0].name) + errstr = ( + "cannot accept %s with host_redirect after mappings without host_redirect (eg %s)" + % (mapping.name, self.mappings[0].name) + ) aconf.post_error(RichStatus.fromError(errstr), resource=self) else: # All good. Save it. @@ -202,8 +224,10 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: # in a group have host_redirect and some do not, so make sure that that can't happen. if self.host_redirect: - aconf.post_error("cannot accept %s without host_redirect after %s with host_redirect" % - (mapping.name, typecast(IRBaseMapping, self.host_redirect).name)) + aconf.post_error( + "cannot accept %s without host_redirect after %s with host_redirect" + % (mapping.name, typecast(IRBaseMapping, self.host_redirect).name) + ) else: # All good. Save this mapping. self.mappings.append(mapping) @@ -215,11 +239,14 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: # self.ir.logger.debug("%s: group now %s" % (self, self.as_json())) - def add_cluster_for_mapping(self, mapping: IRBaseMapping, - marker: Optional[str] = None) -> IRCluster: + def add_cluster_for_mapping( + self, mapping: IRBaseMapping, marker: Optional[str] = None + ) -> IRCluster: # Find or create the cluster for this Mapping... - self.ir.logger.debug(f"IRHTTPMappingGroup: {self.group_id} adding cluster for Mapping {mapping.name} (key {mapping.cluster_key})") + self.ir.logger.debug( + f"IRHTTPMappingGroup: {self.group_id} adding cluster for Mapping {mapping.name} (key {mapping.cluster_key})" + ) cluster: Optional[IRCluster] = None @@ -230,34 +257,41 @@ def add_cluster_for_mapping(self, mapping: IRBaseMapping, if cached_cluster is not None: # We know a priori that anything in the cache under a cluster key must be # an IRCluster, but let's assert that rather than casting. - assert(isinstance(cached_cluster, IRCluster)) + assert isinstance(cached_cluster, IRCluster) cluster = cached_cluster - self.ir.logger.debug(f"IRHTTPMappingGroup: got Cluster from cache for {mapping.cluster_key}") + self.ir.logger.debug( + f"IRHTTPMappingGroup: got Cluster from cache for {mapping.cluster_key}" + ) if not cluster: # OK, we have to actually do some work. self.ir.logger.debug(f"IRHTTPMappingGroup: synthesizing Cluster for {mapping.name}") - cluster = IRCluster(ir=self.ir, aconf=self.ir.aconf, - parent_ir_resource=mapping, - location=mapping.location, - service=mapping.service, - resolver=mapping.resolver, - ctx_name=mapping.get('tls', None), - dns_type=mapping.get('dns_type', 'strict_dns'), - host_rewrite=mapping.get('host_rewrite', False), - enable_ipv4=mapping.get('enable_ipv4', None), - enable_ipv6=mapping.get('enable_ipv6', None), - grpc=mapping.get('grpc', False), - load_balancer=mapping.get('load_balancer', None), - keepalive=mapping.get('keepalive', None), - connect_timeout_ms=mapping.get('connect_timeout_ms', 3000), - cluster_idle_timeout_ms=mapping.get('cluster_idle_timeout_ms', None), - cluster_max_connection_lifetime_ms=mapping.get('cluster_max_connection_lifetime_ms', None), - circuit_breakers=mapping.get('circuit_breakers', None), - marker=marker, - stats_name=mapping.get('stats_name'), - respect_dns_ttl=mapping.get('respect_dns_ttl', False)) + cluster = IRCluster( + ir=self.ir, + aconf=self.ir.aconf, + parent_ir_resource=mapping, + location=mapping.location, + service=mapping.service, + resolver=mapping.resolver, + ctx_name=mapping.get("tls", None), + dns_type=mapping.get("dns_type", "strict_dns"), + host_rewrite=mapping.get("host_rewrite", False), + enable_ipv4=mapping.get("enable_ipv4", None), + enable_ipv6=mapping.get("enable_ipv6", None), + grpc=mapping.get("grpc", False), + load_balancer=mapping.get("load_balancer", None), + keepalive=mapping.get("keepalive", None), + connect_timeout_ms=mapping.get("connect_timeout_ms", 3000), + cluster_idle_timeout_ms=mapping.get("cluster_idle_timeout_ms", None), + cluster_max_connection_lifetime_ms=mapping.get( + "cluster_max_connection_lifetime_ms", None + ), + circuit_breakers=mapping.get("circuit_breakers", None), + marker=marker, + stats_name=mapping.get("stats_name"), + respect_dns_ttl=mapping.get("respect_dns_ttl", False), + ) # Make sure that the cluster is actually in our IR... stored = self.ir.add_cluster(cluster) @@ -285,10 +319,15 @@ def add_cluster_for_mapping(self, mapping: IRBaseMapping, mapping.cluster_key = stored.cache_key # Finally, return the stored cluster. Done. - self.ir.logger.debug(f"IRHTTPMappingGroup: %s returning cluster %s for Mapping %s", self.group_id, stored, mapping.name) + self.ir.logger.debug( + f"IRHTTPMappingGroup: %s returning cluster %s for Mapping %s", + self.group_id, + stored, + mapping.name, + ) return stored - def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: + def finalize(self, ir: "IR", aconf: Config) -> List[IRCluster]: """ Finalize a MappingGroup based on the attributes of its Mappings. Core elements get lifted into the Group so we can more easily build Envoy routes; host-redirect and shadow get handled, etc. @@ -309,7 +348,11 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: # self.ir.logger.debug("%s mapping %s" % (self, mapping.as_json())) for k in mapping.keys(): - if k.startswith('_') or mapping.skip_key(k) or (k in IRHTTPMappingGroup.DoNotFlattenKeys): + if ( + k.startswith("_") + or mapping.skip_key(k) + or (k in IRHTTPMappingGroup.DoNotFlattenKeys) + ): # if verbose: # self.ir.logger.debug("%s: don't flatten %s" % (self, k)) continue @@ -319,12 +362,12 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: self[k] = mapping[k] - add_request_headers.update(mapping.get('add_request_headers', {})) - add_response_headers.update(mapping.get('add_response_headers', {})) + add_request_headers.update(mapping.get("add_request_headers", {})) + add_response_headers.update(mapping.get("add_response_headers", {})) # Should we have higher weights win over lower if there are conflicts? # Should we disallow conflicts? - metadata_labels.update(mapping.get('metadata_labels') or {}) + metadata_labels.update(mapping.get("metadata_labels") or {}) if add_request_headers: self.add_request_headers = add_request_headers @@ -334,8 +377,8 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: if metadata_labels: self.metadata_labels = metadata_labels - if self.get('load_balancer', None) is None: - self['load_balancer'] = ir.ambassador_module.load_balancer + if self.get("load_balancer", None) is None: + self["load_balancer"] = ir.ambassador_module.load_balancer # if verbose: # self.ir.logger.debug("%s after flattening %s" % (self, self.as_json())) @@ -350,16 +393,16 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: # If they did give a rewrite, leave it alone so that the Envoy config can correctly # handle an empty rewrite as no rewriting at all. - if 'rewrite' not in self: + if "rewrite" not in self: self.rewrite = "/" # OK. Save some typing with local variables for default labels and our labels... - labels: Dict[str, Any] = self.get('labels', None) + labels: Dict[str, Any] = self.get("labels", None) - if self.get('keepalive', None) is None: - keepalive_default = ir.ambassador_module.get('keepalive', None) + if self.get("keepalive", None) is None: + keepalive_default = ir.ambassador_module.get("keepalive", None) if keepalive_default: - self['keepalive'] = keepalive_default + self["keepalive"] = keepalive_default if not labels: # No labels. Use the default label domain to see if we have some valid defaults. @@ -368,13 +411,7 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: if defaults: domain = ir.ambassador_module.get_default_label_domain() - self.labels = { - domain: [ - { - 'defaults': defaults - } - ] - } + self.labels = {domain: [{"defaults": defaults}]} else: # Walk all the domains in our labels, and prepend the defaults, if any. # ir.logger.info("%s: labels %s" % (self.as_json(), labels)) @@ -391,13 +428,14 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: lkeys = label.keys() if len(lkeys) > 1: - err = RichStatus.fromError("label has multiple entries (%s) instead of just one" % - lkeys) + err = RichStatus.fromError( + "label has multiple entries (%s) instead of just one" % lkeys + ) aconf.post_error(err, self) lkey = list(lkeys)[0] - if lkey.startswith('v0_ratelimit_'): + if lkey.startswith("v0_ratelimit_"): # Don't prepend defaults, as this was imported from a V0 rate_limit. continue @@ -408,14 +446,16 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: shadow = self.shadows[0] # The shadow is an IRMapping. Save the cluster for it. - shadow.cluster = self.add_cluster_for_mapping(shadow, marker='shadow') + shadow.cluster = self.add_cluster_for_mapping(shadow, marker="shadow") # We don't need a cluster for host_redirect: it's just a name to redirect to. - redir = self.get('host_redirect', None) + redir = self.get("host_redirect", None) if not redir: - self.ir.logger.debug(f"IRHTTPMappingGroup: checking mapping clusters for %s", self.group_id) + self.ir.logger.debug( + f"IRHTTPMappingGroup: checking mapping clusters for %s", self.group_id + ) for mapping in self.mappings: mapping.cluster = self.add_cluster_for_mapping(mapping, mapping.cluster_tag) @@ -426,10 +466,10 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: self.post_error(f"Could not normalize mapping weights, ignoring...") return [] - return list([ mapping.cluster for mapping in self.mappings ]) + return list([mapping.cluster for mapping in self.mappings]) else: # Flatten the case_sensitive field for host_redirect if it exists - if 'case_sensitive' in redir: - self['case_sensitive'] = redir['case_sensitive'] + if "case_sensitive" in redir: + self["case_sensitive"] = redir["case_sensitive"] return [] diff --git a/python/ambassador/ir/iripallowdeny.py b/python/ambassador/ir/iripallowdeny.py index 5ab037979e..0e97118513 100644 --- a/python/ambassador/ir/iripallowdeny.py +++ b/python/ambassador/ir/iripallowdeny.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from ..envoy.v3.v3cidrrange import CIDRRange - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover class IRIPAllowDeny(IRFilter): @@ -19,20 +19,21 @@ class IRIPAllowDeny(IRFilter): parent: IRResource action: str - principals: List[Tuple[str, 'CIDRRange']] - - EnvoyTypeMap: ClassVar[Dict[str, str]] = { - "remote": "remote_ip", - "peer": "direct_remote_ip" - } - - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.ipallowdeny", - name: str="ir.ipallowdeny", - kind: str="IRIPAllowDeny", - parent: IRResource=None, - action: str=None, - **kwargs) -> None: + principals: List[Tuple[str, "CIDRRange"]] + + EnvoyTypeMap: ClassVar[Dict[str, str]] = {"remote": "remote_ip", "peer": "direct_remote_ip"} + + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.ipallowdeny", + name: str = "ir.ipallowdeny", + kind: str = "IRIPAllowDeny", + parent: IRResource = None, + action: str = None, + **kwargs, + ) -> None: """ Initialize an IRIPAllowDeny. In addition to the usual IRFilter parameters, parent and action are required: @@ -49,10 +50,17 @@ def __init__(self, ir: 'IR', aconf: Config, assert action is not None super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, - parent=parent, action=action, **kwargs) - - def setup(self, ir: 'IR', aconf: Config) -> bool: + ir=ir, + aconf=aconf, + rkey=rkey, + kind=kind, + name=name, + parent=parent, + action=action, + **kwargs, + ) + + def setup(self, ir: "IR", aconf: Config) -> bool: """ Set up an IRIPAllowDeny based on the action and principals passed into __init__. @@ -90,6 +98,7 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # IP match, and the value is a CIDRRange spec. from ..envoy.v3.v3cidrrange import CIDRRange + for pdict in principals: # If we have more than one thing in the dict, that's an error. @@ -97,7 +106,9 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: for kind, spec in pdict.items(): if not first: - self.parent.post_error(f"ip{self.action.lower()} principals must be separate list elements") + self.parent.post_error( + f"ip{self.action.lower()} principals must be separate list elements" + ) break first = False @@ -105,7 +116,9 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: envoy_kind = IRIPAllowDeny.EnvoyTypeMap.get(kind, None) if not envoy_kind: - self.parent.post_error(f"ip{self.action.lower()} principal type {kind} unknown: must be peer or remote") + self.parent.post_error( + f"ip{self.action.lower()} principal type {kind} unknown: must be peer or remote" + ) continue cidrrange = CIDRRange(spec) @@ -113,7 +126,9 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: if cidrrange: self.principals.append((envoy_kind, cidrrange)) else: - self.parent.post_error(f"ip_{self.action.lower()} principal {spec} is not valid: {cidrrange.error}") + self.parent.post_error( + f"ip_{self.action.lower()} principal {spec} is not valid: {cidrrange.error}" + ) if len(self.principals) > 0: return True @@ -121,11 +136,11 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return False def __str__(self) -> str: - pstrs = [ str(x) for x in self.principals ] + pstrs = [str(x) for x in self.principals] return f"" def as_dict(self) -> dict: return { "action": self.action, - "principals": [ { kind: block.as_dict() } for kind, block in self.principals ] + "principals": [{kind: block.as_dict()} for kind, block in self.principals], } diff --git a/python/ambassador/ir/irlistener.py b/python/ambassador/ir/irlistener.py index aebaa800bb..a45bfa1ecc 100644 --- a/python/ambassador/ir/irlistener.py +++ b/python/ambassador/ir/irlistener.py @@ -13,88 +13,89 @@ from .irutils import selector_matches if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRListener (IRResource): +class IRListener(IRResource): """ IRListener is a pretty direct translation of the Ambassador Listener resource. """ socket_protocol: Literal["TCP", "UDP"] - bind_address: str # Often "0.0.0.0", but can be overridden. + bind_address: str # Often "0.0.0.0", but can be overridden. service_port: int use_proxy_proto: bool hostname: str - http3_enabled: bool # indicates Listener will support http3 + http3_enabled: bool # indicates Listener will support http3 context: Optional[IRTLSContext] - insecure_only: bool # Was this synthesized solely due to an insecure_addl_port? + insecure_only: bool # Was this synthesized solely due to an insecure_addl_port? namespace_literal: str # Literal namespace to be matched namespace_selector: Dict[str, str] # Namespace selector - host_selector: Dict[str, str] # Host selector + host_selector: Dict[str, str] # Host selector AllowedKeys = { - 'bind_address', - 'l7Depth', - 'hostBinding', # Note that hostBinding gets processed and deleted in setup. - 'port', - 'protocol', - 'protocolStack', - 'securityModel', - 'statsPrefix', + "bind_address", + "l7Depth", + "hostBinding", # Note that hostBinding gets processed and deleted in setup. + "port", + "protocol", + "protocolStack", + "securityModel", + "statsPrefix", } ProtocolStacks: Dict[str, List[str]] = { # HTTP: accepts cleartext HTTP/1.1 sessions over TCP. - "HTTP": [ "HTTP", "TCP" ], - + "HTTP": ["HTTP", "TCP"], # HTTPS: accepts encrypted HTTP/1.1 or HTTP/2 sessions using TLS over TCP. - "HTTPS": [ "TLS", "HTTP", "TCP" ], - + "HTTPS": ["TLS", "HTTP", "TCP"], # HTTPPROXY: accepts cleartext HTTP/1.1 sessions using the HAProxy PROXY protocol over TCP. - "HTTPPROXY": [ "PROXY", "HTTP", "TCP" ], - + "HTTPPROXY": ["PROXY", "HTTP", "TCP"], # HTTPSPROXY: accepts encrypted HTTP/1.1 or HTTP/2 sessions using the HAProxy PROXY protocol over TLS over TCP. - "HTTPSPROXY": [ "TLS", "PROXY", "HTTP", "TCP" ], - + "HTTPSPROXY": ["TLS", "PROXY", "HTTP", "TCP"], # TCP: accepts raw TCP sessions. - "TCP": [ "TCP" ], - + "TCP": ["TCP"], # TLS: accepts TLS over TCP. - "TLS": [ "TLS", "TCP" ], - + "TLS": ["TLS", "TCP"], # # UDP: accepts UDP packets. # "UDP": [ "UDP" ], } - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, # REQUIRED - name: str, # REQUIRED - location: str, # REQUIRED - namespace: Optional[str]=None, - kind: str="IRListener", - apiVersion: str="getambassador.io/v3alpha1", - insecure_only: bool=False, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, # REQUIRED + name: str, # REQUIRED + location: str, # REQUIRED + namespace: Optional[str] = None, + kind: str = "IRListener", + apiVersion: str = "getambassador.io/v3alpha1", + insecure_only: bool = False, + **kwargs, + ) -> None: ir.logger.debug("IRListener __init__ (%s %s %s)" % (kind, name, kwargs)) # A note: we copy hostBinding from kwargs in this loop, but we end up processing # and deleting it in setup(). This is arranged this way because __init__ can't # return an error, but setup() can. - new_args = { - x: kwargs[x] for x in kwargs.keys() - if x in IRListener.AllowedKeys - } + new_args = {x: kwargs[x] for x in kwargs.keys() if x in IRListener.AllowedKeys} super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, - kind=kind, name=name, namespace=namespace, apiVersion=apiVersion, + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + kind=kind, + name=name, + namespace=namespace, + apiVersion=apiVersion, insecure_only=insecure_only, - **new_args + **new_args, ) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # Default hostBinding information early, so that we don't have to worry about it # ever being unset. We default to only looking for Hosts in our own namespace, and # to not using selectors beyond that. @@ -103,7 +104,7 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: self.host_selector = {} # Was a bind address specified? - if not self.get('bind_address', None): + if not self.get("bind_address", None): # Nope, use the default. self.bind_address = Config.envoy_bind_address @@ -117,7 +118,9 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: ir.logger.debug(f"Listener {self.name} has pstack {pstack}") # It's an error to specify both protocol and protocolStack. if protocol: - self.post_error("protocol and protocolStack may not both be specified; using protocolStack and ignoring protocol") + self.post_error( + "protocol and protocolStack may not both be specified; using protocolStack and ignoring protocol" + ) self.protocol = None elif not protocol: # It's also an error to specify neither protocol nor protocolStack. @@ -142,19 +145,23 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: self.socket_protocol = self.protocolStack[-1] if self.socket_protocol not in {"TCP", "UDP"}: - self.post_error(f"TCP or UDP should be the last protocol in the protocolStack. Unable to determine socket protocol for listener {self.name}") + self.post_error( + f"TCP or UDP should be the last protocol in the protocolStack. Unable to determine socket protocol for listener {self.name}" + ) return False self.http3_enabled = False - + # To allow for future raw UDP support we must check whether the protocolStack includes HTTP. # Having both HTTP and UDP will indicate that the QUIC protocol should be used vs raw UDP forwarding if self.socket_protocol == "UDP": if "HTTP" in self.protocolStack: self.http3_enabled = True else: - self.post_error(f"UDP is only supported with HTTP. Invalid protocolStack for listener {self.name}") - return False + self.post_error( + f"UDP is only supported with HTTP. Invalid protocolStack for listener {self.name}" + ) + return False if not securityModel: self.post_error("securityModel is required") @@ -189,7 +196,7 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # XXX You can't do del(self.hostBinding) here, because underneath everything, an # IRListener is a Resource, and Resources are really much more like dicts than we # like to admit. - del(self["hostBinding"]) + del self["hostBinding"] # We are going to require at least one of 'namespace' and 'selector' in the # hostBinding. (Really, K8s validation should be enforcing this before we get @@ -211,11 +218,11 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: self.post_error("hostBinding.namespace.from is required") return False - if nsfrom.lower() == 'all': - self.namespace_literal = "*" # Special, obviously. - elif nsfrom.lower() == 'self': + if nsfrom.lower() == "all": + self.namespace_literal = "*" # Special, obviously. + elif nsfrom.lower() == "self": self.namespace_literal = self.namespace - elif nsfrom.lower() == 'selector': + elif nsfrom.lower() == "selector": # Augh. We can't actually support this yet, since the Python side of # Ambassador has no sense of Namespace objects, so it can't look at the # namespace labels! @@ -243,7 +250,9 @@ def matches_host(self, host: IRHost) -> bool: nsmatch = (self.namespace_literal == "*") or (self.namespace_literal == host.namespace) if not nsmatch: - self.ir.logger.debug(" namespace mismatch (we're %s), DROP %s", self.namespace_literal, host) + self.ir.logger.debug( + " namespace mismatch (we're %s), DROP %s", self.namespace_literal, host + ) return False if not selector_matches(self.ir.logger, self.host_selector, host.metadata_labels): @@ -264,20 +273,22 @@ def __str__(self) -> str: hsstr = "" if self.host_selector: - hsstr = "; ".join([ f"{k}={v}" for k, v in self.host_selector.items() ]) + hsstr = "; ".join([f"{k}={v}" for k, v in self.host_selector.items()]) nsstr = "" if self.namespace_selector: - nsstr = "; ".join([ f"{k}={v}" for k, v in self.namespace_selector.items() ]) + nsstr = "; ".join([f"{k}={v}" for k, v in self.namespace_selector.items()]) socket_protocol = "" if self.get("socket_protocol"): socket_protocol = self.socket_protocol.lower() - return f'' + return ( + f"' + ) # Deliberately matches IRTCPMappingGroup.bind_to() def bind_to(self) -> str: @@ -286,14 +297,16 @@ def bind_to(self) -> str: class ListenerFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: + def load_all(cls, ir: "IR", aconf: Config) -> None: amod = ir.ambassador_module - listeners = aconf.get_config('listeners') + listeners = aconf.get_config("listeners") if listeners: for config in listeners.values(): - ir.logger.debug("ListenerFactory: creating Listener for %s" % repr(config.as_dict())) + ir.logger.debug( + "ListenerFactory: creating Listener for %s" % repr(config.as_dict()) + ) listener = IRListener(ir, aconf, **config) @@ -307,7 +320,7 @@ def load_all(cls, ir: 'IR', aconf: Config) -> None: ir.logger.debug(f"ListenerFactory: not saving inactive Listener {listener}") @classmethod - def finalize(cls, ir: 'IR', aconf: Config) -> None: + def finalize(cls, ir: "IR", aconf: Config) -> None: # Finally, cycle over our TCPMappingGroups and make sure we have # Listeners for all of them, too. for group in ir.ordered_groups(): @@ -324,27 +337,30 @@ def finalize(cls, ir: 'IR', aconf: Config) -> None: if group_key not in ir.listeners: # Nothing already exists, so fab one up. Use TLS if and only if a host match is specified; # with no host match, use TCP. - group_host = group.get('host', None) + group_host = group.get("host", None) protocol = "TLS" if group_host else "TCP" - bind_address = group.get('address') or Config.envoy_bind_address + bind_address = group.get("address") or Config.envoy_bind_address name = f"listener-{bind_address}-{group.port}" - ir.logger.debug("ListenerFactory: synthesizing %s listener for TCPMappingGroup on %s:%d" % - (protocol, bind_address, group.port)) + ir.logger.debug( + "ListenerFactory: synthesizing %s listener for TCPMappingGroup on %s:%d" + % (protocol, bind_address, group.port) + ) # The securityModel of a TCP listener is kind of a no-op at this point. We'll set it # to SECURE because that seems more rational than anything else. I guess. - ir.save_listener(IRListener( - ir, aconf, '-internal-', name, '-internal-', - bind_address=bind_address, - port=group.port, - protocol=protocol, - securityModel="SECURE", # See above. - hostBinding={ - "namespace": { - "from": "SELF" - } - } - )) - + ir.save_listener( + IRListener( + ir, + aconf, + "-internal-", + name, + "-internal-", + bind_address=bind_address, + port=group.port, + protocol=protocol, + securityModel="SECURE", # See above. + hostBinding={"namespace": {"from": "SELF"}}, + ) + ) diff --git a/python/ambassador/ir/irlogservice.py b/python/ambassador/ir/irlogservice.py index 6ef663052e..c04381548c 100644 --- a/python/ambassador/ir/irlogservice.py +++ b/python/ambassador/ir/irlogservice.py @@ -7,60 +7,64 @@ from .ircluster import IRCluster if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover class IRLogService(IRResource): cluster: Optional[IRCluster] service: str - protocol_version: Literal['v2', 'v3'] + protocol_version: Literal["v2", "v3"] driver: str driver_config: dict flush_interval_byte_size: int flush_interval_time: int grpc: bool - def __init__(self, ir: 'IR', config, - rkey: str = "ir.logservice", - kind: str = "ir.logservice", - name: str = "logservice", - namespace: Optional[str] = None, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + config, + rkey: str = "ir.logservice", + kind: str = "ir.logservice", + name: str = "logservice", + namespace: Optional[str] = None, + **kwargs, + ) -> None: del kwargs # silence unused-variable warning - super().__init__( - ir=ir, aconf=config, rkey=rkey, kind=kind, name=name, namespace=namespace - ) + super().__init__(ir=ir, aconf=config, rkey=rkey, kind=kind, name=name, namespace=namespace) - def setup(self, ir: 'IR', config) -> bool: - self.service = config.get('service') + def setup(self, ir: "IR", config) -> bool: + self.service = config.get("service") if not self.service: self.post_error("service must be present for a remote log service!") return False self.namespace = config.get("namespace", self.namespace) self.cluster = None - self.grpc = config.get('grpc', False) + self.grpc = config.get("grpc", False) - self.protocol_version = config.get('protocol_version', 'v2') + self.protocol_version = config.get("protocol_version", "v2") if self.protocol_version == "v2": - self.post_error(f'LogService: protocol_version {self.protocol_version} is unsupported, protocol_version must be "v3"') + self.post_error( + f'LogService: protocol_version {self.protocol_version} is unsupported, protocol_version must be "v3"' + ) return False - self.driver = config.get('driver') + self.driver = config.get("driver") # These defaults come from Envoy: # https://www.envoyproxy.io/docs/envoy/v1.22.2/api-v3/extensions/access_loggers/grpc/v3/als.proto#extensions-access-loggers-grpc-v3-commongrpcaccesslogconfig - self.flush_interval_byte_size = config.get('flush_interval_byte_size', 16384) - self.flush_interval_time = config.get('flush_interval_time', 1) + self.flush_interval_byte_size = config.get("flush_interval_byte_size", 16384) + self.flush_interval_time = config.get("flush_interval_time", 1) - self.driver_config = config.get('driver_config') - if 'additional_log_headers' in self.driver_config: - if self.driver != 'http' and self.driver_config['additional_log_headers']: + self.driver_config = config.get("driver_config") + if "additional_log_headers" in self.driver_config: + if self.driver != "http" and self.driver_config["additional_log_headers"]: self.post_error("additional_log_headers are not supported in tcp mode") return False for header_obj in self.get_additional_headers(): - if header_obj.get('header_name', '') == '': + if header_obj.get("header_name", "") == "": self.post_error("Please provide a header name for every additional log header!") return False @@ -69,7 +73,7 @@ def setup(self, ir: 'IR', config) -> bool: return True - def add_mappings(self, ir: 'IR', aconf: Config): + def add_mappings(self, ir: "IR", aconf: Config): self.cluster = ir.add_cluster( IRCluster( ir=ir, @@ -77,10 +81,10 @@ def add_mappings(self, ir: 'IR', aconf: Config): parent_ir_resource=self, location=self.location, service=self.service, - host_rewrite=self.get('host_rewrite', None), - marker='logging', + host_rewrite=self.get("host_rewrite", None), + marker="logging", grpc=self.grpc, - stats_name=self.get("stats_name", None) + stats_name=self.get("stats_name", None), ) ) @@ -91,38 +95,37 @@ def get_common_config(self) -> dict: # is called (by ir.walk_saved_resources). So we can assert that # self.cluster isn't None here, both to make mypy happier and out # of paranoia. - assert(self.cluster) + assert self.cluster return { "transport_api_version": self.protocol_version.upper(), "log_name": self.name, - "grpc_service": { - "envoy_grpc": { - "cluster_name": self.cluster.envoy_name - } - }, + "grpc_service": {"envoy_grpc": {"cluster_name": self.cluster.envoy_name}}, "buffer_flush_interval": "%ds" % self.flush_interval_time, "buffer_size_bytes": self.flush_interval_byte_size, } def get_additional_headers(self) -> list: - if 'additional_log_headers' in self.driver_config: - return self.driver_config.get('additional_log_headers', []) + if "additional_log_headers" in self.driver_config: + return self.driver_config.get("additional_log_headers", []) else: return [] class IRLogServiceFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: - services = aconf.get_config('log_services') + def load_all(cls, ir: "IR", aconf: Config) -> None: + services = aconf.get_config("log_services") if services is not None: for config in services.values(): srv = IRLogService(ir, config) extant_srv = ir.log_services.get(srv.name, None) if extant_srv: - ir.post_error("Duplicate LogService %s; keeping definition from %s" % (srv.name, extant_srv.location)) + ir.post_error( + "Duplicate LogService %s; keeping definition from %s" + % (srv.name, extant_srv.location) + ) elif srv.is_active(): ir.log_services[srv.name] = srv ir.save_resource(srv) diff --git a/python/ambassador/ir/irmappingfactory.py b/python/ambassador/ir/irmappingfactory.py index dab5a2a441..99a8ab4f13 100644 --- a/python/ambassador/ir/irmappingfactory.py +++ b/python/ambassador/ir/irmappingfactory.py @@ -7,12 +7,12 @@ from .irtcpmapping import IRTCPMapping if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover def unique_mapping_name(aconf: Config, name: str) -> str: - http_mappings = aconf.get_config('mappings') or {} - tcp_mappings = aconf.get_config('tcpmappings') or {} + http_mappings = aconf.get_config("mappings") or {} + tcp_mappings = aconf.get_config("tcpmappings") or {} basename = name counter = 0 @@ -26,19 +26,25 @@ def unique_mapping_name(aconf: Config, name: str) -> str: class MappingFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: + def load_all(cls, ir: "IR", aconf: Config) -> None: cls.load_config(ir, aconf, "Mapping", "mappings", IRHTTPMapping) cls.load_config(ir, aconf, "TCPMapping", "tcpmappings", IRTCPMapping) @classmethod - def load_config(cls, ir: 'IR', aconf: Config, - kind: str, config_name: str, mapping_class: Type[IRBaseMapping]) -> None: + def load_config( + cls, + ir: "IR", + aconf: Config, + kind: str, + config_name: str, + mapping_class: Type[IRBaseMapping], + ) -> None: config_info = aconf.get_config(config_name) if not config_info: return - assert(len(config_info) > 0) # really rank paranoia on my part... + assert len(config_info) > 0 # really rank paranoia on my part... live_mappings: List[IRBaseMapping] = [] @@ -58,7 +64,7 @@ def load_config(cls, ir: 'IR', aconf: Config, else: # Cache hit. We know a priori that anything in the cache under a Mapping # key must be an IRBaseMapping, but let's assert that rather than casting. - assert(isinstance(cached_mapping, IRBaseMapping)) + assert isinstance(cached_mapping, IRBaseMapping) mapping = cached_mapping if mapping: @@ -70,7 +76,9 @@ def load_config(cls, ir: 'IR', aconf: Config, for mapping in live_mappings: if mapping.cache_key in ir.invalidate_groups_for: group_key = mapping.group_class().key_for_id(mapping.group_id) - ir.logger.debug("IR: MappingFactory invalidating %s for %s", group_key, mapping.name) + ir.logger.debug( + "IR: MappingFactory invalidating %s for %s", group_key, mapping.name + ) ir.cache.invalidate(group_key) ir.logger.debug("IR: MappingFactory adding live mappings") @@ -82,7 +90,7 @@ def load_config(cls, ir: 'IR', aconf: Config, ir.cache.dump("MappingFactory") @classmethod - def finalize(cls, ir: 'IR', aconf: Config) -> None: + def finalize(cls, ir: "IR", aconf: Config) -> None: # OK. We've created whatever IRMappings we need. Time to create the clusters # they need. diff --git a/python/ambassador/ir/irratelimit.py b/python/ambassador/ir/irratelimit.py index 894a72cba2..132ed7c0d0 100644 --- a/python/ambassador/ir/irratelimit.py +++ b/python/ambassador/ir/irratelimit.py @@ -7,25 +7,28 @@ from .ircluster import IRCluster if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRRateLimit (IRFilter): - protocol_version: Literal['v2', 'v3'] - - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.ratelimit", - kind: str="IRRateLimit", - name: str="rate_limit", # This is a key for Envoy! You can't just change it. - namespace: Optional[str] = None, - **kwargs) -> None: +class IRRateLimit(IRFilter): + protocol_version: Literal["v2", "v3"] + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.ratelimit", + kind: str = "IRRateLimit", + name: str = "rate_limit", # This is a key for Envoy! You can't just change it. + namespace: Optional[str] = None, + **kwargs, + ) -> None: super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, namespace=namespace, type='decoder' + ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, namespace=namespace, type="decoder" ) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: config_info = aconf.get_config("ratelimit_configs") if not config_info: @@ -44,8 +47,9 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: service = config.get("service", None) if not service: - self.post_error(RichStatus.fromError("service is required in RateLimitService", - module=config)) + self.post_error( + RichStatus.fromError("service is required in RateLimitService", module=config) + ) return False ir.logger.debug("IRRateLimit: ratelimit using service %s" % service) @@ -53,18 +57,20 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # OK, we have a valid config. self.service = service - self.ctx_name = config.get('tls', None) - self.name = "rate_limit" # Force this, just in case. + self.ctx_name = config.get("tls", None) + self.name = "rate_limit" # Force this, just in case. self.namespace = config.get("namespace", self.namespace) - self.domain = config.get('domain', ir.ambassador_module.default_label_domain) + self.domain = config.get("domain", ir.ambassador_module.default_label_domain) self.protocol_version = config.get("protocol_version", "v2") if self.protocol_version == "v2": - self.post_error(f'RateLimitService: protocol_version {self.protocol_version} is unsupported, protocol_version must be "v3"') + self.post_error( + f'RateLimitService: protocol_version {self.protocol_version} is unsupported, protocol_version must be "v3"' + ) return False # XXX host_rewrite actually isn't in the schema right now. - self.host_rewrite = config.get('host_rewrite', None) + self.host_rewrite = config.get("host_rewrite", None) # Should we use the shiny new data_plane_proto? Default false right now. # XXX Needs to be configurable. @@ -73,8 +79,8 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # Filter config. self.config = { "domain": self.domain, - "timeout_ms": config.get('timeout_ms', 20), - "request_type": "both" # XXX configurability! + "timeout_ms": config.get("timeout_ms", 20), + "request_type": "both", # XXX configurability! } self.sourced_by(config) @@ -82,7 +88,7 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return True - def add_mappings(self, ir: 'IR', aconf: Config): + def add_mappings(self, ir: "IR", aconf: Config): cluster = ir.add_cluster( IRCluster( ir=ir, @@ -91,9 +97,9 @@ def add_mappings(self, ir: 'IR', aconf: Config): location=self.location, service=self.service, grpc=True, - host_rewrite=self.get('host_rewrite', None), - ctx_name=self.get('ctx_name', None), - stats_name=self.get("stats_name", None) + host_rewrite=self.get("host_rewrite", None), + ctx_name=self.get("ctx_name", None), + stats_name=self.get("stats_name", None), ) ) diff --git a/python/ambassador/ir/irresource.py b/python/ambassador/ir/irresource.py index 259cb0f845..0487ac4536 100644 --- a/python/ambassador/ir/irresource.py +++ b/python/ambassador/ir/irresource.py @@ -8,55 +8,61 @@ from ..utils import RichStatus if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRResource (Resource): +class IRResource(Resource): """ A resource within the IR. """ @staticmethod - def helper_sort_keys(res: 'IRResource', k: str) -> Tuple[str, List[str]]: + def helper_sort_keys(res: "IRResource", k: str) -> Tuple[str, List[str]]: return k, list(sorted(res[k].keys())) @staticmethod - def helper_rkey(res: 'IRResource', k: str) -> Tuple[str, str]: - return '_rkey', res[k] + def helper_rkey(res: "IRResource", k: str) -> Tuple[str, str]: + return "_rkey", res[k] @staticmethod - def helper_list(res: 'IRResource', k: str) -> Tuple[str, list]: - return k, list([ x.as_dict() for x in res[k] ]) + def helper_list(res: "IRResource", k: str) -> Tuple[str, list]: + return k, list([x.as_dict() for x in res[k]]) - __as_dict_helpers: Dict[str, Any] = { - "apiVersion": "drop", - "logger": "drop", - "ir": "drop" - } + __as_dict_helpers: Dict[str, Any] = {"apiVersion": "drop", "logger": "drop", "ir": "drop"} _active: bool _errored: bool _cache_key: Optional[str] - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, - kind: str, - name: str, - namespace: Optional[str]=None, - metadata_labels: Optional[Dict[str, str]]=None, - location: str = "--internal--", - apiVersion: str="ambassador/ir", - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, + kind: str, + name: str, + namespace: Optional[str] = None, + metadata_labels: Optional[Dict[str, str]] = None, + location: str = "--internal--", + apiVersion: str = "ambassador/ir", + **kwargs + ) -> None: # print("IRResource __init__ (%s %s)" % (kind, name)) if not namespace: namespace = ir.ambassador_namespace self.namespace = namespace - super().__init__(rkey=rkey, location=location, - kind=kind, name=name, namespace=namespace, metadata_labels=metadata_labels, - apiVersion=apiVersion, - **kwargs) + super().__init__( + rkey=rkey, + location=location, + kind=kind, + name=name, + namespace=namespace, + metadata_labels=metadata_labels, + apiVersion=apiVersion, + **kwargs + ) self.ir = ir self.logger = ir.logger @@ -79,13 +85,15 @@ def __init__(self, ir: 'IR', aconf: Config, # XXX WTFO, I hear you cry. Why is this "type: ignore here?" So here's the deal: # mypy doesn't like it if you override just the getter of a property that has a # setter, too, and I cannot figure out how else to shut it up. - @property # type: ignore + @property # type: ignore def cache_key(self) -> str: # If you ask for the cache key and it's not set, that is an error. - assert(self._cache_key is not None) + assert self._cache_key is not None return self._cache_key - def lookup_default(self, key: str, default_value: Optional[Any]=None, lookup_class: Optional[str]=None) -> Any: + def lookup_default( + self, key: str, default_value: Optional[Any] = None, lookup_class: Optional[str] = None + ) -> Any: """ Look up a key in the Ambassador module's "defaults" element. @@ -111,14 +119,14 @@ def lookup_default(self, key: str, default_value: Optional[Any]=None, lookup_cla :return: Any """ - defaults = self.ir.ambassador_module.get('defaults', {}) + defaults = self.ir.ambassador_module.get("defaults", {}) lclass = lookup_class if not lclass: - lclass = self.get('default_class', None) + lclass = self.get("default_class", None) - if lclass and (lclass != '/'): + if lclass and (lclass != "/"): # Case 1. classdict = defaults.get(lclass, None) @@ -132,7 +140,13 @@ def lookup_default(self, key: str, default_value: Optional[Any]=None, lookup_cla # We didn't find anything in either case. Return the default value. return default_value - def lookup(self, key: str, *args, default_class: Optional[str]=None, default_key: Optional[str]=None) -> Any: + def lookup( + self, + key: str, + *args, + default_class: Optional[str] = None, + default_key: Optional[str] = None + ) -> Any: """ Look up a key in this IRResource, with a fallback to the Ambassador module's "defaults" element. @@ -162,7 +176,9 @@ def lookup(self, key: str, *args, default_class: Optional[str]=None, default_key if not default_key: default_key = key - value = self.lookup_default(default_key, default_value=default_value, lookup_class=default_class) + value = self.lookup_default( + default_key, default_value=default_value, lookup_class=default_class + ) return value @@ -178,11 +194,11 @@ def is_active(self) -> bool: def __bool__(self) -> bool: return self._active and not self._errored - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # If you don't override setup, you end up with an IRResource that's always active. return True - def add_mappings(self, ir: 'IR', aconf: Config) -> None: + def add_mappings(self, ir: "IR", aconf: Config) -> None: # If you don't override add_mappings, uh, no mappings will get added. pass @@ -195,10 +211,10 @@ def post_error(self, error: Union[str, RichStatus], log_level=logging.INFO): self.ir.post_error(error, resource=self, log_level=log_level) def skip_key(self, k: str) -> bool: - if k.startswith('__') or k.startswith("_IRResource__"): + if k.startswith("__") or k.startswith("_IRResource__"): return True - if self.__as_dict_helpers.get(k, None) == 'drop': + if self.__as_dict_helpers.get(k, None) == "drop": return True return False @@ -229,8 +245,8 @@ def normalize_service(service: str) -> str: normalized_service = service if service.lower().startswith("http://"): - normalized_service = service[len("http://"):] + normalized_service = service[len("http://") :] elif service.lower().startswith("https://"): - normalized_service = service[len("https://"):] + normalized_service = service[len("https://") :] return normalized_service diff --git a/python/ambassador/ir/irretrypolicy.py b/python/ambassador/ir/irretrypolicy.py index 4e7ee5fff5..5d7f9cfe93 100644 --- a/python/ambassador/ir/irretrypolicy.py +++ b/python/ambassador/ir/irretrypolicy.py @@ -6,23 +6,24 @@ from .irresource import IRResource if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRRetryPolicy (IRResource): - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.retrypolicy", - kind: str="IRRetryPolicy", - name: str="ir.retrypolicy", - **kwargs) -> None: +class IRRetryPolicy(IRResource): + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.retrypolicy", + kind: str = "IRRetryPolicy", + name: str = "ir.retrypolicy", + **kwargs + ) -> None: # print("IRRetryPolicy __init__ (%s %s %s)" % (kind, name, kwargs)) - super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, - **kwargs - ) + super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: if not self.validate_retry_policy(): self.post_error("Invalid retry policy specified: {}".format(self)) return False @@ -30,10 +31,17 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: return True def validate_retry_policy(self) -> bool: - retry_on = self.get('retry_on', None) + retry_on = self.get("retry_on", None) is_valid = False - if retry_on in {'5xx', 'gateway-error', 'connect-failure', 'retriable-4xx', 'refused-stream', 'retriable-status-codes'}: + if retry_on in { + "5xx", + "gateway-error", + "connect-failure", + "retriable-4xx", + "refused-stream", + "retriable-status-codes", + }: is_valid = True return is_valid @@ -42,8 +50,17 @@ def as_dict(self) -> dict: raw_dict = super().as_dict() for key in list(raw_dict): - if key in ["_active", "_errored", "_referenced_by", "_rkey", - "kind", "location", "name", "namespace", "metadata_labels"]: + if key in [ + "_active", + "_errored", + "_referenced_by", + "_rkey", + "kind", + "location", + "name", + "namespace", + "metadata_labels", + ]: raw_dict.pop(key, None) return raw_dict diff --git a/python/ambassador/ir/irserviceresolver.py b/python/ambassador/ir/irserviceresolver.py index b13541cbab..be969b1bfa 100644 --- a/python/ambassador/ir/irserviceresolver.py +++ b/python/ambassador/ir/irserviceresolver.py @@ -14,9 +14,9 @@ from .irtlscontext import IRTLSContext if TYPE_CHECKING: - from .ir import IR # pragma: no cover - from .ircluster import IRCluster # pragma: no cover - from .irbasemapping import IRBaseMapping # pragma: no cover + from .ir import IR # pragma: no cover + from .ircluster import IRCluster # pragma: no cover + from .irbasemapping import IRBaseMapping # pragma: no cover ############################################################################# ## irserviceresolver.py -- resolve endpoints for services @@ -39,36 +39,40 @@ SvcEndpointSet = List[SvcEndpoint] ClustermapEntry = Dict[str, Union[int, str]] + class IRServiceResolver(IRResource): - def __init__(self, ir: 'IR', aconf: Config, - rkey: str = "ir.resolver", - kind: str = "IRServiceResolver", - name: str = "ir.resolver", - location: str = "--internal--", - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.resolver", + kind: str = "IRServiceResolver", + name: str = "ir.resolver", + location: str = "--internal--", + **kwargs, + ) -> None: super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, - location=location, - **kwargs) + ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, location=location, **kwargs + ) - def setup(self, ir: 'IR', aconf: Config) -> bool: - if self.kind == 'ConsulResolver': - self.resolve_with = 'consul' + def setup(self, ir: "IR", aconf: Config) -> bool: + if self.kind == "ConsulResolver": + self.resolve_with = "consul" - if not self.get('datacenter'): + if not self.get("datacenter"): self.post_error("ConsulResolver is required to have a datacenter") return False - elif self.kind == 'KubernetesServiceResolver': - self.resolve_with = 'k8s' - elif self.kind == 'KubernetesEndpointResolver': - self.resolve_with = 'k8s' + elif self.kind == "KubernetesServiceResolver": + self.resolve_with = "k8s" + elif self.kind == "KubernetesEndpointResolver": + self.resolve_with = "k8s" else: self.post_error(f"Resolver kind {self.kind} unknown") return False return True - def valid_mapping(self, ir: 'IR', mapping: 'IRBaseMapping') -> bool: + def valid_mapping(self, ir: "IR", mapping: "IRBaseMapping") -> bool: fn = { "KubernetesServiceResolver": self._k8s_svc_valid_mapping, "KubernetesEndpointResolver": self._k8s_valid_mapping, @@ -77,36 +81,42 @@ def valid_mapping(self, ir: 'IR', mapping: 'IRBaseMapping') -> bool: return fn(ir, mapping) - def _k8s_svc_valid_mapping(self, ir: 'IR', mapping: 'IRBaseMapping'): + def _k8s_svc_valid_mapping(self, ir: "IR", mapping: "IRBaseMapping"): # You're not allowed to specific a load balancer with a KubernetesServiceResolver. - if mapping.get('load_balancer'): - mapping.post_error('No load_balancer setting is allowed with the KubernetesServiceResolver') + if mapping.get("load_balancer"): + mapping.post_error( + "No load_balancer setting is allowed with the KubernetesServiceResolver" + ) return False return True - def _k8s_valid_mapping(self, ir: 'IR', mapping: 'IRBaseMapping'): + def _k8s_valid_mapping(self, ir: "IR", mapping: "IRBaseMapping"): # There's no real validation to do here beyond what the Mapping already does. return True - def _consul_valid_mapping(self, ir: 'IR', mapping: 'IRBaseMapping'): + def _consul_valid_mapping(self, ir: "IR", mapping: "IRBaseMapping"): # Mappings using the Consul resolver can't use service names with '.', or port # override. We currently do this the cheap & sleazy way. valid = True - if mapping.service.find('.') >= 0: - mapping.post_error('The Consul resolver does not allow dots in service names') + if mapping.service.find(".") >= 0: + mapping.post_error("The Consul resolver does not allow dots in service names") valid = False - if mapping.service.find(':') >= 0: + if mapping.service.find(":") >= 0: # This is not an _error_ per se -- we'll accept the mapping and just ignore the port. - ir.aconf.post_notice('The Consul resolver does not allow overriding service port; ignoring requested port', - resource=mapping) + ir.aconf.post_notice( + "The Consul resolver does not allow overriding service port; ignoring requested port", + resource=mapping, + ) return valid - def resolve(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> Optional[SvcEndpointSet]: + def resolve( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> Optional[SvcEndpointSet]: fn = { "KubernetesServiceResolver": self._k8s_svc_resolver, "KubernetesEndpointResolver": self._k8s_resolver, @@ -115,25 +125,25 @@ def resolve(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: return fn(ir, cluster, svc_name, svc_namespace, port) - def _k8s_svc_resolver(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> Optional[SvcEndpointSet]: + def _k8s_svc_resolver( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> Optional[SvcEndpointSet]: # The K8s service resolver always returns a single endpoint. - return [ { - 'ip': svc_name, - 'port': port, - 'target_kind': 'DNSname' - } ] + return [{"ip": svc_name, "port": port, "target_kind": "DNSname"}] - def _k8s_resolver(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> Optional[SvcEndpointSet]: + def _k8s_resolver( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> Optional[SvcEndpointSet]: svc, namespace = self.parse_service(ir, svc_name, svc_namespace) # Find endpoints, and try for a port match! - return self.get_endpoints(ir, f'k8s-{svc}-{namespace}', port) + return self.get_endpoints(ir, f"k8s-{svc}-{namespace}", port) - def parse_service(self, ir: 'IR', svc_name: str, svc_namespace: str) -> Tuple[str, str]: + def parse_service(self, ir: "IR", svc_name: str, svc_namespace: str) -> Tuple[str, str]: # K8s service names can be 'svc' or 'svc.namespace'. Which does this look like? svc = svc_name namespace = Config.ambassador_namespace - if '.' in svc and not is_ip_address(svc): + if "." in svc and not is_ip_address(svc): # OK, cool. Peel off the service and the namespace. # # Note that some people may use service.namespace.cluster.svc.local or @@ -141,57 +151,72 @@ def parse_service(self, ir: 'IR', svc_name: str, svc_namespace: str) -> Tuple[st # elements if there are more, but still work if there are not. (svc, namespace) = svc.split(".", 2)[0:2] - elif not ir.ambassador_module.use_ambassador_namespace_for_service_resolution and svc_namespace: + elif ( + not ir.ambassador_module.use_ambassador_namespace_for_service_resolution + and svc_namespace + ): namespace = svc_namespace - ir.logger.debug("KubernetesEndpointResolver use_ambassador_namespace_for_service_resolution %s, upstream key %s" % (ir.ambassador_module.use_ambassador_namespace_for_service_resolution, f'{svc}-{namespace}')) + ir.logger.debug( + "KubernetesEndpointResolver use_ambassador_namespace_for_service_resolution %s, upstream key %s" + % ( + ir.ambassador_module.use_ambassador_namespace_for_service_resolution, + f"{svc}-{namespace}", + ) + ) return svc, namespace - def _consul_resolver(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> Optional[SvcEndpointSet]: + def _consul_resolver( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> Optional[SvcEndpointSet]: # For Consul, we look things up with the service name and the datacenter at present. # We ignore the port in the lookup (we should've already posted a warning about the port # being present, actually). - return self.get_endpoints(ir, f'consul-{svc_name}-{self.datacenter}', None) + return self.get_endpoints(ir, f"consul-{svc_name}-{self.datacenter}", None) - def get_endpoints(self, ir: 'IR', key: str, port: Optional[int]) -> Optional[SvcEndpointSet]: + def get_endpoints(self, ir: "IR", key: str, port: Optional[int]) -> Optional[SvcEndpointSet]: # OK. Do we have a Service by this key? service = ir.services.get(key) if not service: - self.logger.debug(f'Resolver {self.name}: {key} matches no Service for endpoints') + self.logger.debug(f"Resolver {self.name}: {key} matches no Service for endpoints") return None - self.logger.debug(f'Resolver {self.name}: {key} matches %s' % service.as_json()) + self.logger.debug(f"Resolver {self.name}: {key} matches %s" % service.as_json()) - endpoints = service.get('endpoints') + endpoints = service.get("endpoints") if not endpoints: - self.logger.debug(f'Resolver {self.name}: {key} has no endpoints') + self.logger.debug(f"Resolver {self.name}: {key} has no endpoints") return None # Do we have a match for the port they're asking for (y'know, if they're asking for one)? - targets = endpoints.get(port or '*') + targets = endpoints.get(port or "*") if targets: # Yes! - tstr = ", ".join([ f'{x["ip"]}:{x["port"]}' for x in targets ]) + tstr = ", ".join([f'{x["ip"]}:{x["port"]}' for x in targets]) - self.logger.debug(f'Resolver {self.name}: {key}:{port} matches {tstr}') + self.logger.debug(f"Resolver {self.name}: {key}:{port} matches {tstr}") return targets else: - hrtype = 'Kubernetes' if (self.resolve_with == 'k8s') else self.resolve_with + hrtype = "Kubernetes" if (self.resolve_with == "k8s") else self.resolve_with # This is ugly. We're almost certainly being called from _within_ the initialization # of the cluster here -- so I guess we'll report the error against the service. Sigh. - self.ir.aconf.post_error(f'Service {service.name}: {key}:{port} matches no endpoints from {hrtype}', - resource=service) + self.ir.aconf.post_error( + f"Service {service.name}: {key}:{port} matches no endpoints from {hrtype}", + resource=service, + ) return None - def clustermap_entry(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> ClustermapEntry: + def clustermap_entry( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> ClustermapEntry: fn = { "KubernetesServiceResolver": self._k8s_svc_clustermap_entry, "KubernetesEndpointResolver": self._k8s_clustermap_entry, @@ -200,24 +225,23 @@ def clustermap_entry(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_na return fn(ir, cluster, svc_name, svc_namespace, port) - def _k8s_svc_clustermap_entry(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> ClustermapEntry: + def _k8s_svc_clustermap_entry( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> ClustermapEntry: # The K8s service resolver always returns a single endpoint. svc, namespace = self.parse_service(ir, svc_name, svc_namespace) - return { - 'port': port, - 'kind': self.kind, - 'service': svc, - 'namespace': namespace - } + return {"port": port, "kind": self.kind, "service": svc, "namespace": namespace} - def _k8s_clustermap_entry(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> ClustermapEntry: + def _k8s_clustermap_entry( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> ClustermapEntry: # Fallback to the KubernetesServiceResolver for IP addresses or if the service doesn't exist. if is_ip_address(svc_name): return { - 'service': svc_name, - 'namespace': svc_namespace, - 'port': port, - 'kind': "KubernetesServiceResolver", + "service": svc_name, + "namespace": svc_namespace, + "port": port, + "kind": "KubernetesServiceResolver", } if port: @@ -227,110 +251,119 @@ def _k8s_clustermap_entry(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, s svc, namespace = self.parse_service(ir, svc_name, svc_namespace) # Find endpoints, and try for a port match! return { - 'service': svc, - 'namespace': namespace, - 'port': port, - 'kind': self.kind, - 'endpoint_path': 'k8s/%s/%s%s' % (namespace, svc, portstr) + "service": svc, + "namespace": namespace, + "port": port, + "kind": self.kind, + "endpoint_path": "k8s/%s/%s%s" % (namespace, svc, portstr), } - def _consul_clustermap_entry(self, ir: 'IR', cluster: 'IRCluster', svc_name: str, svc_namespace: str, port: int) -> ClustermapEntry: + def _consul_clustermap_entry( + self, ir: "IR", cluster: "IRCluster", svc_name: str, svc_namespace: str, port: int + ) -> ClustermapEntry: # Fallback to the KubernetesServiceResolver for ip addresses. if is_ip_address(svc_name): return { - 'service': svc_name, - 'namespace': svc_namespace, - 'port': port, - 'kind': "KubernetesServiceResolver", + "service": svc_name, + "namespace": svc_namespace, + "port": port, + "kind": "KubernetesServiceResolver", } # For Consul, we look things up with the service name and the datacenter at present. # We ignore the port in the lookup (we should've already posted a warning about the port # being present, actually). return { - 'service': svc_name, - 'datacenter': self.datacenter, - 'kind': self.kind, - 'endpoint_path': 'consul/%s/%s' % (self.datacenter, svc_name) + "service": svc_name, + "datacenter": self.datacenter, + "kind": self.kind, + "endpoint_path": "consul/%s/%s" % (self.datacenter, svc_name), } + class IRServiceResolverFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: - config_info = aconf.get_config('resolvers') + def load_all(cls, ir: "IR", aconf: Config) -> None: + config_info = aconf.get_config("resolvers") if config_info: - assert(len(config_info) > 0) # really rank paranoia on my part... + assert len(config_info) > 0 # really rank paranoia on my part... for config in config_info.values(): cdict = config.as_dict() - cdict['rkey'] = config.rkey - cdict['location'] = config.location + cdict["rkey"] = config.rkey + cdict["location"] = config.location ir.add_resolver(IRServiceResolver(ir, aconf, **cdict)) - if not ir.get_resolver('kubernetes-service'): + if not ir.get_resolver("kubernetes-service"): # Default the K8s service resolver. resolver_config = { - 'apiVersion': 'getambassador.io/v3alpha1', - 'kind': 'KubernetesServiceResolver', - 'name': 'kubernetes-service' + "apiVersion": "getambassador.io/v3alpha1", + "kind": "KubernetesServiceResolver", + "name": "kubernetes-service", } if Config.single_namespace: - resolver_config['namespace'] = Config.ambassador_namespace + resolver_config["namespace"] = Config.ambassador_namespace ir.add_resolver(IRServiceResolver(ir, aconf, **resolver_config)) # Ugh, the aliasing for the K8s and Consul endpoint resolvers is annoying. - res_e = ir.get_resolver('endpoint') - res_k_e = ir.get_resolver('kubernetes-endpoint') + res_e = ir.get_resolver("endpoint") + res_k_e = ir.get_resolver("kubernetes-endpoint") if not res_e and not res_k_e: # Neither exists. Create them from scratch. resolver_config = { - 'apiVersion': 'getambassador.io/v3alpha1', - 'kind': 'KubernetesEndpointResolver', - 'name': 'kubernetes-endpoint' + "apiVersion": "getambassador.io/v3alpha1", + "kind": "KubernetesEndpointResolver", + "name": "kubernetes-endpoint", } if Config.single_namespace: - resolver_config['namespace'] = Config.ambassador_namespace + resolver_config["namespace"] = Config.ambassador_namespace ir.add_resolver(IRServiceResolver(ir, aconf, **resolver_config)) - resolver_config['name'] = 'endpoint' + resolver_config["name"] = "endpoint" ir.add_resolver(IRServiceResolver(ir, aconf, **resolver_config)) else: - cls.check_aliases(ir, aconf, 'endpoint', res_e, 'kubernetes-endpoint', res_k_e) + cls.check_aliases(ir, aconf, "endpoint", res_e, "kubernetes-endpoint", res_k_e) - res_c = ir.get_resolver('consul') - res_c_e = ir.get_resolver('consul-endpoint') + res_c = ir.get_resolver("consul") + res_c_e = ir.get_resolver("consul-endpoint") if not res_c and not res_c_e: # Neither exists. Create them from scratch. resolver_config = { - 'apiVersion': 'getambassador.io/v3alpha1', - 'kind': 'ConsulResolver', - 'name': 'consul-endpoint', - 'datacenter': 'dc1' + "apiVersion": "getambassador.io/v3alpha1", + "kind": "ConsulResolver", + "name": "consul-endpoint", + "datacenter": "dc1", } ir.add_resolver(IRServiceResolver(ir, aconf, **resolver_config)) - resolver_config['name'] = 'consul' + resolver_config["name"] = "consul" ir.add_resolver(IRServiceResolver(ir, aconf, **resolver_config)) else: - cls.check_aliases(ir, aconf, 'consul', res_c, 'consul-endpoint', res_c_e) + cls.check_aliases(ir, aconf, "consul", res_c, "consul-endpoint", res_c_e) @classmethod - def check_aliases(cls, ir: 'IR', aconf: Config, - n1: str, r1: Optional[IRServiceResolver], - n2: str, r2: Optional[IRServiceResolver]) -> None: + def check_aliases( + cls, + ir: "IR", + aconf: Config, + n1: str, + r1: Optional[IRServiceResolver], + n2: str, + r2: Optional[IRServiceResolver], + ) -> None: source = None name = None @@ -347,15 +380,16 @@ def check_aliases(cls, ir: 'IR', aconf: Config, config = dict(**source.as_dict()) # Fix up this dict. Sigh. - config['rkey'] = config.pop('_rkey', config.get('rkey', None)) # Kludge, I know... - config.pop('_errored', None) - config.pop('_active', None) - config.pop('resolve_with', None) + config["rkey"] = config.pop("_rkey", config.get("rkey", None)) # Kludge, I know... + config.pop("_errored", None) + config.pop("_active", None) + config.pop("resolve_with", None) - config['name'] = name + config["name"] = name ir.add_resolver(IRServiceResolver(ir, aconf, **config)) + def is_ip_address(addr: str) -> bool: try: x = ip_address(addr) diff --git a/python/ambassador/ir/irtcpmapping.py b/python/ambassador/ir/irtcpmapping.py index 63d08a32ce..13506aa3f9 100644 --- a/python/ambassador/ir/irtcpmapping.py +++ b/python/ambassador/ir/irtcpmapping.py @@ -10,10 +10,10 @@ import hashlib if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover -class IRTCPMapping (IRBaseMapping): +class IRTCPMapping(IRBaseMapping): binding: str service: str group_id: str @@ -36,30 +36,35 @@ class IRTCPMapping (IRBaseMapping): "serialization": True, } - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, # REQUIRED - name: str, # REQUIRED - location: str, # REQUIRED - service: str, # REQUIRED - namespace: Optional[str] = None, - metadata_labels: Optional[Dict[str, str]] = None, - - kind: str="IRTCPMapping", - apiVersion: str="getambassador.io/v3alpha1", # Not a typo! See below. - precedence: int=0, - cluster_tag: Optional[str]=None, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, # REQUIRED + name: str, # REQUIRED + location: str, # REQUIRED + service: str, # REQUIRED + namespace: Optional[str] = None, + metadata_labels: Optional[Dict[str, str]] = None, + kind: str = "IRTCPMapping", + apiVersion: str = "getambassador.io/v3alpha1", # Not a typo! See below. + precedence: int = 0, + cluster_tag: Optional[str] = None, + **kwargs, + ) -> None: # OK, this is a bit of a pain. We want to preserve the name and rkey and # such here, unlike most kinds of IRResource. So. Shallow copy the keys # we're going to allow from the incoming kwargs... - new_args = { x: kwargs[x] for x in kwargs.keys() if x in IRTCPMapping.AllowedKeys } + new_args = {x: kwargs[x] for x in kwargs.keys() if x in IRTCPMapping.AllowedKeys} # XXX The resolver lookup code is duplicated from IRBaseMapping.setup -- # needs to be fixed after 1.6.1. - resolver_name = kwargs.get('resolver') or ir.ambassador_module.get('resolver', 'kubernetes-service') + resolver_name = kwargs.get("resolver") or ir.ambassador_module.get( + "resolver", "kubernetes-service" + ) - assert(resolver_name) # for mypy -- resolver_name cannot be None at this point + assert resolver_name # for mypy -- resolver_name cannot be None at this point resolver = ir.get_resolver(resolver_name) if resolver: @@ -68,17 +73,26 @@ def __init__(self, ir: 'IR', aconf: Config, # In IRBaseMapping.setup, we post an error if the resolver is unknown. # Here, we just don't bother; we're only using it for service # qualification. - resolver_kind = 'KubernetesBogusResolver' + resolver_kind = "KubernetesBogusResolver" service = normalize_service_name(ir, service, namespace, resolver_kind, rkey=rkey) ir.logger.debug(f"TCPMapping {name} service normalized to {repr(service)}") # ...and then init the superclass. super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, service=service, - kind=kind, name=name, namespace=namespace, metadata_labels=metadata_labels, - apiVersion=apiVersion, precedence=precedence, cluster_tag=cluster_tag, - **new_args + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + service=service, + kind=kind, + name=name, + namespace=namespace, + metadata_labels=metadata_labels, + apiVersion=apiVersion, + precedence=precedence, + cluster_tag=cluster_tag, + **new_args, ) ir.logger.debug("IRTCPMapping %s: self.host = %s", name, self.get("host") or "i'*'") @@ -88,28 +102,28 @@ def group_class() -> Type[IRBaseMappingGroup]: return IRTCPMappingGroup def bind_to(self) -> str: - bind_addr = self.get('address') or '0.0.0.0' + bind_addr = self.get("address") or "0.0.0.0" return f"tcp-{bind_addr}-{self.port}" def _group_id(self) -> str: # Yes, we're using a cryptographic hash here. Cope. [ :) ] - h = hashlib.new('sha1') + h = hashlib.new("sha1") # This is a TCP mapping. - h.update('TCP-'.encode('utf-8')) + h.update("TCP-".encode("utf-8")) - address = self.get('address') or '*' - h.update(address.encode('utf-8')) + address = self.get("address") or "*" + h.update(address.encode("utf-8")) port = str(self.port) - h.update(port.encode('utf-8')) + h.update(port.encode("utf-8")) - host = self.get('host') or '*' - h.update(host.encode('utf-8')) + host = self.get("host") or "*" + h.update(host.encode("utf-8")) return h.hexdigest() def _route_weight(self) -> List[Union[str, int]]: # These aren't order-dependent? or are they? - return [ 0 ] + return [0] diff --git a/python/ambassador/ir/irtcpmappinggroup.py b/python/ambassador/ir/irtcpmappinggroup.py index 4734c6573d..136a6796cb 100644 --- a/python/ambassador/ir/irtcpmappinggroup.py +++ b/python/ambassador/ir/irtcpmappinggroup.py @@ -10,68 +10,75 @@ from .irbasemapping import IRBaseMapping if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover ######## ## IRTCPMappingGroup is a collection of IRTCPMappings. We'll use it to build Envoy routes later, ## so the group itself ends up with some of the group-wide attributes of its Mappings. -class IRTCPMappingGroup (IRBaseMappingGroup): + +class IRTCPMappingGroup(IRBaseMappingGroup): CoreMappingKeys: ClassVar[Dict[str, bool]] = { - 'address': True, - 'circuit_breakers': True, - 'enable_ipv4': True, - 'enable_ipv6': True, - 'group_id': True, - 'host': True, - 'idle_timeout_ms': True, + "address": True, + "circuit_breakers": True, + "enable_ipv4": True, + "enable_ipv6": True, + "group_id": True, + "host": True, + "idle_timeout_ms": True, # 'labels' doesn't appear in the TransparentKeys list for IRMapping, but it's still # a CoreMappingKey -- if it appears, it can't have multiple values within an IRTCPMappingGroup. - 'labels': True, - 'port': True, - 'tls': True, + "labels": True, + "port": True, + "tls": True, } DoNotFlattenKeys: ClassVar[Dict[str, bool]] = dict(CoreMappingKeys) - DoNotFlattenKeys.update({ - 'cluster': True, - 'cluster_key': True, - 'kind': True, - 'location': True, - 'name': True, - 'rkey': True, - 'route_weight': True, - 'service': True, - 'weight': True, - }) + DoNotFlattenKeys.update( + { + "cluster": True, + "cluster_key": True, + "kind": True, + "location": True, + "name": True, + "rkey": True, + "route_weight": True, + "service": True, + "weight": True, + } + ) @staticmethod def helper_mappings(res: IRResource, k: str) -> Tuple[str, List[dict]]: - return k, list(reversed(sorted([ x.as_dict() for x in res.mappings ], - key=lambda x: x['route_weight']))) - - def __init__(self, ir: 'IR', aconf: Config, - location: str, - mapping: IRBaseMapping, - rkey: str="ir.mappinggroup", - kind: str="IRTCPMappingGroup", - name: str="ir.mappinggroup", - **kwargs) -> None: + return k, list( + reversed(sorted([x.as_dict() for x in res.mappings], key=lambda x: x["route_weight"])) + ) + + def __init__( + self, + ir: "IR", + aconf: Config, + location: str, + mapping: IRBaseMapping, + rkey: str = "ir.mappinggroup", + kind: str = "IRTCPMappingGroup", + name: str = "ir.mappinggroup", + **kwargs, + ) -> None: # print("IRTCPMappingGroup __init__ (%s %s %s)" % (kind, name, kwargs)) - del rkey # silence unused-variable warning + del rkey # silence unused-variable warning super().__init__( - ir=ir, aconf=aconf, rkey=mapping.rkey, location=location, - kind=kind, name=name, **kwargs + ir=ir, aconf=aconf, rkey=mapping.rkey, location=location, kind=kind, name=name, **kwargs ) - self.add_dict_helper('mappings', IRTCPMappingGroup.helper_mappings) + self.add_dict_helper("mappings", IRTCPMappingGroup.helper_mappings) # Time to lift a bunch of core stuff from the first mapping up into the # group. - if ('group_weight' not in self) and ('route_weight' in mapping): + if ("group_weight" not in self) and ("route_weight" in mapping): self.group_weight = mapping.route_weight for k in IRTCPMappingGroup.CoreMappingKeys: @@ -84,15 +91,14 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: mismatches = [] for k in IRTCPMappingGroup.CoreMappingKeys: - if (k in mapping) and ((k not in self) or - (mapping[k] != self[k])): - mismatches.append((k, mapping[k], self.get(k, '-unset-'))) + if (k in mapping) and ((k not in self) or (mapping[k] != self[k])): + mismatches.append((k, mapping[k], self.get(k, "-unset-"))) if mismatches: - self.post_error("cannot accept new mapping %s with mismatched %s" % ( - mapping.name, - ", ".join([ "%s: %s != %s" % (x, y, z) for x, y, z in mismatches ]) - )) + self.post_error( + "cannot accept new mapping %s with mismatched %s" + % (mapping.name, ", ".join(["%s: %s != %s" % (x, y, z) for x, y, z in mismatches])) + ) return self.mappings.append(mapping) @@ -104,11 +110,12 @@ def add_mapping(self, aconf: Config, mapping: IRBaseMapping) -> None: # Deliberately matches IRListener.bind_to() def bind_to(self) -> str: - bind_addr = self.get('address') or Config.envoy_bind_address + bind_addr = self.get("address") or Config.envoy_bind_address return f"tcp-{bind_addr}-{self.port}" - def add_cluster_for_mapping(self, mapping: IRBaseMapping, - marker: Optional[str] = None) -> IRCluster: + def add_cluster_for_mapping( + self, mapping: IRBaseMapping, marker: Optional[str] = None + ) -> IRCluster: cluster: Optional[IRCluster] = None if mapping.cluster_key: @@ -118,24 +125,29 @@ def add_cluster_for_mapping(self, mapping: IRBaseMapping, if cached_cluster is not None: # We know a priori that anything in the cache under a cluster key must be # an IRCluster, but let's assert that rather than casting. - assert(isinstance(cached_cluster, IRCluster)) + assert isinstance(cached_cluster, IRCluster) cluster = cached_cluster - self.ir.logger.debug(f"IRTCPMappingGroup: got Cluster from cache for {mapping.cluster_key}") + self.ir.logger.debug( + f"IRTCPMappingGroup: got Cluster from cache for {mapping.cluster_key}" + ) if not cluster: # Find or create the cluster for this Mapping... - cluster = IRCluster(ir=self.ir, aconf=self.ir.aconf, parent_ir_resource=mapping, - location=mapping.location, - service=mapping.service, - resolver=mapping.resolver, - ctx_name=mapping.get('tls', None), - host_rewrite=mapping.get('host_rewrite', False), - enable_ipv4=mapping.get('enable_ipv4', None), - enable_ipv6=mapping.get('enable_ipv6', None), - circuit_breakers=mapping.get('circuit_breakers', None), - marker=marker, - stats_name=self.get("stats_name", None) + cluster = IRCluster( + ir=self.ir, + aconf=self.ir.aconf, + parent_ir_resource=mapping, + location=mapping.location, + service=mapping.service, + resolver=mapping.resolver, + ctx_name=mapping.get("tls", None), + host_rewrite=mapping.get("host_rewrite", False), + enable_ipv4=mapping.get("enable_ipv4", None), + enable_ipv6=mapping.get("enable_ipv6", None), + circuit_breakers=mapping.get("circuit_breakers", None), + marker=marker, + stats_name=self.get("stats_name", None), ) # Make sure that the cluster is really in our IR... @@ -166,7 +178,7 @@ def add_cluster_for_mapping(self, mapping: IRBaseMapping, # Finally, return the stored cluster. Done. return stored - def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: + def finalize(self, ir: "IR", aconf: Config) -> List[IRCluster]: """ Finalize a MappingGroup based on the attributes of its Mappings. Core elements get lifted into the Group so we can more easily build Envoy routes; host-redirect and shadow get handled, etc. @@ -182,7 +194,11 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: self.ir.logger.debug("%s mapping %s" % (self, mapping.as_json())) for k in mapping.keys(): - if k.startswith('_') or mapping.skip_key(k) or (k in IRTCPMappingGroup.DoNotFlattenKeys): + if ( + k.startswith("_") + or mapping.skip_key(k) + or (k in IRTCPMappingGroup.DoNotFlattenKeys) + ): # self.ir.logger.debug("%s: don't flatten %s" % (self, k)) continue @@ -192,7 +208,7 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: # Should we have higher weights win over lower if there are conflicts? # Should we disallow conflicts? - metadata_labels.update(mapping.get('metadata_labels') or {}) + metadata_labels.update(mapping.get("metadata_labels") or {}) if metadata_labels: self.metadata_labels = metadata_labels @@ -255,4 +271,4 @@ def finalize(self, ir: 'IR', aconf: Config) -> List[IRCluster]: self.post_error(f"Could not normalize mapping weights, ignoring...") return [] - return list([ mapping.cluster for mapping in self.mappings ]) + return list([mapping.cluster for mapping in self.mappings]) diff --git a/python/ambassador/ir/irtls.py b/python/ambassador/ir/irtls.py index 5499a9d602..81a6b99851 100644 --- a/python/ambassador/ir/irtls.py +++ b/python/ambassador/ir/irtls.py @@ -20,7 +20,7 @@ from ambassador.utils import RichStatus if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover ############################################################################# @@ -37,14 +37,17 @@ ## migrate here, or this class should go away. -class IRAmbassadorTLS (IRResource): - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.tlsmodule", - kind: str="IRTLSModule", - name: str="ir.tlsmodule", - enabled: bool=True, - - **kwargs) -> None: +class IRAmbassadorTLS(IRResource): + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.tlsmodule", + kind: str = "IRTLSModule", + name: str = "ir.tlsmodule", + enabled: bool = True, + **kwargs + ) -> None: """ Initialize an IRAmbassadorTLS from the raw fields of its Resource. """ @@ -52,18 +55,16 @@ def __init__(self, ir: 'IR', aconf: Config, ir.logger.debug("IRAmbassadorTLS __init__ (%s %s %s)" % (kind, name, kwargs)) super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, - enabled=enabled, - **kwargs + ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, enabled=enabled, **kwargs ) class TLSModuleFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: + def load_all(cls, ir: "IR", aconf: Config) -> None: assert ir - tls_module = aconf.get_module('tls') + tls_module = aconf.get_module("tls") if tls_module: # ir.logger.debug("TLSModuleFactory saving TLS module: %s" % tls_module.as_json()) @@ -71,17 +72,20 @@ def load_all(cls, ir: 'IR', aconf: Config) -> None: # XXX What a hack. IRAmbassadorTLS.from_resource() should be able to make # this painless. new_args = dict(tls_module.as_dict()) - new_rkey = new_args.pop('rkey', tls_module.rkey) - new_kind = new_args.pop('kind', tls_module.kind) - new_name = new_args.pop('name', tls_module.name) - new_location = new_args.pop('location', tls_module.location) - - ir.tls_module = IRAmbassadorTLS(ir, aconf, - rkey=new_rkey, - kind=new_kind, - name=new_name, - location=new_location, - **new_args) + new_rkey = new_args.pop("rkey", tls_module.rkey) + new_kind = new_args.pop("kind", tls_module.kind) + new_name = new_args.pop("name", tls_module.name) + new_location = new_args.pop("location", tls_module.location) + + ir.tls_module = IRAmbassadorTLS( + ir, + aconf, + rkey=new_rkey, + kind=new_kind, + name=new_name, + location=new_location, + **new_args + ) ir.logger.debug("TLSModuleFactory saved TLS module: %s" % ir.tls_module.as_json()) @@ -92,31 +96,40 @@ def load_all(cls, ir: 'IR', aconf: Config) -> None: ir.ambassador_module.sourced_by(amod) ir.ambassador_module.referenced_by(amod) - amod_tls = amod.get('tls', None) + amod_tls = amod.get("tls", None) # Check for an Ambassador module tls field so that we can warn the user that this field is deprecated! if amod_tls: - ir.post_error("The 'tls' field on the Ambassador module is deprecated! Please use a TLSContext instead https://www.getambassador.io/docs/edge-stack/latest/topics/running/tls/#tlscontext") + ir.post_error( + "The 'tls' field on the Ambassador module is deprecated! Please use a TLSContext instead https://www.getambassador.io/docs/edge-stack/latest/topics/running/tls/#tlscontext" + ) # Finally, if we have a TLS Module, turn it into a TLSContext. if ir.tls_module: ir.logger.debug("TLSModuleFactory translating TLS module to TLSContext") # Stash a sane rkey and location for contexts we create. - ctx_rkey = ir.tls_module.get('rkey', ir.ambassador_module.rkey) - ctx_location = ir.tls_module.get('location', ir.ambassador_module.location) + ctx_rkey = ir.tls_module.get("rkey", ir.ambassador_module.rkey) + ctx_location = ir.tls_module.get("location", ir.ambassador_module.location) # The TLS module 'server' and 'client' blocks are actually a _single_ TLSContext # to Ambassador. - server = ir.tls_module.pop('server', None) - client = ir.tls_module.pop('client', None) + server = ir.tls_module.pop("server", None) + client = ir.tls_module.pop("client", None) - if server and server.get('enabled', True): + if server and server.get("enabled", True): # We have a server half. Excellent. - ctx = IRTLSContext.from_legacy(ir, 'server', ctx_rkey, ctx_location, - cert=server, termination=True, validation_ca=client) + ctx = IRTLSContext.from_legacy( + ir, + "server", + ctx_rkey, + ctx_location, + cert=server, + termination=True, + validation_ca=client, + ) if ctx.is_active(): ir.save_tls_context(ctx) @@ -125,22 +138,30 @@ def load_all(cls, ir: 'IR', aconf: Config) -> None: # that they're a factor... but, weirdly, we have a test for them... for legacy_name, legacy_ctx in ir.tls_module.as_dict().items(): - if (legacy_name.startswith('_') or - (legacy_name == 'name') or - (legacy_name == 'namespace') or - (legacy_name == 'metadata_labels') or - (legacy_name == 'location') or - (legacy_name == 'kind') or - (legacy_name == 'enabled')): + if ( + legacy_name.startswith("_") + or (legacy_name == "name") + or (legacy_name == "namespace") + or (legacy_name == "metadata_labels") + or (legacy_name == "location") + or (legacy_name == "kind") + or (legacy_name == "enabled") + ): continue - ctx = IRTLSContext.from_legacy(ir, legacy_name, ctx_rkey, ctx_location, - cert=legacy_ctx, termination=False, validation_ca=None) + ctx = IRTLSContext.from_legacy( + ir, + legacy_name, + ctx_rkey, + ctx_location, + cert=legacy_ctx, + termination=False, + validation_ca=None, + ) if ctx.is_active(): ir.save_tls_context(ctx) - @classmethod - def finalize(cls, ir: 'IR', aconf: Config) -> None: + def finalize(cls, ir: "IR", aconf: Config) -> None: pass diff --git a/python/ambassador/ir/irtlscontext.py b/python/ambassador/ir/irtlscontext.py index bf13b92423..7743657042 100644 --- a/python/ambassador/ir/irtlscontext.py +++ b/python/ambassador/ir/irtlscontext.py @@ -9,26 +9,24 @@ from .irresource import IRResource if TYPE_CHECKING: - from .ir import IR # pragma: no cover - from .irtls import IRAmbassadorTLS # pragma: no cover + from .ir import IR # pragma: no cover + from .irtls import IRAmbassadorTLS # pragma: no cover class IRTLSContext(IRResource): CertKeys: ClassVar = { - 'secret', - 'cert_chain_file', - 'private_key_file', - - 'ca_secret', - 'cacert_chain_file', - - 'crl_secret', - 'crl_file', + "secret", + "cert_chain_file", + "private_key_file", + "ca_secret", + "cacert_chain_file", + "crl_secret", + "crl_file", } AllowedKeys: ClassVar = { - '_ambassador_enabled', - '_legacy', + "_ambassador_enabled", + "_legacy", "alpn_protocols", "cert_required", "cipher_suites", @@ -61,51 +59,63 @@ class IRTLSContext(IRResource): _ambassador_enabled: bool _legacy: bool - def __init__(self, ir: 'IR', aconf: Config, - rkey: str, # REQUIRED - name: str, # REQUIRED - location: str, # REQUIRED - namespace: Optional[str]=None, - metadata_labels: Optional[Dict[str, str]]=None, - kind: str="IRTLSContext", - apiVersion: str = "getambassador.io/v3alpha1", - is_fallback: Optional[bool]=False, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str, # REQUIRED + name: str, # REQUIRED + location: str, # REQUIRED + namespace: Optional[str] = None, + metadata_labels: Optional[Dict[str, str]] = None, + kind: str = "IRTLSContext", + apiVersion: str = "getambassador.io/v3alpha1", + is_fallback: Optional[bool] = False, + **kwargs, + ) -> None: new_args = { - x: kwargs[x] for x in kwargs.keys() + x: kwargs[x] + for x in kwargs.keys() if x in IRTLSContext.AllowedKeys.union(IRTLSContext.CertKeys) } super().__init__( - ir=ir, aconf=aconf, rkey=rkey, location=location, - kind=kind, name=name, namespace=namespace, metadata_labels=metadata_labels, - is_fallback=is_fallback, apiVersion=apiVersion, - **new_args + ir=ir, + aconf=aconf, + rkey=rkey, + location=location, + kind=kind, + name=name, + namespace=namespace, + metadata_labels=metadata_labels, + is_fallback=is_fallback, + apiVersion=apiVersion, + **new_args, ) def pretty(self) -> str: - secret_name = self.secret_info.get('secret', '-no secret-') + secret_name = self.secret_info.get("secret", "-no secret-") hoststr = getattr(self, "hosts", "-any-") fbstr = " (fallback)" if self.is_fallback else "" - rcf = self.get('redirect_cleartext_from', None) + rcf = self.get("redirect_cleartext_from", None) rcfstr = f" rcf {rcf}" if (rcf is not None) else "" return f"" - def setup(self, ir: 'IR', aconf: Config) -> bool: - if not self.get('_ambassador_enabled', False): + def setup(self, ir: "IR", aconf: Config) -> bool: + if not self.get("_ambassador_enabled", False): spec_count = 0 errors = 0 - if self.get('secret', None): + if self.get("secret", None): spec_count += 1 - if self.get('cert_chain_file', None): + if self.get("cert_chain_file", None): spec_count += 1 - if not self.get('private_key_file', None): + if not self.get("private_key_file", None): err_msg = f"TLSContext {self.name}: 'cert_chain_file' requires 'private_key_file' as well" self.post_error(err_msg) @@ -124,7 +134,7 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: # self.referenced_by(config) # Assume that we have no redirect_cleartext_from... - rcf = self.get('redirect_cleartext_from', None) + rcf = self.get("redirect_cleartext_from", None) if rcf is not None: try: @@ -156,18 +166,21 @@ def resolve_secret(self, secret_name: str) -> SavedSecret: # you can set secret_namespacing to False in a TLSContext or tls_secret_namespacing False # in the Ambassador module's defaults to prevent that. - secret_namespacing = self.lookup('secret_namespacing', True, - default_key='tls_secret_namespacing') + secret_namespacing = self.lookup( + "secret_namespacing", True, default_key="tls_secret_namespacing" + ) - self.ir.logger.debug(f"TLSContext.resolve_secret {secret_name}, namespace {namespace}: namespacing is {secret_namespacing}") + self.ir.logger.debug( + f"TLSContext.resolve_secret {secret_name}, namespace {namespace}: namespacing is {secret_namespacing}" + ) if "." in secret_name and secret_namespacing: - secret_name, namespace = secret_name.rsplit('.', 1) + secret_name, namespace = secret_name.rsplit(".", 1) return self.ir.resolve_secret(self, secret_name, namespace) def resolve(self) -> bool: - if self.get('_ambassador_enabled', False): + if self.get("_ambassador_enabled", False): self.ir.logger.debug("IRTLSContext skipping resolution of null context") return True @@ -176,17 +189,20 @@ def resolve(self) -> bool: # If redirect_cleartext_from or alpn_protocols is specified, the TLS Context is # valid anyway, even if secret config is invalid - if self.get('redirect_cleartext_from', False) or self.get('alpn_protocols', False): + if self.get("redirect_cleartext_from", False) or self.get("alpn_protocols", False): is_valid = True # If we don't have secret info, it's worth posting an error. if not self.secret_info: - self.post_error("TLSContext %s has no certificate information at all?" % self.name, log_level=logging.DEBUG) + self.post_error( + "TLSContext %s has no certificate information at all?" % self.name, + log_level=logging.DEBUG, + ) self.ir.logger.debug("resolve_secrets working on: %s" % self.as_json()) # OK. Do we have a secret name? - secret_name = self.secret_info.get('secret') + secret_name = self.secret_info.get("secret") secret_valid = True if secret_name: @@ -200,87 +216,110 @@ def resolve(self) -> bool: if not ss: # This is definitively an error: they mentioned a secret, it can't be loaded, # post an error. - self.post_error("TLSContext %s found no certificate in %s, ignoring..." % (self.name, ss.name)) - self.secret_info.pop('secret') + self.post_error( + "TLSContext %s found no certificate in %s, ignoring..." % (self.name, ss.name) + ) + self.secret_info.pop("secret") secret_valid = False else: # If they only gave a public key, that's an error if not ss.key_path: - self.post_error("TLSContext %s found no private key in %s" % (self.name, ss.name)) + self.post_error( + "TLSContext %s found no private key in %s" % (self.name, ss.name) + ) return False # So far, so good. self.ir.logger.debug("TLSContext %s saved secret %s" % (self.name, ss.name)) # Update paths for this cert. - self.secret_info['cert_chain_file'] = ss.cert_path - self.secret_info['private_key_file'] = ss.key_path + self.secret_info["cert_chain_file"] = ss.cert_path + self.secret_info["private_key_file"] = ss.key_path if ss.root_cert_path: - self.secret_info['cacert_chain_file'] = ss.root_cert_path + self.secret_info["cacert_chain_file"] = ss.root_cert_path - self.ir.logger.debug("TLSContext - successfully processed the cert_chain_file, private_key_file, and cacert_chain_file: %s" % self.secret_info) + self.ir.logger.debug( + "TLSContext - successfully processed the cert_chain_file, private_key_file, and cacert_chain_file: %s" + % self.secret_info + ) # OK. Repeat for the crl_secret. - crl_secret = self.secret_info.get('crl_secret') + crl_secret = self.secret_info.get("crl_secret") if crl_secret: # They gave a secret name for the certificate revocation list. Try loading it. crls = self.resolve_secret(crl_secret) - self.ir.logger.debug("resolve_secrets: IR returned secret %s as %s" % (crl_secret, crls)) + self.ir.logger.debug( + "resolve_secrets: IR returned secret %s as %s" % (crl_secret, crls) + ) if not crls: # This is definitively an error: they mentioned a secret, it can't be loaded, # give up. - self.post_error("TLSContext %s found no certificate revocation list in %s" % (self.name, crls.name)) + self.post_error( + "TLSContext %s found no certificate revocation list in %s" + % (self.name, crls.name) + ) secret_valid = False else: - self.ir.logger.debug("TLSContext %s saved certificate revocation list secret %s" % (self.name, crls.name)) - self.secret_info['crl_file'] = crls.user_path + self.ir.logger.debug( + "TLSContext %s saved certificate revocation list secret %s" + % (self.name, crls.name) + ) + self.secret_info["crl_file"] = crls.user_path # OK. Repeat for the ca_secret_name. - ca_secret_name = self.secret_info.get('ca_secret') + ca_secret_name = self.secret_info.get("ca_secret") if ca_secret_name: - if not self.secret_info.get('cert_chain_file'): + if not self.secret_info.get("cert_chain_file"): # DUPLICATED BELOW: This is an error: validation without termination isn't meaningful. # (This is duplicated for the case where they gave a validation path.) - self.post_error("TLSContext %s cannot validate client certs without TLS termination" % - self.name) + self.post_error( + "TLSContext %s cannot validate client certs without TLS termination" % self.name + ) return False # They gave a secret name for the validation cert. Try loading it. ss = self.resolve_secret(ca_secret_name) - self.ir.logger.debug("resolve_secrets: IR returned secret %s as %s" % (ca_secret_name, ss)) + self.ir.logger.debug( + "resolve_secrets: IR returned secret %s as %s" % (ca_secret_name, ss) + ) if not ss: # This is definitively an error: they mentioned a secret, it can't be loaded, # give up. - self.post_error("TLSContext %s found no validation certificate in %s" % (self.name, ss.name)) + self.post_error( + "TLSContext %s found no validation certificate in %s" % (self.name, ss.name) + ) secret_valid = False else: # Validation certs don't need the private key, but it's not an error if they gave # one. We're good to go here. self.ir.logger.debug("TLSContext %s saved CA secret %s" % (self.name, ss.name)) - self.secret_info['cacert_chain_file'] = ss.cert_path + self.secret_info["cacert_chain_file"] = ss.cert_path # While we're here, did they set cert_required _in the secret_? if ss.cert_data: - cert_required = ss.cert_data.get('cert_required') + cert_required = ss.cert_data.get("cert_required") if cert_required is not None: - decoded = base64.b64decode(cert_required).decode('utf-8').lower() == 'true' + decoded = base64.b64decode(cert_required).decode("utf-8").lower() == "true" # cert_required is at toplevel, _not_ in secret_info! - self['cert_required'] = decoded + self["cert_required"] = decoded else: # No secret is named; did they provide a file location instead? - if self.secret_info.get('cacert_chain_file') and not self.secret_info.get('cert_chain_file'): + if self.secret_info.get("cacert_chain_file") and not self.secret_info.get( + "cert_chain_file" + ): # DUPLICATED ABOVE: This is an error: validation without termination isn't meaningful. # (This is duplicated for the case where they gave a validation secret.) - self.post_error("TLSContext %s cannot validate client certs without TLS termination" % - self.name) + self.post_error( + "TLSContext %s cannot validate client certs without TLS termination" % self.name + ) return False # If the secret has been invalidated above, then we do not need to check for paths down under. @@ -292,15 +331,17 @@ def resolve(self) -> bool: errors = 0 # self.ir.logger.debug("resolve_secrets before path checks: %s" % self.as_json()) - for key in [ 'cert_chain_file', 'private_key_file', 'cacert_chain_file', 'crl_file' ]: + for key in ["cert_chain_file", "private_key_file", "cacert_chain_file", "crl_file"]: path = self.secret_info.get(key, None) if path: - fc = getattr(self.ir, 'file_checker') + fc = getattr(self.ir, "file_checker") if not fc(path): self.post_error("TLSContext %s found no %s '%s'" % (self.name, key, path)) errors += 1 - elif (not(key == 'cacert_chain_file' or key == 'crl_file')) and self.get('hosts', None): + elif (not (key == "cacert_chain_file" or key == "crl_file")) and self.get( + "hosts", None + ): self.post_error("TLSContext %s is missing %s" % (self.name, key)) errors += 1 @@ -313,9 +354,9 @@ def has_secret(self) -> bool: # Safely verify that self.secret_info['secret'] exists -- in other words, verify # that this IRTLSContext is based on a Secret we load from elsewhere, rather than # on files in the filesystem. - si = self.get('secret_info', {}) + si = self.get("secret_info", {}) - return 'secret' in si + return "secret" in si def secret_name(self) -> Optional[str]: # Return the name of the Secret we're based on, or None if we're based on files @@ -325,34 +366,44 @@ def secret_name(self) -> Optional[str]: # this later. if self.has_secret(): - return self.secret_info['secret'] + return self.secret_info["secret"] else: return None def set_secret_name(self, secret_name: str) -> None: # Set the name of the Secret we're based on. - self.secret_info['secret'] = secret_name + self.secret_info["secret"] = secret_name @classmethod - def null_context(cls, ir: 'IR') -> 'IRTLSContext': + def null_context(cls, ir: "IR") -> "IRTLSContext": ctx = ir.get_tls_context("no-cert-upstream") if not ctx: - ctx = IRTLSContext(ir, ir.aconf, - rkey="ir.no-cert-upstream", - name="no-cert-upstream", - location="ir.no-cert-upstream", - kind="null-TLS-context", - _ambassador_enabled=True) + ctx = IRTLSContext( + ir, + ir.aconf, + rkey="ir.no-cert-upstream", + name="no-cert-upstream", + location="ir.no-cert-upstream", + kind="null-TLS-context", + _ambassador_enabled=True, + ) ir.save_tls_context(ctx) return ctx @classmethod - def from_legacy(cls, ir: 'IR', name: str, rkey: str, location: str, - cert: 'IRAmbassadorTLS', termination: bool, - validation_ca: Optional['IRAmbassadorTLS']) -> 'IRTLSContext': + def from_legacy( + cls, + ir: "IR", + name: str, + rkey: str, + location: str, + cert: "IRAmbassadorTLS", + termination: bool, + validation_ca: Optional["IRAmbassadorTLS"], + ) -> "IRTLSContext": """ Create an IRTLSContext from a legacy TLS-module style definition. @@ -375,56 +426,65 @@ def from_legacy(cls, ir: 'IR', name: str, rkey: str, location: str, """ new_args = {} - for key in [ 'secret', 'cert_chain_file', 'private_key_file', - 'alpn_protocols', 'redirect_cleartext_from' ]: + for key in [ + "secret", + "cert_chain_file", + "private_key_file", + "alpn_protocols", + "redirect_cleartext_from", + ]: value = cert.get(key, None) if value: new_args[key] = value - if (('secret' not in new_args) and - ('cert_chain_file' not in new_args) and - ('private_key_file' not in new_args)): + if ( + ("secret" not in new_args) + and ("cert_chain_file" not in new_args) + and ("private_key_file" not in new_args) + ): # Assume they want the 'ambassador-certs' secret. - new_args['secret'] = 'ambassador-certs' + new_args["secret"] = "ambassador-certs" if termination: - new_args['hosts'] = [ '*' ] + new_args["hosts"] = ["*"] - if validation_ca and validation_ca.get('enabled', True): - for key in [ 'secret', 'cacert_chain_file', 'cert_required' ]: + if validation_ca and validation_ca.get("enabled", True): + for key in ["secret", "cacert_chain_file", "cert_required"]: value = validation_ca.get(key, None) if value: - if key == 'secret': - new_args['ca_secret'] = value + if key == "secret": + new_args["ca_secret"] = value else: new_args[key] = value - if (('ca_secret' not in new_args) and - ('cacert_chain_file' not in new_args)): + if ("ca_secret" not in new_args) and ("cacert_chain_file" not in new_args): # Assume they want the 'ambassador-cacert' secret. - new_args['secret'] = 'ambassador-cacert' - - ctx = IRTLSContext(ir, ir.aconf, - rkey=rkey, - name=name, - location=location, - kind="synthesized-TLS-context", - _legacy=True, - **new_args) + new_args["secret"] = "ambassador-cacert" + + ctx = IRTLSContext( + ir, + ir.aconf, + rkey=rkey, + name=name, + location=location, + kind="synthesized-TLS-context", + _legacy=True, + **new_args, + ) return ctx class TLSContextFactory: @classmethod - def load_all(cls, ir: 'IR', aconf: Config) -> None: + def load_all(cls, ir: "IR", aconf: Config) -> None: assert ir # Save TLS contexts from the aconf into the IR. Note that the contexts in the aconf # are just ACResources; they need to be turned into IRTLSContexts. - tls_contexts = aconf.get_config('tls_contexts') + tls_contexts = aconf.get_config("tls_contexts") if tls_contexts is not None: for config in tls_contexts.values(): diff --git a/python/ambassador/ir/irtracing.py b/python/ambassador/ir/irtracing.py index 2826a0012d..263ceee46b 100644 --- a/python/ambassador/ir/irtracing.py +++ b/python/ambassador/ir/irtracing.py @@ -6,7 +6,7 @@ from ..utils import RichStatus if TYPE_CHECKING: - from .ir import IR # pragma: no cover + from .ir import IR # pragma: no cover class IRTracing(IRResource): @@ -18,23 +18,25 @@ class IRTracing(IRResource): host_rewrite: Optional[str] sampling: dict - def __init__(self, ir: 'IR', aconf: Config, - rkey: str = "ir.tracing", - kind: str = "ir.tracing", - name: str = "tracing", - namespace: Optional[str] = None, - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.tracing", + kind: str = "ir.tracing", + name: str = "tracing", + namespace: Optional[str] = None, + **kwargs + ) -> None: del kwargs # silence unused-variable warning - super().__init__( - ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, namespace=namespace - ) + super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, namespace=namespace) self.cluster = None - def setup(self, ir: 'IR', aconf: Config) -> bool: + def setup(self, ir: "IR", aconf: Config) -> bool: # Some of the validations might go away if JSON Schema is doing the validations, but need to check on that - config_info = aconf.get_config('tracing_configs') + config_info = aconf.get_config("tracing_configs") if not config_info: ir.logger.debug("IRTracing: no tracing config, bailing") @@ -45,18 +47,21 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: number_configs = len(configs) if number_configs != 1: self.post_error( - RichStatus.fromError("exactly one TracingService is supported, got {}".format(number_configs), - module=aconf)) + RichStatus.fromError( + "exactly one TracingService is supported, got {}".format(number_configs), + module=aconf, + ) + ) return False config = list(configs)[0] - service = config.get('service') + service = config.get("service") if not service: self.post_error(RichStatus.fromError("service field is required in TracingService")) return False - driver = config.get('driver') + driver = config.get("driver") if not driver: self.post_error(RichStatus.fromError("driver field is required in TracingService")) return False @@ -77,16 +82,20 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: if driver == "zipkin": # fill zipkin defaults - if not driver_config.get('collector_endpoint'): - driver_config['collector_endpoint'] = '/api/v2/spans' - if not driver_config.get('collector_endpoint_version'): - driver_config['collector_endpoint_version'] = 'HTTP_JSON' - if not 'trace_id_128bit' in driver_config: + if not driver_config.get("collector_endpoint"): + driver_config["collector_endpoint"] = "/api/v2/spans" + if not driver_config.get("collector_endpoint_version"): + driver_config["collector_endpoint_version"] = "HTTP_JSON" + if not "trace_id_128bit" in driver_config: # Make 128-bit traceid the default - driver_config['trace_id_128bit'] = True + driver_config["trace_id_128bit"] = True # validate - if driver_config['collector_endpoint_version'] not in ['HTTP_JSON', 'HTTP_PROTO']: - self.post_error(RichStatus.fromError("collector_endpoint_version must be one of HTTP_JSON, HTTP_PROTO'")) + if driver_config["collector_endpoint_version"] not in ["HTTP_JSON", "HTTP_PROTO"]: + self.post_error( + RichStatus.fromError( + "collector_endpoint_version must be one of HTTP_JSON, HTTP_PROTO'" + ) + ) return False # OK, we have a valid config. @@ -97,18 +106,18 @@ def setup(self, ir: 'IR', aconf: Config) -> bool: self.grpc = grpc self.cluster = None self.driver_config = driver_config - self.tag_headers = config.get('tag_headers', []) - self.sampling = config.get('sampling', {}) + self.tag_headers = config.get("tag_headers", []) + self.sampling = config.get("sampling", {}) # XXX host_rewrite actually isn't in the schema right now. - self.host_rewrite = config.get('host_rewrite', None) + self.host_rewrite = config.get("host_rewrite", None) # Remember that the config references us. self.referenced_by(config) return True - def add_mappings(self, ir: 'IR', aconf: Config): + def add_mappings(self, ir: "IR", aconf: Config): cluster = ir.add_cluster( IRCluster( ir=ir, @@ -116,10 +125,10 @@ def add_mappings(self, ir: 'IR', aconf: Config): parent_ir_resource=self, location=self.location, service=self.service, - host_rewrite=self.get('host_rewrite', None), - marker='tracing', + host_rewrite=self.get("host_rewrite", None), + marker="tracing", grpc=self.grpc, - stats_name=self.get("stats_name", None) + stats_name=self.get("stats_name", None), ) ) @@ -129,4 +138,4 @@ def add_mappings(self, ir: 'IR', aconf: Config): def finalize(self): assert self.cluster self.ir.logger.debug("tracing cluster envoy name: %s" % self.cluster.envoy_name) - self.driver_config['collector_cluster'] = self.cluster.envoy_name + self.driver_config["collector_cluster"] = self.cluster.envoy_name diff --git a/python/ambassador/ir/irutils.py b/python/ambassador/ir/irutils.py index 41daf05b41..9106354e8c 100644 --- a/python/ambassador/ir/irutils.py +++ b/python/ambassador/ir/irutils.py @@ -22,6 +22,7 @@ # hostglob_matches_start has g1 starting with '*' and g2 not ending with '*'; # it's OK for g2 to start with a wilcard too. + def hostglob_matches_start(g1: str, g2: str, g2start: bool) -> bool: # Leading "*" cannot match an empty string, so unless we have a wildcard # for g2, we have to have g1 longer than g2. @@ -48,6 +49,7 @@ def hostglob_matches_start(g1: str, g2: str, g2start: bool) -> bool: # hostglob_matches_end has g1 ending with '*' and g2 not starting with '*'; # it's OK for g2 to end with a wilcard too. + def hostglob_matches_end(g1: str, g2: str, g2end: bool) -> bool: # Leading "*" cannot match an empty string, so unless we have a wildcard # for g2, we have to have g1 longer than g2. @@ -72,6 +74,7 @@ def hostglob_matches_end(g1: str, g2: str, g2end: bool) -> bool: ################ + def hostglob_matches(g1: str, g2: str) -> bool: """ hostglob_matches determines whether or not two given DNS globs are @@ -102,10 +105,10 @@ def hostglob_matches(g1: str, g2: str) -> bool: # OK, we don't have the simple-"*" case, so any wildcards must be at # the start or end, and they must be a component alone. - g1start = (g1[0] == "*") - g1end = (g1[-1] == "*") - g2start = (g2[0] == "*") - g2end = (g2[-1] == "*") + g1start = g1[0] == "*" + g1end = g1[-1] == "*" + g2start = g2[0] == "*" + g2end = g2[-1] == "*" # logging.debug(" g1start=%s g1end=%s g2start=%s g2end=%s", g1start, g1end, g2start, g2end) @@ -122,8 +125,8 @@ def hostglob_matches(g1: str, g2: str) -> bool: return False # OK, if we're here, we have a wildcard to check. There are a few cases - # here, so we'll start with the easy one: one value starts with "*" and - # the other ends with "*", because those can always overlap as long as + # here, so we'll start with the easy one: one value starts with "*" and + # the other ends with "*", because those can always overlap as long as # the overlap between isn't empty -- and in this method, we only need to # concern ourselves with being sure that there is a possibility of a match # to both. @@ -132,13 +135,13 @@ def hostglob_matches(g1: str, g2: str) -> bool: return True # OK, now we have to actually do some work. Again, we really only have to - # be convinced that it's possible for something to match, so e.g. + # be convinced that it's possible for something to match, so e.g. # # *example.com, example.com # # is not a valid pair, because that "*" must never match an empty string. # However, - # + # # *example.com, *.example.com # # is fine, because e.g. "foo.example.com" matches both. @@ -162,7 +165,10 @@ def hostglob_matches(g1: str, g2: str) -> bool: ################ ## selector_matches is a utility for doing K8s label selector matching. -def selector_matches(logger: logging.Logger, selector: Dict[str, Any], labels: Dict[str, str]) -> bool: + +def selector_matches( + logger: logging.Logger, selector: Dict[str, Any], labels: Dict[str, str] +) -> bool: match: Dict[str, str] = selector.get("matchLabels") or {} if not match: diff --git a/python/ambassador/reconfig_stats.py b/python/ambassador/reconfig_stats.py index 7a7d8c3984..fa056c0267 100644 --- a/python/ambassador/reconfig_stats.py +++ b/python/ambassador/reconfig_stats.py @@ -22,6 +22,7 @@ PerfCounter = float + class ReconfigStats: """ Track metrics for reconfigurations, whether complete or incremental. @@ -29,10 +30,14 @@ class ReconfigStats: before messing with this! """ - def __init__(self, logger: logging.Logger, - max_incr_between_checks=100, max_time_between_checks=600, - max_config_between_timers=10, max_time_between_timers=120 - ) -> None: + def __init__( + self, + logger: logging.Logger, + max_incr_between_checks=100, + max_time_between_checks=600, + max_config_between_timers=10, + max_time_between_timers=120, + ) -> None: """ Initialize this ReconfigStats. @@ -55,10 +60,7 @@ def __init__(self, logger: logging.Logger, # self.counts tracks how many of each kind of reconfiguration have # happened, for metrics. - self.counts = { - "incremental": 0, - "complete": 0 - } + self.counts = {"incremental": 0, "complete": 0} # In many cases, the previous complete reconfigure will have fallen out # of self.reconfigures, so we remember its timestamp separately. @@ -84,7 +86,7 @@ def __init__(self, logger: logging.Logger, self.checks = 0 self.errors = 0 - def mark(self, what: str, when: Optional[PerfCounter]=None) -> None: + def mark(self, what: str, when: Optional[PerfCounter] = None) -> None: """ Mark that a reconfigure has occurred. The 'what' parameter is one of "complete" for a complete reconfigure, "incremental" for an incremental, @@ -98,15 +100,15 @@ def mark(self, what: str, when: Optional[PerfCounter]=None) -> None: if not when: when = time.perf_counter() - if (what == 'incremental') and not self.last_complete: + if (what == "incremental") and not self.last_complete: # You can't have an incremental without a complete to start. # If this is the first reconfigure, it's a complete reconfigure. - what = 'complete' + what = "complete" # Should we update all the counters? update_counters = True - if what == 'complete': + if what == "complete": # For a complete reconfigure, we need to clear all the outstanding # incrementals, and also remember when it happened. self.incrementals_outstanding = 0 @@ -118,13 +120,13 @@ def mark(self, what: str, when: Optional[PerfCounter]=None) -> None: self.last_check = when self.logger.debug(f"MARK COMPLETE @ {when}") - elif what == 'incremental': + elif what == "incremental": # For an incremental reconfigure, we need to remember that we have # one more incremental outstanding. self.incrementals_outstanding += 1 self.logger.debug(f"MARK INCREMENTAL @ {when}") - elif what == 'diag': + elif what == "diag": # Don't update all the counters for a diagnostic update. update_counters = False else: @@ -143,7 +145,7 @@ def mark(self, what: str, when: Optional[PerfCounter]=None) -> None: # trigger timer logging for diagnostics updates. self.configs_outstanding += 1 - def needs_check(self, when: Optional[PerfCounter]=None) -> bool: + def needs_check(self, when: Optional[PerfCounter] = None) -> bool: """ Determine if we need to do a complete reconfigure to doublecheck our incrementals. The logic here is that we need a check every 100 incrementals @@ -164,7 +166,7 @@ def needs_check(self, when: Optional[PerfCounter]=None) -> bool: # Grab information about our last reconfiguration. what, _ = self.reconfigures[-1] - if what == 'complete': + if what == "complete": # Last reconfiguration was a complete reconfiguration, so # no need to check. # self.logger.debug(f"NEEDS_CHECK @ {when}: last was complete, skip") @@ -189,7 +191,7 @@ def needs_check(self, when: Optional[PerfCounter]=None) -> bool: # We're good for outstanding incrementals. How about the max time between checks? # (We must have a last check time - which may be the time of the last complete # reconfigure, of course - to go on at this point.) - assert(self.last_check is not None) + assert self.last_check is not None delta = when - self.last_check @@ -201,7 +203,7 @@ def needs_check(self, when: Optional[PerfCounter]=None) -> bool: # self.logger.debug(f"NEEDS_CHECK @ {when}: delta {delta}, skip") return False - def needs_timers(self, when: Optional[PerfCounter]=None) -> bool: + def needs_timers(self, when: Optional[PerfCounter] = None) -> bool: """ Determine if we need to log the timers or not. The logic here is that we need to log every max_configs_between_timers incrementals or every @@ -237,7 +239,7 @@ def needs_timers(self, when: Optional[PerfCounter]=None) -> bool: # the time of our last complete reconfigure, which must always be set, as a # baseline. - assert(self.last_complete is not None) + assert self.last_complete is not None baseline = self.last_timer_log or self.last_complete delta = when - baseline @@ -250,7 +252,7 @@ def needs_timers(self, when: Optional[PerfCounter]=None) -> bool: # self.logger.debug(f"NEEDS_TIMERS @ {when}: delta {delta}, skip") return False - def mark_checked(self, result: bool, when: Optional[PerfCounter]=None) -> None: + def mark_checked(self, result: bool, when: Optional[PerfCounter] = None) -> None: """ Mark that we have done a check, and note the results. This resets our outstanding incrementals to 0, and also resets our last check time. @@ -269,7 +271,7 @@ def mark_checked(self, result: bool, when: Optional[PerfCounter]=None) -> None: self.last_check = when or time.perf_counter() - def mark_timers_logged(self, when: Optional[PerfCounter]=None) -> None: + def mark_timers_logged(self, when: Optional[PerfCounter] = None) -> None: """ Mark that we have logged timers. This resets our outstanding configurations to 0, and also resets our last timer log time. @@ -299,12 +301,10 @@ def dump(self) -> None: for what, when in self.reconfigures: self.logger.info(f"CACHE: {what} reconfigure at {self.isofmt(when, now_pc, now_dt)}") - for what in [ "incremental", "complete" ]: + for what in ["incremental", "complete"]: self.logger.info(f"CACHE: {what} count: {self.counts[what]}") self.logger.info(f"CACHE: incrementals outstanding: {self.incrementals_outstanding}") self.logger.info(f"CACHE: incremental checks: {self.checks}, errors {self.errors}") self.logger.info(f"CACHE: last_complete {self.isofmt(self.last_complete, now_pc, now_dt)}") self.logger.info(f"CACHE: last_check {self.isofmt(self.last_check, now_pc, now_dt)}") - - diff --git a/python/ambassador/resource.py b/python/ambassador/resource.py index 1e2a7b4a1d..03c59c5597 100644 --- a/python/ambassador/resource.py +++ b/python/ambassador/resource.py @@ -8,10 +8,10 @@ from .cache import Cacheable -R = TypeVar('R', bound='Resource') +R = TypeVar("R", bound="Resource") -class Resource (Cacheable): +class Resource(Cacheable): """ A resource that's part of the overall Ambassador configuration world. This is the base class for IR resources, Ambassador-config resources, etc. @@ -49,12 +49,11 @@ class Resource (Cacheable): # _errors: List[RichStatus] _errored: bool - _referenced_by: Dict[str, 'Resource'] + _referenced_by: Dict[str, "Resource"] - def __init__(self, rkey: str, location: str, *, - kind: str, - serialization: Optional[str]=None, - **kwargs) -> None: + def __init__( + self, rkey: str, location: str, *, kind: str, serialization: Optional[str] = None, **kwargs + ) -> None: if not rkey: raise Exception("Resource requires rkey") @@ -64,22 +63,25 @@ def __init__(self, rkey: str, location: str, *, # print("Resource __init__ (%s %s)" % (kind, name)) - super().__init__(rkey=rkey, location=location, - kind=kind, serialization=serialization, - # _errors=[], - _referenced_by={}, - **kwargs) - - def sourced_by(self, other: 'Resource'): + super().__init__( + rkey=rkey, + location=location, + kind=kind, + serialization=serialization, + # _errors=[], + _referenced_by={}, + **kwargs + ) + + def sourced_by(self, other: "Resource"): self.rkey = other.rkey self.location = other.location - - def referenced_by(self, other: 'Resource') -> None: + def referenced_by(self, other: "Resource") -> None: # print("%s %s REF BY %s %s" % (self.kind, self.name, other.kind, other.rkey)) self._referenced_by[other.location] = other - def is_referenced_by(self, other_location) -> Optional['Resource']: + def is_referenced_by(self, other_location) -> Optional["Resource"]: return self._referenced_by.get(other_location, None) def __getattr__(self, key: str) -> Any: @@ -92,16 +94,16 @@ def __setattr__(self, key: str, value: Any) -> None: self[key] = value def __str__(self) -> str: - return("<%s %s>" % (self.kind, self.rkey)) + return "<%s %s>" % (self.kind, self.rkey) def as_dict(self) -> Dict[str, Any]: ad = dict(self) - ad.pop('rkey', None) - ad.pop('serialization', None) - ad.pop('location', None) - ad.pop('_referenced_by', None) - ad.pop('_errored', None) + ad.pop("rkey", None) + ad.pop("serialization", None) + ad.pop("location", None) + ad.pop("_referenced_by", None) + ad.pop("_errored", None) return ad @@ -109,12 +111,15 @@ def as_json(self): return dump_json(self.as_dict(), pretty=True) @classmethod - def from_resource(cls: Type[R], other: R, - rkey: Optional[str]=None, - location: Optional[str]=None, - kind: Optional[str]=None, - serialization: Optional[str]=None, - **kwargs) -> R: + def from_resource( + cls: Type[R], + other: R, + rkey: Optional[str] = None, + location: Optional[str] = None, + kind: Optional[str] = None, + serialization: Optional[str] = None, + **kwargs + ) -> R: """ Create a Resource by copying another Resource, possibly overriding elements along the way. @@ -139,29 +144,31 @@ def from_resource(cls: Type[R], other: R, # Don't include kind unless it comes in on this call. if kind: - new_attrs['kind'] = kind + new_attrs["kind"] = kind else: - new_attrs.pop('kind', None) + new_attrs.pop("kind", None) # Don't include serialization at all if we don't have one. if serialization: - new_attrs['serialization'] = serialization + new_attrs["serialization"] = serialization elif other.serialization: - new_attrs['serialization'] = other.serialization + new_attrs["serialization"] = other.serialization # Make sure that things that shouldn't propagate are gone... - new_attrs.pop('rkey', None) - new_attrs.pop('location', None) - new_attrs.pop('_errors', None) - new_attrs.pop('_errored', None) - new_attrs.pop('_referenced_by', None) + new_attrs.pop("rkey", None) + new_attrs.pop("location", None) + new_attrs.pop("_errors", None) + new_attrs.pop("_errored", None) + new_attrs.pop("_referenced_by", None) # ...and finally, use new_attrs for all the keyword args when we set up # the new instance. return cls(new_rkey, new_location, **new_attrs) @classmethod - def from_dict(cls: Type[R], rkey: str, location: str, serialization: Optional[str], attrs: Dict) -> R: + def from_dict( + cls: Type[R], rkey: str, location: str, serialization: Optional[str], attrs: Dict + ) -> R: """ Create a Resource or subclass thereof from a dictionary. The new Resource's rkey and location must be handed in explicitly. @@ -179,12 +186,12 @@ def from_dict(cls: Type[R], rkey: str, location: str, serialization: Optional[st # So this is a touch odd but here we go. We want to use the Kind here to find # the correct type. - ambassador = sys.modules['ambassador'] + ambassador = sys.modules["ambassador"] - resource_class: Optional[Type[R]] = getattr(ambassador, attrs['kind'], None) + resource_class: Optional[Type[R]] = getattr(ambassador, attrs["kind"], None) if not resource_class: - resource_class = getattr(ambassador, 'AC' + attrs[ 'kind' ], cls) + resource_class = getattr(ambassador, "AC" + attrs["kind"], cls) assert resource_class # print("%s.from_dict: %s => %s" % (cls, attrs['kind'], resource_class)) diff --git a/python/ambassador/scout.py b/python/ambassador/scout.py index 178f94d3e6..fadc9ced44 100644 --- a/python/ambassador/scout.py +++ b/python/ambassador/scout.py @@ -13,9 +13,16 @@ class Scout: - - def __init__(self, app, version, install_id=None, - id_plugin=None, id_plugin_args={}, scout_host="metriton.datawire.io", **kwargs): + def __init__( + self, + app, + version, + install_id=None, + id_plugin=None, + id_plugin_args={}, + scout_host="metriton.datawire.io", + **kwargs + ): """ Create a new Scout instance for later reports. @@ -72,7 +79,7 @@ def __init__(self, app, version, install_id=None, if plugin_response: if "install_id" in plugin_response: self.install_id = plugin_response["install_id"] - del(plugin_response["install_id"]) + del plugin_response["install_id"] if plugin_response: self.metadata = Scout.__merge_dicts(self.metadata, plugin_response) @@ -88,28 +95,28 @@ def __init__(self, app, version, install_id=None, self.disabled = Scout.__is_disabled() def report(self, **kwargs): - result = {'latest_version': self.version} + result = {"latest_version": self.version} if self.disabled: return result merged_metadata = Scout.__merge_dicts(self.metadata, kwargs) - headers = { - 'User-Agent': self.user_agent - } + headers = {"User-Agent": self.user_agent} payload = { - 'application': self.app, - 'version': self.version, - 'install_id': self.install_id, - 'user_agent': self.create_user_agent(), - 'metadata': merged_metadata + "application": self.app, + "version": self.version, + "install_id": self.install_id, + "user_agent": self.create_user_agent(), + "metadata": merged_metadata, } self.logger.debug("Scout: report payload: %s" % json.dumps(payload, indent=4)) - url = ("https://" if self.use_https else "http://") + "{}/scout".format(self.scout_host).lower() + url = ("https://" if self.use_https else "http://") + "{}/scout".format( + self.scout_host + ).lower() try: resp = requests.post(url, json=payload, headers=headers, timeout=1) @@ -120,14 +127,14 @@ def report(self, **kwargs): result = Scout.__merge_dicts(result, resp.json()) except OSError as e: self.logger.warning("Scout: could not post report: %s" % e) - result['exception'] = 'could not post report: %s' % e + result["exception"] = "could not post report: %s" % e except Exception as e: # If scout is down or we are getting errors just proceed as if nothing happened. It should not impact the # user at all. tb = "\n".join(traceback.format_exception(*sys.exc_info())) - result['exception'] = e - result['traceback'] = tb + result["exception"] = e + result["traceback"] = tb if "new_install" in self.metadata: del self.metadata["new_install"] @@ -136,11 +143,8 @@ def report(self, **kwargs): def create_user_agent(self): result = "{0}/{1} ({2}; {3}; python {4})".format( - self.app, - self.version, - platform.system(), - platform.release(), - platform.python_version()).lower() + self.app, self.version, platform.system(), platform.release(), platform.python_version() + ).lower() return result @@ -156,12 +160,12 @@ def __filesystem_install_id(self, app): id_file = os.path.join(config_root, "id") if not os.path.isfile(id_file): - with open(id_file, 'w') as f: + with open(id_file, "w") as f: install_id = str(uuid4()) self.metadata["new_install"] = True f.write(install_id) else: - with open(id_file, 'r') as f: + with open(id_file, "r") as f: install_id = f.read() return install_id @@ -211,17 +215,17 @@ def configmap_install_id_plugin(scout, app, map_name=None, namespace="default"): if not map_name: map_name = "scout.config.{0}".format(app) - kube_host = os.environ.get('KUBERNETES_SERVICE_HOST', None) + kube_host = os.environ.get("KUBERNETES_SERVICE_HOST", None) try: - kube_port = int(os.environ.get('KUBERNETES_SERVICE_PORT', 443)) + kube_port = int(os.environ.get("KUBERNETES_SERVICE_PORT", 443)) except ValueError: scout.logger.debug("Scout: KUBERNETES_SERVICE_PORT isn't numeric, defaulting to 443") kube_port = 443 kube_proto = "https" if (kube_port == 443) else "http" - kube_token = os.environ.get('KUBERNETES_ACCESS_TOKEN', None) + kube_token = os.environ.get("KUBERNETES_ACCESS_TOKEN", None) if not kube_host: # We're not running in Kubernetes. Fall back to the usual filesystem stuff. @@ -243,7 +247,7 @@ def configmap_install_id_plugin(scout, app, map_name=None, namespace="default"): base_url = "%s://%s:%s" % (kube_proto, kube_host, kube_port) url_path = "api/v1/namespaces/%s/configmaps" % namespace - auth_headers = { "Authorization": "Bearer " + kube_token } + auth_headers = {"Authorization": "Bearer " + kube_token} install_id = None cm_url = "%s/%s" % (base_url, url_path) @@ -269,25 +273,25 @@ def configmap_install_id_plugin(scout, app, map_name=None, namespace="default"): if install_id: scout.logger.debug("Scout: got install_id %s from map" % install_id) - plugin_response = { "install_id": install_id } + plugin_response = {"install_id": install_id} except OSError as e: - scout.logger.debug("Scout: could not read configmap (map %s, namespace %s): %s" % - (map_name, namespace, e)) + scout.logger.debug( + "Scout: could not read configmap (map %s, namespace %s): %s" + % (map_name, namespace, e) + ) if not install_id: # No extant install_id. Try to create a new one. install_id = str(uuid4()) cm = { - "apiVersion":"v1", - "kind":"ConfigMap", - "metadata":{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { "name": map_name, "namespace": namespace, }, - "data": { - "install_id": install_id - } + "data": {"install_id": install_id}, } scout.logger.debug("Scout: saving new install_id %s" % install_id) @@ -300,15 +304,16 @@ def configmap_install_id_plugin(scout, app, map_name=None, namespace="default"): saved = True scout.logger.debug("Scout: saved install_id %s" % install_id) - plugin_response = { - "install_id": install_id, - "new_install": True - } + plugin_response = {"install_id": install_id, "new_install": True} else: - scout.logger.error("Scout: could not save install_id: {0}, {1}".format(r.status_code, r.text)) + scout.logger.error( + "Scout: could not save install_id: {0}, {1}".format(r.status_code, r.text) + ) except OSError as e: - logging.debug("Scout: could not write configmap (map %s, namespace %s): %s" % - (map_name, namespace, e)) + logging.debug( + "Scout: could not write configmap (map %s, namespace %s): %s" + % (map_name, namespace, e) + ) scout.logger.debug("Scout: plugin_response %s" % json.dumps(plugin_response)) return plugin_response diff --git a/python/ambassador/utils.py b/python/ambassador/utils.py index dc30ccd86a..e20133abdf 100644 --- a/python/ambassador/utils.py +++ b/python/ambassador/utils.py @@ -39,9 +39,9 @@ from prometheus_client import Gauge if TYPE_CHECKING: - from .ir import IRResource # pragma: no cover - from .ir.irtlscontext import IRTLSContext # pragma: no cover - from .config.acresource import ACResource # pragma: no cover + from .ir import IRResource # pragma: no cover + from .ir.irtlscontext import IRTLSContext # pragma: no cover + from .config.acresource import ACResource # pragma: no cover logger = logging.getLogger("utils") logger.setLevel(logging.INFO) @@ -95,12 +95,18 @@ def parse_json(serialization: str) -> Any: def dump_json(obj: Any, pretty=False) -> str: # There's a nicer way to do this in python, I'm sure. if pretty: - return bytes.decode(orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS|orjson.OPT_SORT_KEYS|orjson.OPT_INDENT_2)) + return bytes.decode( + orjson.dumps( + obj, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS | orjson.OPT_INDENT_2 + ) + ) else: return bytes.decode(orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS)) -def _load_url_contents(logger: logging.Logger, url: str, stream1: TextIO, stream2: Optional[TextIO]=None) -> bool: +def _load_url_contents( + logger: logging.Logger, url: str, stream1: TextIO, stream2: Optional[TextIO] = None +) -> bool: saved = False try: @@ -108,7 +114,7 @@ def _load_url_contents(logger: logging.Logger, url: str, stream1: TextIO, stream if r.status_code == 200: # All's well, pull the config down. - encoded = b'' + encoded = b"" try: for chunk in r.iter_content(chunk_size=65536): @@ -117,7 +123,7 @@ def _load_url_contents(logger: logging.Logger, url: str, stream1: TextIO, stream # and WATT hands us application/json... encoded += chunk - decoded = encoded.decode('utf-8') + decoded = encoded.decode("utf-8") stream1.write(decoded) if stream2: @@ -134,12 +140,16 @@ def _load_url_contents(logger: logging.Logger, url: str, stream1: TextIO, stream return saved -def save_url_contents(logger: logging.Logger, url: str, path: str, stream2: Optional[TextIO]=None) -> bool: - with open(path, 'w', encoding='utf-8') as stream: +def save_url_contents( + logger: logging.Logger, url: str, path: str, stream2: Optional[TextIO] = None +) -> bool: + with open(path, "w", encoding="utf-8") as stream: return _load_url_contents(logger, url, stream, stream2=stream2) -def load_url_contents(logger: logging.Logger, url: str, stream2: Optional[TextIO]=None) -> Optional[str]: +def load_url_contents( + logger: logging.Logger, url: str, stream2: Optional[TextIO] = None +) -> Optional[str]: stream = io.StringIO() saved = _load_url_contents(logger, url, stream, stream2=stream2) @@ -169,16 +179,16 @@ def parse_bool(s: Optional[Union[str, bool]]) -> bool: # OK, we got _something_, so try strtobool. try: - return bool(strtobool(s)) # the linter does not like a Literal[0, 1] being returned here + return bool(strtobool(s)) # the linter does not like a Literal[0, 1] being returned here except ValueError: return False class SystemInfo: - MyHostName = os.environ.get('HOSTNAME', None) + MyHostName = os.environ.get("HOSTNAME", None) if not MyHostName: - MyHostName = 'localhost' + MyHostName = "localhost" try: MyHostName = socket.gethostname() @@ -190,8 +200,8 @@ class RichStatus: def __init__(self, ok, **kwargs): self.ok = ok self.info = kwargs - self.info['hostname'] = SystemInfo.MyHostName - self.info['version'] = Version + self.info["hostname"] = SystemInfo.MyHostName + self.info["version"] = Version # Remember that __getattr__ is called only as a last resort if the key # isn't a normal attr. @@ -217,7 +227,7 @@ def __str__(self): return "" % ("OK" if self else "BAD", astr) def as_dict(self): - d = { 'ok': self.ok } + d = {"ok": self.ok} for key in self.info.keys(): d[key] = self.info[key] @@ -226,7 +236,7 @@ def as_dict(self): @classmethod def fromError(self, error, **kwargs): - kwargs['error'] = error + kwargs["error"] = error return RichStatus(False, **kwargs) @classmethod @@ -269,9 +279,9 @@ class Timer: _maximum: float _running: bool _faketime: float - _gauge: Optional[Gauge]=None + _gauge: Optional[Gauge] = None - def __init__(self, name: str, prom_metrics_registry: Optional[Any]=None) -> None: + def __init__(self, name: str, prom_metrics_registry: Optional[Any] = None) -> None: """ Create a Timer, given a name. The Timer is initially stopped. """ @@ -279,9 +289,13 @@ def __init__(self, name: str, prom_metrics_registry: Optional[Any]=None) -> None self.name = name if prom_metrics_registry: - metric_prefix = re.sub(r'\s+', '_', name).lower() - self._gauge = Gauge(f'{metric_prefix}_time_seconds', f'Elapsed time on {name} operations', - namespace='ambassador', registry=prom_metrics_registry) + metric_prefix = re.sub(r"\s+", "_", name).lower() + self._gauge = Gauge( + f"{metric_prefix}_time_seconds", + f"Elapsed time on {name} operations", + namespace="ambassador", + registry=prom_metrics_registry, + ) self.reset() @@ -308,7 +322,7 @@ def __bool__(self) -> bool: """ return self._cycles > 0 - def start(self, when: Optional[float]=None) -> None: + def start(self, when: Optional[float] = None) -> None: """ Start a Timer running. @@ -326,7 +340,7 @@ def start(self, when: Optional[float]=None) -> None: self._starttime = when or time.perf_counter() self._running = True - def stop(self, when: Optional[float]=None) -> float: + def stop(self, when: Optional[float] = None) -> float: """ Stop a Timer, increment the cycle count, and update the accumulated time with the amount of time since the Timer @@ -444,10 +458,15 @@ def summary(self) -> str: """ return "TIMER %s: %d, %.3f/%.3f/%.3f" % ( - self.name, self.cycles, self.minimum, self.average, self.maximum - ) + self.name, + self.cycles, + self.minimum, + self.average, + self.maximum, + ) -class DelayTrigger (threading.Thread): + +class DelayTrigger(threading.Thread): def __init__(self, onfired, timeout=5, name=None): super().__init__() @@ -463,7 +482,7 @@ def __init__(self, onfired, timeout=5, name=None): self.start() def trigger(self): - self.trigger_source.sendall(b'X') + self.trigger_source.sendall(b"X") def run(self): while True: @@ -508,9 +527,18 @@ class SecretInfo: ciphertext elements. Pretty much everything in Ambassador that worries about secrets uses a SecretInfo. """ - def __init__(self, name: str, namespace: str, secret_type: str, - tls_crt: Optional[str]=None, tls_key: Optional[str]=None, user_key: Optional[str]=None, - root_crt: Optional[str]=None, decode_b64=True) -> None: + + def __init__( + self, + name: str, + namespace: str, + secret_type: str, + tls_crt: Optional[str] = None, + tls_key: Optional[str] = None, + user_key: Optional[str] = None, + root_crt: Optional[str] = None, + decode_b64=True, + ) -> None: self.name = name self.namespace = namespace self.secret_type = secret_type @@ -542,8 +570,7 @@ def is_decodable(b64_pem: Optional[str]) -> bool: if not b64_pem: return False - return not (b64_pem.startswith('-----BEGIN') or - b64_pem.startswith('-sanitized-')) + return not (b64_pem.startswith("-----BEGIN") or b64_pem.startswith("-sanitized-")) @staticmethod def decode(b64_pem: str) -> Optional[str]: @@ -562,7 +589,7 @@ def decode(b64_pem: str) -> Optional[str]: return None try: - pem = utf8_pem.decode('utf-8') + pem = utf8_pem.decode("utf-8") except UnicodeDecodeError: return None @@ -580,15 +607,15 @@ def fingerprint(pem: Optional[str]) -> str: :return: fingerprint string """ if not pem: - return '' + return "" - h = hashlib.new('sha1') - h.update(pem.encode('utf-8')) + h = hashlib.new("sha1") + h.update(pem.encode("utf-8")) hd = h.hexdigest()[0:16].upper() - keytype = 'PEM' if pem.startswith('-----BEGIN') else 'RAW' + keytype = "PEM" if pem.startswith("-----BEGIN") else "RAW" - return f'{keytype}: {hd}' + return f"{keytype}: {hd}" def to_dict(self) -> Dict[str, Any]: """ @@ -597,17 +624,17 @@ def to_dict(self) -> Dict[str, Any]: :return: dict """ return { - 'name': self.name, - 'namespace': self.namespace, - 'secret_type': self.secret_type, - 'tls_crt': self.fingerprint(self.tls_crt), - 'tls_key': self.fingerprint(self.tls_key), - 'user_key': self.fingerprint(self.user_key), - 'root_crt': self.fingerprint(self.root_crt) + "name": self.name, + "namespace": self.namespace, + "secret_type": self.secret_type, + "tls_crt": self.fingerprint(self.tls_crt), + "tls_key": self.fingerprint(self.tls_key), + "user_key": self.fingerprint(self.user_key), + "root_crt": self.fingerprint(self.root_crt), } @classmethod - def from_aconf_secret(cls, aconf_object: 'ACResource') -> 'SecretInfo': + def from_aconf_secret(cls, aconf_object: "ACResource") -> "SecretInfo": """ Convert an ACResource containing a secret into a SecretInfo. This is used by the IR.save_secret_info() to convert saved secrets into SecretInfos. @@ -616,18 +643,18 @@ def from_aconf_secret(cls, aconf_object: 'ACResource') -> 'SecretInfo': :return: SecretInfo """ - tls_crt = aconf_object.get('tls_crt', None) + tls_crt = aconf_object.get("tls_crt", None) if not tls_crt: - tls_crt = aconf_object.get('cert-chain_pem') + tls_crt = aconf_object.get("cert-chain_pem") - tls_key = aconf_object.get('tls_key', None) + tls_key = aconf_object.get("tls_key", None) if not tls_key: - tls_key = aconf_object.get('key_pem') + tls_key = aconf_object.get("key_pem") - user_key = aconf_object.get('user_key', None) + user_key = aconf_object.get("user_key", None) if not user_key: # We didn't have a 'user_key', do we have a `crl_pem` instead? - user_key = aconf_object.get('crl_pem', None) + user_key = aconf_object.get("crl_pem", None) return SecretInfo( aconf_object.name, @@ -636,13 +663,19 @@ def from_aconf_secret(cls, aconf_object: 'ACResource') -> 'SecretInfo': tls_crt, tls_key, user_key, - aconf_object.get('root-cert_pem', None) + aconf_object.get("root-cert_pem", None), ) @classmethod - def from_dict(cls, resource: 'IRResource', - secret_name: str, namespace: str, source: str, - cert_data: Optional[Dict[str, Any]], secret_type="kubernetes.io/tls") -> Optional['SecretInfo']: + def from_dict( + cls, + resource: "IRResource", + secret_name: str, + namespace: str, + source: str, + cert_data: Optional[Dict[str, Any]], + secret_type="kubernetes.io/tls", + ) -> Optional["SecretInfo"]: """ Given a secret's name and namespace, and a dictionary of configuration elements, return a SecretInfo for the secret. @@ -667,33 +700,43 @@ def from_dict(cls, resource: 'IRResource', user_key = None if not cert_data: - resource.ir.logger.error(f"{resource.kind} {resource.name}: found no certificate in {source}?") + resource.ir.logger.error( + f"{resource.kind} {resource.name}: found no certificate in {source}?" + ) return None - if secret_type == 'kubernetes.io/tls': + if secret_type == "kubernetes.io/tls": # OK, we have something to work with. Hopefully. - tls_crt = cert_data.get('tls.crt', None) + tls_crt = cert_data.get("tls.crt", None) if not tls_crt: # Having no public half is definitely an error. Having no private half given a public half # might be OK, though -- that's up to our caller to decide. - resource.ir.logger.error(f"{resource.kind} {resource.name}: found data but no certificate in {source}?") + resource.ir.logger.error( + f"{resource.kind} {resource.name}: found data but no certificate in {source}?" + ) return None - tls_key = cert_data.get('tls.key', None) - elif secret_type == 'Opaque': - user_key = cert_data.get('user.key', None) + tls_key = cert_data.get("tls.key", None) + elif secret_type == "Opaque": + user_key = cert_data.get("user.key", None) if not user_key: # The opaque keys we support must have user.key, but will likely have nothing else. - resource.ir.logger.error(f"{resource.kind} {resource.name}: found data but no user.key in {source}?") + resource.ir.logger.error( + f"{resource.kind} {resource.name}: found data but no user.key in {source}?" + ) return None cert = None - elif secret_type == 'istio.io/key-and-cert': - resource.ir.logger.error(f"{resource.kind} {resource.name}: found data but handler for istio key not finished yet") + elif secret_type == "istio.io/key-and-cert": + resource.ir.logger.error( + f"{resource.kind} {resource.name}: found data but handler for istio key not finished yet" + ) - return SecretInfo(secret_name, namespace, secret_type, tls_crt=tls_crt, tls_key=tls_key, user_key=user_key) + return SecretInfo( + secret_name, namespace, secret_type, tls_crt=tls_crt, tls_key=tls_key, user_key=user_key + ) class SavedSecret: @@ -705,10 +748,17 @@ class SavedSecret: found no information. SavedSecret will evaluate True as a boolean if - and only if - it has the minimal information needed to represent a real secret. """ - def __init__(self, secret_name: str, namespace: str, - cert_path: Optional[str], key_path: Optional[str], - user_path: Optional[str], root_cert_path: Optional[str], - cert_data: Optional[Dict]) -> None: + + def __init__( + self, + secret_name: str, + namespace: str, + cert_path: Optional[str], + key_path: Optional[str], + user_path: Optional[str], + root_cert_path: Optional[str], + cert_data: Optional[Dict], + ) -> None: self.secret_name = secret_name self.namespace = namespace self.cert_path = cert_path @@ -725,10 +775,18 @@ def __bool__(self) -> bool: return bool((bool(self.cert_path) or bool(self.user_path)) and (self.cert_data is not None)) def __str__(self) -> str: - return "" % ( - self.secret_name, self.namespace, self.cert_path, self.key_path, self.user_path, self.root_cert_path, - "present" if self.cert_data else "absent" - ) + return ( + "" + % ( + self.secret_name, + self.namespace, + self.cert_path, + self.key_path, + self.user_path, + self.root_cert_path, + "present" if self.cert_data else "absent", + ) + ) class SecretHandler: @@ -760,13 +818,17 @@ class SecretHandler: source_root: str cache_dir: str - def __init__(self, logger: logging.Logger, source_root: str, cache_dir: str, version: str) -> None: + def __init__( + self, logger: logging.Logger, source_root: str, cache_dir: str, version: str + ) -> None: self.logger = logger self.source_root = source_root self.cache_dir = cache_dir self.version = version - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: """ load_secret: given a secret’s name and namespace, pull it from wherever it really lives, write it to disk, and return a SecretInfo telling the rest of Ambassador where it got written. @@ -782,12 +844,14 @@ def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) :return: Optional[SecretInfo] """ - self.logger.debug("SecretHandler (%s %s): load secret %s in namespace %s" % - (resource.kind, resource.name, secret_name, namespace)) + self.logger.debug( + "SecretHandler (%s %s): load secret %s in namespace %s" + % (resource.kind, resource.name, secret_name, namespace) + ) return None - def still_needed(self, resource: 'IRResource', secret_name: str, namespace: str) -> None: + def still_needed(self, resource: "IRResource", secret_name: str, namespace: str) -> None: """ still_needed: remember that a given secret is still needed, so that we can tell watt to keep paying attention to it. @@ -802,10 +866,12 @@ def still_needed(self, resource: 'IRResource', secret_name: str, namespace: str) :return: None """ - self.logger.debug("SecretHandler (%s %s): secret %s in namespace %s is still needed" % - (resource.kind, resource.name, secret_name, namespace)) + self.logger.debug( + "SecretHandler (%s %s): secret %s in namespace %s is still needed" + % (resource.kind, resource.name, secret_name, namespace) + ) - def cache_secret(self, resource: 'IRResource', secret_info: SecretInfo) -> SavedSecret: + def cache_secret(self, resource: "IRResource", secret_info: SecretInfo) -> SavedSecret: """ cache_secret: stash the SecretInfo from load_secret into Ambassador’s internal cache, so that we don’t have to call load_secret again if we need it again. @@ -826,10 +892,16 @@ def cache_secret(self, resource: 'IRResource', secret_info: SecretInfo) -> Saved return self.cache_internal(name, namespace, tls_crt, tls_key, user_key, root_crt) - def cache_internal(self, name: str, namespace: str, - tls_crt: Optional[str], tls_key: Optional[str], - user_key: Optional[str], root_crt: Optional[str]) -> SavedSecret: - h = hashlib.new('sha1') + def cache_internal( + self, + name: str, + namespace: str, + tls_crt: Optional[str], + tls_key: Optional[str], + user_key: Optional[str], + root_crt: Optional[str], + ) -> SavedSecret: + h = hashlib.new("sha1") tls_crt_path = None tls_key_path = None @@ -841,7 +913,7 @@ def cache_internal(self, name: str, namespace: str, if tls_crt or user_key or root_crt: for el in [tls_crt, tls_key, user_key]: if el: - h.update(el.encode('utf-8')) + h.update(el.encode("utf-8")) hd = h.hexdigest().upper() @@ -853,35 +925,44 @@ def cache_internal(self, name: str, namespace: str, pass if tls_crt: - tls_crt_path = os.path.join(secret_dir, f'{hd}.crt') + tls_crt_path = os.path.join(secret_dir, f"{hd}.crt") open(tls_crt_path, "w").write(tls_crt) if tls_key: - tls_key_path = os.path.join(secret_dir, f'{hd}.key') + tls_key_path = os.path.join(secret_dir, f"{hd}.key") open(tls_key_path, "w").write(tls_key) if user_key: - user_key_path = os.path.join(secret_dir, f'{hd}.user') + user_key_path = os.path.join(secret_dir, f"{hd}.user") open(user_key_path, "w").write(user_key) if root_crt: - root_crt_path = os.path.join(secret_dir, f'{hd}.root.crt') + root_crt_path = os.path.join(secret_dir, f"{hd}.root.crt") open(root_crt_path, "w").write(root_crt) cert_data = { - 'tls_crt': tls_crt, - 'tls_key': tls_key, - 'user_key': user_key, - 'root_crt': root_crt, + "tls_crt": tls_crt, + "tls_key": tls_key, + "user_key": user_key, + "root_crt": root_crt, } - self.logger.debug(f"saved secret {name}.{namespace}: {tls_crt_path}, {tls_key_path}, {root_crt_path}") + self.logger.debug( + f"saved secret {name}.{namespace}: {tls_crt_path}, {tls_key_path}, {root_crt_path}" + ) - return SavedSecret(name, namespace, tls_crt_path, tls_key_path, user_key_path, root_crt_path, cert_data) + return SavedSecret( + name, namespace, tls_crt_path, tls_key_path, user_key_path, root_crt_path, cert_data + ) - def secret_info_from_k8s(self, resource: 'IRResource', - secret_name: str, namespace: str, source: str, - serialization: Optional[str]) -> Optional[SecretInfo]: + def secret_info_from_k8s( + self, + resource: "IRResource", + secret_name: str, + namespace: str, + source: str, + serialization: Optional[str], + ) -> Optional[SecretInfo]: """ secret_info_from_k8s is NO LONGER USED. """ @@ -909,85 +990,124 @@ def secret_info_from_k8s(self, resource: 'IRResource', for obj in objects: ocount += 1 - kind = obj.get('kind', None) + kind = obj.get("kind", None) if kind != "Secret": - self.logger.error("%s %s: found K8s %s at %s.%d?" % - (resource.kind, resource.name, kind, source, ocount)) + self.logger.error( + "%s %s: found K8s %s at %s.%d?" + % (resource.kind, resource.name, kind, source, ocount) + ) errors += 1 continue - metadata = obj.get('metadata', None) + metadata = obj.get("metadata", None) if not metadata: - self.logger.error("%s %s: found K8s Secret with no metadata at %s.%d?" % - (resource.kind, resource.name, source, ocount)) + self.logger.error( + "%s %s: found K8s Secret with no metadata at %s.%d?" + % (resource.kind, resource.name, source, ocount) + ) errors += 1 continue - secret_type = metadata.get('type', 'kubernetes.io/tls') + secret_type = metadata.get("type", "kubernetes.io/tls") - if 'data' in obj: + if "data" in obj: if cert_data: - self.logger.error("%s %s: found multiple Secrets in %s?" % - (resource.kind, resource.name, source)) + self.logger.error( + "%s %s: found multiple Secrets in %s?" + % (resource.kind, resource.name, source) + ) errors += 1 continue - cert_data = obj['data'] + cert_data = obj["data"] if errors: # Bzzt. return None - return SecretInfo.from_dict(resource, secret_name, namespace, source, - cert_data=cert_data, secret_type=secret_type) + return SecretInfo.from_dict( + resource, secret_name, namespace, source, cert_data=cert_data, secret_type=secret_type + ) class NullSecretHandler(SecretHandler): - def __init__(self, logger: logging.Logger, source_root: Optional[str], cache_dir: Optional[str], version: str) -> None: + def __init__( + self, + logger: logging.Logger, + source_root: Optional[str], + cache_dir: Optional[str], + version: str, + ) -> None: """ Returns a valid SecretInfo (with fake keys) for any requested secret. Also, you can pass None for source_root and cache_dir to use random temporary directories for them. """ if not source_root: - self.tempdir_source = tempfile.TemporaryDirectory(prefix="null-secret-", suffix="-source") + self.tempdir_source = tempfile.TemporaryDirectory( + prefix="null-secret-", suffix="-source" + ) source_root = self.tempdir_source.name if not cache_dir: self.tempdir_cache = tempfile.TemporaryDirectory(prefix="null-secret-", suffix="-cache") cache_dir = self.tempdir_cache.name - logger.info(f'NullSecretHandler using source_root {source_root}, cache_dir {cache_dir}') + logger.info(f"NullSecretHandler using source_root {source_root}, cache_dir {cache_dir}") super().__init__(logger, source_root, cache_dir, version) - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: # In the Real World, the secret loader should, y'know, load secrets.. # Here we're just gonna fake it. - self.logger.debug("NullSecretHandler (%s %s): load secret %s in namespace %s" % - (resource.kind, resource.name, secret_name, namespace)) + self.logger.debug( + "NullSecretHandler (%s %s): load secret %s in namespace %s" + % (resource.kind, resource.name, secret_name, namespace) + ) + + return SecretInfo( + secret_name, + namespace, + "fake-secret", + "fake-tls-crt", + "fake-tls-key", + "fake-user-key", + decode_b64=False, + ) - return SecretInfo(secret_name, namespace, "fake-secret", "fake-tls-crt", "fake-tls-key", "fake-user-key", - decode_b64=False) class EmptySecretHandler(SecretHandler): - def __init__(self, logger: logging.Logger, source_root: Optional[str], cache_dir: Optional[str], version: str) -> None: + def __init__( + self, + logger: logging.Logger, + source_root: Optional[str], + cache_dir: Optional[str], + version: str, + ) -> None: """ Returns a None to simulate no provided secrets """ super().__init__(logger, "", "", version) - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: return None class FSSecretHandler(SecretHandler): # XXX NO LONGER USED - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: - self.logger.debug("FSSecretHandler (%s %s): load secret %s in namespace %s" % - (resource.kind, resource.name, secret_name, namespace)) + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: + self.logger.debug( + "FSSecretHandler (%s %s): load secret %s in namespace %s" + % (resource.kind, resource.name, secret_name, namespace) + ) source = os.path.join(self.source_root, namespace, "secrets", "%s.yaml" % secret_name) @@ -996,8 +1116,9 @@ def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) try: serialization = open(source, "r").read() except IOError as e: - self.logger.error("%s %s: FSSecretHandler could not open %s" % - (resource.kind, resource.name, source)) + self.logger.error( + "%s %s: FSSecretHandler could not open %s" % (resource.kind, resource.name, source) + ) # Yes, this duplicates part of self.secret_info_from_k8s, but whatever. objects: Optional[List[Any]] = None @@ -1007,28 +1128,34 @@ def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) try: objects = parse_yaml(serialization) except yaml.error.YAMLError as e: - self.logger.error("%s %s: could not parse %s: %s" % - (resource.kind, resource.name, source, e)) + self.logger.error( + "%s %s: could not parse %s: %s" % (resource.kind, resource.name, source, e) + ) if not objects: # Nothing in the serialization, we're done. return None if len(objects) != 1: - self.logger.error("%s %s: found %d objects in %s instead of exactly 1" % - (resource.kind, resource.name, len(objects), source)) + self.logger.error( + "%s %s: found %d objects in %s instead of exactly 1" + % (resource.kind, resource.name, len(objects), source) + ) return None obj = objects[0] - version = obj.get('apiVersion', None) - kind = obj.get('kind', None) + version = obj.get("apiVersion", None) + kind = obj.get("kind", None) - if (kind == 'Secret') and (version.startswith('ambassador') or version.startswith('getambassador.io')): + if (kind == "Secret") and ( + version.startswith("ambassador") or version.startswith("getambassador.io") + ): # It's an Ambassador Secret. It should have a public key and maybe a private key. - secret_type = obj.get('type', 'kubernetes.io/tls') - return SecretInfo.from_dict(resource, secret_name, namespace, source, - cert_data=obj, secret_type=secret_type) + secret_type = obj.get("type", "kubernetes.io/tls") + return SecretInfo.from_dict( + resource, secret_name, namespace, source, cert_data=obj, secret_type=secret_type + ) # Didn't look like an Ambassador object. Try K8s. return self.secret_info_from_k8s(resource, secret_name, namespace, source, serialization) @@ -1036,62 +1163,75 @@ def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) class KubewatchSecretHandler(SecretHandler): # XXX NO LONGER USED - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: - self.logger.debug("FSSecretHandler (%s %s): load secret %s in namespace %s" % - (resource.kind, resource.name, secret_name, namespace)) + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: + self.logger.debug( + "FSSecretHandler (%s %s): load secret %s in namespace %s" + % (resource.kind, resource.name, secret_name, namespace) + ) source = "%s/secrets/%s/%s" % (self.source_root, namespace, secret_name) serialization = load_url_contents(self.logger, source) if not serialization: - self.logger.error("%s %s: SCC.url_reader could not load %s" % (resource.kind, resource.name, source)) + self.logger.error( + "%s %s: SCC.url_reader could not load %s" % (resource.kind, resource.name, source) + ) return self.secret_info_from_k8s(resource, secret_name, namespace, source, serialization) + # TODO(gsagula): This duplicates code from ircluster.py. class ParsedService: - def __init__(self, logger, service: str, allow_scheme=True, ctx_name: str=None) -> None: + def __init__(self, logger, service: str, allow_scheme=True, ctx_name: str = None) -> None: original_service = service originate_tls = False - self.scheme = 'http' + self.scheme = "http" self.errors: List[str] = [] self.name_fields: List[str] = [] self.ctx_name = ctx_name if allow_scheme and service.lower().startswith("https://"): - service = service[len("https://"):] + service = service[len("https://") :] originate_tls = True - self.name_fields.append('otls') + self.name_fields.append("otls") elif allow_scheme and service.lower().startswith("http://"): - service = service[ len("http://"): ] + service = service[len("http://") :] if ctx_name: - self.errors.append(f'Originate-TLS context {ctx_name} being used even though service {service} lists HTTP') + self.errors.append( + f"Originate-TLS context {ctx_name} being used even though service {service} lists HTTP" + ) originate_tls = True - self.name_fields.append('otls') + self.name_fields.append("otls") else: originate_tls = False elif ctx_name: # No scheme (or schemes are ignored), but we have a context. originate_tls = True - self.name_fields.append('otls') + self.name_fields.append("otls") self.name_fields.append(ctx_name) - if '://' in service: - idx = service.index('://') + if "://" in service: + idx = service.index("://") scheme = service[0:idx] if allow_scheme: - self.errors.append(f'service {service} has unknown scheme {scheme}, assuming {self.scheme}') + self.errors.append( + f"service {service} has unknown scheme {scheme}, assuming {self.scheme}" + ) else: - self.errors.append(f'ignoring scheme {scheme} for service {service}, since it is being used for a non-HTTP mapping') + self.errors.append( + f"ignoring scheme {scheme} for service {service}, since it is being used for a non-HTTP mapping" + ) - service = service[idx + 3:] + service = service[idx + 3 :] # # XXX Should this be checking originate_tls? Why does it do that? # if originate_tls and host_rewrite: @@ -1100,13 +1240,17 @@ def __init__(self, logger, service: str, allow_scheme=True, ctx_name: str=None) # Parse the service as a URL. Note that we have to supply a scheme to urllib's # parser, because it's kind of stupid. - logger.debug(f'Service: {original_service} otls {originate_tls} ctx {ctx_name} -> {self.scheme}, {service}') - p = urlparse('random://' + service) + logger.debug( + f"Service: {original_service} otls {originate_tls} ctx {ctx_name} -> {self.scheme}, {service}" + ) + p = urlparse("random://" + service) # Is there any junk after the host? if p.path or p.params or p.query or p.fragment: - self.errors.append(f'service {service} has extra URL components; ignoring everything but the host and port') + self.errors.append( + f"service {service} has extra URL components; ignoring everything but the host and port" + ) # p is read-only, so break stuff out. @@ -1114,11 +1258,15 @@ def __init__(self, logger, service: str, allow_scheme=True, ctx_name: str=None) try: self.port = p.port except ValueError as e: - self.errors.append("found invalid port for service {}. Please specify a valid port between 0 and 65535 - {}. Service {} cluster will be ignored, please re-configure".format(service, e, service)) + self.errors.append( + "found invalid port for service {}. Please specify a valid port between 0 and 65535 - {}. Service {} cluster will be ignored, please re-configure".format( + service, e, service + ) + ) self.port = 0 # If the port is unset, fix it up. if not self.port: self.port = 443 if originate_tls else 80 - self.hostname_port = f'{self.hostname}:{self.port}' + self.hostname_port = f"{self.hostname}:{self.port}" diff --git a/python/ambassador_cli/ambassador.py b/python/ambassador_cli/ambassador.py index 9d5c930348..3e95ac464d 100644 --- a/python/ambassador_cli/ambassador.py +++ b/python/ambassador_cli/ambassador.py @@ -39,17 +39,25 @@ from ambassador.fetch import ResourceFetcher from ambassador.envoy import EnvoyConfig, V3Config -from ambassador.utils import RichStatus, SecretHandler, SecretInfo, NullSecretHandler, Timer, parse_json, dump_json +from ambassador.utils import ( + RichStatus, + SecretHandler, + SecretInfo, + NullSecretHandler, + Timer, + parse_json, + dump_json, +) if TYPE_CHECKING: - from ambassador.ir import IRResource # pragma: no cover + from ambassador.ir import IRResource # pragma: no cover __version__ = Version logging.basicConfig( level=logging.INFO, format="%%(asctime)s ambassador-cli %s %%(levelname)s: %%(message)s" % __version__, - datefmt="%Y-%m-%d %H:%M:%S" + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -61,8 +69,7 @@ def handle_exception(what, e, **kwargs): scout = Scout() result = scout.report(action=what, mode="cli", exception=str(e), traceback=tb, **kwargs) - logger.debug("Scout %s, result: %s" % - ("enabled" if scout._scout else "disabled", result)) + logger.debug("Scout %s, result: %s" % ("enabled" if scout._scout else "disabled", result)) logger.error("%s: %s\n%s" % (what, e, tb)) @@ -70,12 +77,12 @@ def handle_exception(what, e, **kwargs): def show_notices(result: dict, printer=logger.log): - notices = result.get('notices', []) + notices = result.get("notices", []) for notice in notices: - lvl = logging.getLevelName(notice.get('level', 'ERROR')) + lvl = logging.getLevelName(notice.get("level", "ERROR")) - printer(lvl, notice.get('message', '?????')) + printer(lvl, notice.get("message", "?????")) def stdout_printer(lvl, msg): @@ -107,7 +114,7 @@ def showid(): print("Ambassador Scout installation ID %s" % scout.install_id) - result= scout.report(action="showid", mode="cli") + result = scout.report(action="showid", mode="cli") show_notices(result, printer=stdout_printer) @@ -126,41 +133,68 @@ class CLISecretHandler(SecretHandler): # "ssl-certificate.mynamespace" ) - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: # Only allow a secret to be _loaded_ if it's marked Loadable. key = f"{secret_name}.{namespace}" if key in CLISecretHandler.LoadableSecrets: self.logger.info(f"CLISecretHandler: loading {key}") - return SecretInfo(secret_name, namespace, "mocked-loadable-secret", - "-mocked-cert-", "-mocked-key-", decode_b64=False) + return SecretInfo( + secret_name, + namespace, + "mocked-loadable-secret", + "-mocked-cert-", + "-mocked-key-", + decode_b64=False, + ) self.logger.debug(f"CLISecretHandler: cannot load {key}") return None @click.command() -@click.argument('config_dir_path', type=click.Path()) -@click.option('--secret-dir-path', type=click.Path(), help="Directory into which to save secrets") -@click.option('--watt', is_flag=True, help="If set, input must be a WATT snapshot") -@click.option('--debug', is_flag=True, help="If set, generate debugging output") -@click.option('--debug_scout', is_flag=True, help="If set, generate debugging output") -@click.option('--k8s', is_flag=True, help="If set, assume configuration files are annotated K8s manifests") -@click.option('--recurse', is_flag=True, help="If set, recurse into directories below config_dir_path") -@click.option('--stats', is_flag=True, help="If set, dump statistics to stderr") -@click.option('--nopretty', is_flag=True, help="If set, do not pretty print the dumped JSON") -@click.option('--aconf', is_flag=True, help="If set, dump the Ambassador config") -@click.option('--ir', is_flag=True, help="If set, dump the IR") -@click.option('--xds', is_flag=True, help="If set, dump the Envoy config") -@click.option('--diag', is_flag=True, help="If set, dump the Diagnostics overview") -@click.option('--everything', is_flag=True, help="If set, dump everything") -@click.option('--features', is_flag=True, help="If set, dump the feature set") -@click.option('--profile', is_flag=True, help="If set, profile with the cProfile module") -def dump(config_dir_path: str, *, - secret_dir_path=None, watt=False, debug=False, debug_scout=False, k8s=False, recurse=False, - stats=False, nopretty=False, everything=False, aconf=False, ir=False, xds=False, diag=False, - features=False, profile=False): +@click.argument("config_dir_path", type=click.Path()) +@click.option("--secret-dir-path", type=click.Path(), help="Directory into which to save secrets") +@click.option("--watt", is_flag=True, help="If set, input must be a WATT snapshot") +@click.option("--debug", is_flag=True, help="If set, generate debugging output") +@click.option("--debug_scout", is_flag=True, help="If set, generate debugging output") +@click.option( + "--k8s", is_flag=True, help="If set, assume configuration files are annotated K8s manifests" +) +@click.option( + "--recurse", is_flag=True, help="If set, recurse into directories below config_dir_path" +) +@click.option("--stats", is_flag=True, help="If set, dump statistics to stderr") +@click.option("--nopretty", is_flag=True, help="If set, do not pretty print the dumped JSON") +@click.option("--aconf", is_flag=True, help="If set, dump the Ambassador config") +@click.option("--ir", is_flag=True, help="If set, dump the IR") +@click.option("--xds", is_flag=True, help="If set, dump the Envoy config") +@click.option("--diag", is_flag=True, help="If set, dump the Diagnostics overview") +@click.option("--everything", is_flag=True, help="If set, dump everything") +@click.option("--features", is_flag=True, help="If set, dump the feature set") +@click.option("--profile", is_flag=True, help="If set, profile with the cProfile module") +def dump( + config_dir_path: str, + *, + secret_dir_path=None, + watt=False, + debug=False, + debug_scout=False, + k8s=False, + recurse=False, + stats=False, + nopretty=False, + everything=False, + aconf=False, + ir=False, + xds=False, + diag=False, + features=False, + profile=False, +): """ Dump various forms of an Ambassador configuration for debugging @@ -180,7 +214,7 @@ def dump(config_dir_path: str, *, logger.setLevel(logging.DEBUG) if debug_scout: - logging.getLogger('ambassador.scout').setLevel(logging.DEBUG) + logging.getLogger("ambassador.scout").setLevel(logging.DEBUG) if everything: aconf = True @@ -242,19 +276,19 @@ def dump(config_dir_path: str, *, aconf_timer = Timer("aconf") with aconf_timer: if dump_aconf: - od['aconf'] = aconf.as_dict() + od["aconf"] = aconf.as_dict() ir_timer = Timer("ir") with ir_timer: if dump_ir: - od['ir'] = ir.as_dict() + od["ir"] = ir.as_dict() xds_timer = Timer("xds") with xds_timer: if dump_xds: config = V3Config(ir) diagconfig = config - od['xds'] = config.as_dict() + od["xds"] = config.as_dict() diag_timer = Timer("diag") with diag_timer: if dump_diag: @@ -262,13 +296,13 @@ def dump(config_dir_path: str, *, diagconfig = V3Config(ir) econf = typecast(EnvoyConfig, diagconfig) diag = Diagnostics(ir, econf) - od['diag'] = diag.as_dict() - od['elements'] = econf.elements + od["diag"] = diag.as_dict() + od["elements"] = econf.elements features_timer = Timer("features") with features_timer: if dump_features: - od['features'] = ir.features() + od["features"] = ir.features() # scout = Scout() # scout_args = {} @@ -296,15 +330,15 @@ def dump(config_dir_path: str, *, vhost_count = 0 filter_chain_count = 0 filter_count = 0 - if 'xds' in od: - for listener in od['xds']['static_resources']['listeners']: - for fc in listener['filter_chains']: + if "xds" in od: + for listener in od["xds"]["static_resources"]["listeners"]: + for fc in listener["filter_chains"]: filter_chain_count += 1 - for f in fc['filters']: + for f in fc["filters"]: filter_count += 1 - for vh in f['typed_config']['route_config']['virtual_hosts']: + for vh in f["typed_config"]["route_config"]["virtual_hosts"]: vhost_count += 1 - route_count += len(vh['routes']) + route_count += len(vh["routes"]) if stats: sys.stderr.write("STATS:\n") @@ -313,7 +347,9 @@ def dump(config_dir_path: str, *, sys.stderr.write(" filter chains: %d\n" % filter_chain_count) sys.stderr.write(" filters: %d\n" % filter_count) sys.stderr.write(" routes: %d\n" % route_count) - sys.stderr.write(" routes/vhosts: %.3f\n" % float(float(route_count)/float(vhost_count))) + sys.stderr.write( + " routes/vhosts: %.3f\n" % float(float(route_count) / float(vhost_count)) + ) sys.stderr.write("TIMERS:\n") sys.stderr.write(" fetch resources: %.3fs\n" % fetch_timer.average) sys.stderr.write(" load resources: %.3fs\n" % load_timer.average) @@ -327,8 +363,7 @@ def dump(config_dir_path: str, *, sys.stderr.write(" ----------------------\n") sys.stderr.write(" total: %.3fs\n" % total_timer.average) except Exception as e: - handle_exception("EXCEPTION from dump", e, - config_dir_path=config_dir_path) + handle_exception("EXCEPTION from dump", e, config_dir_path=config_dir_path) _rc = 1 if _profile: @@ -339,7 +374,7 @@ def dump(config_dir_path: str, *, @click.command() -@click.argument('config_dir_path', type=click.Path()) +@click.argument("config_dir_path", type=click.Path()) def validate(config_dir_path: str): """ Validate an Ambassador configuration. This is an extension of "config" that @@ -351,18 +386,43 @@ def validate(config_dir_path: str): @click.command() -@click.argument('config_dir_path', type=click.Path()) -@click.argument('output_json_path', type=click.Path()) -@click.option('--debug', is_flag=True, help="If set, generate debugging output") -@click.option('--debug-scout', is_flag=True, help="If set, generate debugging output when talking to Scout") -@click.option('--check', is_flag=True, help="If set, generate configuration only if it doesn't already exist") -@click.option('--k8s', is_flag=True, help="If set, assume configuration files are annotated K8s manifests") -@click.option('--exit-on-error', is_flag=True, help="If set, will exit with status 1 on any configuration error") -@click.option('--ir', type=click.Path(), help="Pathname to which to dump the IR (not dumped if not present)") -@click.option('--aconf', type=click.Path(), help="Pathname to which to dump the aconf (not dumped if not present)") -def config(config_dir_path: str, output_json_path: str, *, - debug=False, debug_scout=False, check=False, k8s=False, ir=None, aconf=None, - exit_on_error=False): +@click.argument("config_dir_path", type=click.Path()) +@click.argument("output_json_path", type=click.Path()) +@click.option("--debug", is_flag=True, help="If set, generate debugging output") +@click.option( + "--debug-scout", is_flag=True, help="If set, generate debugging output when talking to Scout" +) +@click.option( + "--check", is_flag=True, help="If set, generate configuration only if it doesn't already exist" +) +@click.option( + "--k8s", is_flag=True, help="If set, assume configuration files are annotated K8s manifests" +) +@click.option( + "--exit-on-error", + is_flag=True, + help="If set, will exit with status 1 on any configuration error", +) +@click.option( + "--ir", type=click.Path(), help="Pathname to which to dump the IR (not dumped if not present)" +) +@click.option( + "--aconf", + type=click.Path(), + help="Pathname to which to dump the aconf (not dumped if not present)", +) +def config( + config_dir_path: str, + output_json_path: str, + *, + debug=False, + debug_scout=False, + check=False, + k8s=False, + ir=None, + aconf=None, + exit_on_error=False, +): """ Generate an Envoy configuration @@ -375,7 +435,7 @@ def config(config_dir_path: str, output_json_path: str, *, logger.setLevel(logging.DEBUG) if debug_scout: - logging.getLogger('ambassador.scout').setLevel(logging.DEBUG) + logging.getLogger("ambassador.scout").setLevel(logging.DEBUG) try: logger.debug("CHECK MODE %s" % check) @@ -427,7 +487,7 @@ def config(config_dir_path: str, output_json_path: str, *, # If exit_on_error is set, log _errors and exit with status 1 if exit_on_error and aconf.errors: - raise Exception("errors in: {0}".format(', '.join(aconf.errors.keys()))) + raise Exception("errors in: {0}".format(", ".join(aconf.errors.keys()))) secret_handler = NullSecretHandler(logger, config_dir_path, config_dir_path, "0") @@ -453,8 +513,12 @@ def config(config_dir_path: str, output_json_path: str, *, result = scout.report(action="config", mode="cli") show_notices(result) except Exception as e: - handle_exception("EXCEPTION from config", e, - config_dir_path=config_dir_path, output_json_path=output_json_path) + handle_exception( + "EXCEPTION from config", + e, + config_dir_path=config_dir_path, + output_json_path=output_json_path, + ) # This is fatal. sys.exit(1) @@ -478,8 +542,20 @@ def showid_callback(ctx: click.core.Context, param: click.Parameter, value: bool no_args_is_help=False, commands=[config, dump, validate], ) -@click.option('--version', is_flag=True, expose_value=False, callback=version_callback, help="Show the Emissary version number and exit.") -@click.option('--showid', is_flag=True, expose_value=False, callback=showid_callback, help="Show the cluster ID and exit.") +@click.option( + "--version", + is_flag=True, + expose_value=False, + callback=version_callback, + help="Show the Emissary version number and exit.", +) +@click.option( + "--showid", + is_flag=True, + expose_value=False, + callback=showid_callback, + help="Show the cluster ID and exit.", +) def main(): """Generate an Envoy config, or manage an Ambassador deployment. Use diff --git a/python/ambassador_cli/ert.py b/python/ambassador_cli/ert.py index 94d08b77be..b97382199d 100644 --- a/python/ambassador_cli/ert.py +++ b/python/ambassador_cli/ert.py @@ -31,6 +31,7 @@ import click import dpath.util + def lookup(x: Any, path: str) -> Optional[Any]: try: return dpath.util.get(x, path) @@ -38,14 +39,15 @@ def lookup(x: Any, path: str) -> Optional[Any]: return None -reSecret = re.compile(r'^.*/snapshots/([^/]+)/secrets-decoded/([^/]+)/[0-9A-F]+\.(.+)$') +reSecret = re.compile(r"^.*/snapshots/([^/]+)/secrets-decoded/([^/]+)/[0-9A-F]+\.(.+)$") # Use this instead of click.option click_option = functools.partial(click.option, show_default=True) click_option_no_default = functools.partial(click.option, show_default=False) + @click.command(help="Show a simplified Envoy config breakdown") -@click.argument('envoy-config-path', type=click.Path(exists=True, readable=True)) +@click.argument("envoy-config-path", type=click.Path(exists=True, readable=True)) def main(envoy_config_path: str) -> None: econf = json.load(open(envoy_config_path, "r")) @@ -60,7 +62,7 @@ def main(envoy_config_path: str) -> None: lfiltstr = "" if lfilters: - lfilter_names = [ x["name"].replace("envoy.listener.", "") for x in lfilters ] + lfilter_names = [x["name"].replace("envoy.listener.", "") for x in lfilters] lfiltstr = f" [using {', '.join(lfilter_names)}]" print(f"LISTENER on {proto} {bind_addr}:{port}{lfiltstr}") @@ -70,7 +72,7 @@ def main(envoy_config_path: str) -> None: match_proto = lookup(chain, "/filter_chain_match/transport_protocol") match_domains = lookup(chain, "/filter_chain_match/server_names") - match_domain_str = '*' + match_domain_str = "*" if match_domains: match_domain_str = "/".join(match_domains) @@ -135,7 +137,9 @@ def main(envoy_config_path: str) -> None: if headers: for hdr in headers: - if (hdr["name"] == "x-forwarded-proto") and (hdr.get("exact_match") == "https"): + if (hdr["name"] == "x-forwarded-proto") and ( + hdr.get("exact_match") == "https" + ): security = "secure" elif hdr["name"] == ":authority": authority = f"@{hdr['exact_match']}" @@ -162,5 +166,5 @@ def main(envoy_config_path: str) -> None: print(f"... ... ... {target}: {'; '.join(action_list)}") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/python/ambassador_cli/grab_snapshots.py b/python/ambassador_cli/grab_snapshots.py index 063104c0f3..b59b6d2021 100644 --- a/python/ambassador_cli/grab_snapshots.py +++ b/python/ambassador_cli/grab_snapshots.py @@ -41,7 +41,7 @@ def sanitize_snapshot(snapshot: dict): sanitized = {} # Consul is pretty easy. Just sort, using service-dc as the sort key. - consul_elements = snapshot.get('Consul') + consul_elements = snapshot.get("Consul") if consul_elements: csorted = {} @@ -49,16 +49,16 @@ def sanitize_snapshot(snapshot: dict): for key, value in consul_elements.items(): csorted[key] = sorted(value, key=lambda x: f'{x["Service"]-x["Id"]}') - sanitized['Consul'] = csorted + sanitized["Consul"] = csorted # Make sure we grab Deltas and Invalid -- these should really be OK as-is. - for key in [ 'Deltas', 'Invalid' ]: + for key in ["Deltas", "Invalid"]: if key in snapshot: sanitized[key] = snapshot[key] # Kube is harder because we need to sanitize Kube secrets. - kube_elements = snapshot.get('Kubernetes') + kube_elements = snapshot.get("Kubernetes") if kube_elements: ksorted = {} @@ -67,41 +67,43 @@ def sanitize_snapshot(snapshot: dict): if not value: continue - if key == 'secret': + if key == "secret": for secret in value: if "data" in secret: data = secret["data"] for k in data.keys(): - data[k] = f'-sanitized-{k}-' + data[k] = f"-sanitized-{k}-" - metadata = secret.get('metadata', {}) - annotations = metadata.get('annotations', {}) + metadata = secret.get("metadata", {}) + annotations = metadata.get("annotations", {}) # Wipe the last-applied-configuration annotation, too, because it # often contains the secret data. - if 'kubectl.kubernetes.io/last-applied-configuration' in annotations: - annotations['kubectl.kubernetes.io/last-applied-configuration'] = '--sanitized--' + if "kubectl.kubernetes.io/last-applied-configuration" in annotations: + annotations[ + "kubectl.kubernetes.io/last-applied-configuration" + ] = "--sanitized--" # All the sanitization above happened in-place in value, so we can just # sort it. - ksorted[key] = sorted(value, key=lambda x: x.get('metadata',{}).get('name')) + ksorted[key] = sorted(value, key=lambda x: x.get("metadata", {}).get("name")) - sanitized['Kubernetes'] = ksorted + sanitized["Kubernetes"] = ksorted return sanitized # Helper to open a snapshot.yaml and sanitize it. def helper_snapshot(path: str) -> str: - snapshot = json.loads(open(path, "r"). read()) + snapshot = json.loads(open(path, "r").read()) return dump_json(sanitize_snapshot(snapshot)) # Helper to open a problems.json and sanitize the snapshot it contains. def helper_problems(path: str) -> str: - bad_dict = json.loads(open(path, "r"). read()) + bad_dict = json.loads(open(path, "r").read()) bad_dict["snapshot"] = sanitize_snapshot(bad_dict["snapshot"]) @@ -115,12 +117,22 @@ def helper_copy(path: str) -> str: # Open a tarfile for output... @click.command(help="Grab, and sanitize, Ambassador snapshots for later debugging") -@click_option('--debug/--no-debug', default=True, - help="enable debugging") -@click_option('-o', '--output-path', '--output', type=click.Path(writable=True), default="sanitized.tgz", - help="output path") -@click_option('-s', '--snapshot-dir', '--snapshot', type=click.Path(exists=True, dir_okay=True, file_okay=False), - help="snapshot directory to read") +@click_option("--debug/--no-debug", default=True, help="enable debugging") +@click_option( + "-o", + "--output-path", + "--output", + type=click.Path(writable=True), + default="sanitized.tgz", + help="output path", +) +@click_option( + "-s", + "--snapshot-dir", + "--snapshot", + type=click.Path(exists=True, dir_okay=True, file_okay=False), + help="snapshot directory to read", +) def main(snapshot_dir: str, debug: bool, output_path: str) -> None: if not snapshot_dir: config_base_dir = os.environ.get("AMBASSADOR_CONFIG_BASE_DIR", "/ambassador") @@ -129,7 +141,7 @@ def main(snapshot_dir: str, debug: bool, output_path: str) -> None: if debug: print(f"Saving sanitized snapshots from {snapshot_dir} to {output_path}") - with tarfile.open(output_path, 'w:gz') as archive: + with tarfile.open(output_path, "w:gz") as archive: # ...then iterate any snapshots, sanitize, and stuff 'em in the tarfile. # Note that the '.yaml' on the snapshot file name is a misnomer: when # watt is involved, they're actually JSON. It's a long story. @@ -137,10 +149,10 @@ def main(snapshot_dir: str, debug: bool, output_path: str) -> None: some_found = False interesting_things = [ - ( "snap*yaml", helper_snapshot ), - ( "problems*json", helper_problems ), - ( "econf*json", helper_copy ), - ( "diff*txt", helper_copy ) + ("snap*yaml", helper_snapshot), + ("problems*json", helper_problems), + ("econf*json", helper_copy), + ("diff*txt", helper_copy), ] for pattern, helper in interesting_things: @@ -160,7 +172,7 @@ def main(snapshot_dir: str, debug: bool, output_path: str) -> None: _, ext = os.path.splitext(path) sanitized_name = f"sanitized{ext}" - with open(sanitized_name, 'w') as tmp: + with open(sanitized_name, "w") as tmp: tmp.write(sanitized) archive.add(sanitized_name, arcname=b) @@ -172,5 +184,6 @@ def main(snapshot_dir: str, debug: bool, output_path: str) -> None: sys.exit(0) + if __name__ == "__main__": main() diff --git a/python/ambassador_cli/madness.py b/python/ambassador_cli/madness.py index 2a0a3a0a76..454f23d52f 100644 --- a/python/ambassador_cli/madness.py +++ b/python/ambassador_cli/madness.py @@ -13,6 +13,7 @@ # Types OptionalStats = Optional[pstats.Stats] + class Profiler: def __init__(self): self.pr = cProfile.Profile() @@ -42,20 +43,22 @@ def stats(self) -> OptionalStats: class Madness: - def __init__(self, - watt_path: Optional[str]=None, - yaml_path: Optional[str]=None, - logger: Optional[logging.Logger]=None, - secret_handler: Optional[SecretHandler]=None, - file_checker: Optional[IRFileChecker]=None) -> None: + def __init__( + self, + watt_path: Optional[str] = None, + yaml_path: Optional[str] = None, + logger: Optional[logging.Logger] = None, + secret_handler: Optional[SecretHandler] = None, + file_checker: Optional[IRFileChecker] = None, + ) -> None: if not logger: logging.basicConfig( level=logging.INFO, format="%(asctime)s madness %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" + datefmt="%Y-%m-%d %H:%M:%S", ) - logger = logging.getLogger('mockery') + logger = logging.getLogger("mockery") self.logger = logger @@ -103,8 +106,7 @@ def summarize(self) -> None: if timer: self.logger.info(timer.summary()) - def build_ir(self, - cache=True, profile=False, summarize=True) -> Tuple[IR, OptionalStats]: + def build_ir(self, cache=True, profile=False, summarize=True) -> Tuple[IR, OptionalStats]: self.ir_timer.reset() _cache = self.cache if cache else None @@ -112,16 +114,16 @@ def build_ir(self, with self.ir_timer: with _pr: - ir = IR(self.aconf, cache=_cache, - secret_handler=self.secret_handler) + ir = IR(self.aconf, cache=_cache, secret_handler=self.secret_handler) if summarize: self.summarize() return (ir, _pr.stats()) - def build_econf(self, ir: Union[IR, Tuple[IR, OptionalStats]], - cache=True, profile=False, summarize=True) -> Tuple[EnvoyConfig, OptionalStats]: + def build_econf( + self, ir: Union[IR, Tuple[IR, OptionalStats]], cache=True, profile=False, summarize=True + ) -> Tuple[EnvoyConfig, OptionalStats]: self.econf_timer.reset() _cache = self.cache if cache else None @@ -159,15 +161,15 @@ def build(self, cache=True, profile=False) -> Tuple[IR, EnvoyConfig, OptionalSta return (ir, econf, _pr.stats()) def diff(self, *rsrcs) -> None: - jsons = [ rsrc.as_json() for rsrc in rsrcs ] + jsons = [rsrc.as_json() for rsrc in rsrcs] if len(set(jsons)) == 1: return for i in range(len(rsrcs) - 1): - if jsons[i] != jsons[i+1]: + if jsons[i] != jsons[i + 1]: l1 = jsons[i].split("\n") - l2 = jsons[i+1].split("\n") + l2 = jsons[i + 1].split("\n") n1 = f"rsrcs[{i}]" n2 = f"rsrcs[{i+1}]" diff --git a/python/ambassador_cli/mockery.py b/python/ambassador_cli/mockery.py index bc02607ed4..bc508b2453 100644 --- a/python/ambassador_cli/mockery.py +++ b/python/ambassador_cli/mockery.py @@ -55,12 +55,13 @@ KubeList = List[KubeResource] WattDict = Dict[str, KubeList] + class LabelSpec: def __init__(self, serialization: str) -> None: - if '=' not in serialization: + if "=" not in serialization: raise Exception(f"label serialization must be key=value, not {serialization}") - (key, value) = serialization.split('=', 1) + (key, value) = serialization.split("=", 1) self.key = key self.value = value @@ -74,12 +75,12 @@ def match(self, labels: Dict[str, str]) -> bool: class FieldSpec: def __init__(self, serialization: str) -> None: - if '=' not in serialization: + if "=" not in serialization: raise Exception(f"field serialization must be key=value, not {serialization}") - (key, value) = serialization.split('=', 1) + (key, value) = serialization.split("=", 1) - self.elements = key.split('.') + self.elements = key.split(".") self.value = value def __str__(self) -> str: @@ -102,32 +103,39 @@ def __init__(self, kind: str, watch_id: str) -> None: self.kind = kind self.watch_id = watch_id + class WatchSpec: - def __init__(self, logger: logging.Logger, kind: str, namespace: Optional[str], - labels: Optional[str], fields: Optional[str]=None, - bootstrap: Optional[bool]=False): + def __init__( + self, + logger: logging.Logger, + kind: str, + namespace: Optional[str], + labels: Optional[str], + fields: Optional[str] = None, + bootstrap: Optional[bool] = False, + ): self.logger = logger self.kind = kind - self.match_kinds = { self.kind.lower(): True } + self.match_kinds = {self.kind.lower(): True} self.namespace = namespace self.labels: Optional[List[LabelSpec]] = None self.fields: Optional[List[FieldSpec]] = None self.bootstrap = bootstrap - if self.kind == 'ingresses': - self.match_kinds['ingress'] = True + if self.kind == "ingresses": + self.match_kinds["ingress"] = True if labels: - self.labels = [ LabelSpec(l) for l in labels.split(',') ] + self.labels = [LabelSpec(l) for l in labels.split(",")] if fields: - self.fields = [ FieldSpec(f) for f in fields.split(',') ] + self.fields = [FieldSpec(f) for f in fields.split(",")] def _labelstr(self) -> str: - return ",".join([ str(x) for x in self.labels or [] ]) + return ",".join([str(x) for x in self.labels or []]) def _fieldstr(self) -> str: - return ",".join([ str(x) for x in self.fields or [] ]) + return ",".join([str(x) for x in self.fields or []]) @staticmethod def _star(s: Optional[str]) -> str: @@ -137,7 +145,7 @@ def __repr__(self) -> str: s = f"{self.kind}|{self._star(self.namespace)}|{self._star(self._fieldstr())}|{self._star(self._labelstr())}" if self.bootstrap: - s += ' (bootstrap)' + s += " (bootstrap)" return f"<{s}>" @@ -148,18 +156,18 @@ def __str__(self) -> str: return f"{self.kind}|{self._star(self.namespace)}|{self._star(self._fieldstr())}|{self._star(self._labelstr())}" def match(self, obj: KubeResource) -> Optional[WatchResult]: - kind: Optional[str] = obj.get('kind') or None - metadata: Dict[str, Any] = obj.get('metadata') or {} - name: Optional[str] = metadata.get('name') or None - namespace: str = metadata.get('namespace') or 'default' - labels: Dict[str, str] = metadata.get('labels') or {} + kind: Optional[str] = obj.get("kind") or None + metadata: Dict[str, Any] = obj.get("metadata") or {} + name: Optional[str] = metadata.get("name") or None + namespace: str = metadata.get("namespace") or "default" + labels: Dict[str, str] = metadata.get("labels") or {} if not kind or not name: self.logger.error(f"K8s object requires kind and name: {obj}") return None # self.logger.debug(f"match {self}: check {obj}") - match_kind_str = ','.join(sorted(self.match_kinds.keys())) + match_kind_str = ",".join(sorted(self.match_kinds.keys())) # OK. Does the kind match up? if kind.lower() not in self.match_kinds: @@ -194,8 +202,15 @@ def match(self, obj: KubeResource) -> Optional[WatchResult]: class Mockery: - def __init__(self, logger: logging.Logger, debug: bool, sources: List[str], - labels: Optional[str], namespace: Optional[str], watch: str) -> None: + def __init__( + self, + logger: logging.Logger, + debug: bool, + sources: List[str], + labels: Optional[str], + namespace: Optional[str], + watch: str, + ) -> None: self.logger = logger self.debug = debug self.sources = sources @@ -211,7 +226,7 @@ def __init__(self, logger: logging.Logger, debug: bool, sources: List[str], kind=source, namespace=self.namespace, labels=labels, - bootstrap=True + bootstrap=True, ) if not self.maybe_add(bootstrap_watch): @@ -238,8 +253,8 @@ def load(self, manifest: KubeList) -> WattDict: self.logger.debug(f"{repr(spec)}") for obj in manifest: - metadata = obj.get('metadata') or {} - name = metadata.get('name') + metadata = obj.get("metadata") or {} + name = metadata.get("name") if not name: self.logger.debug(f"skipping unnamed object {obj}") @@ -262,7 +277,7 @@ def load(self, manifest: KubeList) -> WattDict: for kind in collected.keys(): watt_k8s[kind] = list(collected[kind].values()) - self.snapshot = dump_json({ 'Consul': {}, 'Kubernetes': watt_k8s }, pretty=True) + self.snapshot = dump_json({"Consul": {}, "Kubernetes": watt_k8s}, pretty=True) return watt_k8s @@ -279,11 +294,11 @@ def run_hook(self) -> Tuple[bool, bool]: for w in wh.watchset.get("kubernetes-watches") or []: potential = WatchSpec( logger=self.logger, - kind=w['kind'], - namespace=w.get('namespace'), - labels=w.get('label-selector'), - fields=w.get('field-selector'), - bootstrap=False + kind=w["kind"], + namespace=w.get("namespace"), + labels=w.get("label-selector"), + fields=w.get("field-selector"), + bootstrap=False, ) if self.maybe_add(potential): @@ -293,76 +308,132 @@ def run_hook(self) -> Tuple[bool, bool]: class MockSecretHandler(SecretHandler): - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: # Allow an environment variable to state whether we're in Edge Stack. But keep the # existing condition as sufficient, so that there is less of a chance of breaking # things running in a container with this file present. - if parse_bool(os.environ.get('EDGE_STACK', 'false')) or os.path.exists('/ambassador/.edge_stack'): - if ((secret_name == "fallback-self-signed-cert") and - (namespace == Config.ambassador_namespace)): + if parse_bool(os.environ.get("EDGE_STACK", "false")) or os.path.exists( + "/ambassador/.edge_stack" + ): + if (secret_name == "fallback-self-signed-cert") and ( + namespace == Config.ambassador_namespace + ): # This is Edge Stack. Force the fake TLS secret. - self.logger.info(f"MockSecretHandler: mocking fallback secret {secret_name}.{namespace}") - return SecretInfo(secret_name, namespace, "mocked-fallback-secret", - "-fallback-cert-", "-fallback-key-", decode_b64=False) + self.logger.info( + f"MockSecretHandler: mocking fallback secret {secret_name}.{namespace}" + ) + return SecretInfo( + secret_name, + namespace, + "mocked-fallback-secret", + "-fallback-cert-", + "-fallback-key-", + decode_b64=False, + ) self.logger.debug(f"MockSecretHandler: cannot load {secret_name}.{namespace}") return None -@click.command(help="Mock the watt/watch_hook/diagd cycle to generate an IR from a Kubernetes YAML manifest.") -@click_option('--debug/--no-debug', default=True, - help="enable debugging") -@click_option('-n', '--namespace', type=click.STRING, - help="namespace to watch [default: all namespaces])") -@click_option('-s', '--source', type=click.STRING, multiple=True, - help="define initial source types [default: all Ambassador resources]") -@click_option('--labels', type=click.STRING, multiple=True, - help="define initial label selector") -@click_option('--force-pod-labels/--no-force-pod-labels', default=True, - help="copy initial label selector to /tmp/ambassador-pod-info/labels") -@click_option('--kat-name', '--kat', type=click.STRING, - help="emulate a running KAT test with this name") -@click_option('-w', '--watch', type=click.STRING, default="python /ambassador/watch_hook.py", - help="define a watch hook") -@click_option('--diff-path', '--diff', type=click.STRING, - help="directory to diff against") -@click_option('--include-ir/--no-include-ir', '--ir/--no-ir', default=False, - help="include IR in diff when using --diff-path") -@click_option('--include-aconf/--no-include-aconf', '--aconf/--no-aconf', default=False, - help="include AConf in diff when using --diff-path") -@click_option('--update/--no-update', default=False, - help="update the diff path when finished") -@click.argument('k8s-yaml-paths', nargs=-1) -def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: bool, - source: List[str], labels: List[str], namespace: Optional[str], watch: str, - include_ir: bool, include_aconf: bool, - diff_path: Optional[str]=None, kat_name: Optional[str]=None) -> None: + +@click.command( + help="Mock the watt/watch_hook/diagd cycle to generate an IR from a Kubernetes YAML manifest." +) +@click_option("--debug/--no-debug", default=True, help="enable debugging") +@click_option( + "-n", "--namespace", type=click.STRING, help="namespace to watch [default: all namespaces])" +) +@click_option( + "-s", + "--source", + type=click.STRING, + multiple=True, + help="define initial source types [default: all Ambassador resources]", +) +@click_option("--labels", type=click.STRING, multiple=True, help="define initial label selector") +@click_option( + "--force-pod-labels/--no-force-pod-labels", + default=True, + help="copy initial label selector to /tmp/ambassador-pod-info/labels", +) +@click_option( + "--kat-name", "--kat", type=click.STRING, help="emulate a running KAT test with this name" +) +@click_option( + "-w", + "--watch", + type=click.STRING, + default="python /ambassador/watch_hook.py", + help="define a watch hook", +) +@click_option("--diff-path", "--diff", type=click.STRING, help="directory to diff against") +@click_option( + "--include-ir/--no-include-ir", + "--ir/--no-ir", + default=False, + help="include IR in diff when using --diff-path", +) +@click_option( + "--include-aconf/--no-include-aconf", + "--aconf/--no-aconf", + default=False, + help="include AConf in diff when using --diff-path", +) +@click_option("--update/--no-update", default=False, help="update the diff path when finished") +@click.argument("k8s-yaml-paths", nargs=-1) +def main( + k8s_yaml_paths: List[str], + debug: bool, + force_pod_labels: bool, + update: bool, + source: List[str], + labels: List[str], + namespace: Optional[str], + watch: str, + include_ir: bool, + include_aconf: bool, + diff_path: Optional[str] = None, + kat_name: Optional[str] = None, +) -> None: loglevel = logging.DEBUG if debug else logging.INFO logging.basicConfig( level=loglevel, format="%(asctime)s mockery %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" + datefmt="%Y-%m-%d %H:%M:%S", ) - logger = logging.getLogger('mockery') + logger = logging.getLogger("mockery") logger.debug(f"reading from {k8s_yaml_paths}") if not source: source = [ - "Host", "service", "ingresses", - "AuthService", "Listener", "LogService", "Mapping", "Module", "RateLimitService", - "TCPMapping", "TLSContext", "TracingService", - "ConsulResolver", "KubernetesEndpointResolver", "KubernetesServiceResolver" + "Host", + "service", + "ingresses", + "AuthService", + "Listener", + "LogService", + "Mapping", + "Module", + "RateLimitService", + "TCPMapping", + "TLSContext", + "TracingService", + "ConsulResolver", + "KubernetesEndpointResolver", + "KubernetesServiceResolver", ] if namespace: - os.environ['AMBASSADOR_NAMESPACE'] = namespace + os.environ["AMBASSADOR_NAMESPACE"] = namespace # Make labels a list, instead of a tuple. labels = list(labels) - labels_to_force = { l: True for l in labels or [] } + labels_to_force = {l: True for l in labels or []} if kat_name: logger.debug(f"KAT name {kat_name}") @@ -378,8 +449,8 @@ def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: labels_to_force[kat_amb_id_label] = True labels.append(kat_amb_id_label) - os.environ['AMBASSADOR_ID'] = kat_name - os.environ['AMBASSADOR_LABEL_SELECTOR'] = kat_amb_id_label + os.environ["AMBASSADOR_ID"] = kat_name + os.environ["AMBASSADOR_LABEL_SELECTOR"] = kat_amb_id_label # Forcibly override the cached ambassador_id. Config.ambassador_id = kat_name @@ -391,7 +462,7 @@ def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: logger.debug(f"sources {', '.join(source)}") for key in sorted(os.environ.keys()): - if key.startswith('AMBASSADOR'): + if key.startswith("AMBASSADOR"): logger.debug(f"${key}={os.environ[key]}") if force_pod_labels: @@ -407,7 +478,7 @@ def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: outfile.write("\n") # Pull in the YAML. - input_yaml = ''.join([ open(x, "r").read() for x in k8s_yaml_paths ]) + input_yaml = "".join([open(x, "r").read() for x in k8s_yaml_paths]) manifest = parse_yaml(input_yaml) w = Mockery(logger, debug, source, ",".join(labels), namespace, watch) @@ -468,11 +539,13 @@ def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: econf = EnvoyConfig.generate(ir) bootstrap_config, ads_config, clustermap = econf.split_config() - ads_config.pop('@type', None) + ads_config.pop("@type", None) with open("/tmp/ambassador/snapshots/econf.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(ads_config, pretty=True)) - with open(f"/tmp/ambassador/snapshots/econf-{Config.ambassador_id}.json", "w", encoding="utf-8") as outfile: + with open( + f"/tmp/ambassador/snapshots/econf-{Config.ambassador_id}.json", "w", encoding="utf-8" + ) as outfile: outfile.write(dump_json(ads_config, pretty=True)) with open("/tmp/ambassador/snapshots/bootstrap.json", "w", encoding="utf-8") as outfile: @@ -487,18 +560,30 @@ def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: diffs = False pairs_to_check = [ - (os.path.join(diff_path, 'snapshots', 'econf.json'), '/tmp/ambassador/snapshots/econf.json'), - (os.path.join(diff_path, 'bootstrap-ads.json'), '/tmp/ambassador/snapshots/bootstrap.json') + ( + os.path.join(diff_path, "snapshots", "econf.json"), + "/tmp/ambassador/snapshots/econf.json", + ), + ( + os.path.join(diff_path, "bootstrap-ads.json"), + "/tmp/ambassador/snapshots/bootstrap.json", + ), ] if include_ir: pairs_to_check.append( - ( os.path.join(diff_path, 'snapshots', 'ir.json'), '/tmp/ambassador/snapshots/ir.json' ) + ( + os.path.join(diff_path, "snapshots", "ir.json"), + "/tmp/ambassador/snapshots/ir.json", + ) ) if include_aconf: pairs_to_check.append( - ( os.path.join(diff_path, 'snapshots', 'aconf.json'), '/tmp/ambassador/snapshots/aconf.json' ) + ( + os.path.join(diff_path, "snapshots", "aconf.json"), + "/tmp/ambassador/snapshots/aconf.json", + ) ) for gold_path, check_path in pairs_to_check: @@ -511,12 +596,14 @@ def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: gold_lines = open(gold_path, "r", encoding="utf-8").readlines() check_lines = open(check_path, "r", encoding="utf-8").readlines() - for line in difflib.unified_diff(gold_lines, check_lines, fromfile=gold_path, tofile=check_path): + for line in difflib.unified_diff( + gold_lines, check_lines, fromfile=gold_path, tofile=check_path + ): sys.stdout.write(line) if diffs: sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/python/ambassador_diag/diagd.py b/python/ambassador_diag/diagd.py index e5d24dfabe..5aa4866f03 100644 --- a/python/ambassador_diag/diagd.py +++ b/python/ambassador_diag/diagd.py @@ -56,7 +56,16 @@ from ambassador.reconfig_stats import ReconfigStats from ambassador.ir.irambassador import IRAmbassador from ambassador.ir.irbasemapping import IRBaseMapping -from ambassador.utils import SystemInfo, Timer, PeriodicTrigger, SavedSecret, load_url_contents, parse_json, dump_json, parse_bool +from ambassador.utils import ( + SystemInfo, + Timer, + PeriodicTrigger, + SavedSecret, + load_url_contents, + parse_json, + dump_json, + parse_bool, +) from ambassador.utils import SecretHandler, KubewatchSecretHandler, FSSecretHandler, parse_bool from ambassador.fetch import ResourceFetcher @@ -65,7 +74,7 @@ from ambassador.constants import Constants if TYPE_CHECKING: - from ambassador.ir.irtlscontext import IRTLSContext # pragma: no cover + from ambassador.ir.irtlscontext import IRTLSContext # pragma: no cover __version__ = Version @@ -76,7 +85,9 @@ logHandler = None if parse_bool(os.environ.get("AMBASSADOR_JSON_LOGGING", "false")): - jsonFormatter = jsonlogger.JsonFormatter("%%(asctime)s %%(filename)s %%(lineno)d %%(process)d (threadName)s %%(levelname)s %%(message)s") + jsonFormatter = jsonlogger.JsonFormatter( + "%%(asctime)s %%(filename)s %%(lineno)d %%(process)d (threadName)s %%(levelname)s %%(message)s" + ) logHandler = logging.StreamHandler() logHandler.setFormatter(jsonFormatter) @@ -86,7 +97,7 @@ logger.addHandler(logHandler) # Update all of the other loggers to also use the new log handler. - loggingManager = getattr(logging.root, 'manager', None) + loggingManager = getattr(logging.root, "manager", None) if loggingManager is not None: for name in loggingManager.loggerDict: logging.getLogger(name).addHandler(logHandler) @@ -106,8 +117,9 @@ # Set defauts for all loggers logging.basicConfig( level=level, - format="%%(asctime)s diagd %s [P%%(process)dT%%(threadName)s] %%(levelname)s: %%(message)s" % __version__, - datefmt="%Y-%m-%d %H:%M:%S" + format="%%(asctime)s diagd %s [P%%(process)dT%%(threadName)s] %%(levelname)s: %%(message)s" + % __version__, + datefmt="%Y-%m-%d %H:%M:%S", ) # Shut up Werkzeug's standard request logs -- they're just too noisy. @@ -118,8 +130,8 @@ logging.getLogger("requests").setLevel(logging.WARNING) ambassador_targets = { - 'mapping': 'https://www.getambassador.io/reference/configuration#mappings', - 'module': 'https://www.getambassador.io/reference/configuration#modules', + "mapping": "https://www.getambassador.io/reference/configuration#mappings", + "module": "https://www.getambassador.io/reference/configuration#modules", } # envoy_targets = { @@ -132,7 +144,7 @@ def number_of_workers(): return (multiprocessing.cpu_count() * 2) + 1 -class DiagApp (Flask): +class DiagApp(Flask): cache: Optional[Cache] ambex_pid: int kick: Optional[str] @@ -155,9 +167,9 @@ class DiagApp (Flask): econf: Optional[EnvoyConfig] # self.diag is actually a property _diag: Optional[Diagnostics] - notices: 'Notices' + notices: "Notices" scout: Scout - watcher: 'AmbassadorEventWatcher' + watcher: "AmbassadorEventWatcher" stats_updater: Optional[PeriodicTrigger] scout_checker: Optional[PeriodicTrigger] timer_logger: Optional[PeriodicTrigger] @@ -178,11 +190,30 @@ class DiagApp (Flask): config_lock: threading.Lock diag_lock: threading.Lock - def setup(self, snapshot_path: str, bootstrap_path: str, ads_path: str, - config_path: Optional[str], ambex_pid: int, kick: Optional[str], banner_endpoint: Optional[str], - metrics_endpoint: Optional[str], k8s=False, do_checks=True, no_envoy=False, reload=False, debug=False, - verbose=False, notices=None, validation_retries=5, allow_fs_commands=False, local_scout=False, - report_action_keys=False, enable_fast_reconfigure=False, clustermap_path=None): + def setup( + self, + snapshot_path: str, + bootstrap_path: str, + ads_path: str, + config_path: Optional[str], + ambex_pid: int, + kick: Optional[str], + banner_endpoint: Optional[str], + metrics_endpoint: Optional[str], + k8s=False, + do_checks=True, + no_envoy=False, + reload=False, + debug=False, + verbose=False, + notices=None, + validation_retries=5, + allow_fs_commands=False, + local_scout=False, + report_action_keys=False, + enable_fast_reconfigure=False, + clustermap_path=None, + ): self.health_checks = do_checks self.no_envoy = no_envoy self.debugging = reload @@ -230,21 +261,32 @@ def setup(self, snapshot_path: str, bootstrap_path: str, ads_path: str, self.diag_timer = Timer("Diagnostics", self.metrics_registry) # Use gauges to keep some metrics on active config - self.diag_errors = Gauge(f'diagnostics_errors', f'Number of configuration errors', - namespace='ambassador', registry=self.metrics_registry) - self.diag_notices = Gauge(f'diagnostics_notices', f'Number of configuration notices', - namespace='ambassador', registry=self.metrics_registry) - self.diag_log_level = Gauge(f'log_level', f'Debug log level enabled or not', - ["level"], - namespace='ambassador', registry=self.metrics_registry) + self.diag_errors = Gauge( + f"diagnostics_errors", + f"Number of configuration errors", + namespace="ambassador", + registry=self.metrics_registry, + ) + self.diag_notices = Gauge( + f"diagnostics_notices", + f"Number of configuration notices", + namespace="ambassador", + registry=self.metrics_registry, + ) + self.diag_log_level = Gauge( + f"log_level", + f"Debug log level enabled or not", + ["level"], + namespace="ambassador", + registry=self.metrics_registry, + ) if debug: self.logger.setLevel(logging.DEBUG) - self.diag_log_level.labels('debug').set(1) - logging.getLogger('ambassador').setLevel(logging.DEBUG) + self.diag_log_level.labels("debug").set(1) + logging.getLogger("ambassador").setLevel(logging.DEBUG) else: - self.diag_log_level.labels('debug').set(0) - + self.diag_log_level.labels("debug").set(0) # Assume that we will NOT update Mapping status. ksclass: Type[KubeStatus] = KubeStatusNoMappings @@ -261,7 +303,9 @@ def setup(self, snapshot_path: str, bootstrap_path: str, ads_path: str, self.bootstrap_path = bootstrap_path self.ads_path = ads_path self.snapshot_path = snapshot_path - self.clustermap_path = clustermap_path or os.path.join(os.path.dirname(self.bootstrap_path), "clustermap.json") + self.clustermap_path = clustermap_path or os.path.join( + os.path.dirname(self.bootstrap_path), "clustermap.json" + ) # You must hold config_lock when updating config elements (including diag!). self.config_lock = threading.Lock() @@ -278,9 +322,9 @@ def setup(self, snapshot_path: str, bootstrap_path: str, ads_path: str, # be wrong. with self.config_lock: - self.ir = None # don't update unless you hold config_lock - self.econf = None # don't update unless you hold config_lock - self.diag = None # don't update unless you hold config_lock + self.ir = None # don't update unless you hold config_lock + self.econf = None # don't update unless you hold config_lock + self.diag = None # don't update unless you hold config_lock self.stats_updater = None self.scout_checker = None @@ -292,14 +336,23 @@ def setup(self, snapshot_path: str, bootstrap_path: str, ads_path: str, self.scout = Scout(local_only=self.local_scout) ProcessCollector(namespace="ambassador", registry=self.metrics_registry) - metrics_info = Info(name='diagnostics', namespace='ambassador', documentation='Ambassador diagnostic info', registry=self.metrics_registry) - metrics_info.info({ - "version": __version__, - "ambassador_id": Config.ambassador_id, - "cluster_id": os.environ.get('AMBASSADOR_CLUSTER_ID', - os.environ.get('AMBASSADOR_SCOUT_ID', "00000000-0000-0000-0000-000000000000")), - "single_namespace": str(Config.single_namespace), - }) + metrics_info = Info( + name="diagnostics", + namespace="ambassador", + documentation="Ambassador diagnostic info", + registry=self.metrics_registry, + ) + metrics_info.info( + { + "version": __version__, + "ambassador_id": Config.ambassador_id, + "cluster_id": os.environ.get( + "AMBASSADOR_CLUSTER_ID", + os.environ.get("AMBASSADOR_SCOUT_ID", "00000000-0000-0000-0000-000000000000"), + ), + "single_namespace": str(Config.single_namespace), + } + ) @property def diag(self) -> Optional[Diagnostics]: @@ -409,12 +462,14 @@ def check_timers(self) -> None: if self.reconf_stats.needs_timers(): # OK! Log the timers... - for t in [ self.config_timer, - self.fetcher_timer, - self.aconf_timer, - self.ir_timer, - self.econf_timer, - self.diag_timer ]: + for t in [ + self.config_timer, + self.fetcher_timer, + self.aconf_timer, + self.ir_timer, + self.econf_timer, + self.diag_timer, + ]: if t: self.logger.info(t.summary()) @@ -461,15 +516,14 @@ def json_diff(what: str, j1: str, j2: str) -> str: return output - def check_cache(self) -> bool: # We're going to build a shiny new IR and econf from our existing aconf, and make # sure everything matches. We will _not_ use the existing cache for this. # # For this, make sure we have an IR already... - assert(self.aconf) - assert(self.ir) - assert(self.econf) + assert self.aconf + assert self.ir + assert self.econf # Compute this IR/econf with a new empty cache. It saves a lot of trouble with # having to delete cache keys from the JSON. @@ -510,7 +564,7 @@ def check_cache(self) -> bool: open(err_path, "w").write(errors) - snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4")) + snapcount = int(os.environ.get("AMBASSADOR_SNAPSHOT_COUNT", "4")) snaplist: List[Tuple[str, str]] = [] if snapcount > 0: @@ -519,13 +573,13 @@ def check_cache(self) -> bool: # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]... # which is the list of suffixes to rename to rotate the snapshots. - snaplist += [ (str(x+1), str(x)) for x in range(-1 * snapcount, -1) ] + snaplist += [(str(x + 1), str(x)) for x in range(-1 * snapcount, -1)] # After dealing with that, we need to rotate the current file into -1. - snaplist.append(( '', '-1' )) + snaplist.append(("", "-1")) # Whether or not we do any rotation, we need to cycle in the '-tmp' file. - snaplist.append(( '-tmp', '' )) + snaplist.append(("-tmp", "")) for from_suffix, to_suffix in snaplist: from_path = os.path.join(app.snapshot_path, "diff{}.txt".format(from_suffix)) @@ -554,10 +608,7 @@ def get_templates_dir(): except: pass - maybe_dirs = [ - res_dir, - os.path.join(os.path.dirname(__file__), "..", "templates") - ] + maybe_dirs = [res_dir, os.path.join(os.path.dirname(__file__), "..", "templates")] for d in maybe_dirs: if d and os.path.isdir(d): return d @@ -570,13 +621,14 @@ def get_templates_dir(): ######## DECORATORS + def standard_handler(f): - func_name = getattr(f, '__name__', '') + func_name = getattr(f, "__name__", "") @functools.wraps(f) def wrapper(*args, **kwds): reqid = str(uuid.uuid4()).upper() - prefix = "%s: %s \"%s %s\"" % (reqid, request.remote_addr, request.method, request.path) + prefix = '%s: %s "%s %s"' % (reqid, request.remote_addr, request.method, request.path) app.logger.debug("%s START" % prefix) @@ -613,9 +665,11 @@ def wrapper(*args, **kwds): app.logger.exception(e) end = datetime.datetime.now() - ms = int(((end - start).total_seconds() * 1000) + .5) + ms = int(((end - start).total_seconds() * 1000) + 0.5) - app.logger.log(result_log_level, "%s %dms %d %s" % (prefix, ms, status_to_log, result_to_log)) + app.logger.log( + result_log_level, "%s %dms %d %s" % (prefix, ms, status_to_log, result_to_log) + ) return result @@ -627,7 +681,7 @@ def internal_handler(f): Reject requests where the remote address is not localhost. See the docstring for _is_local_request() for important caveats! """ - func_name = getattr(f, '__name__', '') + func_name = getattr(f, "__name__", "") @functools.wraps(f) def wrapper(*args, **kwds): @@ -676,7 +730,7 @@ def _allow_diag_ui() -> bool: allow traffic only from localhost clients """ enabled = False - allow_non_local= False + allow_non_local = False ir = app.ir if ir: enabled = ir.ambassador_module.diagnostics.get("enabled", False) @@ -693,7 +747,7 @@ def __init__(self, local_config_path: str) -> None: def reset(self): local_notices: List[Dict[str, str]] = [] - local_data = '' + local_data = "" try: local_stream = open(self.local_path, "r") @@ -702,7 +756,9 @@ def reset(self): except OSError: pass except: - local_notices.append({ 'level': 'ERROR', 'message': 'bad local notices: %s' % local_data }) + local_notices.append( + {"level": "ERROR", "message": "bad local notices: %s" % local_data} + ) self.notices = local_notices # app.logger.info("Notices: after RESET: %s" % dump_json(self.notices)) @@ -725,12 +781,12 @@ def extend(self, notices): def td_format(td_object): seconds = int(td_object.total_seconds()) periods = [ - ('year', 60*60*24*365), - ('month', 60*60*24*30), - ('day', 60*60*24), - ('hour', 60*60), - ('minute', 60), - ('second', 1) + ("year", 60 * 60 * 24 * 365), + ("month", 60 * 60 * 24 * 30), + ("day", 60 * 60 * 24), + ("hour", 60 * 60), + ("minute", 60), + ("second", 1), ] strings = [] @@ -738,8 +794,9 @@ def td_format(td_object): if seconds > period_seconds: period_value, seconds = divmod(seconds, period_seconds) - strings.append("%d %s%s" % - (period_value, period_name, "" if (period_value == 1) else "s")) + strings.append( + "%d %s%s" % (period_value, period_name, "" if (period_value == 1) else "s") + ) formatted = ", ".join(strings) @@ -762,13 +819,13 @@ def system_info(app): if ir: amod = ir.ambassador_module - debug_mode = amod.get('debug_mode', False) + debug_mode = amod.get("debug_mode", False) - app.logger.debug(f'DEBUG_MODE {debug_mode}') + app.logger.debug(f"DEBUG_MODE {debug_mode}") - status_dict = {'config failure': [False, 'no configuration loaded']} + status_dict = {"config failure": [False, "no configuration loaded"]} - env_status = getattr(app.watcher, 'env_status', None) + env_status = getattr(app.watcher, "env_status", None) if env_status: status_dict = env_status.to_dict() @@ -780,18 +837,20 @@ def system_info(app): "ambassador_id": Config.ambassador_id, "ambassador_namespace": Config.ambassador_namespace, "single_namespace": Config.single_namespace, - "knative_enabled": os.environ.get('AMBASSADOR_KNATIVE_SUPPORT', '').lower() == 'true', - "statsd_enabled": os.environ.get('STATSD_ENABLED', '').lower() == 'true', + "knative_enabled": os.environ.get("AMBASSADOR_KNATIVE_SUPPORT", "").lower() == "true", + "statsd_enabled": os.environ.get("STATSD_ENABLED", "").lower() == "true", "endpoints_enabled": Config.enable_endpoints, - "cluster_id": os.environ.get('AMBASSADOR_CLUSTER_ID', - os.environ.get('AMBASSADOR_SCOUT_ID', "00000000-0000-0000-0000-000000000000")), + "cluster_id": os.environ.get( + "AMBASSADOR_CLUSTER_ID", + os.environ.get("AMBASSADOR_SCOUT_ID", "00000000-0000-0000-0000-000000000000"), + ), "boot_time": boot_time, "hr_uptime": td_format(datetime.datetime.now() - boot_time), "latest_snapshot": app.latest_snapshot, - "env_good": getattr(app.watcher, 'env_good', False), - "env_failures": getattr(app.watcher, 'failure_list', [ 'no IR loaded' ]), + "env_good": getattr(app.watcher, "env_good", False), + "env_failures": getattr(app.watcher, "failure_list", ["no IR loaded"]), "env_status": status_dict, - "debug_mode": debug_mode + "debug_mode": debug_mode, } @@ -801,13 +860,15 @@ def envoy_status(estats: EnvoyStats): since_update = "Never updated" if estats.time_since_update(): - since_update = interval_format(estats.time_since_update(), "%s ago", "within the last second") + since_update = interval_format( + estats.time_since_update(), "%s ago", "within the last second" + ) return { "alive": estats.is_alive(), "ready": estats.is_ready(), "uptime": since_boot, - "since_update": since_update + "since_update": since_update, } @@ -823,29 +884,40 @@ def drop_serializer_key(d: Dict[Any, Any]) -> Dict[Any, Any]: def filter_keys(d: Dict[Any, Any], keys_to_keep): unwanted_keys = set(d) - set(keys_to_keep) - for unwanted_key in unwanted_keys: del d[unwanted_key] + for unwanted_key in unwanted_keys: + del d[unwanted_key] def filter_webui(d: Dict[Any, Any]): - filter_keys(d, ['system', 'route_info', 'source_map', - 'ambassador_resolvers', 'ambassador_services', - 'envoy_status', 'cluster_stats', 'loginfo', 'errors']) - for ambassador_resolver in d['ambassador_resolvers']: - filter_keys(ambassador_resolver, ['_source', 'kind']) - for route_info in d['route_info']: - filter_keys(route_info, ['diag_class', 'key', 'headers', - 'precedence', 'clusters']) - for cluster in route_info['clusters']: - filter_keys(cluster, ['_hcolor', 'type_label', 'service', 'weight']) - - -@app.route('/_internal/v0/ping', methods=[ 'GET' ]) + filter_keys( + d, + [ + "system", + "route_info", + "source_map", + "ambassador_resolvers", + "ambassador_services", + "envoy_status", + "cluster_stats", + "loginfo", + "errors", + ], + ) + for ambassador_resolver in d["ambassador_resolvers"]: + filter_keys(ambassador_resolver, ["_source", "kind"]) + for route_info in d["route_info"]: + filter_keys(route_info, ["diag_class", "key", "headers", "precedence", "clusters"]) + for cluster in route_info["clusters"]: + filter_keys(cluster, ["_hcolor", "type_label", "service", "weight"]) + + +@app.route("/_internal/v0/ping", methods=["GET"]) @internal_handler def handle_ping(): return "ACK\n", 200 -@app.route("/_internal/v0/features", methods=[ 'GET' ]) +@app.route("/_internal/v0/features", methods=["GET"]) @internal_handler def handle_features(): # If we don't have an IR yet, do nothing. @@ -855,16 +927,16 @@ def handle_features(): # the first configure is a race anyway. If it fails, that's not a big deal, # they can try again. if not app.ir: - app.logger.debug("Features: configuration required first") - return "Can't do features before configuration", 503 + app.logger.debug("Features: configuration required first") + return "Can't do features before configuration", 503 return jsonify(app.ir.features()), 200 -@app.route('/_internal/v0/watt', methods=[ 'POST' ]) +@app.route("/_internal/v0/watt", methods=["POST"]) @internal_handler def handle_watt_update(): - url = request.args.get('url', None) + url = request.args.get("url", None) if not url: app.logger.error("error: watt update requested with no URL") @@ -872,15 +944,15 @@ def handle_watt_update(): app.logger.debug("Update requested: watt, %s" % url) - status, info = app.watcher.post('CONFIG', ( 'watt', url )) + status, info = app.watcher.post("CONFIG", ("watt", url)) return info, status -@app.route('/_internal/v0/fs', methods=[ 'POST' ]) +@app.route("/_internal/v0/fs", methods=["POST"]) @internal_handler def handle_fs(): - path = request.args.get('path', None) + path = request.args.get("path", None) if not path: app.logger.error("error: update requested with no PATH") @@ -888,58 +960,58 @@ def handle_fs(): app.logger.debug("Update requested from %s" % path) - status, info = app.watcher.post('CONFIG_FS', path) + status, info = app.watcher.post("CONFIG_FS", path) return info, status -@app.route('/_internal/v0/events', methods=[ 'GET' ]) +@app.route("/_internal/v0/events", methods=["GET"]) @internal_handler def handle_events(): if not app.local_scout: - return 'Local Scout is not enabled\n', 400 + return "Local Scout is not enabled\n", 400 assert isinstance(app.scout._scout, LocalScout) event_dump = [ - ( x['local_scout_timestamp'], x['mode'], x['action'], x ) for x in app.scout._scout.events + (x["local_scout_timestamp"], x["mode"], x["action"], x) for x in app.scout._scout.events ] - app.logger.debug(f'Event dump {event_dump}') + app.logger.debug(f"Event dump {event_dump}") return jsonify(event_dump) -@app.route('/ambassador/v0/favicon.ico', methods=[ 'GET' ]) +@app.route("/ambassador/v0/favicon.ico", methods=["GET"]) def favicon(): template_path = resource_filename(Requirement.parse("ambassador"), "templates") return send_from_directory(template_path, "favicon.ico") -@app.route('/ambassador/v0/check_alive', methods=[ 'GET' ]) +@app.route("/ambassador/v0/check_alive", methods=["GET"]) def check_alive(): status = envoy_status(app.estatsmgr.get_stats()) - if status['alive']: - return "ambassador liveness check OK (%s)\n" % status['uptime'], 200 + if status["alive"]: + return "ambassador liveness check OK (%s)\n" % status["uptime"], 200 else: - return "ambassador seems to have died (%s)\n" % status['uptime'], 503 + return "ambassador seems to have died (%s)\n" % status["uptime"], 503 -@app.route('/ambassador/v0/check_ready', methods=[ 'GET' ]) +@app.route("/ambassador/v0/check_ready", methods=["GET"]) def check_ready(): if not app.ir: return "ambassador waiting for config\n", 503 status = envoy_status(app.estatsmgr.get_stats()) - if status['ready']: - return "ambassador readiness check OK (%s)\n" % status['since_update'], 200 + if status["ready"]: + return "ambassador readiness check OK (%s)\n" % status["since_update"], 200 else: - return "ambassador not ready (%s)\n" % status['since_update'], 503 + return "ambassador not ready (%s)\n" % status["since_update"], 503 -@app.route('/ambassador/v0/diag/', methods=[ 'GET' ]) +@app.route("/ambassador/v0/diag/", methods=["GET"]) @standard_handler def show_overview(reqid=None): # If we don't have an IR yet, do nothing. @@ -949,8 +1021,8 @@ def show_overview(reqid=None): # the first configure is a race anyway. If it fails, that's not a big deal, # they can try again. if not app.ir: - app.logger.debug("OV %s - can't do overview before configuration" % reqid) - return "Can't do overview before configuration", 503 + app.logger.debug("OV %s - can't do overview before configuration" % reqid) + return "Can't do overview before configuration", 503 if not _allow_diag_ui(): return Response("Not found\n", 404) @@ -986,18 +1058,21 @@ def show_overview(reqid=None): except Exception as e: app.logger.error("could not get banner_content: %s" % e) - tvars = dict(system=system_info(app), - envoy_status=envoy_status(estats), - loginfo=app.estatsmgr.loginfo, - notices=app.notices.notices, - banner_content=banner_content, - **ov, **ddict) + tvars = dict( + system=system_info(app), + envoy_status=envoy_status(estats), + loginfo=app.estatsmgr.loginfo, + notices=app.notices.notices, + banner_content=banner_content, + **ov, + **ddict, + ) - patch_client = request.args.get('patch_client', None) - if request.args.get('json', None): - filter_key = request.args.get('filter', None) + patch_client = request.args.get("patch_client", None) + if request.args.get("json", None): + filter_key = request.args.get("filter", None) - if filter_key == 'webui': + if filter_key == "webui": filter_webui(tvars) elif filter_key: return jsonify(tvars.get(filter_key, None)) @@ -1030,15 +1105,15 @@ def show_overview(reqid=None): def collect_errors_and_notices(request, reqid, what: str, diag: Diagnostics) -> Dict: - loglevel = request.args.get('loglevel', None) + loglevel = request.args.get("loglevel", None) notice = None if loglevel: app.logger.debug("%s %s -- requesting loglevel %s" % (what, reqid, loglevel)) - app.diag_log_level.labels('debug').set(1 if loglevel == 'debug' else 0) + app.diag_log_level.labels("debug").set(1 if loglevel == "debug" else 0) if not app.estatsmgr.update_log_levels(time.time(), level=loglevel): - notice = { 'level': 'WARNING', 'message': "Could not update log level!" } + notice = {"level": "WARNING", "message": "Could not update log level!"} # else: # return redirect("/ambassador/v0/diag/", code=302) @@ -1050,7 +1125,7 @@ def collect_errors_and_notices(request, reqid, what: str, diag: Diagnostics) -> # app.logger.debug("ddict %s" % dump_json(ddict, pretty=True)) - derrors = ddict.pop('errors', {}) + derrors = ddict.pop("errors", {}) errors = [] @@ -1059,9 +1134,9 @@ def collect_errors_and_notices(request, reqid, what: str, diag: Diagnostics) -> err_key = "" for err in err_list: - errors.append((err_key, err[ 'error' ])) + errors.append((err_key, err["error"])) - dnotices = ddict.pop('notices', {}) + dnotices = ddict.pop("notices", {}) # Make sure that anything about the loglevel gets folded into this set. if notice: @@ -1069,14 +1144,14 @@ def collect_errors_and_notices(request, reqid, what: str, diag: Diagnostics) -> for notice_key, notice_list in dnotices.items(): for notice in notice_list: - app.notices.post({'level': 'NOTICE', 'message': "%s: %s" % (notice_key, notice)}) + app.notices.post({"level": "NOTICE", "message": "%s: %s" % (notice_key, notice)}) - ddict['errors'] = errors + ddict["errors"] = errors return ddict -@app.route('/ambassador/v0/diag/', methods=[ 'GET' ]) +@app.route("/ambassador/v0/diag/", methods=["GET"]) @standard_handler def show_intermediate(source=None, reqid=None): # If we don't have an IR yet, do nothing. @@ -1086,8 +1161,10 @@ def show_intermediate(source=None, reqid=None): # the first configure is a race anyway. If it fails, that's not a big deal, # they can try again. if not app.ir: - app.logger.debug("SRC %s - can't do intermediate for %s before configuration" % (reqid, source)) - return "Can't do overview before configuration", 503 + app.logger.debug( + "SRC %s - can't do intermediate for %s before configuration" % (reqid, source) + ) + return "Can't do overview before configuration", 503 if not _allow_diag_ui(): return Response("Not found\n", 404) @@ -1100,8 +1177,8 @@ def show_intermediate(source=None, reqid=None): diag = app.diag assert diag - method = request.args.get('method', None) - resource = request.args.get('resource', None) + method = request.args.get("method", None) + resource = request.args.get("resource", None) estats = app.estatsmgr.get_stats() result = diag.lookup(request, source, estats) @@ -1113,18 +1190,18 @@ def show_intermediate(source=None, reqid=None): ddict = collect_errors_and_notices(request, reqid, "detail %s" % source, diag) tvars = { - 'system': system_info(app), - 'envoy_status': envoy_status(estats), - 'loginfo': app.estatsmgr.loginfo, - 'notices': app.notices.notices, - 'method': method, - 'resource': resource, + "system": system_info(app), + "envoy_status": envoy_status(estats), + "loginfo": app.estatsmgr.loginfo, + "notices": app.notices.notices, + "method": method, + "resource": resource, **result, **ddict, } - if request.args.get('json', None): - key = request.args.get('filter', None) + if request.args.get("json", None): + key = request.args.get("filter", None) if key: return jsonify(tvars.get(key, None)) @@ -1135,31 +1212,31 @@ def show_intermediate(source=None, reqid=None): return Response(render_template("diag.html", **tvars)) -@app.template_filter('sort_by_key') +@app.template_filter("sort_by_key") def sort_by_key(objects): - return sorted(objects, key=lambda x: x['key']) + return sorted(objects, key=lambda x: x["key"]) -@app.template_filter('pretty_json') +@app.template_filter("pretty_json") def pretty_json(obj): if isinstance(obj, dict): obj = dict(**obj) - keys_to_drop = [ key for key in obj.keys() if key.startswith('_') ] + keys_to_drop = [key for key in obj.keys() if key.startswith("_")] for key in keys_to_drop: - del(obj[key]) + del obj[key] return dump_json(obj, pretty=True) -@app.template_filter('sort_clusters_by_service') +@app.template_filter("sort_clusters_by_service") def sort_clusters_by_service(clusters): - return sorted(clusters, key=lambda x: x['service']) + return sorted(clusters, key=lambda x: x["service"]) # return sorted([ c for c in clusters.values() ], key=lambda x: x['service']) -@app.template_filter('source_lookup') +@app.template_filter("source_lookup") def source_lookup(name, sources): app.logger.debug("%s => sources %s" % (name, sources)) @@ -1167,20 +1244,20 @@ def source_lookup(name, sources): app.logger.debug("%s => source %s" % (name, source)) - return source.get('_source', name) + return source.get("_source", name) -@app.route('/metrics', methods=['GET']) +@app.route("/metrics", methods=["GET"]) @standard_handler def get_prometheus_metrics(*args, **kwargs): # Envoy metrics envoy_metrics = app.estatsmgr.get_prometheus_stats() # Ambassador OSS metrics - ambassador_metrics = generate_latest(registry=app.metrics_registry).decode('utf-8') + ambassador_metrics = generate_latest(registry=app.metrics_registry).decode("utf-8") # Extra metrics endpoint - extra_metrics_content = '' + extra_metrics_content = "" if app.metrics_endpoint and app.ir and app.ir.edge_stack_allowed: try: response = requests.get(app.metrics_endpoint) @@ -1189,12 +1266,15 @@ def get_prometheus_metrics(*args, **kwargs): except Exception as e: app.logger.error("could not get metrics_endpoint: %s" % e) - return Response(''.join([envoy_metrics, ambassador_metrics, extra_metrics_content]).encode('utf-8'), - 200, mimetype="text/plain") + return Response( + "".join([envoy_metrics, ambassador_metrics, extra_metrics_content]).encode("utf-8"), + 200, + mimetype="text/plain", + ) def bool_fmt(b: bool) -> str: - return 'T' if b else 'F' + return "T" if b else "F" class StatusInfo: @@ -1210,10 +1290,8 @@ def OK(self, message: str) -> None: self.specifics.append((True, message)) def to_dict(self) -> Dict[str, Union[bool, List[Tuple[bool, str]]]]: - return { - 'status': self.status, - 'specifics': self.specifics - } + return {"status": self.status, "specifics": self.specifics} + class SystemStatus: def __init__(self) -> None: @@ -1232,7 +1310,7 @@ def info_for_key(self, key) -> StatusInfo: return self.status[key] def to_dict(self) -> Dict[str, Dict[str, Union[bool, List[Tuple[bool, str]]]]]: - return { key: info.to_dict() for key, info in self.status.items() } + return {key: info.to_dict() for key, info in self.status.items()} class KubeStatus: @@ -1241,7 +1319,7 @@ class KubeStatus: def __init__(self, app) -> None: self.app = app self.logger = app.logger - self.live: Dict[str, bool] = {} + self.live: Dict[str, bool] = {} self.current_status: Dict[str, str] = {} self.pool = concurrent.futures.ProcessPoolExecutor(max_workers=5) @@ -1260,7 +1338,7 @@ def prune(self) -> None: for key in drop: # self.logger.debug(f"KubeStatus MASTER {os.getpid()}: prune {key}") - del(self.current_status[key]) + del self.current_status[key] self.live = {} @@ -1283,7 +1361,7 @@ def post(self, kind: str, name: str, namespace: str, text: str) -> None: # The KubeStatusNoMappings class clobbers the mark_live() method of the # KubeStatus class, so that updates to Mappings don't actually have any # effect, but updates to Ingress (for example) do. -class KubeStatusNoMappings (KubeStatus): +class KubeStatusNoMappings(KubeStatus): def mark_live(self, kind: str, name: str, namespace: str) -> None: pass @@ -1293,17 +1371,34 @@ def post(self, kind: str, name: str, namespace: str, text: str) -> None: # straight here without mark_live being involved -- so short-circuit # here for Mappings, too. - if kind == 'Mapping': + if kind == "Mapping": return super().post(kind, name, namespace, text) + def kubestatus_update(kind: str, name: str, namespace: str, text: str) -> str: - cmd = [ 'kubestatus', '--cache-dir', '/tmp/client-go-http-cache', kind, name, '-n', namespace, '-u', '/dev/fd/0' ] + cmd = [ + "kubestatus", + "--cache-dir", + "/tmp/client-go-http-cache", + kind, + name, + "-n", + namespace, + "-u", + "/dev/fd/0", + ] # print(f"KubeStatus UPDATE {os.getpid()}: running command: {cmd}") try: - rc = subprocess.run(cmd, input=text.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=5) + rc = subprocess.run( + cmd, + input=text.encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=5, + ) if rc.returncode == 0: return f"{name}.{namespace}: update OK" else: @@ -1312,26 +1407,28 @@ def kubestatus_update(kind: str, name: str, namespace: str, text: str) -> str: except subprocess.TimeoutExpired as e: return f"{name}.{namespace}: timed out\n\n{e.output}" + def kubestatus_update_done(f: concurrent.futures.Future) -> None: # print(f"KubeStatus DONE {os.getpid()}: result {f.result()}") pass + class AmbassadorEventWatcher(threading.Thread): # The key for 'Actions' is chimed - chimed_ok - env_good. This will make more sense # if you read through the _load_ir method. Actions = { - 'F-F-F': ( 'unhealthy', True ), # make sure the first chime always gets out - 'F-F-T': ( 'now-healthy', True ), # make sure the first chime always gets out - 'F-T-F': ( 'now-unhealthy', True ), # this is actually impossible - 'F-T-T': ( 'healthy', True ), # this is actually impossible - 'T-F-F': ( 'unhealthy', False ), - 'T-F-T': ( 'now-healthy', True ), - 'T-T-F': ( 'now-unhealthy', True ), - 'T-T-T': ( 'update', False ), + "F-F-F": ("unhealthy", True), # make sure the first chime always gets out + "F-F-T": ("now-healthy", True), # make sure the first chime always gets out + "F-T-F": ("now-unhealthy", True), # this is actually impossible + "F-T-T": ("healthy", True), # this is actually impossible + "T-F-F": ("unhealthy", False), + "T-F-T": ("now-healthy", True), + "T-T-F": ("now-unhealthy", True), + "T-T-T": ("update", False), } - reCompressed = re.compile(r'-\d+$') + reCompressed = re.compile(r"-\d+$") def __init__(self, app: DiagApp) -> None: super().__init__(name="AEW", daemon=True) @@ -1339,12 +1436,16 @@ def __init__(self, app: DiagApp) -> None: self.logger = self.app.logger self.events: queue.Queue = queue.Queue() - self.chimed = False # Have we ever sent a chime about the environment? - self.last_chime = False # What was the status of our last chime? (starts as False) - self.env_good = False # Is our environment currently believed to be OK? - self.failure_list: List[str] = [ 'unhealthy at boot' ] # What's making our environment not OK? + self.chimed = False # Have we ever sent a chime about the environment? + self.last_chime = False # What was the status of our last chime? (starts as False) + self.env_good = False # Is our environment currently believed to be OK? + self.failure_list: List[str] = [ + "unhealthy at boot" + ] # What's making our environment not OK? - def post(self, cmd: str, arg: Optional[Union[str, Tuple[str, Optional[IR]], Tuple[str, str]]]) -> Tuple[int, str]: + def post( + self, cmd: str, arg: Optional[Union[str, Tuple[str, Optional[IR]], Tuple[str, str]]] + ) -> Tuple[int, str]: rqueue: queue.Queue = queue.Queue() self.events.put((cmd, arg, rqueue)) @@ -1356,7 +1457,9 @@ def update_estats(self) -> None: def run(self): self.logger.info("starting Scout checker and timer logger") - self.app.scout_checker = PeriodicTrigger(lambda: self.check_scout("checkin"), period=86400) # Yup, one day. + self.app.scout_checker = PeriodicTrigger( + lambda: self.check_scout("checkin"), period=86400 + ) # Yup, one day. self.app.timer_logger = PeriodicTrigger(self.app.post_timer_event, period=1) self.logger.info("starting event watcher") @@ -1365,36 +1468,36 @@ def run(self): cmd, arg, rqueue = self.events.get() # self.logger.info("EVENT: %s" % cmd) - if cmd == 'CONFIG_FS': + if cmd == "CONFIG_FS": try: self.load_config_fs(rqueue, arg) except Exception as e: self.logger.error("could not reconfigure: %s" % e) self.logger.exception(e) - self._respond(rqueue, 500, 'configuration from filesystem failed') - elif cmd == 'CONFIG': + self._respond(rqueue, 500, "configuration from filesystem failed") + elif cmd == "CONFIG": version, url = arg try: - if version == 'watt': + if version == "watt": self.load_config_watt(rqueue, url) else: raise RuntimeError("config from %s not supported" % version) except Exception as e: self.logger.error("could not reconfigure: %s" % e) self.logger.exception(e) - self._respond(rqueue, 500, 'configuration failed') - elif cmd == 'SCOUT': + self._respond(rqueue, 500, "configuration failed") + elif cmd == "SCOUT": try: - self._respond(rqueue, 200, 'checking Scout') + self._respond(rqueue, 200, "checking Scout") self.check_scout(*arg) except Exception as e: self.logger.error("could not reconfigure: %s" % e) self.logger.exception(e) - self._respond(rqueue, 500, 'scout check failed') - elif cmd == 'TIMER': + self._respond(rqueue, 500, "scout check failed") + elif cmd == "TIMER": try: - self._respond(rqueue, 200, 'done') + self._respond(rqueue, 200, "done") self.app.check_timers() except Exception as e: self.logger.error("could not check timers? %s" % e) @@ -1403,7 +1506,7 @@ def run(self): self.logger.error(f"unknown event type: '{cmd}' '{arg}'") self._respond(rqueue, 400, f"unknown event type '{cmd}' '{arg}'") - def _respond(self, rqueue: queue.Queue, status: int, info='') -> None: + def _respond(self, rqueue: queue.Queue, status: int, info="") -> None: # self.logger.debug("responding to query with %s %s" % (status, info)) rqueue.put((status, info)) @@ -1417,23 +1520,23 @@ def load_config_fs(self, rqueue: queue.Queue, path: str) -> None: # The "path" here can just be a path, but it can also be a command for testing, # if the user has chosen to allow that. - if self.app.allow_fs_commands and (':' in path): - pfx, rest = path.split(':', 1) + if self.app.allow_fs_commands and (":" in path): + pfx, rest = path.split(":", 1) - if pfx.lower() == 'cmd': - fields = rest.split(':', 1) + if pfx.lower() == "cmd": + fields = rest.split(":", 1) cmd = fields[0].upper() args = fields[1:] if (len(fields) > 1) else None - if cmd.upper() == 'CHIME': - self.logger.info('CMD: Chiming') + if cmd.upper() == "CHIME": + self.logger.info("CMD: Chiming") self.chime() - self._respond(rqueue, 200, 'Chimed') - elif cmd.upper() == 'CHIME_RESET': + self._respond(rqueue, 200, "Chimed") + elif cmd.upper() == "CHIME_RESET": self.chimed = False self.last_chime = False self.env_good = False @@ -1441,25 +1544,25 @@ def load_config_fs(self, rqueue: queue.Queue, path: str) -> None: self.app.scout.reset_events() self.app.scout.report(mode="boot", action="boot1", no_cache=True) - self.logger.info('CMD: Reset chime state') - self._respond(rqueue, 200, 'CMD: Reset chime state') - elif cmd.upper() == 'SCOUT_CACHE_RESET': + self.logger.info("CMD: Reset chime state") + self._respond(rqueue, 200, "CMD: Reset chime state") + elif cmd.upper() == "SCOUT_CACHE_RESET": self.app.scout.reset_cache_time() - self.logger.info('CMD: Reset Scout cache time') - self._respond(rqueue, 200, 'CMD: Reset Scout cache time') - elif cmd.upper() == 'ENV_OK': + self.logger.info("CMD: Reset Scout cache time") + self._respond(rqueue, 200, "CMD: Reset Scout cache time") + elif cmd.upper() == "ENV_OK": self.env_good = True self.failure_list = [] - self.logger.info('CMD: Marked environment good') - self._respond(rqueue, 200, 'CMD: Marked environment good') - elif cmd.upper() == 'ENV_BAD': + self.logger.info("CMD: Marked environment good") + self._respond(rqueue, 200, "CMD: Marked environment good") + elif cmd.upper() == "ENV_BAD": self.env_good = False - self.failure_list = [ 'failure forced' ] + self.failure_list = ["failure forced"] - self.logger.info('CMD: Marked environment bad') - self._respond(rqueue, 200, 'CMD: Marked environment bad') + self.logger.info("CMD: Marked environment bad") + self._respond(rqueue, 200, "CMD: Marked environment bad") else: self.logger.info(f'CMD: no such command "{cmd}"') self._respond(rqueue, 400, f'CMD: no such command "{cmd}"') @@ -1475,7 +1578,7 @@ def load_config_fs(self, rqueue: queue.Queue, path: str) -> None: # BEFORE YOU RESPOND TO THE CALLER. self.app.config_timer.start() - snapshot = re.sub(r'[^A-Za-z0-9_-]', '_', path) + snapshot = re.sub(r"[^A-Za-z0-9_-]", "_", path) scc = FSSecretHandler(app.logger, path, app.snapshot_path, "0") with self.app.fetcher_timer: @@ -1497,7 +1600,7 @@ def load_config_fs(self, rqueue: queue.Queue, path: str) -> None: # # BE CAREFUL ABOUT STOPPING THE RECONFIGURATION TIMER ONCE IT IS STARTED. def load_config_watt(self, rqueue: queue.Queue, url: str): - snapshot = url.split('/')[-1] + snapshot = url.split("/")[-1] ss_path = os.path.join(app.snapshot_path, "snapshot-tmp.yaml") # OK, we're starting a reconfiguration. BE CAREFUL TO STOP THE TIMER @@ -1542,8 +1645,14 @@ def load_config_watt(self, rqueue: queue.Queue, url: str): # # AT THE POINT OF ENTRY, THE RECONFIGURATION TIMER IS RUNNING. DO NOT LEAVE # THIS METHOD WITHOUT STOPPING THE RECONFIGURATION TIMER. - def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, - secret_handler: SecretHandler, snapshot: str) -> None: + def _load_ir( + self, + rqueue: queue.Queue, + aconf: Config, + fetcher: ResourceFetcher, + secret_handler: SecretHandler, + snapshot: str, + ) -> None: with self.app.aconf_timer: aconf.load_all(fetcher.sorted()) @@ -1558,15 +1667,21 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, open(aconf_path, "w").write(aconf.as_json()) # OK. What kind of reconfiguration are we doing? - config_type, reset_cache, invalidate_groups_for = IR.check_deltas(self.logger, fetcher, self.app.cache) + config_type, reset_cache, invalidate_groups_for = IR.check_deltas( + self.logger, fetcher, self.app.cache + ) if reset_cache: self.logger.debug("RESETTING CACHE") self.app.cache = Cache(self.logger) with self.app.ir_timer: - ir = IR(aconf, secret_handler=secret_handler, - invalidate_groups_for=invalidate_groups_for, cache=self.app.cache) + ir = IR( + aconf, + secret_handler=secret_handler, + invalidate_groups_for=invalidate_groups_for, + cache=self.app.cache, + ) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) @@ -1594,8 +1709,12 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, # # As it happens, Envoy is OK running a config with no listeners, and it'll # answer on port 8001 for readiness checks, so... log a notice, but run with it. - self.logger.warning("No active listeners at all; check your Listener and Host configuration") - elif not self.validate_envoy_config(ir, config=ads_config, retries=self.app.validation_retries): + self.logger.warning( + "No active listeners at all; check your Listener and Host configuration" + ) + elif not self.validate_envoy_config( + ir, config=ads_config, retries=self.app.validation_retries + ): # Invalid Envoy config probably indicates a bug in Emissary itself. Sigh. econf_is_valid = False econf_bad_reason = "invalid envoy configuration generated" @@ -1603,17 +1722,22 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, # OK. Is the config invalid? if not econf_is_valid: # BZzzt. Don't post this update. - self.logger.info("no update performed (%s), continuing with current configuration..." % econf_bad_reason) + self.logger.info( + "no update performed (%s), continuing with current configuration..." + % econf_bad_reason + ) # Don't use app.check_scout; it will deadlock. self.check_scout("attempted bad update") # DO stop the reconfiguration timer before leaving. self.app.config_timer.stop() - self._respond(rqueue, 500, 'ignoring (%s) in snapshot %s' % (econf_bad_reason, snapshot)) + self._respond( + rqueue, 500, "ignoring (%s) in snapshot %s" % (econf_bad_reason, snapshot) + ) return - snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4")) + snapcount = int(os.environ.get("AMBASSADOR_SNAPSHOT_COUNT", "4")) snaplist: List[Tuple[str, str]] = [] if snapcount > 0: @@ -1624,16 +1748,16 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]... # which is the list of suffixes to rename to rotate the snapshots. - snaplist += [ (str(x+1), str(x)) for x in range(-1 * snapcount, -1) ] + snaplist += [(str(x + 1), str(x)) for x in range(-1 * snapcount, -1)] # After dealing with that, we need to rotate the current file into -1. - snaplist.append(( '', '-1' )) + snaplist.append(("", "-1")) # Whether or not we do any rotation, we need to cycle in the '-tmp' file. - snaplist.append(( '-tmp', '' )) + snaplist.append(("-tmp", "")) for from_suffix, to_suffix in snaplist: - for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: + for fmt in ["aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml"]: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) @@ -1681,11 +1805,13 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, os.kill(app.ambex_pid, signal.SIGHUP) # don't worry about TCPMappings yet - mappings = app.aconf.get_config('mappings') + mappings = app.aconf.get_config("mappings") if mappings: for mapping_name, mapping in mappings.items(): - app.kubestatus.mark_live("Mapping", mapping_name, mapping.get('namespace', Config.ambassador_namespace)) + app.kubestatus.mark_live( + "Mapping", mapping_name, mapping.get("namespace", Config.ambassador_namespace) + ) app.kubestatus.prune() @@ -1695,7 +1821,7 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, for name in app.ir.k8s_status_updates.keys(): update_count += 1 # Strip off any namespace in the name. - resource_name = name.split('.', 1)[0] + resource_name = name.split(".", 1)[0] kind, namespace, update = app.ir.k8s_status_updates[name] text = dump_json(update) @@ -1703,17 +1829,19 @@ def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, app.kubestatus.post(kind, resource_name, namespace, text) - group_count = len(app.ir.groups) cluster_count = len(app.ir.clusters) listener_count = len(app.ir.listeners) service_count = len(app.ir.services) - self._respond(rqueue, 200, - 'configuration updated (%s) from snapshot %s' % (config_type, snapshot)) + self._respond( + rqueue, 200, "configuration updated (%s) from snapshot %s" % (config_type, snapshot) + ) - self.logger.info("configuration updated (%s) from snapshot %s (S%d L%d G%d C%d)" % - (config_type, snapshot, service_count, listener_count, group_count, cluster_count)) + self.logger.info( + "configuration updated (%s) from snapshot %s (S%d L%d G%d C%d)" + % (config_type, snapshot, service_count, listener_count, group_count, cluster_count) + ) # Remember that we've reconfigured. self.app.reconf_stats.mark(config_type) @@ -1737,10 +1865,10 @@ def chime(self): now_ok = bool_fmt(self.env_good) # Poor man's state machine... - action_key = f'{already_chimed}-{was_ok}-{now_ok}' + action_key = f"{already_chimed}-{was_ok}-{now_ok}" action, no_cache = AmbassadorEventWatcher.Actions[action_key] - self.logger.debug(f'CHIME: {action_key}') + self.logger.debug(f"CHIME: {action_key}") class ChimeArgs(TypedDict): no_cache: NotRequired[Optional[bool]] @@ -1748,13 +1876,10 @@ class ChimeArgs(TypedDict): failures: NotRequired[Optional[List[str]]] action_key: NotRequired[Optional[str]] - chime_args: ChimeArgs = { - 'no_cache': no_cache, - 'failures': self.failure_list - } + chime_args: ChimeArgs = {"no_cache": no_cache, "failures": self.failure_list} if self.app.report_action_keys: - chime_args['action_key'] = action_key + chime_args["action_key"] = action_key # Don't use app.check_scout; it will deadlock. self.check_scout(action, **chime_args) @@ -1765,7 +1890,7 @@ class ChimeArgs(TypedDict): # ...and remember what we sent for that chime. self.last_chime = self.env_good - def check_environment(self, ir: Optional[IR]=None) -> None: + def check_environment(self, ir: Optional[IR] = None) -> None: env_good = True chime_failures = {} env_status = SystemStatus() @@ -1778,11 +1903,11 @@ def check_environment(self, ir: Optional[IR]=None) -> None: ir = app.ir if not ir: - chime_failures['no config loaded'] = True + chime_failures["no config loaded"] = True env_good = False else: if not ir.aconf: - chime_failures['completely empty config'] = True + chime_failures["completely empty config"] = True env_good = False else: for err_key, err_list in ir.aconf.errors.items(): @@ -1791,22 +1916,24 @@ def check_environment(self, ir: Optional[IR]=None) -> None: for err in err_list: error_count += 1 - err_text = err['error'] + err_text = err["error"] - self.app.logger.info(f'error {err_key} {err_text}') + self.app.logger.info(f"error {err_key} {err_text}") - if err_text.find('CRD') >= 0: - if err_text.find('core') >= 0: - chime_failures['core CRDs'] = True + if err_text.find("CRD") >= 0: + if err_text.find("core") >= 0: + chime_failures["core CRDs"] = True env_status.failure("CRDs", "Core CRD type definitions are missing") else: - chime_failures['other CRDs'] = True - env_status.failure("CRDs", "Resolver CRD type definitions are missing") + chime_failures["other CRDs"] = True + env_status.failure( + "CRDs", "Resolver CRD type definitions are missing" + ) env_good = False - elif err_text.find('TLS') >= 0: - chime_failures['TLS errors'] = True - env_status.failure('TLS', err_text) + elif err_text.find("TLS") >= 0: + chime_failures["TLS errors"] = True + env_status.failure("TLS", err_text) env_good = False @@ -1817,32 +1944,40 @@ def check_environment(self, ir: Optional[IR]=None) -> None: for group in ir.groups.values(): for mapping in group.mappings: - pfx = mapping.get('prefix', None) - name = mapping.get('name', None) + pfx = mapping.get("prefix", None) + name = mapping.get("name", None) if pfx: - if not pfx.startswith('/ambassador/v0') or not name.startswith('internal_'): + if not pfx.startswith("/ambassador/v0") or not name.startswith("internal_"): mapping_count += 1 if error_count: - env_status.failure('Error check', f'{error_count} total error{"" if (error_count == 1) else "s"} logged') + env_status.failure( + "Error check", + f'{error_count} total error{"" if (error_count == 1) else "s"} logged', + ) env_good = False else: - env_status.OK('Error check', "No errors logged") + env_status.OK("Error check", "No errors logged") if tls_count: - env_status.OK('TLS', f'{tls_count} TLSContext{" is" if (tls_count == 1) else "s are"} active') + env_status.OK( + "TLS", f'{tls_count} TLSContext{" is" if (tls_count == 1) else "s are"} active' + ) else: - chime_failures['no TLS contexts'] = True - env_status.failure('TLS', "No TLSContexts are active") + chime_failures["no TLS contexts"] = True + env_status.failure("TLS", "No TLSContexts are active") env_good = False if mapping_count: - env_status.OK('Mappings', f'{mapping_count} Mapping{" is" if (mapping_count == 1) else "s are"} active') + env_status.OK( + "Mappings", + f'{mapping_count} Mapping{" is" if (mapping_count == 1) else "s are"} active', + ) else: - chime_failures['no Mappings'] = True - env_status.failure('Mappings', "No Mappings are active") + chime_failures["no Mappings"] = True + env_status.failure("Mappings", "No Mappings are active") env_good = False failure_list: List[str] = [] @@ -1854,9 +1989,14 @@ def check_environment(self, ir: Optional[IR]=None) -> None: self.env_status = env_status self.failure_list = failure_list - def check_scout(self, what: str, no_cache: Optional[bool]=False, - ir: Optional[IR]=None, failures: Optional[List[str]]=None, - action_key: Optional[str]=None) -> None: + def check_scout( + self, + what: str, + no_cache: Optional[bool] = False, + ir: Optional[IR] = None, + failures: Optional[List[str]] = None, + action_key: Optional[str] = None, + ) -> None: now = datetime.datetime.now() uptime = now - boot_time hr_uptime = td_format(uptime) @@ -1866,16 +2006,13 @@ def check_scout(self, what: str, no_cache: Optional[bool]=False, self.app.notices.reset() - scout_args = { - "uptime": int(uptime.total_seconds()), - "hr_uptime": hr_uptime - } + scout_args = {"uptime": int(uptime.total_seconds()), "hr_uptime": hr_uptime} if failures: - scout_args['failures'] = failures + scout_args["failures"] = failures if action_key: - scout_args['action_key'] = action_key + scout_args["action_key"] = action_key if ir: self.app.logger.debug("check_scout: we have an IR") @@ -1889,20 +2026,20 @@ def check_scout(self, what: str, no_cache: Optional[bool]=False, if self.app.cache is not None: # Fast reconfigure is on. Supply the real info. - feat['frc_enabled'] = True - feat['frc_cache_hits'] = self.app.cache.hits - feat['frc_cache_misses'] = self.app.cache.misses - feat['frc_inv_calls'] = self.app.cache.invalidate_calls - feat['frc_inv_objects'] = self.app.cache.invalidated_objects + feat["frc_enabled"] = True + feat["frc_cache_hits"] = self.app.cache.hits + feat["frc_cache_misses"] = self.app.cache.misses + feat["frc_inv_calls"] = self.app.cache.invalidate_calls + feat["frc_inv_objects"] = self.app.cache.invalidated_objects else: # Fast reconfigure is off. - feat['frc_enabled'] = False + feat["frc_enabled"] = False # Whether the cache is on or off, we can talk about reconfigurations. - feat['frc_incr_count'] = self.app.reconf_stats.counts["incremental"] - feat['frc_complete_count'] = self.app.reconf_stats.counts["complete"] - feat['frc_check_count'] = self.app.reconf_stats.checks - feat['frc_check_errors'] = self.app.reconf_stats.errors + feat["frc_incr_count"] = self.app.reconf_stats.counts["incremental"] + feat["frc_complete_count"] = self.app.reconf_stats.counts["complete"] + feat["frc_check_count"] = self.app.reconf_stats.checks + feat["frc_check_errors"] = self.app.reconf_stats.errors request_data = app.estatsmgr.get_stats().requests @@ -1912,7 +2049,7 @@ def check_scout(self, what: str, no_cache: Optional[bool]=False, for rkey in request_data.keys(): cur = request_data[rkey] prev = app.last_request_info.get(rkey, 0) - feat[f'request_{rkey}_count'] = max(cur - prev, 0) + feat[f"request_{rkey}_count"] = max(cur - prev, 0) lrt = app.last_request_time or boot_time since_lrt = now - lrt @@ -1922,27 +2059,29 @@ def check_scout(self, what: str, no_cache: Optional[bool]=False, app.last_request_time = now app.last_request_info = request_data - feat['request_elapsed'] = elapsed - feat['request_hr_elapsed'] = hr_elapsed + feat["request_elapsed"] = elapsed + feat["request_hr_elapsed"] = hr_elapsed scout_args["features"] = feat - scout_result = self.app.scout.report(mode="diagd", action=what, no_cache=no_cache, **scout_args) - scout_notices = scout_result.pop('notices', []) + scout_result = self.app.scout.report( + mode="diagd", action=what, no_cache=no_cache, **scout_args + ) + scout_notices = scout_result.pop("notices", []) global_loglevel = self.app.logger.getEffectiveLevel() - self.app.logger.debug(f'Scout section: global loglevel {global_loglevel}') + self.app.logger.debug(f"Scout section: global loglevel {global_loglevel}") for notice in scout_notices: - notice_level_name = notice.get('level') or 'INFO' + notice_level_name = notice.get("level") or "INFO" notice_level = logging.getLevelName(notice_level_name) if notice_level >= global_loglevel: - self.app.logger.debug(f'Scout section: include {notice}') + self.app.logger.debug(f"Scout section: include {notice}") self.app.notices.post(notice) else: - self.app.logger.debug(f'Scout section: skip {notice}') + self.app.logger.debug(f"Scout section: skip {notice}") self.app.logger.debug("Scout reports %s" % dump_json(scout_result)) self.app.logger.debug("Scout notices: %s" % dump_json(scout_notices)) @@ -1957,10 +2096,10 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: validation_config = copy.deepcopy(config) # Envoy fails to validate with @type field in envoy config, so removing that - validation_config.pop('@type') + validation_config.pop("@type") if os.environ.get("AMBASSADOR_DEBUG_CLUSTER_CONFIG", "false").lower() == "true": - vconf_clusters = validation_config['static_resources']['clusters'] + vconf_clusters = validation_config["static_resources"]["clusters"] if len(vconf_clusters) > 10: vconf_clusters.append(copy.deepcopy(vconf_clusters[10])) @@ -1973,8 +2112,8 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: if AmbassadorEventWatcher.reCompressed.search(name): _problems.append(f"IR pre-compressed cluster {name}") - for cluster in validation_config['static_resources']['clusters']: - name = cluster['name'] + for cluster in validation_config["static_resources"]["clusters"]: + name = cluster["name"] if name in _v2_clusters: _problems.append(f"V2 dup cluster {name}") @@ -1984,7 +2123,9 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: self.logger.error("ENVOY CONFIG PROBLEMS:\n%s", "\n".join(_problems)) stamp = datetime.datetime.now().isoformat() - bad_snapshot = open(os.path.join(app.snapshot_path, "snapshot-tmp.yaml"), "r").read() + bad_snapshot = open( + os.path.join(app.snapshot_path, "snapshot-tmp.yaml"), "r" + ).read() cache_dict: Dict[str, Any] = {} cache_links: Dict[str, Any] = {} @@ -1993,12 +2134,12 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: for k, c in self.app.cache.cache.items(): v: Any = c[0] - if getattr(v, 'as_dict', None): + if getattr(v, "as_dict", None): v = v.as_dict() cache_dict[k] = v - cache_links = { k: list(v) for k, v in self.app.cache.links.items() } + cache_links = {k: list(v) for k, v in self.app.cache.links.items()} bad_dict = { "ir": ir.as_dict(), @@ -2007,7 +2148,7 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: "problems": _problems, "snapshot": bad_snapshot, "cache": cache_dict, - "links": cache_links + "links": cache_links, } bad_dict_str = dump_json(bad_dict, pretty=True) @@ -2021,10 +2162,20 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: with open(econf_validation_path, "w") as output: output.write(config_json) - command = ['envoy', '--service-node', 'test-id', '--service-cluster', ir.ambassador_nodename, '--config-path', econf_validation_path, '--mode', 'validate'] + command = [ + "envoy", + "--service-node", + "test-id", + "--service-cluster", + ir.ambassador_nodename, + "--config-path", + econf_validation_path, + "--mode", + "validate", + ] v_exit = 0 - v_encoded = ''.encode('utf-8') + v_encoded = "".encode("utf-8") # Try to validate the Envoy config. Short circuit and fall through # immediately on concrete success or failure, and retry (up to the @@ -2045,7 +2196,9 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: for retry in range(retries): try: - v_encoded = subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout) + v_encoded = subprocess.check_output( + command, stderr=subprocess.STDOUT, timeout=timeout + ) v_exit = 0 break except subprocess.CalledProcessError as e: @@ -2054,26 +2207,36 @@ def validate_envoy_config(self, ir: IR, config, retries) -> bool: break except subprocess.TimeoutExpired as e: v_exit = 1 - v_encoded = e.output or ''.encode('utf-8') - - self.logger.warn("envoy configuration validation timed out after {} seconds{}\n{}".format( - timeout,', retrying...' if retry < retries - 1 else '', v_encoded.decode('utf-8')) + v_encoded = e.output or "".encode("utf-8") + + self.logger.warn( + "envoy configuration validation timed out after {} seconds{}\n{}".format( + timeout, + ", retrying..." if retry < retries - 1 else "", + v_encoded.decode("utf-8"), + ) ) # Don't break here; continue on to the next iteration of the loop. if v_exit == 0: - self.logger.debug("successfully validated the resulting envoy configuration, continuing...") + self.logger.debug( + "successfully validated the resulting envoy configuration, continuing..." + ) return True v_str = typecast(str, v_encoded) try: - v_str = v_encoded.decode('utf-8') + v_str = v_encoded.decode("utf-8") except: pass - self.logger.error("{}\ncould not validate the envoy configuration above after {} retries, failed with error \n{}\n(exit code {})\nAborting update...".format(config_json, retries, v_str, v_exit)) + self.logger.error( + "{}\ncould not validate the envoy configuration above after {} retries, failed with error \n{}\n(exit code {})\nAborting update...".format( + config_json, retries, v_str, v_exit + ) + ) return False @@ -2086,12 +2249,17 @@ def __init__(self, app, options=None): # Boot chime. This is basically the earliest point at which we can consider an Ambassador # to be "running". scout_result = self.application.scout.report(mode="boot", action="boot1", no_cache=True) - self.application.logger.debug(f'BOOT: Scout result {dump_json(scout_result)}') - self.application.logger.info(f'Ambassador {__version__} booted') + self.application.logger.debug(f"BOOT: Scout result {dump_json(scout_result)}") + self.application.logger.info(f"Ambassador {__version__} booted") def load_config(self): - config = dict([ (key, value) for key, value in self.options.items() - if key in self.cfg.settings and value is not None ]) + config = dict( + [ + (key, value) + for key, value in self.options.items() + if key in self.cfg.settings and value is not None + ] + ) for key, value in config.items(): self.cfg.set(key.lower(), value) @@ -2108,36 +2276,101 @@ def load(self): @click.command() -@click.argument('snapshot-path', type=click.Path(), required=False) -@click.argument('bootstrap-path', type=click.Path(), required=False) -@click.argument('ads-path', type=click.Path(), required=False) -@click.option('--config-path', type=click.Path(), help="Optional configuration path to scan for Ambassador YAML files") -@click.option('--k8s', is_flag=True, help="If True, assume config_path contains Kubernetes resources (only relevant with config_path)") -@click.option('--ambex-pid', type=int, default=0, help="Optional PID to signal with HUP after updating Envoy configuration", show_default=True) -@click.option('--kick', type=str, help="Optional command to run after updating Envoy configuration") -@click.option('--banner-endpoint', type=str, default="http://127.0.0.1:8500/banner", help="Optional endpoint of extra banner to include", show_default=True) -@click.option('--metrics-endpoint', type=str, default="http://127.0.0.1:8500/metrics", help="Optional endpoint of extra prometheus metrics to include", show_default=True) -@click.option('--no-checks', is_flag=True, help="If True, don't do Envoy-cluster health checking") -@click.option('--no-envoy', is_flag=True, help="If True, don't interact with Envoy at all") -@click.option('--reload', is_flag=True, help="If True, run Flask in debug mode for live reloading") -@click.option('--debug', is_flag=True, help="If True, do debug logging") -@click.option('--dev-magic', is_flag=True, help="If True, override a bunch of things for Datawire dev-loop stuff") -@click.option('--verbose', is_flag=True, help="If True, do really verbose debug logging") -@click.option('--workers', type=int, help="Number of workers; default is based on the number of CPUs present") -@click.option('--host', type=str, help="Interface on which to listen") -@click.option('--port', type=int, default=-1, help="Port on which to listen", show_default=True) -@click.option('--notices', type=click.Path(), help="Optional file to read for local notices") -@click.option('--validation-retries', type=int, default=5, help="Number of times to retry Envoy configuration validation after a timeout", show_default=True) -@click.option('--allow-fs-commands', is_flag=True, help="If true, allow CONFIG_FS to support debug/testing commands") -@click.option('--local-scout', is_flag=True, help="Don't talk to remote Scout at all; keep everything purely local") -@click.option('--report-action-keys', is_flag=True, help="Report action keys when chiming") -def main(snapshot_path=None, bootstrap_path=None, ads_path=None, - *, dev_magic=False, config_path=None, ambex_pid=0, kick=None, - banner_endpoint="http://127.0.0.1:8500/banner", metrics_endpoint="http://127.0.0.1:8500/metrics", k8s=False, - no_checks=False, no_envoy=False, reload=False, debug=False, verbose=False, - workers=None, port=-1, host="", notices=None, - validation_retries=5, allow_fs_commands=False, local_scout=False, - report_action_keys=False): +@click.argument("snapshot-path", type=click.Path(), required=False) +@click.argument("bootstrap-path", type=click.Path(), required=False) +@click.argument("ads-path", type=click.Path(), required=False) +@click.option( + "--config-path", + type=click.Path(), + help="Optional configuration path to scan for Ambassador YAML files", +) +@click.option( + "--k8s", + is_flag=True, + help="If True, assume config_path contains Kubernetes resources (only relevant with config_path)", +) +@click.option( + "--ambex-pid", + type=int, + default=0, + help="Optional PID to signal with HUP after updating Envoy configuration", + show_default=True, +) +@click.option("--kick", type=str, help="Optional command to run after updating Envoy configuration") +@click.option( + "--banner-endpoint", + type=str, + default="http://127.0.0.1:8500/banner", + help="Optional endpoint of extra banner to include", + show_default=True, +) +@click.option( + "--metrics-endpoint", + type=str, + default="http://127.0.0.1:8500/metrics", + help="Optional endpoint of extra prometheus metrics to include", + show_default=True, +) +@click.option("--no-checks", is_flag=True, help="If True, don't do Envoy-cluster health checking") +@click.option("--no-envoy", is_flag=True, help="If True, don't interact with Envoy at all") +@click.option("--reload", is_flag=True, help="If True, run Flask in debug mode for live reloading") +@click.option("--debug", is_flag=True, help="If True, do debug logging") +@click.option( + "--dev-magic", + is_flag=True, + help="If True, override a bunch of things for Datawire dev-loop stuff", +) +@click.option("--verbose", is_flag=True, help="If True, do really verbose debug logging") +@click.option( + "--workers", type=int, help="Number of workers; default is based on the number of CPUs present" +) +@click.option("--host", type=str, help="Interface on which to listen") +@click.option("--port", type=int, default=-1, help="Port on which to listen", show_default=True) +@click.option("--notices", type=click.Path(), help="Optional file to read for local notices") +@click.option( + "--validation-retries", + type=int, + default=5, + help="Number of times to retry Envoy configuration validation after a timeout", + show_default=True, +) +@click.option( + "--allow-fs-commands", + is_flag=True, + help="If true, allow CONFIG_FS to support debug/testing commands", +) +@click.option( + "--local-scout", + is_flag=True, + help="Don't talk to remote Scout at all; keep everything purely local", +) +@click.option("--report-action-keys", is_flag=True, help="Report action keys when chiming") +def main( + snapshot_path=None, + bootstrap_path=None, + ads_path=None, + *, + dev_magic=False, + config_path=None, + ambex_pid=0, + kick=None, + banner_endpoint="http://127.0.0.1:8500/banner", + metrics_endpoint="http://127.0.0.1:8500/metrics", + k8s=False, + no_checks=False, + no_envoy=False, + reload=False, + debug=False, + verbose=False, + workers=None, + port=-1, + host="", + notices=None, + validation_retries=5, + allow_fs_commands=False, + local_scout=False, + report_action_keys=False, +): """ Run the diagnostic daemon. @@ -2160,21 +2393,21 @@ def main(snapshot_path=None, bootstrap_path=None, ads_path=None, # port = Constants.DIAG_PORT if not host: - host = '0.0.0.0' if not enable_fast_reconfigure else '127.0.0.1' + host = "0.0.0.0" if not enable_fast_reconfigure else "127.0.0.1" if dev_magic: # Override the world. - os.environ['SCOUT_HOST'] = '127.0.0.1:9999' - os.environ['SCOUT_HTTPS'] = 'no' + os.environ["SCOUT_HOST"] = "127.0.0.1:9999" + os.environ["SCOUT_HTTPS"] = "no" no_checks = True no_envoy = True - os.makedirs('/tmp/snapshots', mode=0o755, exist_ok=True) + os.makedirs("/tmp/snapshots", mode=0o755, exist_ok=True) - snapshot_path = '/tmp/snapshots' - bootstrap_path = '/tmp/boot.json' - ads_path = '/tmp/ads.json' + snapshot_path = "/tmp/snapshots" + bootstrap_path = "/tmp/boot.json" + ads_path = "/tmp/ads.json" port = 9998 @@ -2186,21 +2419,41 @@ def main(snapshot_path=None, bootstrap_path=None, ads_path=None, no_checks = True # Create the application itself. - app.setup(snapshot_path, bootstrap_path, ads_path, config_path, ambex_pid, kick, banner_endpoint, - metrics_endpoint, k8s, not no_checks, no_envoy, reload, debug, verbose, notices, - validation_retries, allow_fs_commands, local_scout, report_action_keys, - enable_fast_reconfigure) + app.setup( + snapshot_path, + bootstrap_path, + ads_path, + config_path, + ambex_pid, + kick, + banner_endpoint, + metrics_endpoint, + k8s, + not no_checks, + no_envoy, + reload, + debug, + verbose, + notices, + validation_retries, + allow_fs_commands, + local_scout, + report_action_keys, + enable_fast_reconfigure, + ) if not workers: workers = number_of_workers() gunicorn_config = { - 'bind': '%s:%s' % (host, port), + "bind": "%s:%s" % (host, port), # 'workers': 1, - 'threads': workers, + "threads": workers, } - app.logger.info("thread count %d, listening on %s" % (gunicorn_config['threads'], gunicorn_config['bind'])) + app.logger.info( + "thread count %d, listening on %s" % (gunicorn_config["threads"], gunicorn_config["bind"]) + ) StandaloneApplication(app, gunicorn_config).run() diff --git a/python/kat/harness.py b/python/kat/harness.py index 698a73a3b8..8bcadafe5b 100755 --- a/python/kat/harness.py +++ b/python/kat/harness.py @@ -26,7 +26,12 @@ import tests.integration.manifests as integration_manifests from .parser import dump, load, Tag, SequenceView -from tests.manifests import httpbin_manifests, websocket_echo_server_manifests, cleartext_host_manifest, default_listener_manifest +from tests.manifests import ( + httpbin_manifests, + websocket_echo_server_manifests, + cleartext_host_manifest, + default_listener_manifest, +) from tests.kubeutils import apply_kube_artifacts import yaml as pyyaml @@ -42,10 +47,10 @@ # Run mode can be local (don't do any Envoy stuff), envoy (only do Envoy stuff), # or all (allow both). Default is all. -RUN_MODE = os.environ.get('KAT_RUN_MODE', 'all').lower() +RUN_MODE = os.environ.get("KAT_RUN_MODE", "all").lower() # We may have a SOURCE_ROOT override from the environment -SOURCE_ROOT = os.environ.get('SOURCE_ROOT', '') +SOURCE_ROOT = os.environ.get("SOURCE_ROOT", "") # Figure out if we're running in Edge Stack or what. if os.path.exists("/buildroot/apro.version"): @@ -56,7 +61,7 @@ # If we do not see concrete evidence of running in an apro builder shell, # then try to decide if the user wants us to assume we're running Edge Stack # from an environment variable. And if that isn't set, just assume OSS. - EDGE_STACK = parse_bool(os.environ.get('EDGE_STACK', 'false')) + EDGE_STACK = parse_bool(os.environ.get("EDGE_STACK", "false")) if EDGE_STACK: # Hey look, we're running inside Edge Stack! @@ -83,7 +88,9 @@ def run(cmd): def kube_version_json(): - result = subprocess.Popen('tools/bin/kubectl version -o json', stdout=subprocess.PIPE, shell=True) + result = subprocess.Popen( + "tools/bin/kubectl version -o json", stdout=subprocess.PIPE, shell=True + ) stdout, _ = result.communicate() return json.loads(stdout) @@ -101,7 +108,7 @@ def strip_version(ver: str): return int(ver) except ValueError as e: # GKE returns weird versions with '+' in the end - if ver[-1] == '+': + if ver[-1] == "+": return int(ver[:-1]) # If we still have not taken care of this, raise the error @@ -112,11 +119,11 @@ def kube_server_version(version_json=None): if not version_json: version_json = kube_version_json() - server_json = version_json.get('serverVersion', {}) + server_json = version_json.get("serverVersion", {}) if server_json: - server_major = strip_version(server_json.get('major', None)) - server_minor = strip_version(server_json.get('minor', None)) + server_major = strip_version(server_json.get("major", None)) + server_minor = strip_version(server_json.get("minor", None)) return f"{server_major}.{server_minor}" else: @@ -127,18 +134,20 @@ def kube_client_version(version_json=None): if not version_json: version_json = kube_version_json() - client_json = version_json.get('clientVersion', {}) + client_json = version_json.get("clientVersion", {}) if client_json: - client_major = strip_version(client_json.get('major', None)) - client_minor = strip_version(client_json.get('minor', None)) + client_major = strip_version(client_json.get("major", None)) + client_minor = strip_version(client_json.get("minor", None)) return f"{client_major}.{client_minor}" else: return None -def is_kube_server_client_compatible(debug_desc: str, requested_server_version: str, requested_client_version: str) -> bool: +def is_kube_server_client_compatible( + debug_desc: str, requested_server_version: str, requested_client_version: str +) -> bool: is_cluster_compatible = True kube_json = kube_version_json() @@ -167,20 +176,20 @@ def is_kube_server_client_compatible(debug_desc: str, requested_server_version: def is_ingress_class_compatible() -> bool: - return is_kube_server_client_compatible('IngressClass', '1.18', '1.14') + return is_kube_server_client_compatible("IngressClass", "1.18", "1.14") def is_knative_compatible() -> bool: # Skip KNative immediately for run_mode local. - if RUN_MODE == 'local': + if RUN_MODE == "local": return False - return is_kube_server_client_compatible('Knative', '1.14', '1.14') + return is_kube_server_client_compatible("Knative", "1.14", "1.14") def get_digest(data: str) -> str: s = sha256() - s.update(data.encode('utf-8')) + s.update(data.encode("utf-8")) return s.hexdigest() @@ -192,7 +201,7 @@ def has_changed(data: str, path: str) -> Tuple[bool, str]: prev_data = None changed = True - reason = f'no {path} present' + reason = f"no {path} present" if os.path.exists(path): with open(path) as f: @@ -208,10 +217,10 @@ def has_changed(data: str, path: str) -> Tuple[bool, str]: if data: if data != prev_data: - reason = f'different data in {path}' + reason = f"different data in {path}" else: changed = False - reason = f'same data in {path}' + reason = f"same data in {path}" if changed: # print(f'has_changed: updating {path}') @@ -221,22 +230,24 @@ def has_changed(data: str, path: str) -> Tuple[bool, str]: # For now, we always have to reapply with split testing. if not changed: changed = True - reason = 'always reapply for split test' + reason = "always reapply for split test" return (changed, reason) COUNTERS: Dict[Type, int] = {} -SANITIZATIONS = OrderedDict(( - ("://", "SCHEME"), - (":", "COLON"), - (" ", "SPACE"), - ("/t", "TAB"), - (".", "DOT"), - ("?", "QMARK"), - ("/", "SLASH"), -)) +SANITIZATIONS = OrderedDict( + ( + ("://", "SCHEME"), + (":", "COLON"), + (" ", "SPACE"), + ("/t", "TAB"), + (".", "DOT"), + ("?", "QMARK"), + ("/", "SLASH"), + ) +) def sanitize(obj): @@ -250,8 +261,8 @@ def sanitize(obj): obj = obj.replace(k, "-" + v + "-") return obj elif isinstance(obj, dict): - if 'value' in obj: - return obj['value'] + if "value" in obj: + return obj["value"] else: return "-".join("%s-%s" % (sanitize(k), sanitize(v)) for k, v in sorted(obj.items())) else: @@ -285,7 +296,7 @@ def variants(cls, *args, **kwargs) -> Tuple[Any]: class Name(str): namespace: Optional[str] - def __new__(cls, value, namespace: Optional[str]=None): + def __new__(cls, value, namespace: Optional[str] = None): s = super().__new__(cls, value) s.namespace = namespace return s @@ -298,13 +309,14 @@ def k8s(self): def fqdn(self): r = self.k8s - if self.namespace and (self.namespace != 'default'): - r += '.' + self.namespace + if self.namespace and (self.namespace != "default"): + r += "." + self.namespace return r + class NodeLocal(threading.local): - current: Optional['Node'] + current: Optional["Node"] def __init__(self): self.current = None @@ -328,8 +340,8 @@ def _argprocess(o): class Node(ABC): - parent: Optional['Node'] - children: List['Node'] + parent: Optional["Node"] + children: List["Node"] name: Name ambassador_id: str namespace: str = None # type: ignore @@ -343,8 +355,8 @@ def __init__(self, *args, **kwargs) -> None: name = kwargs.pop("name", None) - if 'namespace' in kwargs: - self.namespace = kwargs.pop('namespace', None) + if "namespace" in kwargs: + self.namespace = kwargs.pop("namespace", None) _clone: Node = kwargs.pop("_clone", None) @@ -390,16 +402,22 @@ def __init__(self, *args, **kwargs) -> None: names = {} # type: ignore for c in self.children: - assert c.name not in names, ("test %s of type %s has duplicate children: %s of type %s, %s" % - (self.name, self.__class__.__name__, c.name, c.__class__.__name__, - names[c.name].__class__.__name__)) + assert ( + c.name not in names + ), "test %s of type %s has duplicate children: %s of type %s, %s" % ( + self.name, + self.__class__.__name__, + c.name, + c.__class__.__name__, + names[c.name].__class__.__name__, + ) names[c.name] = c def clone(self, name=None): return self.__class__(_clone=self, name=name) - def find_local_result(self, stop_at_first_ambassador: bool=False) -> Optional[Dict[str, str]]: - test_name = self.format('{self.path.k8s}') + def find_local_result(self, stop_at_first_ambassador: bool = False) -> Optional[Dict[str, str]]: + test_name = self.format("{self.path.k8s}") # print(f"{test_name} {type(self)} FIND_LOCAL_RESULT") @@ -408,12 +426,12 @@ def find_local_result(self, stop_at_first_ambassador: bool=False) -> Optional[Di n: Optional[Node] = self while n: - node_name = n.format('{self.path.k8s}') + node_name = n.format("{self.path.k8s}") parent = n.parent - parent_name = parent.format('{self.path.k8s}') if parent else "-none-" + parent_name = parent.format("{self.path.k8s}") if parent else "-none-" - end_result = getattr(n, 'local_result', None) - result_str = end_result['result'] if end_result else '-none-' + end_result = getattr(n, "local_result", None) + result_str = end_result["result"] if end_result else "-none-" # print(f"{test_name}: {'ambassador' if n.is_ambassador else 'node'} {node_name}, parent {parent_name}, local_result = {result_str}") if end_result is not None: @@ -428,14 +446,11 @@ def find_local_result(self, stop_at_first_ambassador: bool=False) -> Optional[Di return end_result def check_local(self, gold_root: str, k8s_yaml_path: str) -> Tuple[bool, bool]: - testname = self.format('{self.path.k8s}') + testname = self.format("{self.path.k8s}") if self.xfail: # XFail early -- but still return True, True so that we don't try to run Envoy on it. - self.local_result = { - 'result': 'xfail', - 'reason': self.xfail - } + self.local_result = {"result": "xfail", "reason": self.xfail} # print(f"==== XFAIL: {testname} local: {self.xfail}") return (True, True) @@ -447,11 +462,11 @@ def check_local(self, gold_root: str, k8s_yaml_path: str) -> Tuple[bool, bool]: print(f"{testname} ({type(self)}) is an Ambassador but has no ambassador_id?") return (False, False) - ambassador_namespace = getattr(self, 'namespace', 'default') - ambassador_single_namespace = getattr(self, 'single_namespace', False) + ambassador_namespace = getattr(self, "namespace", "default") + ambassador_single_namespace = getattr(self, "single_namespace", False) - no_local_mode: bool = getattr(self, 'no_local_mode', False) - skip_local_reason: Optional[str] = getattr(self, 'skip_local_instead_of_xfail', None) + no_local_mode: bool = getattr(self, "no_local_mode", False) + skip_local_reason: Optional[str] = getattr(self, "skip_local_instead_of_xfail", None) # print(f"{testname}: ns {ambassador_namespace} ({'single' if ambassador_single_namespace else 'multi'})") @@ -466,16 +481,23 @@ def check_local(self, gold_root: str, k8s_yaml_path: str) -> Tuple[bool, bool]: # it, bring it up-to-date with the environment created in abstract_tests.py envstuff = ["env", f"AMBASSADOR_NAMESPACE={ambassador_namespace}"] - cmd = ["mockery", "--debug", k8s_yaml_path, - "-w", "python /ambassador/watch_hook.py", - "--kat", self.ambassador_id, - "--diff", gold_path] + cmd = [ + "mockery", + "--debug", + k8s_yaml_path, + "-w", + "python /ambassador/watch_hook.py", + "--kat", + self.ambassador_id, + "--diff", + gold_path, + ] if ambassador_single_namespace: envstuff.append("AMBASSADOR_SINGLE_NAMESPACE=yes") cmd += ["-n", ambassador_namespace] - if not getattr(self, 'allow_edge_stack_redirect', False): + if not getattr(self, "allow_edge_stack_redirect", False): envstuff.append("AMBASSADOR_NO_TLS_REDIRECT=yes") cmd = envstuff + cmd @@ -484,15 +506,11 @@ def check_local(self, gold_root: str, k8s_yaml_path: str) -> Tuple[bool, bool]: if w.status(): print(f"==== GOOD: {testname} local against {gold_path}") - self.local_result = {'result': "pass"} + self.local_result = {"result": "pass"} else: print(f"==== FAIL: {testname} local against {gold_path}") - self.local_result = { - 'result': 'fail', - 'stdout': w.stdout, - 'stderr': w.stderr - } + self.local_result = {"result": "fail", "stdout": w.stdout, "stderr": w.stderr} return (True, True) else: @@ -506,8 +524,8 @@ def check_local(self, gold_root: str, k8s_yaml_path: str) -> Tuple[bool, bool]: if local_result: self.local_result = { - 'result': 'skip', - 'reason': f"subsumed by {skip_local_reason} -- {local_result['result']}" + "result": "skip", + "reason": f"subsumed by {skip_local_reason} -- {local_result['result']}", } # print(f"==== {self.local_result['result'].upper()} {testname} {self.local_result['reason']}") return (True, True) @@ -518,16 +536,18 @@ def check_local(self, gold_root: str, k8s_yaml_path: str) -> Tuple[bool, bool]: if RUN_MODE == "local": if skip_local_reason: self.local_result = { - 'result': 'skip', + "result": "skip", # 'reason': f"subsumed by {skip_local_reason} without result in local mode" } - print(f"==== {self.local_result['result'].upper()} {testname} {self.local_result['reason']}") + print( + f"==== {self.local_result['result'].upper()} {testname} {self.local_result['reason']}" + ) return (True, True) else: # XFail -- but still return True, True so that we don't try to run Envoy on it. self.local_result = { - 'result': 'xfail', - 'reason': f"missing local cache {gold_path}" + "result": "xfail", + "reason": f"missing local cache {gold_path}", } # print(f"==== {self.local_result['result'].upper()} {testname} {self.local_result['reason']}") return (True, True) @@ -580,8 +600,8 @@ def format(self, st, **kwargs): return integration_manifests.format(st, self=self, **kwargs) def get_fqdn(self, name: str) -> str: - if self.namespace and (self.namespace != 'default'): - return f'{name}.{self.namespace}' + if self.namespace and (self.namespace != "default"): + return f"{name}.{self.namespace}" else: return name @@ -600,36 +620,38 @@ def requirements(self): # log_kube_artifacts writes various logs about our underlying Kubernetes objects to # a place where the artifact publisher can find them. See run-tests.sh. def log_kube_artifacts(self): - if not getattr(self, 'already_logged', False): + if not getattr(self, "already_logged", False): self.already_logged = True - print(f'logging kube artifacts for {self.path.k8s}') + print(f"logging kube artifacts for {self.path.k8s}") sys.stdout.flush() DEV = os.environ.get("AMBASSADOR_DEV", "0").lower() in ("1", "yes", "true") - log_path = f'/tmp/kat-logs-{self.path.k8s}' + log_path = f"/tmp/kat-logs-{self.path.k8s}" if DEV: - os.system(f'docker logs {self.path.k8s} >{log_path} 2>&1') + os.system(f"docker logs {self.path.k8s} >{log_path} 2>&1") else: - os.system(f'tools/bin/kubectl logs -n {self.namespace} {self.path.k8s} >{log_path} 2>&1') + os.system( + f"tools/bin/kubectl logs -n {self.namespace} {self.path.k8s} >{log_path} 2>&1" + ) - event_path = f'/tmp/kat-events-{self.path.k8s}' + event_path = f"/tmp/kat-events-{self.path.k8s}" - fs1 = f'involvedObject.name={self.path.k8s}' - fs2 = f'involvedObject.namespace={self.namespace}' + fs1 = f"involvedObject.name={self.path.k8s}" + fs2 = f"involvedObject.namespace={self.namespace}" cmd = f'tools/bin/kubectl get events -o json --field-selector "{fs1}" --field-selector "{fs2}"' os.system(f'echo ==== "{cmd}" >{event_path}') - os.system(f'{cmd} >>{event_path} 2>&1') + os.system(f"{cmd} >>{event_path} 2>&1") class Test(Node): - results: List['Result'] = [] - pending: List['Query'] = [] - queried: List['Query'] = [] + results: List["Result"] = [] + pending: List["Query"] = [] + queried: List["Query"] = [] __test__ = False @@ -646,28 +668,28 @@ def check(self): pass def handle_local_result(self) -> bool: - test_name = self.format('{self.path.k8s}') + test_name = self.format("{self.path.k8s}") # print(f"{test_name} {type(self)} HANDLE_LOCAL_RESULT") end_result = self.find_local_result() if end_result is not None: - result_type = end_result['result'] + result_type = end_result["result"] - if result_type == 'pass': + if result_type == "pass": pass - elif result_type == 'skip': - pytest.skip(end_result['reason']) - elif result_type == 'fail': - sys.stdout.write(end_result['stdout']) + elif result_type == "skip": + pytest.skip(end_result["reason"]) + elif result_type == "fail": + sys.stdout.write(end_result["stdout"]) - if os.environ.get('KAT_VERBOSE', None): - sys.stderr.write(end_result['stderr']) + if os.environ.get("KAT_VERBOSE", None): + sys.stderr.write(end_result["stderr"]) pytest.fail("local check failed") - elif result_type == 'xfail': - pytest.xfail(end_result['reason']) + elif result_type == "xfail": + pytest.xfail(end_result["reason"]) return True @@ -685,20 +707,45 @@ def ambassador_id(self): def encode_body(obj): return encode_body(json.dumps(obj)) + @encode_body.register def encode_body_bytes(b: bytes): return base64.encodebytes(b).decode("utf-8") + @encode_body.register def encode_body_str(s: str): return encode_body(s.encode("utf-8")) -class Query: - def __init__(self, url, expected=None, method="GET", headers=None, messages=None, insecure=False, skip=None, - xfail=None, phase=1, debug=False, sni=False, error=None, client_crt=None, client_key=None, - client_cert_required=False, ca_cert=None, grpc_type=None, cookies=None, ignore_result=False, body=None, - minTLSv="", maxTLSv="", cipherSuites=[], ecdhCurves=[]): +class Query: + def __init__( + self, + url, + expected=None, + method="GET", + headers=None, + messages=None, + insecure=False, + skip=None, + xfail=None, + phase=1, + debug=False, + sni=False, + error=None, + client_crt=None, + client_key=None, + client_cert_required=False, + ca_cert=None, + grpc_type=None, + cookies=None, + ignore_result=False, + body=None, + minTLSv="", + maxTLSv="", + cipherSuites=[], + ecdhCurves=[], + ): self.method = method self.url = url self.headers = headers @@ -739,7 +786,7 @@ def as_json(self): "test": self.parent.path, "id": id(self), "url": self.url, - "insecure": self.insecure + "insecure": self.insecure, } if self.sni: result["sni"] = self.sni @@ -811,7 +858,7 @@ def check(self): errors = self.query.error if isinstance(self.query.error, str): - errors = [ self.query.error ] + errors = [self.query.error] if self.error is not None: for error in errors: @@ -820,30 +867,36 @@ def check(self): break assert found, "{}: expected error to contain any of {}; got {} instead".format( - self.query.url, ", ".join([ "'%s'" % x for x in errors ]), - ("'%s'" % self.error) if self.error else "no error" + self.query.url, + ", ".join(["'%s'" % x for x in errors]), + ("'%s'" % self.error) if self.error else "no error", ) else: if self.query.expected != self.status: self.parent.log_kube_artifacts() - assert self.query.expected == self.status, \ - "%s: expected status code %s, got %s instead with error %s" % ( - self.query.url, self.query.expected, self.status, self.error) + assert ( + self.query.expected == self.status + ), "%s: expected status code %s, got %s instead with error %s" % ( + self.query.url, + self.query.expected, + self.status, + self.error, + ) def as_dict(self) -> Dict[str, Any]: od = { - 'query': self.query.as_json(), - 'status': self.status, - 'error': self.error, - 'headers': self.headers, + "query": self.query.as_json(), + "status": self.status, + "error": self.error, + "headers": self.headers, } if self.backend and self.backend.name: - od['backend'] = self.backend.as_dict() + od["backend"] = self.backend.as_dict() else: - od['json'] = self.json - od['text'] = self.text + od["json"] = self.json + od["text"] = self.text return od @@ -887,9 +940,18 @@ def as_dict(self) -> Dict[str, Any]: class BackendURL: - - def __init__(self, fragment=None, host=None, opaque=None, path=None, query=None, rawQuery=None, - scheme=None, username=None, password=None): + def __init__( + self, + fragment=None, + host=None, + opaque=None, + path=None, + query=None, + rawQuery=None, + scheme=None, + username=None, + password=None, + ): self.fragment = fragment self.host = host self.opaque = opaque @@ -900,22 +962,21 @@ def __init__(self, fragment=None, host=None, opaque=None, path=None, query=None, self.username = username self.password = password - def as_dict(self) -> Dict['str', Any]: + def as_dict(self) -> Dict["str", Any]: return { - 'fragment': self.fragment, - 'host': self.host, - 'opaque': self.opaque, - 'path': self.path, - 'query': self.query, - 'rawQuery': self.rawQuery, - 'scheme': self.scheme, - 'username': self.username, - 'password': self.password, + "fragment": self.fragment, + "host": self.host, + "opaque": self.opaque, + "path": self.path, + "query": self.query, + "rawQuery": self.rawQuery, + "scheme": self.scheme, + "username": self.username, + "password": self.password, } class BackendRequest: - def __init__(self, req): self.url = BackendURL(**req.get("url")) self.headers = req.get("headers", {}) @@ -924,21 +985,20 @@ def __init__(self, req): def as_dict(self) -> Dict[str, Any]: od = { - 'headers': self.headers, - 'host': self.host, + "headers": self.headers, + "host": self.host, } if self.url: - od['url'] = self.url.as_dict() + od["url"] = self.url.as_dict() if self.tls: - od['tls'] = self.tls.as_dict() + od["tls"] = self.tls.as_dict() return od class BackendTLS: - def __init__(self, tls): self.enabled = tls["enabled"] self.server_name = tls.get("server-name") @@ -948,21 +1008,20 @@ def __init__(self, tls): def as_dict(self) -> Dict[str, Any]: return { - 'enabled': self.enabled, - 'server_name': self.server_name, - 'version': self.version, - 'negotiated_protocol': self.negotiated_protocol, - 'negotiated_protocol_version': self.negotiated_protocol_version, + "enabled": self.enabled, + "server_name": self.server_name, + "version": self.version, + "negotiated_protocol": self.negotiated_protocol, + "negotiated_protocol_version": self.negotiated_protocol_version, } class BackendResponse: - def __init__(self, resp): self.headers = resp.get("headers", {}) def as_dict(self) -> Dict[str, Any]: - return { 'headers': self.headers } + return {"headers": self.headers} def dictify(obj): @@ -973,7 +1032,6 @@ def dictify(obj): class BackendResult: - def __init__(self, bres): self.name = "raw" self.request = None @@ -985,15 +1043,13 @@ def __init__(self, bres): self.response = BackendResponse(bres["response"]) if "response" in bres else None def as_dict(self) -> Dict[str, Any]: - od = { - 'name': self.name - } + od = {"name": self.name} if self.request: - od['request'] = dictify(self.request) + od["request"] = dictify(self.request) if self.response: - od['response'] = dictify(self.response) + od["response"] = dictify(self.response) return od @@ -1011,6 +1067,7 @@ def label(yaml, scope): CLIENT_GO = "kat_client" + def run_queries(name: str, queries: Sequence[Query]) -> Sequence[Result]: jsonified = [] byid = {} @@ -1019,28 +1076,33 @@ def run_queries(name: str, queries: Sequence[Query]) -> Sequence[Result]: jsonified.append(q.as_json()) byid[id(q)] = q - path_urls = f'/tmp/kat-client-{name}-urls.json' - path_results = f'/tmp/kat-client-{name}-results.json' - path_log = f'/tmp/kat-client-{name}.log' + path_urls = f"/tmp/kat-client-{name}-urls.json" + path_results = f"/tmp/kat-client-{name}-results.json" + path_log = f"/tmp/kat-client-{name}.log" - with open(path_urls, 'w') as f: + with open(path_urls, "w") as f: json.dump(jsonified, f) # run(f"{CLIENT_GO} -input {path_urls} -output {path_results} 2> {path_log}") - res = ShellCommand.run('Running queries', - f"tools/bin/kubectl exec -n default -i kat /work/kat_client < '{path_urls}' > '{path_results}' 2> '{path_log}'", - shell=True) + res = ShellCommand.run( + "Running queries", + f"tools/bin/kubectl exec -n default -i kat /work/kat_client < '{path_urls}' > '{path_results}' 2> '{path_log}'", + shell=True, + ) if not res: - ret = [Result(q, {"error":"Command execution error"}) for q in queries] + ret = [Result(q, {"error": "Command execution error"}) for q in queries] return ret - with open(path_results, 'r') as f: + with open(path_results, "r") as f: content = f.read() try: json_results = json.loads(content) except Exception as e: - ret = [Result(q, {"error":"Could not parse JSON content after running KAT queries"}) for q in queries] + ret = [ + Result(q, {"error": "Could not parse JSON content after running KAT queries"}) + for q in queries + ] return ret results = [] @@ -1063,10 +1125,10 @@ def __init__(self, namespace: str) -> None: self.next_clear = 8080 self.next_tls = 8443 self.service_names: Dict[int, str] = {} - self.name = 'superpod-%s' % (self.namespace or 'default') + self.name = "superpod-%s" % (self.namespace or "default") def allocate(self, service_name) -> List[int]: - ports = [ self.next_clear, self.next_tls ] + ports = [self.next_clear, self.next_tls] self.service_names[self.next_clear] = service_name self.service_names[self.next_tls] = service_name @@ -1076,42 +1138,45 @@ def allocate(self, service_name) -> List[int]: return ports def get_manifest_list(self) -> List[Dict[str, Any]]: - manifest = load('superpod', integration_manifests.format(integration_manifests.load("superpod_pod")), Tag.MAPPING) + manifest = load( + "superpod", + integration_manifests.format(integration_manifests.load("superpod_pod")), + Tag.MAPPING, + ) assert len(manifest) == 1, "SUPERPOD manifest must have exactly one object" m = manifest[0] - template = m['spec']['template'] + template = m["spec"]["template"] ports: List[Dict[str, int]] = [] - envs: List[Dict[str, Union[str, int]]] = template['spec']['containers'][0]['env'] + envs: List[Dict[str, Union[str, int]]] = template["spec"]["containers"][0]["env"] for p in sorted(self.service_names.keys()): - ports.append({ 'containerPort': p }) - envs.append({ 'name': f'BACKEND_{p}', 'value': self.service_names[p] }) + ports.append({"containerPort": p}) + envs.append({"name": f"BACKEND_{p}", "value": self.service_names[p]}) - template['spec']['containers'][0]['ports'] = ports + template["spec"]["containers"][0]["ports"] = ports - if 'metadata' not in m: - m['metadata'] = {} + if "metadata" not in m: + m["metadata"] = {} - metadata = m['metadata'] - metadata['name'] = self.name + metadata = m["metadata"] + metadata["name"] = self.name - m['spec']['selector']['matchLabels']['backend'] = self.name - template['metadata']['labels']['backend'] = self.name + m["spec"]["selector"]["matchLabels"]["backend"] = self.name + template["metadata"]["labels"]["backend"] = self.name if self.namespace: # Fix up the namespace. - if 'namespace' not in metadata: - metadata['namespace'] = self.namespace + if "namespace" not in metadata: + metadata["namespace"] = self.namespace return list(manifest) class Runner: - def __init__(self, *classes, scope=None): self.scope = scope or "-".join(c.__name__ for c in classes) self.roots = tuple(v for c in classes for v in variants(c)) @@ -1128,7 +1193,11 @@ def test(request, capsys, t): if t.xfail: pytest.xfail(t.xfail) else: - selected = set(item.callspec.getparam('t') for item in request.session.items if item.function == test) + selected = set( + item.callspec.getparam("t") + for item in request.session.items + if item.function == test + ) with capsys.disabled(): self.setup(selected) @@ -1207,11 +1276,11 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: while cur: if not nsp: - nsp = getattr(cur, 'namespace', None) + nsp = getattr(cur, "namespace", None) # print(f'... {cur.name} has namespace {nsp}') if not ambassador_id: - ambassador_id = getattr(cur, 'ambassador_id', None) + ambassador_id = getattr(cur, "ambassador_id", None) # print(f'... {cur.name} has ambassador_id {ambassador_id}') if nsp and ambassador_id: @@ -1221,7 +1290,7 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: cur = cur.parent # OK. Does this node want to use a superpod? - if getattr(n, 'use_superpod', False): + if getattr(n, "use_superpod", False): # Yup. OK. Do we already have a superpod for this namespace? superpod = superpods.get(nsp, None) # type: ignore @@ -1236,22 +1305,24 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: yaml = n.format(integration_manifests.load("backend_service")) manifest = load(n.path, yaml, Tag.MAPPING) - assert len(manifest) == 1, "backend_service.yaml manifest must have exactly one object" + assert ( + len(manifest) == 1 + ), "backend_service.yaml manifest must have exactly one object" m = manifest[0] # Update the manifest's selector... - m['spec']['selector']['backend'] = superpod.name + m["spec"]["selector"]["backend"] = superpod.name # ...and labels if needed... if ambassador_id: - m['metadata']['labels'] = { 'kat-ambassador-id': ambassador_id } + m["metadata"]["labels"] = {"kat-ambassador-id": ambassador_id} # ...and target ports. superpod_ports = superpod.allocate(n.path.k8s) - m['spec']['ports'][0]['targetPort'] = superpod_ports[0] - m['spec']['ports'][1]['targetPort'] = superpod_ports[1] + m["spec"]["ports"][0]["targetPort"] = superpod_ports[0] + m["spec"]["ports"][1]["targetPort"] = superpod_ports[1] else: # The non-superpod case... yaml = n.manifests() @@ -1260,9 +1331,9 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: is_plain_test = n.path.k8s.startswith("plain-") if n.is_ambassador and not is_plain_test: - add_default_http_listener = getattr(n, 'add_default_http_listener', True) - add_default_https_listener = getattr(n, 'add_default_https_listener', True) - add_cleartext_host = getattr(n, 'edge_stack_cleartext_host', False) + add_default_http_listener = getattr(n, "add_default_http_listener", True) + add_default_https_listener = getattr(n, "add_default_https_listener", True) + add_cleartext_host = getattr(n, "edge_stack_cleartext_host", False) if add_default_http_listener: # print(f"{n.path.k8s} adding default HTTP Listener") @@ -1270,7 +1341,7 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: "namespace": nsp, "port": 8080, "protocol": "HTTPS", - "securityModel": "XFP" + "securityModel": "XFP", } if add_default_https_listener: @@ -1279,7 +1350,7 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: "namespace": nsp, "port": 8443, "protocol": "HTTPS", - "securityModel": "XFP" + "securityModel": "XFP", } if EDGE_STACK and add_cleartext_host: @@ -1293,7 +1364,7 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: try: manifest = load(n.path, yaml, Tag.MAPPING) except Exception as e: - print(f'parse failure! {e}') + print(f"parse failure! {e}") print(yaml) if manifest: @@ -1301,20 +1372,20 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: # Make sure namespaces and labels are properly set. for m in manifest: - if 'metadata' not in m: - m['metadata'] = {} + if "metadata" not in m: + m["metadata"] = {} - metadata = m['metadata'] + metadata = m["metadata"] - if 'labels' not in metadata: - metadata['labels'] = {} + if "labels" not in metadata: + metadata["labels"] = {} if ambassador_id: - metadata['labels']['kat-ambassador-id'] = ambassador_id + metadata["labels"]["kat-ambassador-id"] = ambassador_id if nsp: - if 'namespace' not in metadata: - metadata['namespace'] = nsp + if "namespace" not in metadata: + metadata["namespace"] = nsp # ...and, finally, save the manifest list. manifests[n] = list(manifest) @@ -1327,7 +1398,7 @@ def get_manifests_and_namespaces(self, selected) -> Tuple[Any, List[str]]: return manifests, namespaces def do_local_checks(self, selected, fname) -> bool: - if RUN_MODE == 'envoy': + if RUN_MODE == "envoy": print("Local mode not allowed, continuing to Envoy mode") return False @@ -1422,7 +1493,7 @@ def _setup_k8s(self, selected): return True # Something didn't work out quite right. - print(f'Continuing with Kube tests...') + print(f"Continuing with Kube tests...") # print(f"ids_to_strip {self.ids_to_strip}") # XXX It is _so stupid_ that we're reparsing the whole manifest here. @@ -1436,31 +1507,31 @@ def _setup_k8s(self, selected): for obj in xxx_crap: keep = True - kind = '-nokind-' - name = '-noname-' + kind = "-nokind-" + name = "-noname-" metadata: Dict[str, Any] = {} labels: Dict[str, str] = {} id_to_check: Optional[str] = None - if 'kind' in obj: - kind = obj['kind'] + if "kind" in obj: + kind = obj["kind"] - if 'metadata' in obj: - metadata = obj['metadata'] + if "metadata" in obj: + metadata = obj["metadata"] - if 'name' in metadata: - name = metadata['name'] + if "name" in metadata: + name = metadata["name"] - if 'labels' in metadata: - labels = metadata['labels'] + if "labels" in metadata: + labels = metadata["labels"] - if 'kat-ambassador-id' in labels: - id_to_check = labels['kat-ambassador-id'] + if "kat-ambassador-id" in labels: + id_to_check = labels["kat-ambassador-id"] # print(f"metadata {metadata} id_to_check {id_to_check} obj {obj}") # Keep namespaces, just in case. - if kind == 'Namespace': + if kind == "Namespace": keep = True else: if id_to_check and (id_to_check in self.ids_to_strip): @@ -1518,15 +1589,15 @@ def _setup_k8s(self, selected): for version in crd["spec"]["versions"]: if "schema" in version: version["schema"] = { - 'openAPIV3Schema': { - 'type': 'object', - 'properties': { - 'apiVersion': { 'type': 'string' }, - 'kind': { 'type': 'string' }, - 'metadata': { 'type': 'object' }, - 'spec': { - 'type': 'object', - 'x-kubernetes-preserve-unknown-fields': True, + "openAPIV3Schema": { + "type": "object", + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "x-kubernetes-preserve-unknown-fields": True, }, }, }, @@ -1541,16 +1612,24 @@ def _setup_k8s(self, selected): changed, reason = has_changed(final_crds, "/tmp/k8s-CRDs.yaml") if changed: - print(f'CRDS changed ({reason}), applying.') + print(f"CRDS changed ({reason}), applying.") if not ShellCommand.run_with_retry( - 'Apply CRDs', - 'tools/bin/kubectl', 'apply', '-f', '/tmp/k8s-CRDs.yaml', - retries=5, sleep_seconds=10): + "Apply CRDs", + "tools/bin/kubectl", + "apply", + "-f", + "/tmp/k8s-CRDs.yaml", + retries=5, + sleep_seconds=10, + ): raise RuntimeError("Failed applying CRDs") tries_left = 10 - while os.system('tools/bin/kubectl get crd mappings.getambassador.io > /dev/null 2>&1') != 0: + while ( + os.system("tools/bin/kubectl get crd mappings.getambassador.io > /dev/null 2>&1") + != 0 + ): tries_left -= 1 if tries_left <= 0: @@ -1559,27 +1638,48 @@ def _setup_k8s(self, selected): print("sleeping for CRDs... (%d)" % tries_left) time.sleep(5) else: - print(f'CRDS unchanged {reason}, skipping apply.') + print(f"CRDS unchanged {reason}, skipping apply.") # Next up: the KAT pod. kat_client_manifests = integration_manifests.load("kat_client_pod") if os.environ.get("DEV_USE_IMAGEPULLSECRET", False): - kat_client_manifests = integration_manifests.namespace_manifest("default") + kat_client_manifests - changed, reason = has_changed(integration_manifests.format(kat_client_manifests), "/tmp/k8s-kat-pod.yaml") + kat_client_manifests = ( + integration_manifests.namespace_manifest("default") + kat_client_manifests + ) + changed, reason = has_changed( + integration_manifests.format(kat_client_manifests), "/tmp/k8s-kat-pod.yaml" + ) if changed: - print(f'KAT pod definition changed ({reason}), applying') - if not ShellCommand.run_with_retry('Apply KAT pod', - 'tools/bin/kubectl', 'apply', '-f' , '/tmp/k8s-kat-pod.yaml', '-n', 'default', - retries=5, sleep_seconds=10): - raise RuntimeError('Could not apply manifest for KAT pod') + print(f"KAT pod definition changed ({reason}), applying") + if not ShellCommand.run_with_retry( + "Apply KAT pod", + "tools/bin/kubectl", + "apply", + "-f", + "/tmp/k8s-kat-pod.yaml", + "-n", + "default", + retries=5, + sleep_seconds=10, + ): + raise RuntimeError("Could not apply manifest for KAT pod") tries_left = 3 time.sleep(1) while True: - if ShellCommand.run("wait for KAT pod", - 'tools/bin/kubectl', '-n', 'default', 'wait', '--timeout=30s', '--for=condition=Ready', 'pod', 'kat'): + if ShellCommand.run( + "wait for KAT pod", + "tools/bin/kubectl", + "-n", + "default", + "wait", + "--timeout=30s", + "--for=condition=Ready", + "pod", + "kat", + ): print("KAT pod ready") break @@ -1591,28 +1691,47 @@ def _setup_k8s(self, selected): print("sleeping for KAT pod... (%d)" % tries_left) time.sleep(5) else: - print(f'KAT pod definition unchanged {reason}, skipping apply.') + print(f"KAT pod definition unchanged {reason}, skipping apply.") # Use a dummy pod to get around the !*@&#$!*@&# DockerHub rate limit. # XXX Better: switch to GCR. dummy_pod = integration_manifests.load("dummy_pod") if os.environ.get("DEV_USE_IMAGEPULLSECRET", False): dummy_pod = integration_manifests.namespace_manifest("default") + dummy_pod - changed, reason = has_changed(integration_manifests.format(dummy_pod), "/tmp/k8s-dummy-pod.yaml") + changed, reason = has_changed( + integration_manifests.format(dummy_pod), "/tmp/k8s-dummy-pod.yaml" + ) if changed: - print(f'Dummy pod definition changed ({reason}), applying') - if not ShellCommand.run_with_retry('Apply dummy pod', - 'tools/bin/kubectl', 'apply', '-f' , '/tmp/k8s-dummy-pod.yaml', '-n', 'default', - retries=5, sleep_seconds=10): - raise RuntimeError('Could not apply manifest for dummy pod') + print(f"Dummy pod definition changed ({reason}), applying") + if not ShellCommand.run_with_retry( + "Apply dummy pod", + "tools/bin/kubectl", + "apply", + "-f", + "/tmp/k8s-dummy-pod.yaml", + "-n", + "default", + retries=5, + sleep_seconds=10, + ): + raise RuntimeError("Could not apply manifest for dummy pod") tries_left = 3 time.sleep(1) while True: - if ShellCommand.run("wait for dummy pod", - 'tools/bin/kubectl', '-n', 'default', 'wait', '--timeout=30s', '--for=condition=Ready', 'pod', 'dummy-pod'): + if ShellCommand.run( + "wait for dummy pod", + "tools/bin/kubectl", + "-n", + "default", + "wait", + "--timeout=30s", + "--for=condition=Ready", + "pod", + "dummy-pod", + ): print("Dummy pod ready") break @@ -1624,29 +1743,53 @@ def _setup_k8s(self, selected): print("sleeping for dummy pod... (%d)" % tries_left) time.sleep(5) else: - print(f'Dummy pod definition unchanged {reason}, skipping apply.') + print(f"Dummy pod definition unchanged {reason}, skipping apply.") # # Clear out old stuff. if os.environ.get("DEV_CLEAN_K8S_RESOURCES", False): print("Clearing cluster...") - ShellCommand.run('clear old Kubernetes namespaces', - 'tools/bin/kubectl', 'delete', 'namespaces', '-l', 'scope=AmbassadorTest', - verbose=True) - ShellCommand.run('clear old Kubernetes pods etc.', - 'tools/bin/kubectl', 'delete', 'all', '-l', 'scope=AmbassadorTest', '--all-namespaces', - verbose=True) + ShellCommand.run( + "clear old Kubernetes namespaces", + "tools/bin/kubectl", + "delete", + "namespaces", + "-l", + "scope=AmbassadorTest", + verbose=True, + ) + ShellCommand.run( + "clear old Kubernetes pods etc.", + "tools/bin/kubectl", + "delete", + "all", + "-l", + "scope=AmbassadorTest", + "--all-namespaces", + verbose=True, + ) # XXX: better prune selector label if manifest_changed: print(f"manifest changed ({manifest_reason}), applying...") - if not ShellCommand.run_with_retry('Applying k8s manifests', - 'tools/bin/kubectl', 'apply', '--prune', '-l', 'scope=%s' % self.scope, '-f', fname, - retries=5, sleep_seconds=10): - raise RuntimeError('Could not apply manifests') + if not ShellCommand.run_with_retry( + "Applying k8s manifests", + "tools/bin/kubectl", + "apply", + "--prune", + "-l", + "scope=%s" % self.scope, + "-f", + fname, + retries=5, + sleep_seconds=10, + ): + raise RuntimeError("Could not apply manifests") self.applied_manifests = True # Finally, install httpbin and the websocket-echo-server. - print(f"applying http_manifests + websocket_echo_server_manifests to namespaces: {namespaces}") + print( + f"applying http_manifests + websocket_echo_server_manifests to namespaces: {namespaces}" + ) for namespace in namespaces: apply_kube_artifacts(namespace, httpbin_manifests) apply_kube_artifacts(namespace, websocket_echo_server_manifests) @@ -1666,7 +1809,7 @@ def _setup_k8s(self, selected): def _req_str(kind, req) -> str: printable = req - if kind == 'url': + if kind == "url": printable = req.url return printable @@ -1679,7 +1822,7 @@ def _wait(self, selected: Sequence[Node]): continue node_name = node.format("{self.path.k8s}") - ambassador_id = getattr(node, 'ambassador_id', None) + ambassador_id = getattr(node, "ambassador_id", None) # print(f"{node_name} {ambassador_id}") @@ -1710,7 +1853,7 @@ def _wait(self, selected: Sequence[Node]): homogenous[kind].append((node, name)) - kinds = [ "pod", "url" ] + kinds = ["pod", "url"] delay = 5 start = time.time() limit = int(os.environ.get("KAT_REQ_LIMIT", "900")) @@ -1738,7 +1881,7 @@ def _wait(self, selected: Sequence[Node]): if not is_ready: holdouts[kind] = _holdouts - delay = int(min(delay*2, 10)) + delay = int(min(delay * 2, 10)) print("sleeping %ss..." % delay) sys.stdout.flush() time.sleep(delay) @@ -1757,10 +1900,10 @@ def _wait(self, selected: Sequence[Node]): _holdouts = holdouts.get(kind, []) if _holdouts: - print(f' {kind}:') + print(f" {kind}:") for node, text in _holdouts: - print(f' {node.path.k8s} ({text})') + print(f" {node.path.k8s} ({text})") node.log_kube_artifacts() assert False, "requirements not satisfied in %s seconds" % limit @@ -1804,21 +1947,34 @@ def _ready_url(self, _, requirements): if not_ready: first = not_ready[0] - print("%d not ready (%s: %s) " % (len(not_ready), first.query.url, first.status or first.error), end="") - return (False, [ (x.query.parent, "%s -- %s" % (x.query.url, x.status or x.error)) for x in not_ready ]) + print( + "%d not ready (%s: %s) " + % (len(not_ready), first.query.url, first.status or first.error), + end="", + ) + return ( + False, + [ + (x.query.parent, "%s -- %s" % (x.query.url, x.status or x.error)) + for x in not_ready + ], + ) else: return (True, None) def _pods(self, scope=None): - scope_for_path = scope if scope else 'global' - label_for_scope = f'-l scope={scope}' if scope else '' - - fname = f'/tmp/pods-{scope_for_path}.json' - if not ShellCommand.run_with_retry('Getting pods', - f'tools/bin/kubectl get pod {label_for_scope} --all-namespaces -o json > {fname}', - shell=True, retries=5, sleep_seconds=10): - raise RuntimeError('Could not get pods') - + scope_for_path = scope if scope else "global" + label_for_scope = f"-l scope={scope}" if scope else "" + + fname = f"/tmp/pods-{scope_for_path}.json" + if not ShellCommand.run_with_retry( + "Getting pods", + f"tools/bin/kubectl get pod {label_for_scope} --all-namespaces -o json > {fname}", + shell=True, + retries=5, + sleep_seconds=10, + ): + raise RuntimeError("Could not get pods") with open(fname) as f: raw_pods = json.load(f) @@ -1833,7 +1989,7 @@ def _pods(self, scope=None): all_ready = True for status in cstats: - ready = status.get('ready', False) + ready = status.get("ready", False) if not ready: all_ready = False @@ -1847,7 +2003,7 @@ def _query(self, selected) -> None: queries = [] for t in self.tests: - t_name = t.format('{self.path.k8s}') + t_name = t.format("{self.path.k8s}") if t in selected: t.pending = [] @@ -1860,7 +2016,7 @@ def _query(self, selected) -> None: # print(f"{t_name}: SKIP QUERY due to local result") continue - ambassador_id = getattr(t, 'ambassador_id', None) + ambassador_id = getattr(t, "ambassador_id", None) if ambassador_id and ambassador_id in self.ids_to_strip: # print(f"{t_name}: SKIP QUERY due to ambassador_id {ambassador_id}") @@ -1878,7 +2034,9 @@ def _query(self, selected) -> None: for phase in phases: if not first: phase_delay = int(os.environ.get("KAT_PHASE_DELAY", 10)) - print("Waiting for {} seconds before starting phase {}...".format(phase_delay, phase)) + print( + "Waiting for {} seconds before starting phase {}...".format(phase_delay, phase) + ) time.sleep(phase_delay) first = False @@ -1888,7 +2046,7 @@ def _query(self, selected) -> None: print("Querying %s urls in phase %s..." % (len(phase_queries), phase), end="") sys.stdout.flush() - results = run_queries(f'phase{phase}', phase_queries) + results = run_queries(f"phase{phase}", phase_queries) print(" done.") @@ -1897,7 +2055,10 @@ def _query(self, selected) -> None: t.queried.append(r.query) if getattr(t, "debug", False) or getattr(r.query, "debug", False): - print("%s result: %s" % (t.name, json.dumps(r.as_dict(), sort_keys=True, indent=4))) + print( + "%s result: %s" + % (t.name, json.dumps(r.as_dict(), sort_keys=True, indent=4)) + ) t.results.append(r) t.pending.remove(r.query) diff --git a/python/kat/parser.py b/python/kat/parser.py index 259a43078d..b019b648ed 100644 --- a/python/kat/parser.py +++ b/python/kat/parser.py @@ -1,14 +1,24 @@ - from enum import Enum, auto from io import StringIO from typing import Any, Callable, Mapping, Sequence, Type -from yaml import compose, compose_all, dump_all, MappingNode, SequenceNode, ScalarNode, Node, add_representer +from yaml import ( + compose, + compose_all, + dump_all, + MappingNode, + SequenceNode, + ScalarNode, + Node, + add_representer, +) + class ViewMode(Enum): PYTHON = auto() STRING = auto() NODE = auto() + class Tag(Enum): SEQUENCE = compose("[]").tag MAPPING = compose("{}").tag @@ -18,8 +28,8 @@ class Tag(Enum): BOOL = compose("true").tag NULL = compose("null").tag -class View: +class View: def __init__(self, node: Node, mode: ViewMode) -> None: self.node = node self.mode = mode @@ -34,8 +44,8 @@ def view(self, obj): def mode_ify(self): return self -class MappingView(View, Mapping): +class MappingView(View, Mapping): def get(self, key, default=None): for k, v in self.node.value: if k.value == key: @@ -84,11 +94,13 @@ def __len__(self): return len(self.node.value) def __repr__(self): - return "{%s}" % ", ".join("%r: %r" % (view(k, ViewMode.PYTHON), view(v, ViewMode.PYTHON)) - for k, v in self.node.value) + return "{%s}" % ", ".join( + "%r: %r" % (view(k, ViewMode.PYTHON), view(v, ViewMode.PYTHON)) + for k, v in self.node.value + ) -class SequenceView(View, Sequence): +class SequenceView(View, Sequence): def __getitem__(self, idx): return view(self.node.value[idx], self.mode) @@ -115,16 +127,17 @@ def merge(self, other): def __repr__(self): return repr([v for v in self]) + PYJECTIONS = { Tag.INT: lambda x: int(x), Tag.FLOAT: lambda x: float(x), Tag.STRING: lambda x: x, Tag.BOOL: lambda x: x.lower() in ("y", "yes", "true", "on"), - Tag.NULL: lambda x: None + Tag.NULL: lambda x: None, } -class ScalarView(View): +class ScalarView(View): def mode_ify(self): if self.mode == ViewMode.PYTHON: return PYJECTIONS[Tag(self.tag)](self.node.value) @@ -136,16 +149,19 @@ def mode_ify(self): def __repr__(self): return self.node.value + VIEWS: Mapping[Type[Node], Type[View]] = { MappingNode: MappingView, SequenceNode: SequenceView, - ScalarNode: ScalarView + ScalarNode: ScalarView, } + def view(value: Any, mode: ViewMode) -> Any: nd = node(value) return VIEWS[type(nd)](nd, mode).mode_ify() + COERCIONS: Mapping[Type, Callable[[Any], Node]] = { MappingNode: lambda n: n, SequenceNode: lambda n: n, @@ -159,12 +175,14 @@ def view(value: Any, mode: ViewMode) -> Any: bool: lambda b: ScalarNode(Tag.BOOL.value, str(b)), int: lambda i: ScalarNode(Tag.INT.value, str(i)), float: lambda f: ScalarNode(Tag.FLOAT.value, str(f)), - dict: lambda d: MappingNode(Tag.MAPPING.value, [(node(k), node(v)) for k, v in d.items()]) + dict: lambda d: MappingNode(Tag.MAPPING.value, [(node(k), node(v)) for k, v in d.items()]), } + def node(value: Any) -> Node: return COERCIONS[type(value)](value) + def load(name: str, value: Any, *allowed: Tag) -> SequenceView: if isinstance(value, str): value = StringIO(value) @@ -172,19 +190,23 @@ def load(name: str, value: Any, *allowed: Tag) -> SequenceView: result = view(SequenceNode(Tag.SEQUENCE.value, list(compose_all(value))), ViewMode.PYTHON) for r in view(result, ViewMode.NODE): if r.tag not in allowed: - raise ValueError("expecting %s, got %s" % (", ".join(t.name for t in allowed), - r.node.tag)) + raise ValueError( + "expecting %s, got %s" % (", ".join(t.name for t in allowed), r.node.tag) + ) return result + def dump(value: SequenceView): st = dump_all(value, default_flow_style=False) - if not st.startswith('---'): - st = '---\n' + st + if not st.startswith("---"): + st = "---\n" + st return st + def view_representer(dumper, data): return data.node + add_representer(SequenceView, view_representer) add_representer(MappingView, view_representer) add_representer(ScalarView, view_representer) diff --git a/python/kat/utils.py b/python/kat/utils.py index a4a0c88ebc..19fcaf1f74 100644 --- a/python/kat/utils.py +++ b/python/kat/utils.py @@ -3,7 +3,8 @@ import time -_quote_pos = re.compile('(?=[^-0-9a-zA-Z_./\n])') +_quote_pos = re.compile("(?=[^-0-9a-zA-Z_./\n])") + def quote(arg): r""" @@ -15,14 +16,14 @@ def quote(arg): # This is the logic emacs uses if arg: - return _quote_pos.sub('\\\\', arg).replace('\n', "'\n'") + return _quote_pos.sub("\\\\", arg).replace("\n", "'\n'") else: return "''" class ShellCommand: def __init__(self, *args, **kwargs) -> None: - self.verbose = kwargs.pop('verbose', False) + self.verbose = kwargs.pop("verbose", False) for arg in "stdout", "stderr": if arg not in kwargs: @@ -31,7 +32,7 @@ def __init__(self, *args, **kwargs) -> None: self.cmdline = " ".join([quote(x) for x in args]) if self.verbose: - print(f'---- running: {self.cmdline}') + print(f"---- running: {self.cmdline}") self.proc = subprocess.run(args, **kwargs) @@ -47,7 +48,7 @@ def check(self, what: str) -> bool: return True else: print(f"==== COMMAND FAILED: {what}") - print(f'---- command line: {self.cmdline}') + print(f"---- command line: {self.cmdline}") if self.stdout: print("---- stdout ----") print(self.stdout) @@ -70,15 +71,15 @@ def stderr(self) -> str: @classmethod def run_with_retry(cls, what: str, *args, **kwargs) -> bool: try_count = 0 - retries = kwargs.pop('retries', 3) - sleep_seconds = kwargs.pop('sleep_seconds', 5) + retries = kwargs.pop("retries", 3) + sleep_seconds = kwargs.pop("sleep_seconds", 5) while try_count < retries: if try_count > 0: print(f"Sleeping for {sleep_seconds} before retrying command") time.sleep(sleep_seconds) if cls.run(what, *args, **kwargs): return True - try_count+=1 + try_count += 1 return False @classmethod diff --git a/python/kubewatch.py b/python/kubewatch.py index fdd733aedf..7b5455030b 100644 --- a/python/kubewatch.py +++ b/python/kubewatch.py @@ -28,14 +28,15 @@ __version__ = Version ambassador_id = os.getenv("AMBASSADOR_ID", "default") -ambassador_namespace = os.environ.get('AMBASSADOR_NAMESPACE', 'default') +ambassador_namespace = os.environ.get("AMBASSADOR_NAMESPACE", "default") ambassador_single_namespace = bool("AMBASSADOR_SINGLE_NAMESPACE" in os.environ) -ambassador_basedir = os.environ.get('AMBASSADOR_CONFIG_BASE_DIR', '/ambassador') +ambassador_basedir = os.environ.get("AMBASSADOR_CONFIG_BASE_DIR", "/ambassador") logging.basicConfig( level=logging.INFO, # if appDebug else logging.INFO, - format="%%(asctime)s kubewatch [%%(process)d T%%(threadName)s] %s %%(levelname)s: %%(message)s" % __version__, - datefmt="%Y-%m-%d %H:%M:%S" + format="%%(asctime)s kubewatch [%%(process)d T%%(threadName)s] %s %%(levelname)s: %%(message)s" + % __version__, + datefmt="%Y-%m-%d %H:%M:%S", ) # logging.getLogger("datawire.scout").setLevel(logging.DEBUG) @@ -104,9 +105,9 @@ def check_crd_type(crd): status = True except client.rest.ApiException as e: if e.status == 404: - logger.debug(f'CRD type definition not found for {crd}') + logger.debug(f"CRD type definition not found for {crd}") else: - logger.debug(f'CRD type definition unreadable for {crd}: {e.reason}') + logger.debug(f"CRD type definition unreadable for {crd}: {e.reason}") return status @@ -124,7 +125,7 @@ def check_ingresses(): k8s_v1b1.list_ingress_for_all_namespaces() status = True except ApiException as e: - logger.debug(f'Ingress check got {e.status}') + logger.debug(f"Ingress check got {e.status}") return status @@ -145,26 +146,31 @@ def check_ingress_classes(): query_params: List[str] = [] header_params: Dict[str, str] = {} - header_params['Accept'] = api_client. \ - select_header_accept(['application/json', - 'application/yaml', - 'application/vnd.kubernetes.protobuf', - 'application/json;stream=watch', - 'application/vnd.kubernetes.protobuf;stream=watch']) + header_params["Accept"] = api_client.select_header_accept( + [ + "application/json", + "application/yaml", + "application/vnd.kubernetes.protobuf", + "application/json;stream=watch", + "application/vnd.kubernetes.protobuf;stream=watch", + ] + ) - header_params['Content-Type'] = api_client. \ - select_header_content_type(['*/*']) + header_params["Content-Type"] = api_client.select_header_content_type(["*/*"]) - auth_settings = ['BearerToken'] + auth_settings = ["BearerToken"] - api_client.call_api('/apis/networking.k8s.io/v1beta1/ingressclasses', 'GET', - path_params, - query_params, - header_params, - auth_settings=auth_settings) + api_client.call_api( + "/apis/networking.k8s.io/v1beta1/ingressclasses", + "GET", + path_params, + query_params, + header_params, + auth_settings=auth_settings, + ) status = True except ApiException as e: - logger.debug(f'IngressClass check got {e.status}') + logger.debug(f"IngressClass check got {e.status}") return status @@ -183,20 +189,22 @@ def get_api_resources(group, version): query_params: List[str] = [] header_params: Dict[str, str] = {} - header_params['Accept'] = api_client. \ - select_header_accept(['application/json']) + header_params["Accept"] = api_client.select_header_accept(["application/json"]) - auth_settings = ['BearerToken'] + auth_settings = ["BearerToken"] - (data) = api_client.call_api(f'/apis/{group}/{version}', 'GET', - path_params, - query_params, - header_params, - auth_settings=auth_settings, - response_type='V1APIResourceList') + (data) = api_client.call_api( + f"/apis/{group}/{version}", + "GET", + path_params, + query_params, + header_params, + auth_settings=auth_settings, + response_type="V1APIResourceList", + ) return data[0] except ApiException as e: - logger.error(f'get_api_resources {e.status}') + logger.error(f"get_api_resources {e.status}") return None @@ -218,7 +226,9 @@ def main(debug): found = None root_id = None - cluster_id = os.environ.get('AMBASSADOR_CLUSTER_ID', os.environ.get('AMBASSADOR_SCOUT_ID', None)) + cluster_id = os.environ.get( + "AMBASSADOR_CLUSTER_ID", os.environ.get("AMBASSADOR_SCOUT_ID", None) + ) wanted = ambassador_namespace if ambassador_single_namespace else "default" # Go ahead and try connecting to Kube. @@ -261,112 +271,110 @@ def main(debug): required_crds = [ ( - '.ambassador_ignore_crds', 'Main CRDs', - [ - 'authservices.getambassador.io', - 'mappings.getambassador.io', - 'modules.getambassador.io', - 'ratelimitservices.getambassador.io', - 'tcpmappings.getambassador.io', - 'tlscontexts.getambassador.io', - 'tracingservices.getambassador.io' - ] - ), - ( - '.ambassador_ignore_crds_2', 'Resolver CRDs', - [ - 'consulresolvers.getambassador.io', - 'kubernetesendpointresolvers.getambassador.io', - 'kubernetesserviceresolvers.getambassador.io' - ] - ), - ( - '.ambassador_ignore_crds_3', 'Host CRDs', + ".ambassador_ignore_crds", + "Main CRDs", [ - 'hosts.getambassador.io' - ] + "authservices.getambassador.io", + "mappings.getambassador.io", + "modules.getambassador.io", + "ratelimitservices.getambassador.io", + "tcpmappings.getambassador.io", + "tlscontexts.getambassador.io", + "tracingservices.getambassador.io", + ], ), ( - '.ambassador_ignore_crds_4', 'LogService CRDs', + ".ambassador_ignore_crds_2", + "Resolver CRDs", [ - 'logservices.getambassador.io' - ] + "consulresolvers.getambassador.io", + "kubernetesendpointresolvers.getambassador.io", + "kubernetesserviceresolvers.getambassador.io", + ], ), - ( - '.ambassador_ignore_crds_5', 'DevPortal CRDs', - [ - 'devportals.getambassador.io' - ] - ) + (".ambassador_ignore_crds_3", "Host CRDs", ["hosts.getambassador.io"]), + (".ambassador_ignore_crds_4", "LogService CRDs", ["logservices.getambassador.io"]), + (".ambassador_ignore_crds_5", "DevPortal CRDs", ["devportals.getambassador.io"]), ] # Flynn would say "Ew.", but we need to patch this till https://github.com/kubernetes-client/python/issues/376 # and https://github.com/kubernetes-client/gen/issues/52 are fixed \_(0.0)_/ - client.models.V1beta1CustomResourceDefinitionStatus.accepted_names = \ - property(hack_accepted_names, hack_accepted_names_setter) + client.models.V1beta1CustomResourceDefinitionStatus.accepted_names = property( + hack_accepted_names, hack_accepted_names_setter + ) - client.models.V1beta1CustomResourceDefinitionStatus.conditions = \ - property(hack_conditions, hack_conditions_setter) + client.models.V1beta1CustomResourceDefinitionStatus.conditions = property( + hack_conditions, hack_conditions_setter + ) - client.models.V1beta1CustomResourceDefinitionStatus.stored_versions = \ - property(hack_stored_versions, hack_stored_versions_setter) + client.models.V1beta1CustomResourceDefinitionStatus.stored_versions = property( + hack_stored_versions, hack_stored_versions_setter + ) known_api_resources = [] api_resources = get_api_resources("getambassador.io", "v2") if api_resources: - known_api_resources = list(map(lambda r: r.name + '.getambassador.io', api_resources.resources)) + known_api_resources = list( + map(lambda r: r.name + ".getambassador.io", api_resources.resources) + ) for touchfile, description, required in required_crds: for crd in required: if not crd in known_api_resources: touch_file(touchfile) - logger.debug(f'{description} are not available.' + - ' To enable CRD support, configure the Ambassador CRD type definitions and RBAC,' + - ' then restart the Ambassador pod.') + logger.debug( + f"{description} are not available." + + " To enable CRD support, configure the Ambassador CRD type definitions and RBAC," + + " then restart the Ambassador pod." + ) # logger.debug(f'touched {touchpath}') if not check_ingress_classes(): - touch_file('.ambassador_ignore_ingress_class') + touch_file(".ambassador_ignore_ingress_class") - logger.debug(f'Ambassador does not have permission to read IngressClass resources.' + - ' To enable IngressClass support, configure RBAC to allow Ambassador to read IngressClass' - ' resources, then restart the Ambassador pod.') + logger.debug( + f"Ambassador does not have permission to read IngressClass resources." + + " To enable IngressClass support, configure RBAC to allow Ambassador to read IngressClass" + " resources, then restart the Ambassador pod." + ) if not check_ingresses(): - touch_file('.ambassador_ignore_ingress') + touch_file(".ambassador_ignore_ingress") - logger.debug(f'Ambassador does not have permission to read Ingress resources.' + - ' To enable Ingress support, configure RBAC to allow Ambassador to read Ingress resources,' + - ' then restart the Ambassador pod.') + logger.debug( + f"Ambassador does not have permission to read Ingress resources." + + " To enable Ingress support, configure RBAC to allow Ambassador to read Ingress resources," + + " then restart the Ambassador pod." + ) # Check for our operator's CRD now - if check_crd_type('ambassadorinstallations.getambassador.io'): - touch_file('.ambassadorinstallations_ok') - logger.debug('ambassadorinstallations.getambassador.io CRD available') + if check_crd_type("ambassadorinstallations.getambassador.io"): + touch_file(".ambassadorinstallations_ok") + logger.debug("ambassadorinstallations.getambassador.io CRD available") else: - logger.debug('ambassadorinstallations.getambassador.io CRD not available') + logger.debug("ambassadorinstallations.getambassador.io CRD not available") # Have we been asked to do Knative support? - if os.environ.get('AMBASSADOR_KNATIVE_SUPPORT', '').lower() == 'true': + if os.environ.get("AMBASSADOR_KNATIVE_SUPPORT", "").lower() == "true": # Yes. Check for their CRD types. - if check_crd_type('clusteringresses.networking.internal.knative.dev'): - touch_file('.knative_clusteringress_ok') - logger.debug('Knative clusteringresses available') + if check_crd_type("clusteringresses.networking.internal.knative.dev"): + touch_file(".knative_clusteringress_ok") + logger.debug("Knative clusteringresses available") else: - logger.debug('Knative clusteringresses not available') + logger.debug("Knative clusteringresses not available") - if check_crd_type('ingresses.networking.internal.knative.dev'): - touch_file('.knative_ingress_ok') - logger.debug('Knative ingresses available') + if check_crd_type("ingresses.networking.internal.knative.dev"): + touch_file(".knative_ingress_ok") + logger.debug("Knative ingresses available") else: - logger.debug('Knative ingresses not available') + logger.debug("Knative ingresses not available") else: # If we couldn't talk to Kube, log that, but broadly we'll expect our caller # to DTRT around CRDs. - logger.debug('Kubernetes is not available, so not doing CRD check') + logger.debug("Kubernetes is not available, so not doing CRD check") # Finally, spit out the cluster ID for our caller. logger.debug("cluster ID is %s (from %s)" % (cluster_id, found)) diff --git a/python/post_update.py b/python/post_update.py index 4481e21f38..caa7d5330d 100644 --- a/python/post_update.py +++ b/python/post_update.py @@ -6,50 +6,53 @@ import requests from ambassador.utils import parse_bool + def usage(program): - sys.stderr.write(f'Usage: {program} [--watt|--k8s|--fs] UPDATE_URL\n') - sys.stderr.write('Notify `diagd` (and `amb-sidecar`, if AES) that a new WATT snapshot is available at UPDATE_URL.\n') + sys.stderr.write(f"Usage: {program} [--watt|--k8s|--fs] UPDATE_URL\n") + sys.stderr.write( + "Notify `diagd` (and `amb-sidecar`, if AES) that a new WATT snapshot is available at UPDATE_URL.\n" + ) sys.exit(1) -base_host = os.environ.get('DEV_AMBASSADOR_EVENT_HOST', 'http://localhost:8877') -base_path = os.environ.get('DEV_AMBASSADOR_EVENT_PATH', '_internal/v0') +base_host = os.environ.get("DEV_AMBASSADOR_EVENT_HOST", "http://localhost:8877") +base_path = os.environ.get("DEV_AMBASSADOR_EVENT_PATH", "_internal/v0") -sidecar_host = os.environ.get('DEV_AMBASSADOR_SIDECAR_HOST', 'http://localhost:8500') -sidecar_path = os.environ.get('DEV_AMBASSADOR_SIDECAR_PATH', '_internal/v0') +sidecar_host = os.environ.get("DEV_AMBASSADOR_SIDECAR_HOST", "http://localhost:8500") +sidecar_path = os.environ.get("DEV_AMBASSADOR_SIDECAR_PATH", "_internal/v0") -url_type = 'update' -arg_key = 'url' +url_type = "update" +arg_key = "url" program = os.path.basename(sys.argv[0]) args = sys.argv[1:] -while args and args[0].startswith('--'): +while args and args[0].startswith("--"): arg = args.pop(0) - if arg == '--k8s': + if arg == "--k8s": # Already set up. pass - elif arg == '--watt': - url_type = 'watt' - elif arg == '--fs': - url_type = 'fs' - arg_key = 'path' + elif arg == "--watt": + url_type = "watt" + elif arg == "--fs": + url_type = "fs" + arg_key = "path" else: usage(program) if len(args) != 1: usage(program) -urls = [ f'{base_host}/{base_path}/{url_type}' ] +urls = [f"{base_host}/{base_path}/{url_type}"] -if parse_bool(os.environ.get('EDGE_STACK', 'false')) or os.path.exists('/ambassador/.edge_stack'): - urls.append(f'{sidecar_host}/{sidecar_path}/{url_type}') +if parse_bool(os.environ.get("EDGE_STACK", "false")) or os.path.exists("/ambassador/.edge_stack"): + urls.append(f"{sidecar_host}/{sidecar_path}/{url_type}") exitcode = 0 for url in urls: - r = requests.post(url, params={ arg_key: args[0] }) + r = requests.post(url, params={arg_key: args[0]}) if r.status_code != 200: sys.stderr.write("failed to update %s: %d: %s" % (r.url, r.status_code, r.text)) diff --git a/python/setup.py b/python/setup.py index cf6a5f191b..626282fd42 100644 --- a/python/setup.py +++ b/python/setup.py @@ -22,20 +22,21 @@ requirements = open("requirements.txt", "r").read().split("\n") + def collect_data_files(dirpath): return [ - (subdirpath, - [ os.path.join(subdirpath, filename) - for filename in filenames ]) + (subdirpath, [os.path.join(subdirpath, filename) for filename in filenames]) for subdirpath, folders, filenames in os.walk(dirpath) ] + template_files = collect_data_files("templates") schema_files = collect_data_files("schemas") kat_files = [ - (subdirpath, - [ os.path.join(subdirpath, filename) - for filename in filenames if filename.endswith("go") ]) + ( + subdirpath, + [os.path.join(subdirpath, filename) for filename in filenames if filename.endswith("go")], + ) for subdirpath, folders, filenames in os.walk("kat") ] @@ -49,29 +50,20 @@ def collect_data_files(dirpath): packages=find_packages(exclude=["tests"]), # include_package_data=True, install_requires=requirements, - data_files=data_files, - entry_points={ - 'console_scripts': [ - 'ambassador=ambassador_cli.ambassador:main', - 'diagd=ambassador_diag.diagd:main', - 'mockery=ambassador_cli.mockery:main', - 'grab-snapshots=ambassador_cli.grab_snapshots:main', - 'ert=ambassador_cli.ert:main' + "console_scripts": [ + "ambassador=ambassador_cli.ambassador:main", + "diagd=ambassador_diag.diagd:main", + "mockery=ambassador_cli.mockery:main", + "grab-snapshots=ambassador_cli.grab_snapshots:main", + "ert=ambassador_cli.ert:main", ] }, - author="datawire.io", author_email="dev@datawire.io", url="https://www.getambassador.io", download_url="https://github.com/datawire/ambassador", - keywords=[ - "kubernetes", - "microservices", - "api gateway", - "envoy", - "ambassador" - ], + keywords=["kubernetes", "microservices", "api gateway", "envoy", "ambassador"], classifiers=[], ) diff --git a/python/tests/integration/manifests.py b/python/tests/integration/manifests.py index 4e6e86c962..7312e41ae7 100644 --- a/python/tests/integration/manifests.py +++ b/python/tests/integration/manifests.py @@ -4,31 +4,37 @@ from base64 import b64encode from typing import Dict, Final, Optional + def _get_images() -> Dict[str, str]: ret: Dict[str, str] = {} # Keep this list in-sync with the 'push-pytest-images' Makefile target. image_names = [ - 'test-auth', - 'test-shadow', - 'test-stats', - 'kat-client', - 'kat-server', + "test-auth", + "test-shadow", + "test-stats", + "kat-client", + "kat-server", ] - if image := os.environ.get('AMBASSADOR_DOCKER_IMAGE'): - ret['emissary'] = image + if image := os.environ.get("AMBASSADOR_DOCKER_IMAGE"): + ret["emissary"] = image else: - image_names.append('emissary') + image_names.append("emissary") try: - subprocess.run(['make']+[f'docker/{name}.docker.push.remote' for name in image_names], - check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + subprocess.run( + ["make"] + [f"docker/{name}.docker.push.remote" for name in image_names], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) except subprocess.CalledProcessError as err: raise Exception(f"{err.stdout}{err}") from err for name in image_names: - with open(f'docker/{name}.docker.push.remote', 'r') as fh: + with open(f"docker/{name}.docker.push.remote", "r") as fh: # file contents: # line 1: image ID # line 2: tag 1 @@ -39,35 +45,39 @@ def _get_images() -> Dict[str, str]: return ret + _image_cache: Optional[Dict[str, str]] = None + def get_images() -> Dict[str, str]: global _image_cache if not _image_cache: _image_cache = _get_images() return _image_cache + _file_cache: Dict[str, str] = {} + def load(manifest_name: str) -> str: if manifest_name in _file_cache: return _file_cache[manifest_name] - manifest_dir = __file__[:-len('.py')] - manifest_file = os.path.join(manifest_dir, manifest_name+'.yaml') - manifest_content = open(manifest_file, 'r').read() + manifest_dir = __file__[: -len(".py")] + manifest_file = os.path.join(manifest_dir, manifest_name + ".yaml") + manifest_content = open(manifest_file, "r").read() _file_cache[manifest_name] = manifest_content return manifest_content + def format(st: str, /, **kwargs): - serviceAccountExtra = '' - if os.environ.get("DEV_USE_IMAGEPULLSECRET", False): - serviceAccountExtra = """ + serviceAccountExtra = "" + if os.environ.get("DEV_USE_IMAGEPULLSECRET", False): + serviceAccountExtra = """ imagePullSecrets: - name: dev-image-pull-secret """ - return st.format(serviceAccountExtra=serviceAccountExtra, - images=get_images(), - **kwargs) + return st.format(serviceAccountExtra=serviceAccountExtra, images=get_images(), **kwargs) + def namespace_manifest(namespace: str) -> str: ret = f""" @@ -81,8 +91,14 @@ def namespace_manifest(namespace: str) -> str: if os.environ.get("DEV_USE_IMAGEPULLSECRET", None): dockercfg = { "auths": { - os.path.dirname(os.environ['DEV_REGISTRY']): { - "auth": b64encode((os.environ['DOCKER_BUILD_USERNAME']+":"+os.environ['DOCKER_BUILD_PASSWORD']).encode("utf-8")).decode("utf-8") + os.path.dirname(os.environ["DEV_REGISTRY"]): { + "auth": b64encode( + ( + os.environ["DOCKER_BUILD_USERNAME"] + + ":" + + os.environ["DOCKER_BUILD_PASSWORD"] + ).encode("utf-8") + ).decode("utf-8") } } } @@ -108,17 +124,18 @@ def namespace_manifest(namespace: str) -> str: return ret + def crd_manifests() -> str: ret = "" - ret += namespace_manifest('emissary-system') + ret += namespace_manifest("emissary-system") # Use .replace instead of .format because there are other '{word}' things in 'description' fields # that would cause KeyErrors when .format erroneously tries to evaluate them. ret += ( - load('crds') - .replace('{images[emissary]}', get_images()['emissary']) - .replace('{serviceAccountExtra}', format('{serviceAccountExtra}')) + load("crds") + .replace("{images[emissary]}", get_images()["emissary"]) + .replace("{serviceAccountExtra}", format("{serviceAccountExtra}")) ) return ret diff --git a/python/tests/integration/test_docker.py b/python/tests/integration/test_docker.py index ace44b5478..239c516008 100644 --- a/python/tests/integration/test_docker.py +++ b/python/tests/integration/test_docker.py @@ -10,8 +10,9 @@ DOCKER_IMAGE = os.environ.get("AMBASSADOR_DOCKER_IMAGE", "") -child: Optional[pexpect.spawnbase.SpawnBase] = None # see docker_start() -ambassador_host: Optional[str] = None # see docker_start() +child: Optional[pexpect.spawnbase.SpawnBase] = None # see docker_start() +ambassador_host: Optional[str] = None # see docker_start() + def docker_start(logfile) -> bool: # Use a global here so that the child process doesn't get killed @@ -19,64 +20,71 @@ def docker_start(logfile) -> bool: global ambassador_host - cmd = f'docker run --rm --name test_docker_ambassador -p 9987:8080 -u8888:0 {DOCKER_IMAGE} --demo' - ambassador_host = 'localhost:9987' + cmd = ( + f"docker run --rm --name test_docker_ambassador -p 9987:8080 -u8888:0 {DOCKER_IMAGE} --demo" + ) + ambassador_host = "localhost:9987" - child = pexpect.spawn(cmd, encoding='utf-8') + child = pexpect.spawn(cmd, encoding="utf-8") child.logfile = logfile - i = child.expect([ pexpect.EOF, pexpect.TIMEOUT, 'AMBASSADOR DEMO RUNNING' ]) + i = child.expect([pexpect.EOF, pexpect.TIMEOUT, "AMBASSADOR DEMO RUNNING"]) if i == 0: - logfile.write('ambassador died?\n') + logfile.write("ambassador died?\n") return False elif i == 1: - logfile.write('ambassador timed out?\n') + logfile.write("ambassador timed out?\n") return False else: - logfile.write('ambassador running\n') + logfile.write("ambassador running\n") return True + def docker_kill(logfile): - cmd = f'docker kill test_docker_ambassador' + cmd = f"docker kill test_docker_ambassador" - child = pexpect.spawn(cmd, encoding='utf-8') + child = pexpect.spawn(cmd, encoding="utf-8") child.logfile = logfile - child.expect([ pexpect.EOF, pexpect.TIMEOUT ]) + child.expect([pexpect.EOF, pexpect.TIMEOUT]) + def check_http(logfile) -> bool: try: logfile.write("QotM: making request\n") - response = requests.get(f'http://{ambassador_host}/qotm/?json=true', headers={ 'Host': 'localhost' }) + response = requests.get( + f"http://{ambassador_host}/qotm/?json=true", headers={"Host": "localhost"} + ) text = response.text logfile.write(f"QotM: got status {response.status_code}, text {text}\n") if response.status_code != 200: - logfile.write(f'QotM: wanted 200 but got {response.status_code} {text}\n') + logfile.write(f"QotM: wanted 200 but got {response.status_code} {text}\n") return False return True except Exception as e: - logfile.write(f'Could not do HTTP: {e}\n') + logfile.write(f"Could not do HTTP: {e}\n") return False + def check_cli() -> bool: child = pexpect.spawn(f"docker run --rm --entrypoint ambassador {DOCKER_IMAGE} --help") - #v_encoded = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - i = child.expect([ pexpect.EOF, pexpect.TIMEOUT, "Usage: ambassador" ]) + # v_encoded = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + i = child.expect([pexpect.EOF, pexpect.TIMEOUT, "Usage: ambassador"]) if i == 0: - print('ambassador died without usage statement?') + print("ambassador died without usage statement?") return False elif i == 1: - print('ambassador timed out without usage statement?') + print("ambassador timed out without usage statement?") return False - i = child.expect([ pexpect.EOF, pexpect.TIMEOUT ]) + i = child.expect([pexpect.EOF, pexpect.TIMEOUT]) if i == 0: return True @@ -88,16 +96,16 @@ def check_cli() -> bool: def check_grab_snapshots() -> bool: child = pexpect.spawn(f"docker run --rm --entrypoint grab-snapshots {DOCKER_IMAGE} --help") - i = child.expect([ pexpect.EOF, pexpect.TIMEOUT, "Usage: grab-snapshots" ]) + i = child.expect([pexpect.EOF, pexpect.TIMEOUT, "Usage: grab-snapshots"]) if i == 0: - print('grab-snapshots died without usage statement?') + print("grab-snapshots died without usage statement?") return False elif i == 1: - print('grab-snapshots timed out without usage statement?') + print("grab-snapshots timed out without usage statement?") return False - i = child.expect([ pexpect.EOF, pexpect.TIMEOUT ]) + i = child.expect([pexpect.EOF, pexpect.TIMEOUT]) if i == 0: return True @@ -109,17 +117,19 @@ def check_grab_snapshots() -> bool: def test_cli(): assert check_cli(), "CLI check failed" + def test_grab_snapshots(): assert check_grab_snapshots(), "grab-snapshots check failed" + def test_demo(): test_status = False # And this tests that the Ambasasdor can run with the `--demo` argument # and run normally with a sample /qotm/ Mapping. - with open('/tmp/test_docker_output', 'w') as logfile: + with open("/tmp/test_docker_output", "w") as logfile: if not DOCKER_IMAGE: - logfile.write('No $AMBASSADOR_DOCKER_IMAGE??\n') + logfile.write("No $AMBASSADOR_DOCKER_IMAGE??\n") else: if docker_start(logfile): logfile.write("Demo started, first check...") @@ -134,11 +144,12 @@ def test_demo(): docker_kill(logfile) if not test_status: - with open('/tmp/test_docker_output', 'r') as logfile: + with open("/tmp/test_docker_output", "r") as logfile: for line in logfile: print(line.rstrip()) - assert test_status, 'test failed' + assert test_status, "test failed" + -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/integration/test_header_case_overrides.py b/python/tests/integration/test_header_case_overrides.py index 159eca0cb6..a5757c7416 100644 --- a/python/tests/integration/test_header_case_overrides.py +++ b/python/tests/integration/test_header_case_overrides.py @@ -14,7 +14,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -25,7 +25,7 @@ from ambassador.fetch import ResourceFetcher from ambassador.utils import NullSecretHandler -headerecho_manifests =""" +headerecho_manifests = """ --- apiVersion: v1 kind: Service @@ -62,6 +62,7 @@ containerPort: 8080 """ + def create_headerecho_mapping(namespace): headerecho_mapping = f""" --- @@ -79,8 +80,9 @@ def create_headerecho_mapping(namespace): apply_kube_artifacts(namespace=namespace, artifacts=headerecho_mapping) + def _ambassador_module_config(): - return ''' + return """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -89,35 +91,51 @@ def _ambassador_module_config(): namespace: default spec: config: -''' +""" + def _ambassador_module_header_case_overrides(overrides, proper_case=False): mod = _ambassador_module_config() if len(overrides) == 0: - mod = mod + ''' + mod = ( + mod + + """ header_case_overrides: [] -''' +""" + ) return mod - mod = mod + ''' + mod = ( + mod + + """ header_case_overrides: -''' +""" + ) for override in overrides: - mod = mod + f''' + mod = ( + mod + + f""" - {override} -''' +""" + ) # proper case isn't valid if header_case_overrides are set, but we do # it here for tests that want to test that this is in fact invalid. if proper_case: - mod = mod + f''' + mod = ( + mod + + f""" proper_case: true -''' +""" + ) return mod + def _test_headercaseoverrides(yaml, expectations, expect_norules=False): aconf = Config() - yaml = yaml + ''' + yaml = ( + yaml + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -142,7 +160,8 @@ def _test_headercaseoverrides(yaml, expectations, expect_norules=False): service: httpbin hostname: "*" prefix: /httpbin/ -''' +""" + ) fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(yaml, k8s=True) @@ -161,27 +180,30 @@ def _test_headercaseoverrides(yaml, expectations, expect_norules=False): found_cluster_rules = False conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - for filter_chain in listener['filter_chains']: - for f in filter_chain['filters']: - typed_config = f['typed_config'] - if 'http_protocol_options' not in typed_config: + for listener in conf["static_resources"]["listeners"]: + for filter_chain in listener["filter_chains"]: + for f in filter_chain["filters"]: + typed_config = f["typed_config"] + if "http_protocol_options" not in typed_config: continue - http_protocol_options = typed_config['http_protocol_options'] + http_protocol_options = typed_config["http_protocol_options"] if expect_norules: - assert 'header_key_format' not in http_protocol_options, \ - f"'header_key_format' found unexpected typed_config {typed_config}" + assert ( + "header_key_format" not in http_protocol_options + ), f"'header_key_format' found unexpected typed_config {typed_config}" continue - assert 'header_key_format' in http_protocol_options, \ - f"'header_key_format' not found, typed_config {typed_config}" + assert ( + "header_key_format" in http_protocol_options + ), f"'header_key_format' not found, typed_config {typed_config}" - header_key_format = http_protocol_options['header_key_format'] - assert 'custom' in header_key_format, \ - f"'custom' not found, typed_config {typed_config}" + header_key_format = http_protocol_options["header_key_format"] + assert ( + "custom" in header_key_format + ), f"'custom' not found, typed_config {typed_config}" - rules = header_key_format['custom']['rules'] + rules = header_key_format["custom"]["rules"] assert len(rules) == len(expectations) for e in expectations: hdr = e.lower() @@ -190,30 +212,32 @@ def _test_headercaseoverrides(yaml, expectations, expect_norules=False): assert rule == e, f"unexpected rule {rule} in {rules}" found_module_rules = True - for cluster in conf['static_resources']['clusters']: - if 'httpbin' not in cluster['name']: + for cluster in conf["static_resources"]["clusters"]: + if "httpbin" not in cluster["name"]: continue - http_protocol_options = cluster.get('http_protocol_options', None) + http_protocol_options = cluster.get("http_protocol_options", None) if not http_protocol_options: if expect_norules: continue - assert 'http_protocol_options' in cluster, \ - f"'http_protocol_options' missing from cluster: {cluster}" + assert ( + "http_protocol_options" in cluster + ), f"'http_protocol_options' missing from cluster: {cluster}" if expect_norules: - assert 'header_key_format' not in http_protocol_options, \ - f"'header_key_format' found unexpected cluster: {cluster}" + assert ( + "header_key_format" not in http_protocol_options + ), f"'header_key_format' found unexpected cluster: {cluster}" continue - assert 'header_key_format' in http_protocol_options, \ - f"'header_key_format' not found, cluster {cluster}" + assert ( + "header_key_format" in http_protocol_options + ), f"'header_key_format' not found, cluster {cluster}" - header_key_format = http_protocol_options['header_key_format'] - assert 'custom' in header_key_format, \ - f"'custom' not found, cluster {cluster}" + header_key_format = http_protocol_options["header_key_format"] + assert "custom" in header_key_format, f"'custom' not found, cluster {cluster}" - rules = header_key_format['custom']['rules'] + rules = header_key_format["custom"]["rules"] assert len(rules) == len(expectations) for e in expectations: hdr = e.lower() @@ -229,6 +253,7 @@ def _test_headercaseoverrides(yaml, expectations, expect_norules=False): assert found_module_rules assert found_cluster_rules + def _test_headercaseoverrides_rules(rules, expected=None, expect_norules=False): if not expected: expected = rules @@ -238,40 +263,43 @@ def _test_headercaseoverrides_rules(rules, expected=None, expect_norules=False): expect_norules=expect_norules, ) + # Test that we throw assertions for obviously wrong cases @pytest.mark.compilertest def test_testsanity(): failed = False try: - _test_headercaseoverrides_rules(['X-ABC'], expected=['X-Wrong']) + _test_headercaseoverrides_rules(["X-ABC"], expected=["X-Wrong"]) except AssertionError as e: failed = True assert failed failed = False try: - _test_headercaseoverrides_rules([], expected=['X-Wrong']) + _test_headercaseoverrides_rules([], expected=["X-Wrong"]) except AssertionError as e: failed = True assert failed + # Test that we can parse a variety of header case override arrays. @pytest.mark.compilertest def test_headercaseoverrides_basic(): _test_headercaseoverrides_rules([], expect_norules=True) _test_headercaseoverrides_rules([{}], expect_norules=True) _test_headercaseoverrides_rules([5], expect_norules=True) - _test_headercaseoverrides_rules(['X-ABC']) - _test_headercaseoverrides_rules(['X-foo', 'X-ABC-Baz']) - _test_headercaseoverrides_rules(['x-goOd', 'X-alSo-good', 'Authorization']) - _test_headercaseoverrides_rules(['x-good', ['hello']], expected=['x-good']) - _test_headercaseoverrides_rules(['X-ABC', 'x-foo', 5, {}], expected=['X-ABC', 'x-foo']) + _test_headercaseoverrides_rules(["X-ABC"]) + _test_headercaseoverrides_rules(["X-foo", "X-ABC-Baz"]) + _test_headercaseoverrides_rules(["x-goOd", "X-alSo-good", "Authorization"]) + _test_headercaseoverrides_rules(["x-good", ["hello"]], expected=["x-good"]) + _test_headercaseoverrides_rules(["X-ABC", "x-foo", 5, {}], expected=["X-ABC", "x-foo"]) + # Test that we always omit header case overrides if proper case is set @pytest.mark.compilertest def test_headercaseoverrides_propercasefail(): _test_headercaseoverrides( - _ambassador_module_header_case_overrides(['My-OPINIONATED-CASING'], proper_case=True), + _ambassador_module_header_case_overrides(["My-OPINIONATED-CASING"], proper_case=True), [], expect_norules=True, ) @@ -324,7 +352,7 @@ def create_listeners(self, namespace): def test_header_case_overrides(self): # Is there any reason not to use the default namespace? - namespace = 'header-case-overrides' + namespace = "header-case-overrides" # Install Ambassador install_ambassador(namespace=namespace) @@ -348,16 +376,40 @@ def test_header_case_overrides(self): create_headerecho_mapping(namespace=namespace) # Now let's wait for ambassador and httpbin pods to become ready - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'service=ambassador', '-n', namespace]) - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'service=httpbin', '-n', namespace]) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "service=ambassador", + "-n", + namespace, + ] + ) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "service=httpbin", + "-n", + namespace, + ] + ) # Assume we can reach Ambassador through telepresence ambassador_host = "ambassador." + namespace # Assert 200 OK at httpbin/status/200 endpoint ready = False - httpbin_url = f'http://{ambassador_host}/httpbin/status/200' - headerecho_url = f'http://{ambassador_host}/headerecho/' + httpbin_url = f"http://{ambassador_host}/httpbin/status/200" + headerecho_url = f"http://{ambassador_host}/headerecho/" loop_limit = 10 while not ready: @@ -387,7 +439,7 @@ def test_header_case_overrides(self): assert ready - httpbin_url = f'http://{ambassador_host}/httpbin/response-headers?x-Hello=1&X-foo-Bar=1&x-Lowercase1=1&x-lowercase2=1' + httpbin_url = f"http://{ambassador_host}/httpbin/response-headers?x-Hello=1&X-foo-Bar=1&x-Lowercase1=1&x-lowercase2=1" resp = requests.get(httpbin_url, timeout=5) code = resp.status_code assert code == 200, f"Expected 200 OK, got {code}" @@ -397,28 +449,23 @@ def test_header_case_overrides(self): # Very important: this test relies on matching case sensitive header keys. # Fortunately it appears that we can convert resp.headers, a case insensitive # dictionary, into a list of case sensitive keys. - keys = [ h for h in resp.headers.keys() ] + keys = [h for h in resp.headers.keys()] for k in keys: print(f"header key: {k}") - assert 'x-hello' not in keys - assert 'X-HELLO' in keys - assert 'x-foo-bar' not in keys - assert 'X-FOO-Bar' in keys - assert 'x-lowercase1' in keys - assert 'x-Lowercase1' not in keys - assert 'x-lowercase2' in keys + assert "x-hello" not in keys + assert "X-HELLO" in keys + assert "x-foo-bar" not in keys + assert "X-FOO-Bar" in keys + assert "x-lowercase1" in keys + assert "x-Lowercase1" not in keys + assert "x-lowercase2" in keys resp.close() # Second, test that the request headers sent to the headerecho server # have the correct case. - headers = { - 'x-Hello': '1', - 'X-foo-Bar': '1', - 'x-Lowercase1': '1', - 'x-lowercase2': '1' - } + headers = {"x-Hello": "1", "X-foo-Bar": "1", "x-Lowercase1": "1", "x-lowercase2": "1"} resp = requests.get(headerecho_url, headers=headers, timeout=5) code = resp.status_code assert code == 200, f"Expected 200 OK, got {code}" @@ -426,16 +473,16 @@ def test_header_case_overrides(self): response_obj = json.loads(resp.text) print(f"response_obj = {response_obj}") assert response_obj - assert 'headers' in response_obj + assert "headers" in response_obj - hdrs = response_obj['headers'] - assert 'x-hello' not in hdrs - assert 'X-HELLO' in hdrs - assert 'x-foo-bar' not in hdrs - assert 'X-FOO-Bar' in hdrs - assert 'x-lowercase1' in hdrs - assert 'x-Lowercase1' not in hdrs - assert 'x-lowercase2' in hdrs + hdrs = response_obj["headers"] + assert "x-hello" not in hdrs + assert "X-HELLO" in hdrs + assert "x-foo-bar" not in hdrs + assert "X-FOO-Bar" in hdrs + assert "x-lowercase1" in hdrs + assert "x-Lowercase1" not in hdrs + assert "x-lowercase2" in hdrs @pytest.mark.flaky(reruns=1, reruns_delay=10) @@ -443,5 +490,6 @@ def test_ambassador_headercaseoverrides(): t = HeaderCaseOverridesTesting() t.test_header_case_overrides() -if __name__ == '__main__': + +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/integration/test_knative.py b/python/tests/integration/test_knative.py index cd7e953bb1..37324fb5fb 100644 --- a/python/tests/integration/test_knative.py +++ b/python/tests/integration/test_knative.py @@ -17,7 +17,7 @@ from tests.runutils import run_with_retry, run_and_assert from tests.manifests import qotm_manifests -logger = logging.getLogger('ambassador') +logger = logging.getLogger("ambassador") # knative_service_example gets applied to the cluster with `kubectl --namespace=knative-testing # apply`; we therefore DO NOT explicitly set the 'namespace:' because --namespace will imply it, and @@ -68,34 +68,120 @@ class KnativeTesting: def test_knative(self): - namespace = 'knative-testing' + namespace = "knative-testing" # Install Knative - apply_kube_artifacts(namespace=None, artifacts=integration_manifests.load("knative_serving_crds")) - apply_kube_artifacts(namespace='knative-serving', artifacts=integration_manifests.load("knative_serving_0.18.0")) - run_and_assert(['tools/bin/kubectl', 'patch', 'configmap/config-network', '--type', 'merge', '--patch', r'{"data": {"ingress.class": "ambassador.ingress.networking.knative.dev"}}', '-n', 'knative-serving']) + apply_kube_artifacts( + namespace=None, artifacts=integration_manifests.load("knative_serving_crds") + ) + apply_kube_artifacts( + namespace="knative-serving", + artifacts=integration_manifests.load("knative_serving_0.18.0"), + ) + run_and_assert( + [ + "tools/bin/kubectl", + "patch", + "configmap/config-network", + "--type", + "merge", + "--patch", + r'{"data": {"ingress.class": "ambassador.ingress.networking.knative.dev"}}', + "-n", + "knative-serving", + ] + ) # Wait for Knative to become ready - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'app=activator', '-n', 'knative-serving']) - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'app=controller', '-n', 'knative-serving']) - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'app=webhook', '-n', 'knative-serving']) - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'app=autoscaler', '-n', 'knative-serving']) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "app=activator", + "-n", + "knative-serving", + ] + ) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "app=controller", + "-n", + "knative-serving", + ] + ) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "app=webhook", + "-n", + "knative-serving", + ] + ) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "app=autoscaler", + "-n", + "knative-serving", + ] + ) # Install Ambassador - install_ambassador(namespace=namespace, envs=[ - { - 'name': 'AMBASSADOR_KNATIVE_SUPPORT', - 'value': 'true' - } - ]) + install_ambassador( + namespace=namespace, envs=[{"name": "AMBASSADOR_KNATIVE_SUPPORT", "value": "true"}] + ) # Install QOTM apply_kube_artifacts(namespace=namespace, artifacts=qotm_manifests) create_qotm_mapping(namespace=namespace) # Now let's wait for ambassador and QOTM pods to become ready - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'service=ambassador', '-n', namespace]) - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'service=qotm', '-n', namespace]) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "service=ambassador", + "-n", + namespace, + ] + ) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "service=qotm", + "-n", + namespace, + ] + ) # Create kservice apply_kube_artifacts(namespace=namespace, artifacts=knative_service_example) @@ -104,32 +190,41 @@ def test_knative(self): qotm_host = "ambassador." + namespace # Assert 200 OK at /qotm/ endpoint - qotm_url = f'http://{qotm_host}/qotm/' + qotm_url = f"http://{qotm_host}/qotm/" code = get_code_with_retry(qotm_url) assert code == 200, f"Expected 200 OK, got {code}" print(f"{qotm_url} is ready") # Assert 200 OK at / with Knative Host header and 404 with other/no header - kservice_url = f'http://{qotm_host}/' + kservice_url = f"http://{qotm_host}/" code = get_code_with_retry(kservice_url) assert code == 404, f"Expected 404, got {code}" print(f"{kservice_url} returns 404 with no host") - code = get_code_with_retry(kservice_url, - headers={'Host': 'random.host.whatever'} - ) + code = get_code_with_retry(kservice_url, headers={"Host": "random.host.whatever"}) assert code == 404, f"Expected 404, got {code}" print(f"{kservice_url} returns 404 with a random host") # Wait for kservice - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'ksvc', 'helloworld-go', '-n', namespace]) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "ksvc", + "helloworld-go", + "-n", + namespace, + ] + ) # kservice pod takes some time to spin up, so let's try a few times code = 000 - host = f'helloworld-go.{namespace}.example.com' + host = f"helloworld-go.{namespace}.example.com" for _ in range(5): - code = get_code_with_retry(kservice_url, headers={'Host': host}) + code = get_code_with_retry(kservice_url, headers={"Host": host}) if code == 200: break @@ -147,8 +242,10 @@ def test_knative_counters(): ir = IR(aconf, secret_handler=secret_handler) feats = ir.features() - assert feats['knative_ingress_count'] == 1, f"Expected a Knative ingress, did not find one" - assert feats['cluster_ingress_count'] == 0, f"Expected no Knative cluster ingresses, found at least one" + assert feats["knative_ingress_count"] == 1, f"Expected a Knative ingress, did not find one" + assert ( + feats["cluster_ingress_count"] == 0 + ), f"Expected no Knative cluster ingresses, found at least one" @pytest.mark.flaky(reruns=1, reruns_delay=10) @@ -163,5 +260,5 @@ def test_knative(): pytest.xfail("Knative is not supported") -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/integration/test_scout.py b/python/tests/integration/test_scout.py index cff025a7b9..f11da18621 100644 --- a/python/tests/integration/test_scout.py +++ b/python/tests/integration/test_scout.py @@ -12,37 +12,26 @@ DOCKER_IMAGE = os.environ.get("AMBASSADOR_DOCKER_IMAGE", None) -child: Optional[pexpect.spawnbase.SpawnBase] = None # see docker_start() -diagd_host: Optional[str] = None # see docker_start() -child_name = "diagd-unset" # see docker_start() and docker_kill() +child: Optional[pexpect.spawnbase.SpawnBase] = None # see docker_start() +diagd_host: Optional[str] = None # see docker_start() +child_name = "diagd-unset" # see docker_start() and docker_kill() SEQUENCES = [ + (["env_ok", "chime"], ["boot1", "now-healthy"]), + (["env_ok", "chime", "scout_cache_reset", "chime"], ["boot1", "now-healthy", "healthy"]), + (["env_ok", "chime", "env_bad", "chime"], ["boot1", "now-healthy", "now-unhealthy"]), + (["env_bad", "chime"], ["boot1", "unhealthy"]), ( - [ 'env_ok', 'chime' ], - [ 'boot1', 'now-healthy' ] + ["env_bad", "chime", "chime", "scout_cache_reset", "chime"], + ["boot1", "unhealthy", "unhealthy"], ), ( - [ 'env_ok', 'chime', 'scout_cache_reset', 'chime' ], - [ 'boot1', 'now-healthy', 'healthy' ] - ), - ( - [ 'env_ok', 'chime', 'env_bad', 'chime' ], - [ 'boot1', 'now-healthy', 'now-unhealthy' ] - ), - ( - [ 'env_bad', 'chime' ], - [ 'boot1', 'unhealthy' ] - ), - ( - [ 'env_bad', 'chime', 'chime', 'scout_cache_reset', 'chime' ], - [ 'boot1', 'unhealthy', 'unhealthy' ] - ), - ( - [ 'chime', 'chime', 'chime', 'env_ok', 'chime', 'chime' ], - [ 'boot1', 'unhealthy', 'now-healthy' ] + ["chime", "chime", "chime", "env_ok", "chime", "chime"], + ["boot1", "unhealthy", "now-healthy"], ), ] + def docker_start(logfile) -> bool: # Use a global here so that the child process doesn't get killed global child @@ -52,19 +41,19 @@ def docker_start(logfile) -> bool: global diagd_host - cmd = f'docker run --name {child_name} --rm -p 9999:9999 {DOCKER_IMAGE} --dev-magic' - diagd_host = 'localhost:9999' + cmd = f"docker run --name {child_name} --rm -p 9999:9999 {DOCKER_IMAGE} --dev-magic" + diagd_host = "localhost:9999" - child = pexpect.spawn(cmd, encoding='utf-8') + child = pexpect.spawn(cmd, encoding="utf-8") child.logfile = logfile - i = child.expect([ pexpect.EOF, pexpect.TIMEOUT, 'LocalScout: mode boot, action boot1' ]) + i = child.expect([pexpect.EOF, pexpect.TIMEOUT, "LocalScout: mode boot, action boot1"]) if i == 0: - print('diagd died?') + print("diagd died?") return False elif i == 1: - print('diagd timed out?') + print("diagd timed out?") return False # Set up port forwarding in the Ambassador container from all:9999, where @@ -73,10 +62,12 @@ def docker_start(logfile) -> bool: # container for security reasons. # Copy the simple port forwarding script into the container - child2 = pexpect.spawn(f'docker cp python/tests/_forward.py {child_name}:/tmp/', encoding='utf-8') + child2 = pexpect.spawn( + f"docker cp python/tests/_forward.py {child_name}:/tmp/", encoding="utf-8" + ) child2.logfile = logfile - if child2.expect([ pexpect.EOF, pexpect.TIMEOUT ]) == 1: + if child2.expect([pexpect.EOF, pexpect.TIMEOUT]) == 1: print("docker cp timed out?") return False @@ -86,10 +77,13 @@ def docker_start(logfile) -> bool: return False # Run the port forwarding script - child2 = pexpect.spawn(f'docker exec -d {child_name} python /tmp/_forward.py localhost 9998 "" 9999', encoding='utf-8') + child2 = pexpect.spawn( + f'docker exec -d {child_name} python /tmp/_forward.py localhost 9998 "" 9999', + encoding="utf-8", + ) child2.logfile = logfile - if child2.expect([ pexpect.EOF, pexpect.TIMEOUT ]) == 1: + if child2.expect([pexpect.EOF, pexpect.TIMEOUT]) == 1: print("docker exec timed out?") return False @@ -100,107 +94,117 @@ def docker_start(logfile) -> bool: return True + def docker_kill(logfile): - cmd = f'docker kill {child_name}' + cmd = f"docker kill {child_name}" - child = pexpect.spawn(cmd, encoding='utf-8') + child = pexpect.spawn(cmd, encoding="utf-8") child.logfile = logfile - child.expect([ pexpect.EOF, pexpect.TIMEOUT ]) + child.expect([pexpect.EOF, pexpect.TIMEOUT]) + def wait_for_diagd(logfile) -> bool: status = False tries_left = 5 while tries_left >= 0: - logfile.write(f'...checking diagd ({tries_left})\n') + logfile.write(f"...checking diagd ({tries_left})\n") try: global diagd_host - response = requests.get(f'http://{diagd_host}/_internal/v0/ping', - headers={ "X-Ambassador-Diag-IP": "127.0.0.1" }) + response = requests.get( + f"http://{diagd_host}/_internal/v0/ping", + headers={"X-Ambassador-Diag-IP": "127.0.0.1"}, + ) if response.status_code == 200: - logfile.write(' got it\n') + logfile.write(" got it\n") status = True break else: - logfile.write(f' failed {response.status_code}\n') + logfile.write(f" failed {response.status_code}\n") except requests.exceptions.RequestException as e: - logfile.write(f' failed {e}\n') + logfile.write(f" failed {e}\n") tries_left -= 1 time.sleep(2) return status + def check_http(logfile, cmd: str) -> bool: try: global diagd_host - response = requests.post(f'http://{diagd_host}/_internal/v0/fs', - headers={ "X-Ambassador-Diag-IP": "127.0.0.1" }, - params={ 'path': f'cmd:{cmd}' }) + response = requests.post( + f"http://{diagd_host}/_internal/v0/fs", + headers={"X-Ambassador-Diag-IP": "127.0.0.1"}, + params={"path": f"cmd:{cmd}"}, + ) text = response.text if response.status_code != 200: - logfile.write(f'{cmd}: wanted 200 but got {response.status_code} {text}\n') + logfile.write(f"{cmd}: wanted 200 but got {response.status_code} {text}\n") return False return True except Exception as e: - logfile.write(f'Could not do HTTP: {e}\n') + logfile.write(f"Could not do HTTP: {e}\n") return False + def fetch_events(logfile) -> Any: try: global diagd_host - response = requests.get(f'http://{diagd_host}/_internal/v0/events', - headers={ "X-Ambassador-Diag-IP": "127.0.0.1" }) + response = requests.get( + f"http://{diagd_host}/_internal/v0/events", + headers={"X-Ambassador-Diag-IP": "127.0.0.1"}, + ) if response.status_code != 200: - logfile.write(f'events: wanted 200 but got {response.status_code} {response.text}\n') + logfile.write(f"events: wanted 200 but got {response.status_code} {response.text}\n") return None data = response.json() return data except Exception as e: - logfile.write(f'events: could not do HTTP: {e}\n') + logfile.write(f"events: could not do HTTP: {e}\n") return None + def check_chimes(logfile) -> bool: result = True i = 0 covered = { - 'F-F-F': False, - 'F-F-T': False, + "F-F-F": False, + "F-F-T": False, # 'F-T-F': False, # This particular key can never be generated # 'F-T-T': False, # This particular key can never be generated - 'T-F-F': False, - 'T-F-T': False, - 'T-T-F': False, - 'T-T-T': False, + "T-F-F": False, + "T-F-T": False, + "T-T-F": False, + "T-T-T": False, } - for cmds, wanted_verdict in SEQUENCES: - logfile.write(f'RESETTING for sequence {i}\n') + logfile.write(f"RESETTING for sequence {i}\n") - if not check_http(logfile, 'chime_reset'): - logfile.write(f'could not reset for sequence {i}\n') + if not check_http(logfile, "chime_reset"): + logfile.write(f"could not reset for sequence {i}\n") result = False continue j = 0 for cmd in cmds: - logfile.write(f' sending {cmd} for sequence {i}.{j}\n') + logfile.write(f" sending {cmd} for sequence {i}.{j}\n") if not check_http(logfile, cmd): - logfile.write(f'could not do {cmd} for sequence {i}.{j}\n') + logfile.write(f"could not do {cmd} for sequence {i}.{j}\n") result = False break @@ -217,23 +221,23 @@ def check_chimes(logfile) -> bool: # logfile.write(json.dumps(events, sort_keys=True, indent=4)) - logfile.write(' ----\n') + logfile.write(" ----\n") verdict = [] for timestamp, mode, action, data in events: verdict.append(action) - action_key = data.get('action_key', None) + action_key = data.get("action_key", None) if action_key: covered[action_key] = True - logfile.write(f' {action} - {action_key}\n') + logfile.write(f" {action} - {action_key}\n") # logfile.write(json.dumps(verdict, sort_keys=True, indent=4\n)) if verdict != wanted_verdict: - logfile.write(f'verdict mismatch for sequence {i}:\n') + logfile.write(f"verdict mismatch for sequence {i}:\n") logfile.write(f' wanted {" ".join(wanted_verdict)}\n') logfile.write(f' got {" ".join(verdict)}\n') @@ -241,18 +245,19 @@ def check_chimes(logfile) -> bool: for key in sorted(covered.keys()): if not covered[key]: - logfile.write(f'missing coverage for {key}\n') + logfile.write(f"missing coverage for {key}\n") result = False return result + @pytest.mark.flaky(reruns=1, reruns_delay=10) def test_scout(): test_status = False - with open('/tmp/test_scout_output', 'w') as logfile: + with open("/tmp/test_scout_output", "w") as logfile: if not DOCKER_IMAGE: - logfile.write('No $AMBASSADOR_DOCKER_IMAGE??\n') + logfile.write("No $AMBASSADOR_DOCKER_IMAGE??\n") else: if docker_start(logfile): if wait_for_diagd(logfile) and check_chimes(logfile): @@ -261,11 +266,12 @@ def test_scout(): docker_kill(logfile) if not test_status: - with open('/tmp/test_scout_output', 'r') as logfile: + with open("/tmp/test_scout_output", "r") as logfile: for line in logfile: print(line.rstrip()) - assert test_status, 'test failed' + assert test_status, "test failed" + -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/integration/test_watt_scaling.py b/python/tests/integration/test_watt_scaling.py index 33a2a10ac4..509666e5e5 100644 --- a/python/tests/integration/test_watt_scaling.py +++ b/python/tests/integration/test_watt_scaling.py @@ -85,7 +85,7 @@ def delete_qotm_mapping(self, namespace): delete_kube_artifacts(namespace=namespace, artifacts=qotm_mapping) def test_rapid_additions_and_deletions(self): - namespace = 'watt-rapid' + namespace = "watt-rapid" # Install Ambassador install_ambassador(namespace=namespace) @@ -100,8 +100,32 @@ def test_rapid_additions_and_deletions(self): self.apply_qotm_endpoint_manifests(namespace=namespace) # Now let's wait for ambassador and QOTM pods to become ready - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'service=ambassador', '-n', namespace]) - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=Ready', 'pod', '-l', 'service=qotm', '-n', namespace]) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "service=ambassador", + "-n", + namespace, + ] + ) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=Ready", + "pod", + "-l", + "service=qotm", + "-n", + namespace, + ] + ) # Assume we can reach Ambassador through telepresence qotm_host = "ambassador." + namespace @@ -144,5 +168,5 @@ def test_watt(): watt_test.test_rapid_additions_and_deletions() -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/integration/utils.py b/python/tests/integration/utils.py index 69d78abdd0..39cdefa1bb 100644 --- a/python/tests/integration/utils.py +++ b/python/tests/integration/utils.py @@ -22,10 +22,14 @@ from tests.runutils import run_and_assert # Assume that both of these are on the PATH if not explicitly set -KUBESTATUS_PATH = os.environ.get('KUBESTATUS_PATH', 'kubestatus') +KUBESTATUS_PATH = os.environ.get("KUBESTATUS_PATH", "kubestatus") + def install_crds() -> None: - apply_kube_artifacts(namespace='emissary-system', artifacts=integration_manifests.crd_manifests()) + apply_kube_artifacts( + namespace="emissary-system", artifacts=integration_manifests.crd_manifests() + ) + def install_ambassador(namespace, single_namespace=True, envs=None, debug=None): """ @@ -61,50 +65,64 @@ def install_ambassador(namespace, single_namespace=True, envs=None, debug=None): install_crds() print("Wait for apiext to be running...") - run_and_assert(['tools/bin/kubectl', 'wait', '--timeout=90s', '--for=condition=available', 'deploy', 'emissary-apiext', '-n', 'emissary-system']) + run_and_assert( + [ + "tools/bin/kubectl", + "wait", + "--timeout=90s", + "--for=condition=available", + "deploy", + "emissary-apiext", + "-n", + "emissary-system", + ] + ) # Proceed to install Ambassador now final_yaml = [] - rbac_manifest_name = 'rbac_namespace_scope' if single_namespace else 'rbac_cluster_scope' + rbac_manifest_name = "rbac_namespace_scope" if single_namespace else "rbac_cluster_scope" # Hackish fakes of actual KAT structures -- it's _far_ too much work to synthesize # actual KAT Nodes and Paths. - fakeNode = namedtuple('fakeNode', [ 'namespace', 'path', 'ambassador_id' ]) - fakePath = namedtuple('fakePath', [ 'k8s' ]) - - ambassador_yaml = list(yaml.safe_load_all( - integration_manifests.format( - "\n".join([ - integration_manifests.load(rbac_manifest_name), - integration_manifests.load('ambassador'), - (cleartext_host_manifest % namespace), - ]), - capabilities_block="", - envs="", - extra_ports="", - self=fakeNode( - namespace=namespace, - ambassador_id='default', - path=fakePath(k8s='ambassador') - ), - ))) + fakeNode = namedtuple("fakeNode", ["namespace", "path", "ambassador_id"]) + fakePath = namedtuple("fakePath", ["k8s"]) + + ambassador_yaml = list( + yaml.safe_load_all( + integration_manifests.format( + "\n".join( + [ + integration_manifests.load(rbac_manifest_name), + integration_manifests.load("ambassador"), + (cleartext_host_manifest % namespace), + ] + ), + capabilities_block="", + envs="", + extra_ports="", + self=fakeNode( + namespace=namespace, ambassador_id="default", path=fakePath(k8s="ambassador") + ), + ) + ) + ) for manifest in ambassador_yaml: - kind = manifest.get('kind', None) - metadata = manifest.get('metadata', {}) - name = metadata.get('name', None) + kind = manifest.get("kind", None) + metadata = manifest.get("metadata", {}) + name = metadata.get("name", None) if (kind == "Pod") and (name == "ambassador"): # Force AMBASSADOR_ID to match ours. # # XXX This is not likely to work without single_namespace=True. - for envvar in manifest['spec']['containers'][0]['env']: - if envvar.get('name', '') == 'AMBASSADOR_ID': - envvar['value'] = 'default' + for envvar in manifest["spec"]["containers"][0]["env"]: + if envvar.get("name", "") == "AMBASSADOR_ID": + envvar["value"] = "default" # add new envs, if any - manifest['spec']['containers'][0]['env'].extend(envs) + manifest["spec"]["containers"][0]["env"].extend(envs) # print("INSTALLING AMBASSADOR: manifests:") # print(yaml.safe_dump_all(ambassador_yaml)) @@ -116,20 +134,19 @@ def update_envs(envs, name, value): found = False for e in envs: - if e['name'] == name: - e['value'] = value + if e["name"] == name: + e["value"] = value found = True break if not found: - envs.append({ - 'name': name, - 'value': value - }) + envs.append({"name": name, "value": value}) def create_namespace(namespace): - apply_kube_artifacts(namespace=namespace, artifacts=integration_manifests.namespace_manifest(namespace)) + apply_kube_artifacts( + namespace=namespace, artifacts=integration_manifests.namespace_manifest(namespace) + ) def create_qotm_mapping(namespace): @@ -148,6 +165,7 @@ def create_qotm_mapping(namespace): apply_kube_artifacts(namespace=namespace, artifacts=qotm_mapping) + def create_httpbin_mapping(namespace): httpbin_mapping = f""" --- diff --git a/python/tests/kat/abstract_tests.py b/python/tests/kat/abstract_tests.py index 70440cc12f..cbb45d5777 100644 --- a/python/tests/kat/abstract_tests.py +++ b/python/tests/kat/abstract_tests.py @@ -12,12 +12,12 @@ # These type: ignores are because, weirdly, the yaml.CSafe* variants don't share # a type with their non-C variants. No clue why not. -yaml_loader = yaml.SafeLoader # type: ignore -yaml_dumper = yaml.SafeDumper # type: ignore +yaml_loader = yaml.SafeLoader # type: ignore +yaml_dumper = yaml.SafeDumper # type: ignore try: - yaml_loader = yaml.CSafeLoader # type: ignore - yaml_dumper = yaml.CSafeDumper # type: ignore + yaml_loader = yaml.CSafeLoader # type: ignore + yaml_dumper = yaml.CSafeDumper # type: ignore except AttributeError: pass @@ -39,20 +39,29 @@ def assert_default_errors(errors, include_ingress_errors=True): default_errors = [ - ["", - "Ambassador could not find core CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored..."], - ["", - "Ambassador could not find Resolver type CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored..."], - ["", - "Ambassador could not find the Host CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored..."], - ["", - "Ambassador could not find the LogService CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored..."] + [ + "", + "Ambassador could not find core CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored...", + ], + [ + "", + "Ambassador could not find Resolver type CRD definitions. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored...", + ], + [ + "", + "Ambassador could not find the Host CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored...", + ], + [ + "", + "Ambassador could not find the LogService CRD definition. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/install/upgrade-to-edge-stack/#5-update-and-restart for more information. You can continue using Ambassador via Kubernetes annotations, any configuration via CRDs will be ignored...", + ], ] if include_ingress_errors: default_errors.append( - ["", - "Ambassador is not permitted to read Ingress resources. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/running/ingress-controller/#ambassador-as-an-ingress-controller for more information. You can continue using Ambassador, but Ingress resources will be ignored..." + [ + "", + "Ambassador is not permitted to read Ingress resources. Please visit https://www.getambassador.io/docs/edge-stack/latest/topics/running/ingress-controller/#ambassador-as-an-ingress-controller for more information. You can continue using Ambassador, but Ingress resources will be ignored...", ] ) @@ -62,7 +71,9 @@ def assert_default_errors(errors, include_ingress_errors=True): assert False, f"default error table mismatch: got\n{errors}" for error in errors[number_of_default_errors:]: - assert 'found invalid port' in error[1], "Could not find 'found invalid port' in the error {}".format(error[1]) + assert ( + "found invalid port" in error[1] + ), "Could not find 'found invalid port' in the error {}".format(error[1]) DEV = os.environ.get("AMBASSADOR_DEV", "0").lower() in ("1", "yes", "true") @@ -104,7 +115,7 @@ def manifests(self) -> str: value: "8500" """ - if os.environ.get('AMBASSADOR_FAST_RECONFIGURE', 'true').lower() == 'false': + if os.environ.get("AMBASSADOR_FAST_RECONFIGURE", "true").lower() == "false": self.manifest_envs += """ - name: AMBASSADOR_FAST_RECONFIGURE value: "false" @@ -121,7 +132,9 @@ def manifests(self) -> str: value: "%s" - name: AES_LOG_LEVEL value: "debug" -""" % ":".join(amb_debug) +""" % ":".join( + amb_debug + ) if self.ambassador_id: self.manifest_envs += f""" @@ -159,13 +172,14 @@ def manifests(self) -> str: """ if DEV: - return self.format(rbac + AMBASSADOR_LOCAL, - extra_ports=eports) + return self.format(rbac + AMBASSADOR_LOCAL, extra_ports=eports) else: - return self.format(rbac + integration_manifests.load('ambassador'), - envs=self.manifest_envs, - extra_ports=eports, - capabilities_block="") + return self.format( + rbac + integration_manifests.load("ambassador"), + envs=self.manifest_envs, + extra_ports=eports, + capabilities_block="", + ) # # Will tear this out of the harness shortly # @property @@ -192,7 +206,7 @@ def post_manifest(self): if not DEV: return - if os.environ.get('KAT_SKIP_DOCKER'): + if os.environ.get("KAT_SKIP_DOCKER"): return image = os.environ["AMBASSADOR_DOCKER_IMAGE"] @@ -202,10 +216,12 @@ def post_manifest(self): if not AmbassadorTest.IMAGE_BUILT: AmbassadorTest.IMAGE_BUILT = True - cmd = ShellCommand('docker', 'ps', '-a', '-f', 'label=kat-family=ambassador', '--format', '{{.ID}}') + cmd = ShellCommand( + "docker", "ps", "-a", "-f", "label=kat-family=ambassador", "--format", "{{.ID}}" + ) - if cmd.check('find old docker container IDs'): - ids = cmd.stdout.split('\n') + if cmd.check("find old docker container IDs"): + ids = cmd.stdout.split("\n") while ids: if ids[-1]: @@ -215,15 +231,25 @@ def post_manifest(self): if ids: print("Killing old containers...") - ShellCommand.run('kill old containers', 'docker', 'kill', *ids, verbose=True) - ShellCommand.run('rm old containers', 'docker', 'rm', *ids, verbose=True) + ShellCommand.run("kill old containers", "docker", "kill", *ids, verbose=True) + ShellCommand.run("rm old containers", "docker", "rm", *ids, verbose=True) context = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) print("Starting docker build...", end="") sys.stdout.flush() - cmd = ShellCommand("docker", "build", "--build-arg", "BASE_PY_IMAGE={}".format(cached_image), "--build-arg", "BASE_GO_IMAGE={}".format(ambassador_base_image), context, "-t", image) + cmd = ShellCommand( + "docker", + "build", + "--build-arg", + "BASE_PY_IMAGE={}".format(cached_image), + "--build-arg", + "BASE_GO_IMAGE={}".format(ambassador_base_image), + context, + "-t", + image, + ) if cmd.check("docker build Ambassador image"): print("done.") @@ -235,17 +261,19 @@ def post_manifest(self): with open(fname) as fd: content = fd.read() else: - nsp = getattr(self, 'namespace', None) or 'default' + nsp = getattr(self, "namespace", None) or "default" - cmd = ShellCommand("tools/bin/kubectl", "get", "-n", nsp, "-o", "yaml", "secret", self.path.k8s) + cmd = ShellCommand( + "tools/bin/kubectl", "get", "-n", nsp, "-o", "yaml", "secret", self.path.k8s + ) - if not cmd.check(f'fetch secret for {self.path.k8s}'): - pytest.exit(f'could not fetch secret for {self.path.k8s}') + if not cmd.check(f"fetch secret for {self.path.k8s}"): + pytest.exit(f"could not fetch secret for {self.path.k8s}") content = cmd.stdout with open(fname, "wb") as fd: - fd.write(content.encode('utf-8')) + fd.write(content.encode("utf-8")) try: secret = yaml.load(content, Loader=yaml_loader) @@ -253,9 +281,9 @@ def post_manifest(self): print("could not parse YAML:\n%s" % content) raise e - data = secret['data'] + data = secret["data"] # secret_dir = tempfile.mkdtemp(prefix=self.path.k8s, suffix="secret") - secret_dir = "/tmp/%s-ambassadormixin-%s" % (self.path.k8s, 'secret') + secret_dir = "/tmp/%s-ambassadormixin-%s" % (self.path.k8s, "secret") shutil.rmtree(secret_dir, ignore_errors=True) os.mkdir(secret_dir, 0o777) @@ -266,14 +294,16 @@ def post_manifest(self): print("Launching %s container." % self.path.k8s) command = ["docker", "run", "-d", "-l", "kat-family=ambassador", "--name", self.path.k8s] - envs = [ "KUBERNETES_SERVICE_HOST=kubernetes", - "KUBERNETES_SERVICE_PORT=443", - "AMBASSADOR_SNAPSHOT_COUNT=1", - "AMBASSADOR_CONFIG_BASE_DIR=/tmp/ambassador", - "POLL_EVERY_SECS=0", - "CONSUL_WATCHER_PORT=8500", - "AMBASSADOR_UPDATE_MAPPING_STATUS=false", - "AMBASSADOR_ID=%s" % self.ambassador_id] + envs = [ + "KUBERNETES_SERVICE_HOST=kubernetes", + "KUBERNETES_SERVICE_PORT=443", + "AMBASSADOR_SNAPSHOT_COUNT=1", + "AMBASSADOR_CONFIG_BASE_DIR=/tmp/ambassador", + "POLL_EVERY_SECS=0", + "CONSUL_WATCHER_PORT=8500", + "AMBASSADOR_UPDATE_MAPPING_STATUS=false", + "AMBASSADOR_ID=%s" % self.ambassador_id, + ] if self.namespace: envs.append("AMBASSADOR_NAMESPACE=%s" % self.namespace) @@ -296,11 +326,16 @@ def post_manifest(self): for env in envs: command.extend(["-e", env]) - ports = ["%s:8877" % (8877 + self.index), "%s:8001" % (8001 + self.index), "%s:8080" % (8080 + self.index), "%s:8443" % (8443 + self.index)] + ports = [ + "%s:8877" % (8877 + self.index), + "%s:8001" % (8001 + self.index), + "%s:8080" % (8080 + self.index), + "%s:8443" % (8443 + self.index), + ] if self.extra_ports: for port in self.extra_ports: - ports.append(f'{port}:{port}') + ports.append(f"{port}:{port}") for port_str in ports: command.extend(["-p", port_str]) @@ -311,26 +346,28 @@ def post_manifest(self): command.append(image) - if os.environ.get('KAT_SHOW_DOCKER'): + if os.environ.get("KAT_SHOW_DOCKER"): print(" ".join(command)) cmd = ShellCommand(*command) - if not cmd.check(f'start container for {self.path.k8s}'): - pytest.exit(f'could not start container for {self.path.k8s}') + if not cmd.check(f"start container for {self.path.k8s}"): + pytest.exit(f"could not start container for {self.path.k8s}") def queries(self): if DEV: cmd = ShellCommand("docker", "ps", "-qf", "name=%s" % self.path.k8s) - if not cmd.check(f'docker check for {self.path.k8s}'): + if not cmd.check(f"docker check for {self.path.k8s}"): if not cmd.stdout.strip(): - log_cmd = ShellCommand("docker", "logs", self.path.k8s, stderr=subprocess.STDOUT) + log_cmd = ShellCommand( + "docker", "logs", self.path.k8s, stderr=subprocess.STDOUT + ) - if log_cmd.check(f'docker logs for {self.path.k8s}'): + if log_cmd.check(f"docker logs for {self.path.k8s}"): print(cmd.stdout) - pytest.exit(f'container failed to start for {self.path.k8s}') + pytest.exit(f"container failed to start for {self.path.k8s}") return () @@ -343,7 +380,7 @@ def url(self, prefix, scheme=None, port=None) -> str: if DEV: if not port: - port = 8443 if scheme == 'https' else 8080 + port = 8443 if scheme == "https" else 8080 port += self.index return "%s://%s/%s" % (scheme, "localhost:%s" % port, prefix) @@ -351,7 +388,7 @@ def url(self, prefix, scheme=None, port=None) -> str: host_and_port = self.path.fqdn if port: - host_and_port += f':{port}' + host_and_port += f":{port}" return "%s://%s/%s" % (scheme, host_and_port, prefix) @@ -367,7 +404,9 @@ class ServiceType(Node): _manifests: Optional[str] use_superpod: bool = True - def __init__(self, service_manifests: str=None, namespace: str=None, *args, **kwargs) -> None: + def __init__( + self, service_manifests: str = None, namespace: str = None, *args, **kwargs + ) -> None: super().__init__(namespace=namespace, *args, **kwargs) self._manifests = service_manifests @@ -397,7 +436,7 @@ class ServiceTypeGrpc(Node): path: Name - def __init__(self, service_manifests: str=None, *args, **kwargs) -> None: + def __init__(self, service_manifests: str = None, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._manifests = service_manifests or integration_manifests.load("backend") @@ -419,6 +458,7 @@ class HTTP(ServiceType): class GRPC(ServiceType): pass + class EGRPC(ServiceType): skip_variant: ClassVar[bool] = True @@ -428,11 +468,16 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) def requirements(self): - yield ("url", Query("http://%s/echo.EchoService/Echo" % self.path.fqdn, - headers={ "content-type": "application/grpc", - "kat-req-echo-requested-status": "0" }, - expected=200, - grpc_type="real")) + yield ( + "url", + Query( + "http://%s/echo.EchoService/Echo" % self.path.fqdn, + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + expected=200, + grpc_type="real", + ), + ) + class AHTTP(ServiceType): skip_variant: ClassVar[bool] = True @@ -446,7 +491,7 @@ def __init__(self, *args, **kwargs) -> None: class AGRPC(ServiceType): skip_variant: ClassVar[bool] = True - def __init__(self, protocol_version: str="v3", *args, **kwargs) -> None: + def __init__(self, protocol_version: str = "v3", *args, **kwargs) -> None: self.protocol_version = protocol_version # Do this unconditionally, because that's the point of this class. @@ -456,10 +501,11 @@ def __init__(self, protocol_version: str="v3", *args, **kwargs) -> None: def requirements(self): yield ("pod", self.path.k8s) + class RLSGRPC(ServiceType): skip_variant: ClassVar[bool] = True - def __init__(self, protocol_version: str="v3", *args, **kwargs) -> None: + def __init__(self, protocol_version: str = "v3", *args, **kwargs) -> None: self.protocol_version = protocol_version # Do this unconditionally, because that's the point of this class. @@ -469,6 +515,7 @@ def __init__(self, protocol_version: str="v3", *args, **kwargs) -> None: def requirements(self): yield ("pod", self.path.k8s) + class ALSGRPC(ServiceType): skip_variant: ClassVar[bool] = True @@ -480,11 +527,12 @@ def __init__(self, *args, **kwargs) -> None: def requirements(self): yield ("pod", self.path.k8s) + @abstract_test class MappingTest(Test): target: ServiceType - options: Sequence['OptionTest'] + options: Sequence["OptionTest"] parent: AmbassadorTest no_local_mode = True @@ -495,6 +543,7 @@ def init(self, target: ServiceType, options=()) -> None: self.options = list(options) self.is_ambassador = True + @abstract_test class OptionTest(Test): diff --git a/python/tests/kat/t_basics.py b/python/tests/kat/t_basics.py index e41786cd49..ad4441d236 100644 --- a/python/tests/kat/t_basics.py +++ b/python/tests/kat/t_basics.py @@ -38,15 +38,16 @@ def check(self): # We shouldn't have any missing-CRD-types errors any more. for source, error in errors: - if (('could not find' in error) and ('CRD definitions' in error)): - assert False, f"Missing CRDs: {error}" + if ("could not find" in error) and ("CRD definitions" in error): + assert False, f"Missing CRDs: {error}" - if 'Ingress resources' in error: - assert False, f"Ingress resource error: {error}" + if "Ingress resources" in error: + assert False, f"Ingress resource error: {error}" # The default errors assume that we have missing CRDs, and that's not correct any more, # so don't try to use assert_default_errors here. + class AmbassadorIDTest(AmbassadorTest): target: ServiceType @@ -63,12 +64,15 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: config: use_ambassador_namespace_for_service_resolution: true """ - for prefix, amb_id in (("findme", "[{self.ambassador_id}]"), - ("findme-array", "[{self.ambassador_id}, missme]"), - ("findme-array2", "[missme, {self.ambassador_id}]"), - ("missme", "[missme]"), - ("missme-array", "[missme1, missme2]")): - yield self.target, self.format(""" + for prefix, amb_id in ( + ("findme", "[{self.ambassador_id}]"), + ("findme-array", "[{self.ambassador_id}, missme]"), + ("findme-array2", "[missme, {self.ambassador_id}]"), + ("missme", "[missme]"), + ("missme-array", "[missme1, missme2]"), + ): + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -77,7 +81,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{prefix}/ service: {self.target.path.fqdn} ambassador_id: {amb_id} - """, prefix=self.format(prefix), amb_id=self.format(amb_id)) + """, + prefix=self.format(prefix), + amb_id=self.format(amb_id), + ) def queries(self): yield Query(self.url("findme/")) @@ -94,7 +101,8 @@ def init(self): self.target = HTTP() self.resource_names = [] - self.models = [ """ + self.models = [ + """ apiVersion: getambassador.io/v3alpha1 kind: AuthService metadata: @@ -102,7 +110,8 @@ def init(self): spec: ambassador_id: ["{self.ambassador_id}"] service_bad: {self.target.path.fqdn} -""",""" +""", + """ apiVersion: getambassador.io/v3alpha1 kind: Mapping metadata: @@ -112,7 +121,8 @@ def init(self): hostname: "*" prefix: /good-<>/ service: {self.target.path.fqdn} -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: Mapping metadata: @@ -122,7 +132,8 @@ def init(self): hostname: "*" prefix_bad: /bad-<>/ service: {self.target.path.fqdn} -""", """ +""", + """ kind: Mapping metadata: name: {self.path.k8s}-m-bad-no-apiversion-<> @@ -131,7 +142,8 @@ def init(self): hostname: "*" prefix_bad: /bad-<>/ service: {self.target.path.fqdn} -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: Module metadata: @@ -139,7 +151,8 @@ def init(self): spec: ambassador_id: ["{self.ambassador_id}"] config_bad: [] -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: RateLimitService metadata: @@ -147,7 +160,8 @@ def init(self): spec: ambassador_id: ["{self.ambassador_id}"] service_bad: {self.target.path.fqdn} -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: TCPMapping metadata: @@ -156,7 +170,8 @@ def init(self): ambassador_id: ["{self.ambassador_id}"] service: {self.target.path.fqdn} port_bad: 8888 -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: TCPMapping metadata: @@ -165,7 +180,8 @@ def init(self): ambassador_id: ["{self.ambassador_id}"] service_bad: {self.target.path.fqdn} port: 8888 -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: TracingService metadata: @@ -174,7 +190,8 @@ def init(self): ambassador_id: ["{self.ambassador_id}"] driver_bad: zipkin service: {self.target.path.fqdn} -""", """ +""", + """ apiVersion: getambassador.io/v3alpha1 kind: TracingService metadata: @@ -183,29 +200,28 @@ def init(self): ambassador_id: ["{self.ambassador_id}"] driver: zipkin service_bad: {self.target.path.fqdn} -""" +""", ] - def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: counter = 0 for m_yaml in self.models: counter += 1 - m = yaml.safe_load(self.format(m_yaml.replace('<>', 'annotation'))) + m = yaml.safe_load(self.format(m_yaml.replace("<>", "annotation"))) for k in m["metadata"].keys(): m[k] = m["metadata"][k] - del(m["metadata"]) + del m["metadata"] for k in m["spec"].keys(): if k == "ambassador_id": continue m[k] = m["spec"][k] - del(m["spec"]) + del m["spec"] - if 'good' not in m["name"]: + if "good" not in m["name"]: # These all show up as "invalidresources.default.N" because they're # annotations. self.resource_names.append(f"invalidresources.default.{counter}") @@ -218,13 +234,13 @@ def manifests(self): for m in self.models: m_yaml = self.format(m.replace("<>", "crd")) m_obj = yaml.safe_load(m_yaml) - if 'apiVersion' not in m_obj: + if "apiVersion" not in m_obj: continue manifests.append("---") manifests.append(m_yaml) - if 'good' not in m_obj["metadata"]["name"]: + if "good" not in m_obj["metadata"]["name"]: self.resource_names.append(m_obj["metadata"]["name"] + ".default.1") return super().manifests() + "\n".join(manifests) @@ -255,11 +271,13 @@ def check(self): # Check that the error is one of the errors that we expect. expected_errors = [ - re.compile(r'^.* in body is required$'), - re.compile(r'^apiVersion None/ unsupported$'), + re.compile(r"^.* in body is required$"), + re.compile(r"^apiVersion None/ unsupported$"), re.compile(r'^spec\.config in body must be of type object: "null"$'), ] - assert any(pat.match(error) for pat in expected_errors), f"error for {name} should match one of the expected errors: {repr(error)}" + assert any( + pat.match(error) for pat in expected_errors + ), f"error for {name} should match one of the expected errors: {repr(error)}" class ServerNameTest(AmbassadorTest): @@ -270,7 +288,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -284,13 +303,14 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /server-name service: {self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("server-name/"), expected=301) def check(self): - assert self.results[0].headers["Server"] == [ "test-server" ] + assert self.results[0].headers["Server"] == ["test-server"] class SafeRegexMapping(AmbassadorTest): @@ -301,7 +321,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -313,17 +334,21 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: regex_headers: X-Foo: "^[a-z].*" service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url(self.name + "/"), headers={"X-Foo": "hello"}) - yield Query(self.url(f'need-normalization/../{self.name}/'), headers={"X-Foo": "hello"}) + yield Query(self.url(f"need-normalization/../{self.name}/"), headers={"X-Foo": "hello"}) yield Query(self.url(self.name + "/"), expected=404) - yield Query(self.url(f'need-normalization/../{self.name}/'), expected=404) + yield Query(self.url(f"need-normalization/../{self.name}/"), expected=404) def check(self): for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" diff --git a/python/tests/kat/t_bufferlimitbytes.py b/python/tests/kat/t_bufferlimitbytes.py index 5b5bcea305..6bec01a873 100644 --- a/python/tests/kat/t_bufferlimitbytes.py +++ b/python/tests/kat/t_bufferlimitbytes.py @@ -4,6 +4,7 @@ from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node import json + class BufferLimitBytesTest(AmbassadorTest): target: ServiceType @@ -12,7 +13,8 @@ def init(self): # Test generating config with an increased buffer and that the lua body() funciton runs to buffer the request body def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -30,7 +32,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /foo/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("foo/")) @@ -38,4 +41,4 @@ def queries(self): def check(self): assert self.results[0].status == 200 - assert self.results[1].status == 200 \ No newline at end of file + assert self.results[1].status == 200 diff --git a/python/tests/kat/t_chunked_length.py b/python/tests/kat/t_chunked_length.py index 673d6aa149..9246ccacec 100644 --- a/python/tests/kat/t_chunked_length.py +++ b/python/tests/kat/t_chunked_length.py @@ -4,6 +4,7 @@ from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node import json + class AllowChunkedLengthTestTrue(AmbassadorTest): target: ServiceType @@ -11,7 +12,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: ambassador kind: Module @@ -25,21 +27,17 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /foo/ hostname: "*" service: {self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("foo/")) yield Query(self.url("ambassador/v0/diag/")) - yield Query(self.url("foo/"), - headers={ - "content-length": "0", - "transfer-encoding": "gzip" - }) - yield Query(self.url("ambassador/v0/diag/"), - headers={ - "content-length": "0", - "transfer-encoding": "gzip" - }) + yield Query(self.url("foo/"), headers={"content-length": "0", "transfer-encoding": "gzip"}) + yield Query( + self.url("ambassador/v0/diag/"), + headers={"content-length": "0", "transfer-encoding": "gzip"}, + ) def check(self): # Not getting a 400 bad request is confirmation that this setting works as long as the request can reach the upstream diff --git a/python/tests/kat/t_circuitbreaker.py b/python/tests/kat/t_circuitbreaker.py index 86ad0de93c..fd18211a24 100644 --- a/python/tests/kat/t_circuitbreaker.py +++ b/python/tests/kat/t_circuitbreaker.py @@ -50,10 +50,11 @@ service: {name} """ + class CircuitBreakingTest(AmbassadorTest): target: ServiceType - TARGET_CLUSTER='cluster_circuitbreakingtest_http_cbdc1p1' + TARGET_CLUSTER = "cluster_circuitbreakingtest_http_cbdc1p1" def init(self): self.target = HTTP() @@ -66,7 +67,8 @@ def manifests(self) -> str: value: 'cbstatsd-sink' """ - return """ + return ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -95,19 +97,24 @@ def manifests(self) -> str: requestPolicy: insecure: action: Route -""" + \ - self.format(integration_manifests.load("rbac_cluster_scope") + integration_manifests.load("ambassador"), - envs=envs, - extra_ports="", - capabilities_block="") + \ - STATSD_MANIFEST.format( - name='cbstatsd-sink', - image=integration_manifests.get_images()['test-stats'], - target=self.__class__.TARGET_CLUSTER) - +""" + + self.format( + integration_manifests.load("rbac_cluster_scope") + + integration_manifests.load("ambassador"), + envs=envs, + extra_ports="", + capabilities_block="", + ) + + STATSD_MANIFEST.format( + name="cbstatsd-sink", + image=integration_manifests.get_images()["test-stats"], + target=self.__class__.TARGET_CLUSTER, + ) + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -137,11 +144,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /dump/ rewrite: /DUMP/ service: cbstatsd-sink -""") +""" + ) def requirements(self): yield from super().requirements() - yield ("url", Query(self.url(self.name) + '-pr/')) + yield ("url", Query(self.url(self.name) + "-pr/")) yield ("url", Query(self.url("RESET/"))) def queries(self): @@ -150,16 +158,24 @@ def queries(self): # Run 200 queries in phase 2, after the reset... for i in range(200): - yield Query(self.url(self.name) + '-pr/', headers={ "Kat-Req-Http-Requested-Backend-Delay": "1000" }, - ignore_result=True, phase=2) + yield Query( + self.url(self.name) + "-pr/", + headers={"Kat-Req-Http-Requested-Backend-Delay": "1000"}, + ignore_result=True, + phase=2, + ) # ...then 200 more queries in phase 3. Why the split? Because we get flakes if we # try to ram 500 through at once (in the middle of the run, we get some connections # that time out). for i in range(200): - yield Query(self.url(self.name) + '-pr/', headers={ "Kat-Req-Http-Requested-Backend-Delay": "1000" }, - ignore_result=True, phase=3) + yield Query( + self.url(self.name) + "-pr/", + headers={"Kat-Req-Http-Requested-Backend-Delay": "1000"}, + ignore_result=True, + phase=3, + ) # Dump the results in phase 4, after the queries. yield Query(self.url("DUMP/"), phase=4) @@ -170,7 +186,7 @@ def check(self): failures = [] if result_count != 402: - failures.append(f'wanted 402 results, got {result_count}') + failures.append(f"wanted 402 results, got {result_count}") else: pending_results = self.results[1:400] stats = self.results[401].json or {} @@ -189,31 +205,38 @@ def check(self): if result.error: error += 1 - elif 'X-Envoy-Overloaded' in result.headers: + elif "X-Envoy-Overloaded" in result.headers: pending_overloaded += 1 failed = False if not 300 < pending_overloaded < 400: - failures.append(f'Expected between 300 and 400 overloaded, got {pending_overloaded}') + failures.append( + f"Expected between 300 and 400 overloaded, got {pending_overloaded}" + ) cluster_stats = stats.get(self.__class__.TARGET_CLUSTER, {}) - rq_completed = cluster_stats.get('upstream_rq_completed', -1) - rq_pending_overflow = cluster_stats.get('upstream_rq_pending_overflow', -1) + rq_completed = cluster_stats.get("upstream_rq_completed", -1) + rq_pending_overflow = cluster_stats.get("upstream_rq_pending_overflow", -1) if error != 0: failures.append(f"Expected no errors but got {error}") if rq_completed != 400: - failures.append(f'Expected 400 completed requests to {self.__class__.TARGET_CLUSTER}, got {rq_completed}') + failures.append( + f"Expected 400 completed requests to {self.__class__.TARGET_CLUSTER}, got {rq_completed}" + ) if abs(pending_overloaded - rq_pending_overflow) >= 2: - failures.append(f'Expected {pending_overloaded} rq_pending_overflow, got {rq_pending_overflow}') + failures.append( + f"Expected {pending_overloaded} rq_pending_overflow, got {rq_pending_overflow}" + ) if failures: print("%s FAILED:\n %s" % (self.name, "\n ".join(failures))) pytest.xfail(f"FFS {self.name}") + class GlobalCircuitBreakingTest(AmbassadorTest): target: ServiceType @@ -221,7 +244,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -259,26 +283,35 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - priority: default max_pending_requests: 5 max_connections: 5 -""") +""" + ) def requirements(self): yield from super().requirements() - yield ("url", Query(self.url(self.name) + '-pr/')) - yield ("url", Query(self.url(self.name) + '-normal/')) + yield ("url", Query(self.url(self.name) + "-pr/")) + yield ("url", Query(self.url(self.name) + "-normal/")) def queries(self): for i in range(200): - yield Query(self.url(self.name) + '-pr/', headers={ "Kat-Req-Http-Requested-Backend-Delay": "1000" }, - ignore_result=True, phase=1) + yield Query( + self.url(self.name) + "-pr/", + headers={"Kat-Req-Http-Requested-Backend-Delay": "1000"}, + ignore_result=True, + phase=1, + ) for i in range(200): - yield Query(self.url(self.name) + '-normal/', headers={ "Kat-Req-Http-Requested-Backend-Delay": "1000" }, - ignore_result=True, phase=1) + yield Query( + self.url(self.name) + "-normal/", + headers={"Kat-Req-Http-Requested-Backend-Delay": "1000"}, + ignore_result=True, + phase=1, + ) def check(self): failures = [] if len(self.results) != 400: - failures.append(f'wanted 400 results, got {len(self.results)}') + failures.append(f"wanted 400 results, got {len(self.results)}") else: cb_mapping_results = self.results[0:200] normal_mapping_results = self.results[200:400] @@ -287,11 +320,11 @@ def check(self): pr_mapping_overloaded = 0 for result in cb_mapping_results: - if 'X-Envoy-Overloaded' in result.headers: + if "X-Envoy-Overloaded" in result.headers: pr_mapping_overloaded += 1 if pr_mapping_overloaded != 0: - failures.append(f'[GCR] expected no -pr overloaded, got {pr_mapping_overloaded}') + failures.append(f"[GCR] expected no -pr overloaded, got {pr_mapping_overloaded}") # '-normal' mapping tests: global configuration should be in effect normal_overloaded = 0 @@ -303,39 +336,45 @@ def check(self): # print(json.dumps(result.as_dict(), sort_keys=True, indent=2)) # printed = True - if 'X-Envoy-Overloaded' in result.headers: + if "X-Envoy-Overloaded" in result.headers: normal_overloaded += 1 if not 100 < normal_overloaded < 200: - failures.append(f'[GCF] expected 100-200 normal_overloaded, got {normal_overloaded}') + failures.append( + f"[GCF] expected 100-200 normal_overloaded, got {normal_overloaded}" + ) if failures: print("%s FAILED:\n %s" % (self.name, "\n ".join(failures))) pytest.xfail(f"FFS {self.name}") + class CircuitBreakingTCPTest(AmbassadorTest): - extra_ports = [ 6789, 6790 ] + extra_ports = [6789, 6790] target1: ServiceType target2: ServiceType def init(self): - self.target1 = HTTP(name="target1") - self.target2 = HTTP(name="target2") + self.target1 = HTTP(name="target1") + self.target2 = HTTP(name="target2") # config() must _yield_ tuples of Node, Ambassador-YAML where the # Ambassador-YAML will be annotated onto the Node. def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target1, self.format(""" + yield self.target1, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TCPMapping name: {self.name}-1 port: 6789 service: {self.target1.path.fqdn}:80 -""") - yield self.target2, self.format(""" +""" + ) + yield self.target2, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TCPMapping @@ -346,21 +385,30 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - priority: default max_pending_requests: 1 max_connections: 1 -""") +""" + ) def queries(self): for i in range(200): - yield Query(self.url(self.name, port=6789) , headers={ "Kat-Req-Http-Requested-Backend-Delay": "1000" }, - ignore_result=True, phase=1) + yield Query( + self.url(self.name, port=6789), + headers={"Kat-Req-Http-Requested-Backend-Delay": "1000"}, + ignore_result=True, + phase=1, + ) for i in range(200): - yield Query(self.url(self.name, port=6790) , headers={ "Kat-Req-Http-Requested-Backend-Delay": "1000" }, - ignore_result=True, phase=1) + yield Query( + self.url(self.name, port=6790), + headers={"Kat-Req-Http-Requested-Backend-Delay": "1000"}, + ignore_result=True, + phase=1, + ) def check(self): failures = [] if len(self.results) != 400: - failures.append(f'wanted 400 results, got {len(self.results)}') + failures.append(f"wanted 400 results, got {len(self.results)}") else: default_limit_result = self.results[0:200] low_limit_results = self.results[200:400] @@ -372,7 +420,9 @@ def check(self): default_limit_failure += 1 if default_limit_failure != 0: - failures.append(f'expected no failure with default limit, got {default_limit_failure}') + failures.append( + f"expected no failure with default limit, got {default_limit_failure}" + ) low_limit_failure = 0 @@ -381,7 +431,7 @@ def check(self): low_limit_failure += 1 if not 100 < low_limit_failure < 200: - failures.append(f'expected 100-200 failure with low limit, got {low_limit_failure}') + failures.append(f"expected 100-200 failure with low limit, got {low_limit_failure}") if failures: print("%s FAILED:\n %s" % (self.name, "\n ".join(failures))) diff --git a/python/tests/kat/t_cluster_tag.py b/python/tests/kat/t_cluster_tag.py index 05df9c9f7e..2e5c7fdda7 100644 --- a/python/tests/kat/t_cluster_tag.py +++ b/python/tests/kat/t_cluster_tag.py @@ -10,7 +10,9 @@ def init(self): self.target_2 = HTTP(name="target2") def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -76,7 +78,10 @@ def manifests(self) -> str: prefix: /mapping-6/ service: {self.target_2.path.fqdn} cluster_tag: some-really-long-tag-that-is-really-long -''') + super().manifests() +""" + ) + + super().manifests() + ) def assert_cluster(self, cluster, target_ip): assert cluster is not None @@ -101,8 +106,12 @@ def check(self): cluster_4 = clusters["cluster_tag_2_clustertagtest_http_target2_default"] self.assert_cluster(cluster_4, "clustertagtest-http-target2") - cluster_5 = clusters["cluster_some_really_long_tag_that_is_really_long_clustertagtest_http_target1_default"] + cluster_5 = clusters[ + "cluster_some_really_long_tag_that_is_really_long_clustertagtest_http_target1_default" + ] self.assert_cluster(cluster_5, "clustertagtest-http-target1") - cluster_6 = clusters["cluster_some_really_long_tag_that_is_really_long_clustertagtest_http_target2_default"] + cluster_6 = clusters[ + "cluster_some_really_long_tag_that_is_really_long_clustertagtest_http_target2_default" + ] self.assert_cluster(cluster_6, "clustertagtest-http-target2") diff --git a/python/tests/kat/t_consul.py b/python/tests/kat/t_consul.py index 93e1f52cd4..b33c6d6d5b 100644 --- a/python/tests/kat/t_consul.py +++ b/python/tests/kat/t_consul.py @@ -5,16 +5,21 @@ from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node from tests.selfsigned import TLSCerts -SECRETS=""" +SECRETS = ( + """ --- apiVersion: v1 metadata: name: {self.path.k8s}-client-cert-secret data: - tls.crt: """+TLSCerts["master.datawire.io"].k8s_crt+""" + tls.crt: """ + + TLSCerts["master.datawire.io"].k8s_crt + + """ kind: Secret type: Opaque """ +) + class ConsulTest(AmbassadorTest): k8s_target: ServiceType @@ -39,10 +44,11 @@ def init(self): # escaping, since this gets passed through self.format (hence two layers of # doubled braces) and JSON decoding (hence backslash-escaped double quotes, # and of course the backslashes themselves have to be escaped...) - self.datacenter_json = f'{{{{\\\"datacenter\\\":\\\"{self.datacenter}\\\"}}}}' + self.datacenter_json = f'{{{{\\"datacenter\\":\\"{self.datacenter}\\"}}}}' def manifests(self) -> str: - consul_manifest = self.format(""" + consul_manifest = self.format( + """ --- apiVersion: v1 kind: Service @@ -75,18 +81,26 @@ def manifests(self) -> str: - name: CONSUL_LOCAL_CONFIG value: "{self.datacenter_json}" restartPolicy: Always -""") +""" + ) # Unlike usual, we have stuff both before and after super().manifests(): # we want the namespace early, but we want the superclass before our other # manifests, because of some magic with ServiceAccounts? - return self.format(""" + return ( + self.format( + """ --- apiVersion: v1 kind: Namespace metadata: name: consul-test-namespace -""") + super().manifests() + consul_manifest + self.format(""" +""" + ) + + super().manifests() + + consul_manifest + + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: ConsulResolver @@ -110,10 +124,14 @@ def manifests(self) -> str: resolver: {self.path.k8s}-resolver load_balancer: policy: round_robin -""" + SECRETS) +""" + + SECRETS + ) + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.k8s_target, self.format(""" + yield self.k8s_target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -154,35 +172,42 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: requestPolicy: insecure: action: Route -""") +""" + ) def requirements(self): yield from super().requirements() - yield("url", Query(self.format("http://{self.path.k8s}-consul:8500/ui/"))) + yield ("url", Query(self.format("http://{self.path.k8s}-consul:8500/ui/"))) def queries(self): # Deregister the Consul services in phase 0. - yield Query(self.format("http://{self.path.k8s}-consul:8500/v1/catalog/deregister"), - method="PUT", - body={ - "Datacenter": self.datacenter, - "Node": self.format("{self.path.k8s}-consul-service") - }, - phase=0) - yield Query(self.format("http://{self.path.k8s}-consul:8500/v1/catalog/deregister"), - method="PUT", - body={ - "Datacenter": self.datacenter, - "Node": self.format("{self.path.k8s}-consul-ns-service") - }, - phase=0) - yield Query(self.format("http://{self.path.k8s}-consul:8500/v1/catalog/deregister"), - method="PUT", - body={ - "Datacenter": self.datacenter, - "Node": self.format("{self.path.k8s}-consul-node") - }, - phase=0) + yield Query( + self.format("http://{self.path.k8s}-consul:8500/v1/catalog/deregister"), + method="PUT", + body={ + "Datacenter": self.datacenter, + "Node": self.format("{self.path.k8s}-consul-service"), + }, + phase=0, + ) + yield Query( + self.format("http://{self.path.k8s}-consul:8500/v1/catalog/deregister"), + method="PUT", + body={ + "Datacenter": self.datacenter, + "Node": self.format("{self.path.k8s}-consul-ns-service"), + }, + phase=0, + ) + yield Query( + self.format("http://{self.path.k8s}-consul:8500/v1/catalog/deregister"), + method="PUT", + body={ + "Datacenter": self.datacenter, + "Node": self.format("{self.path.k8s}-consul-node"), + }, + phase=0, + ) # The K8s service should be OK. The Consul services should 503 since they have no upstreams # in phase 1. @@ -192,35 +217,47 @@ def queries(self): yield Query(self.url(self.format("{self.path.k8s}_consul_node/")), expected=503, phase=1) # Register the Consul services in phase 2. - yield Query(self.format("http://{self.path.k8s}-consul:8500/v1/catalog/register"), - method="PUT", - body={ - "Datacenter": self.datacenter, - "Node": self.format("{self.path.k8s}-consul-service"), - "Address": self.k8s_target.path.k8s, - "Service": {"Service": self.format("{self.path.k8s}-consul-service"), - "Address": self.k8s_target.path.k8s, - "Port": 80}}, - phase=2) - yield Query(self.format("http://{self.path.k8s}-consul:8500/v1/catalog/register"), - method="PUT", - body={ - "Datacenter": self.datacenter, - "Node": self.format("{self.path.k8s}-consul-ns-service"), - "Address": self.format("{self.k8s_ns_target.path.k8s}.consul-test-namespace"), - "Service": {"Service": self.format("{self.path.k8s}-consul-ns-service"), - "Address": self.format("{self.k8s_ns_target.path.k8s}.consul-test-namespace"), - "Port": 80}}, - phase=2) - yield Query(self.format("http://{self.path.k8s}-consul:8500/v1/catalog/register"), - method="PUT", - body={ - "Datacenter": self.datacenter, - "Node": self.format("{self.path.k8s}-consul-node"), - "Address": self.k8s_target.path.k8s, - "Service": {"Service": self.format("{self.path.k8s}-consul-node"), - "Port": 80}}, - phase=2) + yield Query( + self.format("http://{self.path.k8s}-consul:8500/v1/catalog/register"), + method="PUT", + body={ + "Datacenter": self.datacenter, + "Node": self.format("{self.path.k8s}-consul-service"), + "Address": self.k8s_target.path.k8s, + "Service": { + "Service": self.format("{self.path.k8s}-consul-service"), + "Address": self.k8s_target.path.k8s, + "Port": 80, + }, + }, + phase=2, + ) + yield Query( + self.format("http://{self.path.k8s}-consul:8500/v1/catalog/register"), + method="PUT", + body={ + "Datacenter": self.datacenter, + "Node": self.format("{self.path.k8s}-consul-ns-service"), + "Address": self.format("{self.k8s_ns_target.path.k8s}.consul-test-namespace"), + "Service": { + "Service": self.format("{self.path.k8s}-consul-ns-service"), + "Address": self.format("{self.k8s_ns_target.path.k8s}.consul-test-namespace"), + "Port": 80, + }, + }, + phase=2, + ) + yield Query( + self.format("http://{self.path.k8s}-consul:8500/v1/catalog/register"), + method="PUT", + body={ + "Datacenter": self.datacenter, + "Node": self.format("{self.path.k8s}-consul-node"), + "Address": self.k8s_target.path.k8s, + "Service": {"Service": self.format("{self.path.k8s}-consul-node"), "Port": 80}, + }, + phase=2, + ) # All services should work in phase 3. yield Query(self.url(self.format("{self.path.k8s}_k8s/")), expected=200, phase=3) diff --git a/python/tests/kat/t_cors.py b/python/tests/kat/t_cors.py index 28447dd6f4..4f8682dd23 100644 --- a/python/tests/kat/t_cors.py +++ b/python/tests/kat/t_cors.py @@ -14,7 +14,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -40,26 +41,27 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: cors: origins: [http://bar.example.com] methods: [POST, GET, OPTIONS] -""") +""" + ) def queries(self): # 0. No Access-Control-Allow-Origin because no Origin was provided. yield Query(self.url("foo/")) # 1. Access-Control-Allow-Origin because a matching Origin was provided. - yield Query(self.url("foo/"), headers={ "Origin": "http://foo.example.com" }) + yield Query(self.url("foo/"), headers={"Origin": "http://foo.example.com"}) # 2. No Access-Control-Allow-Origin because the provided Origin does not match. - yield Query(self.url("foo/"), headers={ "Origin": "http://wrong.example.com" }) + yield Query(self.url("foo/"), headers={"Origin": "http://wrong.example.com"}) # 3. No Access-Control-Allow-Origin because no Origin was provided. yield Query(self.url("bar/")) # 4. Access-Control-Allow-Origin because a matching Origin was provided. - yield Query(self.url("bar/"), headers={ "Origin": "http://bar.example.com" }) + yield Query(self.url("bar/"), headers={"Origin": "http://bar.example.com"}) # 5. No Access-Control-Allow-Origin because no Origin was provided. - yield Query(self.url("bar/"), headers={ "Origin": "http://wrong.example.com" }) + yield Query(self.url("bar/"), headers={"Origin": "http://wrong.example.com"}) def check(self): assert self.results[0].backend @@ -68,7 +70,7 @@ def check(self): assert self.results[1].backend assert self.results[1].backend.name == self.target.path.k8s - assert self.results[1].headers["Access-Control-Allow-Origin"] == [ "http://foo.example.com" ] + assert self.results[1].headers["Access-Control-Allow-Origin"] == ["http://foo.example.com"] assert self.results[2].backend assert self.results[2].backend.name == self.target.path.k8s @@ -80,7 +82,7 @@ def check(self): assert self.results[4].backend assert self.results[4].backend.name == self.target.path.k8s - assert self.results[4].headers["Access-Control-Allow-Origin"] == [ "http://bar.example.com" ] + assert self.results[4].headers["Access-Control-Allow-Origin"] == ["http://bar.example.com"] assert self.results[5].backend assert self.results[5].backend.name == self.target.path.k8s diff --git a/python/tests/kat/t_dns_type.py b/python/tests/kat/t_dns_type.py index 0ce337fc22..78088cf605 100644 --- a/python/tests/kat/t_dns_type.py +++ b/python/tests/kat/t_dns_type.py @@ -13,7 +13,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -22,7 +23,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: {self.target.path.fqdn} hostname: "*" dns_type: logical_dns -""") +""" + ) def queries(self): yield Query(self.url("foo/")) @@ -31,6 +33,7 @@ def check(self): # Not getting a 400 bad request is confirmation that this setting works as long as the request can reach the upstream assert self.results[0].status == 200 + # this test is just to confirm that when both dns_type and the endpoint resolver are in use, the endpoint resolver wins class LogicalDnsTypeEndpoint(AmbassadorTest): target: ServiceType @@ -39,7 +42,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -49,7 +53,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" dns_type: logical_dns resolver: endpoint -""") +""" + ) def queries(self): yield Query(self.url("foo/")) @@ -58,6 +63,7 @@ def check(self): # Not getting a 400 bad request is confirmation that this setting works as long as the request can reach the upstream assert self.results[0].status == 200 + # Tests Respecting DNS TTL alone class DnsTtl(AmbassadorTest): target: ServiceType @@ -66,7 +72,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -75,7 +82,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" service: {self.target.path.fqdn} respect_dns_ttl: true -""") +""" + ) def queries(self): yield Query(self.url("foo/")) @@ -84,6 +92,7 @@ def check(self): # Not getting a 400 bad request is confirmation that this setting works as long as the request can reach the upstream assert self.results[0].status == 200 + # Tests the DNS TTL with the endpoint resolver class DnsTtlEndpoint(AmbassadorTest): target: ServiceType @@ -92,7 +101,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -102,7 +112,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: {self.target.path.fqdn} respect_dns_ttl: true resolver: endpoint -""") +""" + ) def queries(self): yield Query(self.url("foo/")) @@ -111,6 +122,7 @@ def check(self): # Not getting a 400 bad request is confirmation that this setting works as long as the request can reach the upstream assert self.results[0].status == 200 + # Tests the DNS TTL with Logical DNS type class DnsTtlDnsType(AmbassadorTest): target: ServiceType @@ -119,7 +131,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -129,7 +142,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: {self.target.path.fqdn} respect_dns_ttl: true dns_type: logical_dns -""") +""" + ) def queries(self): yield Query(self.url("foo/")) diff --git a/python/tests/kat/t_envoy_logs.py b/python/tests/kat/t_envoy_logs.py index ed98c8ddf8..b43bc45d5b 100644 --- a/python/tests/kat/t_envoy_logs.py +++ b/python/tests/kat/t_envoy_logs.py @@ -17,11 +17,12 @@ def init(self): self.xfail = "Not yet supported in Edge Stack" self.target = HTTP() - self.log_path = '/tmp/ambassador/ambassador.log' - self.log_format = 'MY_REQUEST %RESPONSE_CODE% \"%REQ(:AUTHORITY)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%UPSTREAM_HOST%\"' + self.log_path = "/tmp/ambassador/ambassador.log" + self.log_format = 'MY_REQUEST %RESPONSE_CODE% "%REQ(:AUTHORITY)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%UPSTREAM_HOST%"' def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -30,17 +31,20 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: config: envoy_log_path: {self.log_path} envoy_log_format: {self.log_format} -""") +""" + ) def check(self): - access_log_entry_regex = re.compile('^MY_REQUEST 200 .*') + access_log_entry_regex = re.compile("^MY_REQUEST 200 .*") cmd = ShellCommand("tools/bin/kubectl", "exec", self.path.k8s, "cat", self.log_path) if not cmd.check("check envoy access log"): pytest.exit("envoy access log does not exist") for line in cmd.stdout.splitlines(): - assert access_log_entry_regex.match(line), f"{line} does not match {access_log_entry_regex}" + assert access_log_entry_regex.match( + line + ), f"{line} does not match {access_log_entry_regex}" class EnvoyLogJSONTest(AmbassadorTest): @@ -49,10 +53,11 @@ class EnvoyLogJSONTest(AmbassadorTest): def init(self): self.target = HTTP() - self.log_path = '/tmp/ambassador/ambassador.log' + self.log_path = "/tmp/ambassador/ambassador.log" def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -64,7 +69,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: protocol: "%PROTOCOL%" duration: "%DURATION%" envoy_log_type: json -""") +""" + ) def check(self): access_log_entry_regex = re.compile('^({"duration":|{"protocol":)') @@ -74,4 +80,6 @@ def check(self): pytest.exit("envoy access log does not exist") for line in cmd.stdout.splitlines(): - assert access_log_entry_regex.match(line), f"{line} does not match {access_log_entry_regex}" + assert access_log_entry_regex.match( + line + ), f"{line} does not match {access_log_entry_regex}" diff --git a/python/tests/kat/t_error_response.py b/python/tests/kat/t_error_response.py index 0193389b9b..98c3265ecc 100644 --- a/python/tests/kat/t_error_response.py +++ b/python/tests/kat/t_error_response.py @@ -9,11 +9,12 @@ class ErrorResponseOnStatusCode(AmbassadorTest): """ Check that we can return a customized error response where the body is built as a formatted string. """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -74,27 +75,43 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - on_status_code: 503 body: text_format: '' -''' +""" def queries(self): # [0] yield Query(self.url("does-not-exist/"), expected=404) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401 + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "403"}, expected=403) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "403"}, expected=403 + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [4] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "418"}, expected=418) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "418"}, expected=418 + ) # [5] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "500"}, expected=500) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "500"}, expected=500 + ) # [6] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "501"}, expected=501) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "501"}, expected=501 + ) # [7] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) # [8] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504 + ) # [9] yield Query(self.url("target/")) # [10] @@ -104,64 +121,77 @@ def queries(self): def check(self): # [0] - assert self.results[0].text == 'cannot find the thing', \ - f"unexpected response body: {self.results[0].text}" + assert ( + self.results[0].text == "cannot find the thing" + ), f"unexpected response body: {self.results[0].text}" # [1] - assert self.results[1].text == 'you get a 401', \ - f"unexpected response body: {self.results[1].text}" + assert ( + self.results[1].text == "you get a 401" + ), f"unexpected response body: {self.results[1].text}" # [2] - assert self.results[2].text == 'and you get a 403', \ - f"unexpected response body: {self.results[2].text}" + assert ( + self.results[2].text == "and you get a 403" + ), f"unexpected response body: {self.results[2].text}" # [3] - assert self.results[3].text == 'cannot find the thing', \ - f"unexpected response body: {self.results[3].text}" + assert ( + self.results[3].text == "cannot find the thing" + ), f"unexpected response body: {self.results[3].text}" # [4] - assert self.results[4].text == '2teapot2reply', \ - f"unexpected response body: {self.results[4].text}" + assert ( + self.results[4].text == "2teapot2reply" + ), f"unexpected response body: {self.results[4].text}" # [5] - assert self.results[5].text == 'a five hundred happened', \ - f"unexpected response body: {self.results[5].text}" + assert ( + self.results[5].text == "a five hundred happened" + ), f"unexpected response body: {self.results[5].text}" # [6] - assert self.results[6].text == 'very not implemented', \ - f"unexpected response body: {self.results[6].text}" + assert ( + self.results[6].text == "very not implemented" + ), f"unexpected response body: {self.results[6].text}" # [7] - assert self.results[7].text == 'the upstream probably died', \ - f"unexpected response body: {self.results[7].text}" + assert ( + self.results[7].text == "the upstream probably died" + ), f"unexpected response body: {self.results[7].text}" # [8] - assert self.results[8].text == 'took too long, sorry', \ - f"unexpected response body: {self.results[8].text}" - assert self.results[8].headers["Content-Type"] == ["apology"], \ - f"unexpected Content-Type: {self.results[8].headers}" + assert ( + self.results[8].text == "took too long, sorry" + ), f"unexpected response body: {self.results[8].text}" + assert self.results[8].headers["Content-Type"] == [ + "apology" + ], f"unexpected Content-Type: {self.results[8].headers}" # [9] should just succeed - assert self.results[9].text == None, \ - f"unexpected response body: {self.results[9].text}" + assert self.results[9].text == None, f"unexpected response body: {self.results[9].text}" # [10] envoy-generated 503, since the upstream is 'invalidservice'. - assert self.results[10].text == 'the upstream probably died', \ - f"unexpected response body: {self.results[10].text}" + assert ( + self.results[10].text == "the upstream probably died" + ), f"unexpected response body: {self.results[10].text}" # [11] envoy-generated 503, with an empty body override - assert self.results[11].text == '', \ - f"unexpected response body: {self.results[11].text}" + assert self.results[11].text == "", f"unexpected response body: {self.results[11].text}" + class ErrorResponseOnStatusCodeMappingCRD(AmbassadorTest): """ Check that we can return a customized error response where the body is built as a formatted string. """ + def init(self): self.target = HTTP() def manifests(self) -> str: - return super().manifests() + f''' + return ( + super().manifests() + + f""" --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -223,27 +253,44 @@ def manifests(self) -> str: body: text_format_source: filename: /etc/issue -''' +""" + ) def queries(self): # [0] yield Query(self.url("does-not-exist/"), expected=404) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401 + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "403"}, expected=403) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "403"}, expected=403 + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [4] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "418"}, expected=418) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "418"}, expected=418 + ) # [5] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "500"}, expected=500) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "500"}, expected=500 + ) # [6] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "501"}, expected=501) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "501"}, expected=501 + ) # [7] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) # [8] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504 + ) # [9] yield Query(self.url("target/")) # [10] @@ -254,70 +301,80 @@ def queries(self): def check(self): # [0] does not match the error response mapping, so no 404 response. # when envoy directly replies with 404, we see it as an empty string. - assert self.results[0].text == '', \ - f"unexpected response body: {self.results[0].text}" + assert self.results[0].text == "", f"unexpected response body: {self.results[0].text}" # [1] - assert self.results[1].text == 'you get a 401', \ - f"unexpected response body: {self.results[1].text}" + assert ( + self.results[1].text == "you get a 401" + ), f"unexpected response body: {self.results[1].text}" # [2] - assert self.results[2].text == 'and you get a 403', \ - f"unexpected response body: {self.results[2].text}" + assert ( + self.results[2].text == "and you get a 403" + ), f"unexpected response body: {self.results[2].text}" # [3] - assert self.results[3].text == 'cannot find the thing', \ - f"unexpected response body: {self.results[3].text}" + assert ( + self.results[3].text == "cannot find the thing" + ), f"unexpected response body: {self.results[3].text}" # [4] - assert self.results[4].text == '2teapot2reply', \ - f"unexpected response body: {self.results[4].text}" + assert ( + self.results[4].text == "2teapot2reply" + ), f"unexpected response body: {self.results[4].text}" # [5] - assert self.results[5].text == 'a five hundred happened', \ - f"unexpected response body: {self.results[5].text}" + assert ( + self.results[5].text == "a five hundred happened" + ), f"unexpected response body: {self.results[5].text}" # [6] - assert self.results[6].text == 'very not implemented', \ - f"unexpected response body: {self.results[6].text}" + assert ( + self.results[6].text == "very not implemented" + ), f"unexpected response body: {self.results[6].text}" # [7] - assert self.results[7].text == 'the upstream probably died', \ - f"unexpected response body: {self.results[7].text}" + assert ( + self.results[7].text == "the upstream probably died" + ), f"unexpected response body: {self.results[7].text}" # [8] - assert self.results[8].text == 'took too long, sorry', \ - f"unexpected response body: {self.results[8].text}" - assert self.results[8].headers["Content-Type"] == ["apology"], \ - f"unexpected Content-Type: {self.results[8].headers}" + assert ( + self.results[8].text == "took too long, sorry" + ), f"unexpected response body: {self.results[8].text}" + assert self.results[8].headers["Content-Type"] == [ + "apology" + ], f"unexpected Content-Type: {self.results[8].headers}" # [9] should just succeed - assert self.results[9].text == None, \ - f"unexpected response body: {self.results[9].text}" + assert self.results[9].text == None, f"unexpected response body: {self.results[9].text}" # [10] envoy-generated 503, since the upstream is 'invalidservice'. # this response body comes unmodified from envoy, since it goes through # a mapping with no error response overrides and there's no overrides # on the Ambassador module - assert self.results[10].text == 'no healthy upstream', \ - f"unexpected response body: {self.results[10].text}" + assert ( + self.results[10].text == "no healthy upstream" + ), f"unexpected response body: {self.results[10].text}" # [11] envoy-generated 503, since the upstream is 'invalidservice'. # this response body should be matched by the `text_format_source` override # sorry for using /etc/issue, by the way. - assert "Welcome to Alpine Linux" in self.results[11].text, \ - f"unexpected response body: {self.results[11].text}" + assert ( + "Welcome to Alpine Linux" in self.results[11].text + ), f"unexpected response body: {self.results[11].text}" class ErrorResponseReturnBodyFormattedText(AmbassadorTest): """ Check that we can return a customized error response where the body is built as a formatted string. """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -344,47 +401,58 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -''' +""" def queries(self): # [0] yield Query(self.url("does-not-exist/"), expected=404) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "429"}, expected=429) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "429"}, expected=429 + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504 + ) def check(self): # [0] - assert self.results[0].text == "there has been an error: 404", \ - f"unexpected response body: {self.results[0].text}" - assert self.results[0].headers["Content-Type"] == ["text/plain"], \ - f"unexpected Content-Type: {self.results[0].headers}" + assert ( + self.results[0].text == "there has been an error: 404" + ), f"unexpected response body: {self.results[0].text}" + assert self.results[0].headers["Content-Type"] == [ + "text/plain" + ], f"unexpected Content-Type: {self.results[0].headers}" # [1] - assert self.results[1].text == "2fast HTTP/1.1", \ - f"unexpected response body: {self.results[1].text}" - assert self.results[1].headers["Content-Type"] == ["text/html"], \ - f"unexpected Content-type: {self.results[1].headers}" + assert ( + self.results[1].text == "2fast HTTP/1.1" + ), f"unexpected response body: {self.results[1].text}" + assert self.results[1].headers["Content-Type"] == [ + "text/html" + ], f"unexpected Content-type: {self.results[1].headers}" # [2] - assert self.results[2].text == "2slow HTTP/1.1", \ - f"unexpected response body: {self.results[2].text}" - assert self.results[2].headers["Content-Type"] == ["text/html; charset=\"utf-8\""], \ - f"unexpected Content-Type: {self.results[2].headers}" + assert ( + self.results[2].text == "2slow HTTP/1.1" + ), f"unexpected response body: {self.results[2].text}" + assert self.results[2].headers["Content-Type"] == [ + 'text/html; charset="utf-8"' + ], f"unexpected Content-Type: {self.results[2].headers}" class ErrorResponseReturnBodyFormattedJson(AmbassadorTest): """ Check that we can return a customized error response where the body is built from a text source. """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -415,44 +483,58 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -''' +""" def queries(self): yield Query(self.url("does-not-exist/"), expected=404) - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "429"}, expected=429) - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "429"}, expected=429 + ) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401 + ) def check(self): # [0] # Strange gotcha: it looks like we always get an integer code here # even though the field specifier above is wrapped in single quotes. - assert self.results[0].json == { "custom_error": "truth", "code": 404 }, \ - f"unexpected response body: {self.results[0].json}" - assert self.results[0].headers["Content-Type"] == ["application/json"], \ - f"unexpected Content-Type: {self.results[0].headers}" + assert self.results[0].json == { + "custom_error": "truth", + "code": 404, + }, f"unexpected response body: {self.results[0].json}" + assert self.results[0].headers["Content-Type"] == [ + "application/json" + ], f"unexpected Content-Type: {self.results[0].headers}" # [1] - assert self.results[1].json == { "custom_error": "yep", "toofast": "definitely", "code": "code was 429" }, \ - f"unexpected response body: {self.results[1].json}" - assert self.results[1].headers["Content-Type"] == ["application/json"], \ - f"unexpected Content-Type: {self.results[1].headers}" + assert self.results[1].json == { + "custom_error": "yep", + "toofast": "definitely", + "code": "code was 429", + }, f"unexpected response body: {self.results[1].json}" + assert self.results[1].headers["Content-Type"] == [ + "application/json" + ], f"unexpected Content-Type: {self.results[1].headers}" # [2] - assert self.results[2].json == { "error": "unauthorized" }, \ - f"unexpected response body: {self.results[2].json}" - assert self.results[2].headers["Content-Type"] == ["application/json"], \ - f"unexpected Content-Type: {self.results[2].headers}" + assert self.results[2].json == { + "error": "unauthorized" + }, f"unexpected response body: {self.results[2].json}" + assert self.results[2].headers["Content-Type"] == [ + "application/json" + ], f"unexpected Content-Type: {self.results[2].headers}" class ErrorResponseReturnBodyTextSource(AmbassadorTest): """ Check that we can return a customized error response where the body is built as a formatted string. """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -482,47 +564,61 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -''' +""" def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "500"}, expected=500) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "500"}, expected=500 + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504 + ) def check(self): # [0] Sorry for using /etc/issue... print("headers = %s" % self.results[0].headers) - assert "Welcome to Alpine Linux" in self.results[0].text, \ - f"unexpected response body: {self.results[0].text}" - assert self.results[0].headers["Content-Type"] == ["application/etcissue"], \ - f"unexpected Content-Type: {self.results[0].headers}" + assert ( + "Welcome to Alpine Linux" in self.results[0].text + ), f"unexpected response body: {self.results[0].text}" + assert self.results[0].headers["Content-Type"] == [ + "application/etcissue" + ], f"unexpected Content-Type: {self.results[0].headers}" # [1] ...and sorry for using /etc/motd... - assert "You may change this message by editing /etc/motd." in self.results[1].text, \ - f"unexpected response body: {self.results[1].text}" - assert self.results[1].headers["Content-Type"] == ["application/motd"], \ - f"unexpected Content-Type: {self.results[1].headers}" + assert ( + "You may change this message by editing /etc/motd." in self.results[1].text + ), f"unexpected response body: {self.results[1].text}" + assert self.results[1].headers["Content-Type"] == [ + "application/motd" + ], f"unexpected Content-Type: {self.results[1].headers}" # [2] ...and sorry for using /etc/shells - assert "# valid login shells" in self.results[2].text, \ - f"unexpected response body: {self.results[2].text}" - assert self.results[2].headers["Content-Type"] == ["text/plain"], \ - f"unexpected Content-Type: {self.results[2].headers}" + assert ( + "# valid login shells" in self.results[2].text + ), f"unexpected response body: {self.results[2].text}" + assert self.results[2].headers["Content-Type"] == [ + "text/plain" + ], f"unexpected Content-Type: {self.results[2].headers}" + class ErrorResponseMappingBypass(AmbassadorTest): """ Check that we can return a bypass custom error responses at the mapping level """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -583,25 +679,39 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /bypass/invalidservice service: {self.target.path.fqdn}-invalidservice bypass_error_response_overrides: true -''' +""" def queries(self): # [0] - yield Query(self.url("bypass/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("bypass/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [2] - yield Query(self.url("target/bypass/"), headers={"kat-req-http-requested-status": "418"}, expected=418) + yield Query( + self.url("target/bypass/"), + headers={"kat-req-http-requested-status": "418"}, + expected=418, + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "418"}, expected=418) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "418"}, expected=418 + ) # [4] yield Query(self.url("target/invalidservice"), expected=503) # [5] yield Query(self.url("bypass/invalidservice"), expected=503) # [6] - yield Query(self.url("bypass/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("bypass/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) # [7] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) # [8] yield Query(self.url("bypass/"), headers={"kat-req-http-requested-status": "200"}) # [9] @@ -609,48 +719,50 @@ def queries(self): def check(self): # [0] - assert self.results[0].text is None, \ - f"unexpected response body: {self.results[0].text}" + assert self.results[0].text is None, f"unexpected response body: {self.results[0].text}" # [1] - assert self.results[1].text == 'this is a custom 404 response', \ - f"unexpected response body: {self.results[1].text}" - assert self.results[1].headers["Content-Type"] == ["text/custom"], \ - f"unexpected Content-Type: {self.results[1].headers}" + assert ( + self.results[1].text == "this is a custom 404 response" + ), f"unexpected response body: {self.results[1].text}" + assert self.results[1].headers["Content-Type"] == [ + "text/custom" + ], f"unexpected Content-Type: {self.results[1].headers}" # [2] - assert self.results[2].text is None, \ - f"unexpected response body: {self.results[2].text}" + assert self.results[2].text is None, f"unexpected response body: {self.results[2].text}" # [3] - assert self.results[3].text == 'bad teapot request', \ - f"unexpected response body: {self.results[3].text}" + assert ( + self.results[3].text == "bad teapot request" + ), f"unexpected response body: {self.results[3].text}" # [4] - assert self.results[4].text == 'the upstream is not happy', \ - f"unexpected response body: {self.results[4].text}" + assert ( + self.results[4].text == "the upstream is not happy" + ), f"unexpected response body: {self.results[4].text}" # [5] - assert self.results[5].text == 'no healthy upstream', \ - f"unexpected response body: {self.results[5].text}" - assert self.results[5].headers["Content-Type"] == ["text/plain"], \ - f"unexpected Content-Type: {self.results[5].headers}" + assert ( + self.results[5].text == "no healthy upstream" + ), f"unexpected response body: {self.results[5].text}" + assert self.results[5].headers["Content-Type"] == [ + "text/plain" + ], f"unexpected Content-Type: {self.results[5].headers}" # [6] - assert self.results[6].text is None, \ - f"unexpected response body: {self.results[6].text}" + assert self.results[6].text is None, f"unexpected response body: {self.results[6].text}" # [7] - assert self.results[7].text == 'the upstream is not happy', \ - f"unexpected response body: {self.results[7].text}" + assert ( + self.results[7].text == "the upstream is not happy" + ), f"unexpected response body: {self.results[7].text}" # [8] - assert self.results[8].text is None, \ - f"unexpected response body: {self.results[8].text}" + assert self.results[8].text is None, f"unexpected response body: {self.results[8].text}" # [9] - assert self.results[9].text is None, \ - f"unexpected response body: {self.results[9].text}" + assert self.results[9].text is None, f"unexpected response body: {self.results[9].text}" class ErrorResponseMappingBypassAlternate(AmbassadorTest): @@ -659,11 +771,12 @@ class ErrorResponseMappingBypassAlternate(AmbassadorTest): serving one. This is a baseline sanity check against Envoy's response map filter incorrectly persisting state across filter chain iterations. """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -700,42 +813,53 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /bypass/ service: {self.target.path.fqdn} bypass_error_response_overrides: true -''' +""" def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [1] - yield Query(self.url("bypass/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("bypass/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) def check(self): # [0] - assert self.results[0].text == 'this is a custom 404 response', \ - f"unexpected response body: {self.results[0].text}" - assert self.results[0].headers["Content-Type"] == ["text/custom"], \ - f"unexpected Content-Type: {self.results[0].headers}" + assert ( + self.results[0].text == "this is a custom 404 response" + ), f"unexpected response body: {self.results[0].text}" + assert self.results[0].headers["Content-Type"] == [ + "text/custom" + ], f"unexpected Content-Type: {self.results[0].headers}" # [1] - assert self.results[1].text is None, \ - f"unexpected response body: {self.results[1].text}" + assert self.results[1].text is None, f"unexpected response body: {self.results[1].text}" # [2] - assert self.results[2].text == 'this is a custom 404 response', \ - f"unexpected response body: {self.results[2].text}" - assert self.results[2].headers["Content-Type"] == ["text/custom"], \ - f"unexpected Content-Type: {self.results[2].headers}" + assert ( + self.results[2].text == "this is a custom 404 response" + ), f"unexpected response body: {self.results[2].text}" + assert self.results[2].headers["Content-Type"] == [ + "text/custom" + ], f"unexpected Content-Type: {self.results[2].headers}" + class ErrorResponseMapping404Body(AmbassadorTest): """ Check that a 404 body is consistent whether error response overrides exist or not """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -776,46 +900,49 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - on_status_code: 503 body: text_format: 'custom 503' -''' +""" def queries(self): # [0] yield Query(self.url("does-not-exist/"), expected=404) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [2] - yield Query(self.url("bypass/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("bypass/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) # [3] - yield Query(self.url("overrides/"), headers={"kat-req-http-requested-status": "404"}, expected=404) + yield Query( + self.url("overrides/"), headers={"kat-req-http-requested-status": "404"}, expected=404 + ) def check(self): # [0] does not match the error response mapping, so no 404 response. # when envoy directly replies with 404, we see it as an empty string. - assert self.results[0].text == '', \ - f"unexpected response body: {self.results[0].text}" + assert self.results[0].text == "", f"unexpected response body: {self.results[0].text}" # [1] - assert self.results[1].text is None, \ - f"unexpected response body: {self.results[1].text}" + assert self.results[1].text is None, f"unexpected response body: {self.results[1].text}" # [2] - assert self.results[2].text is None, \ - f"unexpected response body: {self.results[2].text}" + assert self.results[2].text is None, f"unexpected response body: {self.results[2].text}" # [3] - assert self.results[3].text is None, \ - f"unexpected response body: {self.results[3].text}" + assert self.results[3].text is None, f"unexpected response body: {self.results[3].text}" class ErrorResponseMappingOverride(AmbassadorTest): """ Check that we can return a custom error responses at the mapping level """ + def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, f''' + yield self, f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -869,93 +996,135 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: json_format: "y": "2" status: '%RESPONSE_CODE%' -''' +""" def queries(self): # [0] Should match module's on_response_code 401 - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "401"}, expected=401 + ) # [1] Should match mapping-specific on_response_code 401 - yield Query(self.url("override/401/"), headers={"kat-req-http-requested-status": "401"}, expected=401) + yield Query( + self.url("override/401/"), + headers={"kat-req-http-requested-status": "401"}, + expected=401, + ) # [2] Should match mapping-specific on_response_code 503 - yield Query(self.url("override/503/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("override/503/"), + headers={"kat-req-http-requested-status": "503"}, + expected=503, + ) # [3] Should not match mapping-specific rule, therefore no rewrite - yield Query(self.url("override/401/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("override/401/"), + headers={"kat-req-http-requested-status": "503"}, + expected=503, + ) # [4] Should not match mapping-specific rule, therefore no rewrite - yield Query(self.url("override/503/"), headers={"kat-req-http-requested-status": "401"}, expected=401) + yield Query( + self.url("override/503/"), + headers={"kat-req-http-requested-status": "401"}, + expected=401, + ) # [5] Should not match mapping-specific rule, therefore no rewrite - yield Query(self.url("override/401/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("override/401/"), + headers={"kat-req-http-requested-status": "504"}, + expected=504, + ) # [6] Should not match mapping-specific rule, therefore no rewrite - yield Query(self.url("override/503/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("override/503/"), + headers={"kat-req-http-requested-status": "504"}, + expected=504, + ) # [7] Should match module's on_response_code 503 - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) # [8] Should match module's on_response_code 504 - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "504"}, expected=504 + ) def check(self): # [0] Module's 401 rule with custom header - assert self.results[0].text == 'this is a custom 401 response', \ - f"unexpected response body: {self.results[0].text}" - assert self.results[0].headers["Content-Type"] == ["text/custom"], \ - f"unexpected Content-Type: {self.results[0].headers}" + assert ( + self.results[0].text == "this is a custom 401 response" + ), f"unexpected response body: {self.results[0].text}" + assert self.results[0].headers["Content-Type"] == [ + "text/custom" + ], f"unexpected Content-Type: {self.results[0].headers}" # [1] Mapping's 401 rule with json response - assert self.results[1].json == { "x": "1", "status": 401 }, \ - f"unexpected response body: {self.results[1].json}" - assert self.results[1].headers["Content-Type"] == ["application/json"], \ - f"unexpected Content-Type: {self.results[1].headers}" + assert self.results[1].json == { + "x": "1", + "status": 401, + }, f"unexpected response body: {self.results[1].json}" + assert self.results[1].headers["Content-Type"] == [ + "application/json" + ], f"unexpected Content-Type: {self.results[1].headers}" # [2] Mapping's 503 rule with json response - assert self.results[2].json == { "y": "2", "status": 503 }, \ - f"unexpected response body: {self.results[2].json}" - assert self.results[2].headers["Content-Type"] == ["application/json"], \ - f"unexpected Content-Type: {self.results[2].headers}" + assert self.results[2].json == { + "y": "2", + "status": 503, + }, f"unexpected response body: {self.results[2].json}" + assert self.results[2].headers["Content-Type"] == [ + "application/json" + ], f"unexpected Content-Type: {self.results[2].headers}" # [3] Mapping has 401 rule, but response code is 503, no rewrite. - assert self.results[3].text is None, \ - f"unexpected response body: {self.results[3].text}" + assert self.results[3].text is None, f"unexpected response body: {self.results[3].text}" # [4] Mapping has 503 rule, but response code is 401, no rewrite. - assert self.results[4].text is None, \ - f"unexpected response body: {self.results[4].text}" + assert self.results[4].text is None, f"unexpected response body: {self.results[4].text}" # [5] Mapping has 401 rule, but response code is 504, no rewrite. - assert self.results[5].text is None, \ - f"unexpected response body: {self.results[5].text}" + assert self.results[5].text is None, f"unexpected response body: {self.results[5].text}" # [6] Mapping has 503 rule, but response code is 504, no rewrite. - assert self.results[6].text is None, \ - f"unexpected response body: {self.results[6].text}" + assert self.results[6].text is None, f"unexpected response body: {self.results[6].text}" # [7] Module's 503 rule, no custom header - assert self.results[7].text == 'the upstream is not happy', \ - f"unexpected response body: {self.results[7].text}" - assert self.results[7].headers["Content-Type"] == ["text/plain"], \ - f"unexpected Content-Type: {self.results[7].headers}" + assert ( + self.results[7].text == "the upstream is not happy" + ), f"unexpected response body: {self.results[7].text}" + assert self.results[7].headers["Content-Type"] == [ + "text/plain" + ], f"unexpected Content-Type: {self.results[7].headers}" # [8] Module's 504 rule, no custom header - assert self.results[8].text == 'the upstream took a really long time', \ - f"unexpected response body: {self.results[8].text}" - assert self.results[8].headers["Content-Type"] == ["text/plain"], \ - f"unexpected Content-Type: {self.results[8].headers}" + assert ( + self.results[8].text == "the upstream took a really long time" + ), f"unexpected response body: {self.results[8].text}" + assert self.results[8].headers["Content-Type"] == [ + "text/plain" + ], f"unexpected Content-Type: {self.results[8].headers}" + class ErrorResponseSeveralMappings(AmbassadorTest): """ Check that we can specify separate error response overrides on two mappings with no Module config """ + def init(self): self.target = HTTP() def manifests(self) -> str: - return super().manifests() + f''' + return ( + super().manifests() + + f""" --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1014,32 +1183,37 @@ def manifests(self) -> str: - on_status_code: 500 body: text_format: '500 is a bad status code' -''' +""" + ) _queries = [ - { 'url': "does-not-exist/", 'status': 404, 'text': '' }, - { 'url': "target-one/", 'status': 404, 'text': '404 from first mapping' }, - { 'url': "target-one/", 'status': 429, 'text': None }, - { 'url': "target-one/", 'status': 504, 'text': 'a custom 504 response' }, - { 'url': "target-two/", 'status': 404, 'text': '404 from second mapping' }, - { 'url': "target-two/", 'status': 429, 'text': 'a custom 429 response' }, - { 'url': "target-two/", 'status': 504, 'text': None }, - { 'url': "target-three/", 'status': 404, 'text': None }, - { 'url': "target-three/", 'status': 429, 'text': None }, - { 'url': "target-three/", 'status': 504, 'text': None }, - { 'url': "target-four/", 'status': 404, 'text': None }, - { 'url': "target-four/", 'status': 429, 'text': None }, - { 'url': "target-four/", 'status': 504, 'text': None }, + {"url": "does-not-exist/", "status": 404, "text": ""}, + {"url": "target-one/", "status": 404, "text": "404 from first mapping"}, + {"url": "target-one/", "status": 429, "text": None}, + {"url": "target-one/", "status": 504, "text": "a custom 504 response"}, + {"url": "target-two/", "status": 404, "text": "404 from second mapping"}, + {"url": "target-two/", "status": 429, "text": "a custom 429 response"}, + {"url": "target-two/", "status": 504, "text": None}, + {"url": "target-three/", "status": 404, "text": None}, + {"url": "target-three/", "status": 429, "text": None}, + {"url": "target-three/", "status": 504, "text": None}, + {"url": "target-four/", "status": 404, "text": None}, + {"url": "target-four/", "status": 429, "text": None}, + {"url": "target-four/", "status": 504, "text": None}, ] def queries(self): for x in self._queries: - yield Query(self.url(x['url']), - headers={"kat-req-http-requested-status": str(x['status'])}, - expected=x['status']) + yield Query( + self.url(x["url"]), + headers={"kat-req-http-requested-status": str(x["status"])}, + expected=x["status"], + ) def check(self): for i in range(len(self._queries)): - expected = self._queries[i]['text'] + expected = self._queries[i]["text"] res = self.results[i] - assert res.text == expected, f"unexpected response body on query {i}: \"{res.text}\", wanted \"{expected}\"" + assert ( + res.text == expected + ), f'unexpected response body on query {i}: "{res.text}", wanted "{expected}"' diff --git a/python/tests/kat/t_extauth.py b/python/tests/kat/t_extauth.py index 0e82e6bcdd..da643cbb3c 100644 --- a/python/tests/kat/t_extauth.py +++ b/python/tests/kat/t_extauth.py @@ -24,7 +24,9 @@ def init(self): self.auth = AGRPC(name="auth") def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -38,10 +40,14 @@ def manifests(self) -> str: auth_context_extensions: context: "auth-context-name" data: "auth-data" -''') + super().manifests() +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -50,8 +56,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: timeout_ms: 5000 proto: grpc protocol_version: "v3" -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -69,37 +77,72 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: auth_context_extensions: first: "first element" second: "second element" -""") +""" + ) def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "401", - "baz": "baz", - "request-header": "baz"}, expected=401) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "401", + "baz": "baz", + "request-header": "baz", + }, + expected=401, + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "302", - "kat-req-extauth-requested-location": "foo"}, expected=302) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "302", + "kat-req-extauth-requested-location": "foo", + }, + expected=302, + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "401", - "x-foo": "foo", - "kat-req-extauth-requested-header": "x-foo"}, expected=401) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "401", + "x-foo": "foo", + "kat-req-extauth-requested-header": "x-foo", + }, + expected=401, + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "200", - "authorization": "foo-11111", - "foo": "foo", - "kat-req-extauth-append": "foo=bar;baz=bar", - "kat-req-http-requested-header": "Authorization"}, expected=200) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "200", + "authorization": "foo-11111", + "foo": "foo", + "kat-req-extauth-append": "foo=bar;baz=bar", + "kat-req-http-requested-header": "Authorization", + }, + expected=200, + ) # [4] - yield Query(self.url("context-extensions/"), headers={"request-status": "200", - "authorization": "foo-22222", - "kat-req-http-requested-header": "Authorization"}, - expected=200) + yield Query( + self.url("context-extensions/"), + headers={ + "request-status": "200", + "authorization": "foo-22222", + "kat-req-http-requested-header": "Authorization", + }, + expected=200, + ) # [5] - yield Query(self.url("context-extensions-crd/"), headers={"request-status": "200", - "authorization": "foo-33333", - "kat-req-http-requested-header": "Authorization"}, - expected=200) + yield Query( + self.url("context-extensions-crd/"), + headers={ + "request-status": "200", + "authorization": "foo-33333", + "kat-req-http-requested-header": "Authorization", + }, + expected=200, + ) def check(self): # [0] Verifies all request headers sent to the authorization server. @@ -107,48 +150,62 @@ def check(self): assert self.results[0].backend.name == self.auth.path.k8s assert self.results[0].backend.request assert self.results[0].backend.request.url.path == "/target/" - assert self.results[0].backend.request.headers["x-envoy-internal"]== ["true"] - assert self.results[0].backend.request.headers["x-forwarded-proto"]== ["http"] + assert self.results[0].backend.request.headers["x-envoy-internal"] == ["true"] + assert self.results[0].backend.request.headers["x-forwarded-proto"] == ["http"] assert "user-agent" in self.results[0].backend.request.headers assert "baz" in self.results[0].backend.request.headers assert self.results[0].status == 401 assert self.results[0].headers["Server"] == ["envoy"] - assert self.results[0].headers['Kat-Resp-Extauth-Protocol-Version'] == ['v3'] + assert self.results[0].headers["Kat-Resp-Extauth-Protocol-Version"] == ["v3"] # [1] Verifies that Location header is returned from Envoy. assert self.results[1].backend assert self.results[1].backend.name == self.auth.path.k8s assert self.results[1].backend.request - assert self.results[1].backend.request.headers["kat-req-extauth-requested-status"] == ["302"] - assert self.results[1].backend.request.headers["kat-req-extauth-requested-location"] == ["foo"] + assert self.results[1].backend.request.headers["kat-req-extauth-requested-status"] == [ + "302" + ] + assert self.results[1].backend.request.headers["kat-req-extauth-requested-location"] == [ + "foo" + ] assert self.results[1].status == 302 assert self.results[1].headers["Location"] == ["foo"] - assert self.results[1].headers['Kat-Resp-Extauth-Protocol-Version'] == ['v3'] + assert self.results[1].headers["Kat-Resp-Extauth-Protocol-Version"] == ["v3"] # [2] Verifies Envoy returns whitelisted headers input by the user. assert self.results[2].backend assert self.results[2].backend.name == self.auth.path.k8s assert self.results[2].backend.request - assert self.results[2].backend.request.headers["kat-req-extauth-requested-status"] == ["401"] - assert self.results[2].backend.request.headers["kat-req-extauth-requested-header"] == ["x-foo"] + assert self.results[2].backend.request.headers["kat-req-extauth-requested-status"] == [ + "401" + ] + assert self.results[2].backend.request.headers["kat-req-extauth-requested-header"] == [ + "x-foo" + ] assert self.results[2].backend.request.headers["x-foo"] == ["foo"] assert self.results[2].status == 401 assert self.results[2].headers["Server"] == ["envoy"] assert self.results[2].headers["X-Foo"] == ["foo"] - assert self.results[2].headers['Kat-Resp-Extauth-Protocol-Version'] == ['v3'] + assert self.results[2].headers["Kat-Resp-Extauth-Protocol-Version"] == ["v3"] # [3] Verifies default whitelisted Authorization request header. assert self.results[3].backend assert self.results[3].backend.request - assert self.results[3].backend.request.headers["kat-req-extauth-requested-status"] == ["200"] - assert self.results[3].backend.request.headers["kat-req-http-requested-header"] == ["Authorization"] + assert self.results[3].backend.request.headers["kat-req-extauth-requested-status"] == [ + "200" + ] + assert self.results[3].backend.request.headers["kat-req-http-requested-header"] == [ + "Authorization" + ] assert self.results[3].backend.request.headers["authorization"] == ["foo-11111"] assert self.results[3].backend.request.headers["foo"] == ["foo,bar"] assert self.results[3].backend.request.headers["baz"] == ["bar"] assert self.results[3].status == 200 assert self.results[3].headers["Server"] == ["envoy"] assert self.results[3].headers["Authorization"] == ["foo-11111"] - assert self.results[3].backend.request.headers['kat-resp-extauth-protocol-version'] == ['v3'] + assert self.results[3].backend.request.headers["kat-resp-extauth-protocol-version"] == [ + "v3" + ] # [4] Verifies that auth_context_extension is passed along by Envoy. assert self.results[4].status == 200 @@ -156,7 +213,9 @@ def check(self): assert self.results[4].headers["Authorization"] == ["foo-22222"] assert self.results[4].backend assert self.results[4].backend.request - context_ext = json.loads(self.results[4].backend.request.headers["kat-resp-extauth-context-extensions"][0]) + context_ext = json.loads( + self.results[4].backend.request.headers["kat-resp-extauth-context-extensions"][0] + ) assert context_ext["first"] == "first element" assert context_ext["second"] == "second element" @@ -166,7 +225,9 @@ def check(self): assert self.results[5].headers["Authorization"] == ["foo-33333"] assert self.results[5].backend assert self.results[5].backend.request - context_ext = json.loads(self.results[5].backend.request.headers["kat-resp-extauth-context-extensions"][0]) + context_ext = json.loads( + self.results[5].backend.request.headers["kat-resp-extauth-context-extensions"][0] + ) assert context_ext["context"] == "auth-context-name" assert context_ext["data"] == "auth-data" @@ -183,7 +244,8 @@ def init(self): self.auth = HTTP(name="auth") def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -193,10 +255,13 @@ def manifests(self) -> str: metadata: name: auth-partial-secret type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -225,8 +290,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: include_body: max_bytes: 7 allow_partial: true -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -234,17 +301,33 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "200"}, body="message_body", expected=200) + yield Query( + self.url("target/"), + headers={"kat-req-http-requested-status": "200"}, + body="message_body", + expected=200, + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "200"}, body="body", expected=200) + yield Query( + self.url("target/"), + headers={"kat-req-http-requested-status": "200"}, + body="body", + expected=200, + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401"}, body="body", expected=401) + yield Query( + self.url("target/"), + headers={"kat-req-http-requested-status": "401"}, + body="body", + expected=401, + ) def check(self): # [0] Verifies that the authorization server received the partial message body. @@ -274,6 +357,7 @@ def check(self): assert self.results[2].headers["Server"] == ["envoy"] assert extauth_res2["request"]["headers"]["kat-resp-http-request-body"] == ["body"] + class AuthenticationHTTPBufferedTest(AmbassadorTest): target: ServiceType @@ -286,7 +370,8 @@ def init(self): self.auth = HTTP(name="auth") def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -296,10 +381,13 @@ def manifests(self) -> str: metadata: name: auth-buffered-secret type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -337,8 +425,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: include_body: max_bytes: 4096 allow_partial: true -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -346,30 +436,57 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401", - "Baz": "baz", - "Request-Header": "Baz"}, expected=401) + yield Query( + self.url("target/"), + headers={"kat-req-http-requested-status": "401", "Baz": "baz", "Request-Header": "Baz"}, + expected=401, + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "302", - "location": "foo", - "kat-req-http-requested-cookie": "foo, bar, baz", - "kat-req-http-requested-header": "location"}, expected=302) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "302", + "location": "foo", + "kat-req-http-requested-cookie": "foo, bar, baz", + "kat-req-http-requested-header": "location", + }, + expected=302, + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401", - "X-Foo": "foo", - "kat-req-http-requested-header": "X-Foo"}, expected=401) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "401", + "X-Foo": "foo", + "kat-req-http-requested-header": "X-Foo", + }, + expected=401, + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401", - "X-Bar": "bar", - "kat-req-http-requested-header": "X-Bar"}, expected=401) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "401", + "X-Bar": "bar", + "kat-req-http-requested-header": "X-Bar", + }, + expected=401, + ) # [4] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "200", - "Authorization": "foo-11111", - "kat-req-http-requested-header": "Authorization"}, expected=200) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "200", + "Authorization": "foo-11111", + "kat-req-http-requested-header": "Authorization", + }, + expected=200, + ) def check(self): # [0] Verifies all request headers sent to the authorization server. @@ -377,8 +494,8 @@ def check(self): assert self.results[0].backend.name == self.auth.path.k8s assert self.results[0].backend.request assert self.results[0].backend.request.url.path == "/extauth/target/" - assert self.results[0].backend.request.headers["x-forwarded-proto"]== ["http"] - assert self.results[0].backend.request.headers["content-length"]== ["0"] + assert self.results[0].backend.request.headers["x-forwarded-proto"] == ["http"] + assert self.results[0].backend.request.headers["content-length"] == ["0"] assert "x-forwarded-for" in self.results[0].backend.request.headers assert "user-agent" in self.results[0].backend.request.headers assert "baz" not in self.results[0].backend.request.headers @@ -390,7 +507,9 @@ def check(self): assert self.results[1].backend.name == self.auth.path.k8s assert self.results[1].backend.request assert self.results[1].backend.request.headers["kat-req-http-requested-status"] == ["302"] - assert self.results[1].backend.request.headers["kat-req-http-requested-header"] == ["location"] + assert self.results[1].backend.request.headers["kat-req-http-requested-header"] == [ + "location" + ] assert self.results[1].backend.request.headers["location"] == ["foo"] assert self.results[1].status == 302 assert self.results[1].headers["Server"] == ["envoy"] @@ -423,13 +542,18 @@ def check(self): assert self.results[4].backend assert self.results[4].backend.request assert self.results[4].backend.request.headers["kat-req-http-requested-status"] == ["200"] - assert self.results[4].backend.request.headers["kat-req-http-requested-header"] == ["Authorization"] + assert self.results[4].backend.request.headers["kat-req-http-requested-header"] == [ + "Authorization" + ] assert self.results[4].backend.request.headers["authorization"] == ["foo-11111"] - assert self.results[4].backend.request.headers["l5d-dst-override"] == [ 'authenticationhttpbufferedtest-http:80' ] + assert self.results[4].backend.request.headers["l5d-dst-override"] == [ + "authenticationhttpbufferedtest-http:80" + ] assert self.results[4].status == 200 assert self.results[4].headers["Server"] == ["envoy"] assert self.results[4].headers["Authorization"] == ["foo-11111"] + class AuthenticationHTTPFailureModeAllowTest(AmbassadorTest): target: ServiceType auth: ServiceType @@ -441,7 +565,8 @@ def init(self): self.auth = HTTP(name="auth") def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -451,10 +576,13 @@ def manifests(self) -> str: metadata: name: auth-failure-secret type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -475,8 +603,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - Kat-Req-Http-Requested-Header failure_mode_allow: true -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -484,14 +614,19 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "200"}, expected=200) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "200"}, expected=200 + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503) + yield Query( + self.url("target/"), headers={"kat-req-http-requested-status": "503"}, expected=503 + ) def check(self): # [0] Verifies that the authorization server received the partial message body. @@ -509,6 +644,7 @@ def check(self): assert self.results[1].backend.request.headers["kat-req-http-requested-status"] == ["503"] assert self.results[1].headers["Server"] == ["envoy"] + class AuthenticationTestV1(AmbassadorTest): target: ServiceType @@ -523,7 +659,8 @@ def init(self): self.backend_counts = {} def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -571,8 +708,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: status_on_error: code: 503 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -588,43 +727,80 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /target/unauthed/ service: {self.target.path.fqdn} bypass_auth: true -""") +""" + ) def queries(self): # [0] - yield Query(self.url("target/0"), headers={"kat-req-http-requested-status": "401", - "Baz": "baz", - "Request-Header": "Baz"}, expected=401) + yield Query( + self.url("target/0"), + headers={"kat-req-http-requested-status": "401", "Baz": "baz", "Request-Header": "Baz"}, + expected=401, + ) # [1] - yield Query(self.url("target/1"), headers={"kat-req-http-requested-status": "302", - "location": "foo", - "kat-req-http-requested-header": "location"}, expected=302) + yield Query( + self.url("target/1"), + headers={ + "kat-req-http-requested-status": "302", + "location": "foo", + "kat-req-http-requested-header": "location", + }, + expected=302, + ) # [2] - yield Query(self.url("target/2"), headers={"kat-req-http-requested-status": "401", - "X-Foo": "foo", - "kat-req-http-requested-header": "X-Foo"}, expected=401) + yield Query( + self.url("target/2"), + headers={ + "kat-req-http-requested-status": "401", + "X-Foo": "foo", + "kat-req-http-requested-header": "X-Foo", + }, + expected=401, + ) # [3] - yield Query(self.url("target/3"), headers={"kat-req-http-requested-status": "401", - "X-Bar": "bar", - "kat-req-http-requested-header": "X-Bar"}, expected=401) + yield Query( + self.url("target/3"), + headers={ + "kat-req-http-requested-status": "401", + "X-Bar": "bar", + "kat-req-http-requested-header": "X-Bar", + }, + expected=401, + ) # [4] - yield Query(self.url("target/4"), headers={"kat-req-http-requested-status": "200", - "Authorization": "foo-11111", - "kat-req-http-requested-header": "Authorization"}, expected=200) + yield Query( + self.url("target/4"), + headers={ + "kat-req-http-requested-status": "200", + "Authorization": "foo-11111", + "kat-req-http-requested-header": "Authorization", + }, + expected=200, + ) # [5] yield Query(self.url("target/5"), headers={"X-Forwarded-Proto": "https"}, expected=200) # [6] - yield Query(self.url("target/unauthed/6"), headers={"kat-req-http-requested-status": "200"}, expected=200) + yield Query( + self.url("target/unauthed/6"), + headers={"kat-req-http-requested-status": "200"}, + expected=200, + ) # [7] - yield Query(self.url("target/7"), headers={"kat-req-http-requested-status": "500"}, expected=503) + yield Query( + self.url("target/7"), headers={"kat-req-http-requested-status": "500"}, expected=503 + ) # Create some traffic to make it more likely that both auth services get at least one # request for i in range(20): - yield Query(self.url("target/" + str(8 + i)), headers={"kat-req-http-requested-status": "403"}, expected=403) + yield Query( + self.url("target/" + str(8 + i)), + headers={"kat-req-http-requested-status": "403"}, + expected=403, + ) def check_backend_name(self, result) -> bool: backend_name = result.backend.name @@ -641,8 +817,8 @@ def check(self): assert self.results[0].backend assert self.results[0].backend.request assert self.results[0].backend.request.url.path == "/extauth/target/0" - assert self.results[0].backend.request.headers["x-forwarded-proto"]== ["http"] - assert self.results[0].backend.request.headers["content-length"]== ["0"] + assert self.results[0].backend.request.headers["x-forwarded-proto"] == ["http"] + assert self.results[0].backend.request.headers["content-length"] == ["0"] assert "x-forwarded-for" in self.results[0].backend.request.headers assert "user-agent" in self.results[0].backend.request.headers assert "baz" not in self.results[0].backend.request.headers @@ -654,7 +830,9 @@ def check(self): assert self.results[1].backend assert self.results[1].backend.request assert self.results[1].backend.request.headers["kat-req-http-requested-status"] == ["302"] - assert self.results[1].backend.request.headers["kat-req-http-requested-header"] == ["location"] + assert self.results[1].backend.request.headers["kat-req-http-requested-header"] == [ + "location" + ] assert self.results[1].backend.request.headers["location"] == ["foo"] assert self.results[1].status == 302 assert self.results[1].headers["Server"] == ["envoy"] @@ -684,17 +862,21 @@ def check(self): # [4] Verifies default whitelisted Authorization request header. assert self.results[4].backend - assert self.results[4].backend.name == self.target.path.k8s # this response is from an auth success + assert ( + self.results[4].backend.name == self.target.path.k8s + ) # this response is from an auth success assert self.results[4].backend.request assert self.results[4].backend.request.headers["kat-req-http-requested-status"] == ["200"] - assert self.results[4].backend.request.headers["kat-req-http-requested-header"] == ["Authorization"] + assert self.results[4].backend.request.headers["kat-req-http-requested-header"] == [ + "Authorization" + ] assert self.results[4].backend.request.headers["authorization"] == ["foo-11111"] assert self.results[4].status == 200 assert self.results[4].headers["Server"] == ["envoy"] assert self.results[4].headers["Authorization"] == ["foo-11111"] extauth_req = json.loads(self.results[4].backend.request.headers["extauth"][0]) - assert extauth_req["request"]["headers"]["l5d-dst-override"] == [ 'extauth:80' ] + assert extauth_req["request"]["headers"]["l5d-dst-override"] == ["extauth:80"] # [5] Verify that X-Forwarded-Proto makes it to the auth service. # @@ -703,7 +885,7 @@ def check(self): r5 = self.results[5] assert r5 assert r5.backend - assert r5.backend.name == self.target.path.k8s # this response is from an auth success + assert r5.backend.name == self.target.path.k8s # this response is from an auth success assert r5.status == 200 assert r5.headers["Server"] == ["envoy"] @@ -715,8 +897,12 @@ def check(self): # [6] Verifies that Envoy bypasses external auth when disabled for a mapping. assert self.results[6].backend - assert self.results[6].backend.name == self.target.path.k8s # ensure the request made it to the backend - assert not self.check_backend_name(self.results[6]) # ensure the request did not go to the auth service + assert ( + self.results[6].backend.name == self.target.path.k8s + ) # ensure the request made it to the backend + assert not self.check_backend_name( + self.results[6] + ) # ensure the request did not go to the auth service assert self.results[6].backend.request assert self.results[6].backend.request.headers["kat-req-http-requested-status"] == ["200"] assert self.results[6].status == 200 @@ -727,7 +913,7 @@ def check(self): if eainfo: # Envoy should force this to HTTP, not HTTPS. - assert eainfo['request']['headers']['x-forwarded-proto'] == [ 'http' ] + assert eainfo["request"]["headers"]["x-forwarded-proto"] == ["http"] except ValueError as e: assert False, "could not parse Extauth header '%s': %s" % (eahdr, e) @@ -738,10 +924,10 @@ def check(self): # are overridden, e.g. Authorization. for i in range(20): - assert self.check_backend_name(self.results[8+i]) + assert self.check_backend_name(self.results[8 + i]) - print ("auth1 service got %d requests" % self.backend_counts.get(self.auth1.path.k8s, -1)) - print ("auth2 service got %d requests" % self.backend_counts.get(self.auth2.path.k8s, -1)) + print("auth1 service got %d requests" % self.backend_counts.get(self.auth1.path.k8s, -1)) + print("auth2 service got %d requests" % self.backend_counts.get(self.auth2.path.k8s, -1)) assert self.backend_counts.get(self.auth1.path.k8s, 0) > 0, "auth1 got no requests" assert self.backend_counts.get(self.auth2.path.k8s, 0) > 0, "auth2 got no requests" @@ -757,7 +943,8 @@ def init(self): self.auth = AHTTP(name="auth") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -777,8 +964,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - X-Bar - Extauth -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -786,39 +975,68 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401", - "Baz": "baz", - "Request-Header": "Baz"}, expected=401) + yield Query( + self.url("target/"), + headers={"kat-req-http-requested-status": "401", "Baz": "baz", "Request-Header": "Baz"}, + expected=401, + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "302", - "kat-req-http-requested-location": "foo", - "kat-req-http-requested-header": "location"}, expected=302) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "302", + "kat-req-http-requested-location": "foo", + "kat-req-http-requested-header": "location", + }, + expected=302, + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401", - "X-Foo": "foo", - "kat-req-http-requested-header": "X-Foo"}, expected=401) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "401", + "X-Foo": "foo", + "kat-req-http-requested-header": "X-Foo", + }, + expected=401, + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "401", - "X-Bar": "bar", - "kat-req-http-requested-header": "X-Bar"}, expected=401) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "401", + "X-Bar": "bar", + "kat-req-http-requested-header": "X-Bar", + }, + expected=401, + ) # [4] - yield Query(self.url("target/"), headers={"kat-req-http-requested-status": "200", - "Authorization": "foo-11111", - "kat-req-http-requested-header": "Authorization"}, expected=200) + yield Query( + self.url("target/"), + headers={ + "kat-req-http-requested-status": "200", + "Authorization": "foo-11111", + "kat-req-http-requested-header": "Authorization", + }, + expected=200, + ) # [5] yield Query(self.url("target/"), headers={"X-Forwarded-Proto": "https"}, expected=200) def check(self): # [0] Verifies all request headers sent to the authorization server. assert self.results[0].backend - assert self.results[0].backend.name == self.auth.path.k8s, f'wanted backend {self.auth.path.k8s}, got {self.results[0].backend.name}' + assert ( + self.results[0].backend.name == self.auth.path.k8s + ), f"wanted backend {self.auth.path.k8s}, got {self.results[0].backend.name}" assert self.results[0].backend.request assert self.results[0].backend.request.url.path == "/extauth/target/" - assert self.results[0].backend.request.headers["content-length"]== ["0"] + assert self.results[0].backend.request.headers["content-length"] == ["0"] assert "x-forwarded-for" in self.results[0].backend.request.headers assert "user-agent" in self.results[0].backend.request.headers assert "baz" not in self.results[0].backend.request.headers @@ -830,7 +1048,9 @@ def check(self): assert self.results[1].backend.name == self.auth.path.k8s assert self.results[1].backend.request assert self.results[1].backend.request.headers["kat-req-http-requested-status"] == ["302"] - assert self.results[1].backend.request.headers["kat-req-http-requested-header"] == ["location"] + assert self.results[1].backend.request.headers["kat-req-http-requested-header"] == [ + "location" + ] assert self.results[1].backend.request.headers["kat-req-http-requested-location"] == ["foo"] assert self.results[1].status == 302 assert self.results[1].headers["Server"] == ["envoy"] @@ -862,7 +1082,9 @@ def check(self): assert self.results[4].backend assert self.results[4].backend.request assert self.results[4].backend.request.headers["kat-req-http-requested-status"] == ["200"] - assert self.results[4].backend.request.headers["kat-req-http-requested-header"] == ["Authorization"] + assert self.results[4].backend.request.headers["kat-req-http-requested-header"] == [ + "Authorization" + ] assert self.results[4].backend.request.headers["authorization"] == ["foo-11111"] assert self.results[4].status == 200 assert self.results[4].headers["Server"] == ["envoy"] @@ -889,13 +1111,14 @@ def check(self): if eainfo: # Envoy should force this to HTTP, not HTTPS. - assert eainfo['request']['headers']['x-forwarded-proto'] == [ 'http' ] + assert eainfo["request"]["headers"]["x-forwarded-proto"] == ["http"] except ValueError as e: assert False, "could not parse Extauth header '%s': %s" % (eahdr, e) # TODO(gsagula): Write tests for all UCs which request header headers # are overridden, e.g. Authorization. + class AuthenticationWebsocketTest(AmbassadorTest): auth: ServiceType @@ -906,7 +1129,8 @@ def init(self): self.auth = HTTP(name="auth") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -925,8 +1149,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: websocket-echo-server.default use_websocket: true -""") - +""" + ) def queries(self): yield Query(self.url(self.name + "/"), expected=404) @@ -940,23 +1164,33 @@ def check(self): class AuthenticationGRPCVerTest(AmbassadorTest): target: ServiceType - specified_protocol_version: Literal['v2', 'v3', 'default'] - expected_protocol_version: Literal['v3', 'invalid'] + specified_protocol_version: Literal["v2", "v3", "default"] + expected_protocol_version: Literal["v3", "invalid"] auth: ServiceType @classmethod def variants(cls) -> Generator[Node, None, None]: - for protocol_version in ['v2', 'v3', 'default']: + for protocol_version in ["v2", "v3", "default"]: yield cls(protocol_version, name="{self.specified_protocol_version}") - def init(self, protocol_version: Literal['v2', 'v3', 'default']): + def init(self, protocol_version: Literal["v2", "v3", "default"]): self.target = HTTP() self.specified_protocol_version = protocol_version - self.expected_protocol_version = cast(Literal['v3', 'invalid'], protocol_version if protocol_version in ['v3'] else 'invalid') - self.auth = AGRPC(name="auth", protocol_version=(self.expected_protocol_version if self.expected_protocol_version != 'invalid' else 'v3')) + self.expected_protocol_version = cast( + Literal["v3", "invalid"], protocol_version if protocol_version in ["v3"] else "invalid" + ) + self.auth = AGRPC( + name="auth", + protocol_version=( + self.expected_protocol_version + if self.expected_protocol_version != "invalid" + else "v3" + ), + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -964,9 +1198,15 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: auth_service: "{self.auth.path.fqdn}" timeout_ms: 5000 proto: grpc -""") + ("" if self.specified_protocol_version == "default" else f"protocol_version: '{self.specified_protocol_version}'") - - yield self, self.format(""" +""" + ) + ( + "" + if self.specified_protocol_version == "default" + else f"protocol_version: '{self.specified_protocol_version}'" + ) + + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -974,43 +1214,72 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # TODO add more # [0] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "401", - "baz": "baz", - "kat-req-extauth-request-header": "baz"}, - expected=(500 if self.expected_protocol_version == 'invalid' else 401)) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "401", + "baz": "baz", + "kat-req-extauth-request-header": "baz", + }, + expected=(500 if self.expected_protocol_version == "invalid" else 401), + ) # [1] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "302", - "kat-req-extauth-requested-location": "foo"}, - expected=(500 if self.expected_protocol_version == 'invalid' else 302)) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "302", + "kat-req-extauth-requested-location": "foo", + }, + expected=(500 if self.expected_protocol_version == "invalid" else 302), + ) # [2] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "401", - "x-foo": "foo", - "kat-req-extauth-requested-header": "x-foo"}, - expected=(500 if self.expected_protocol_version == 'invalid' else 401)) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "401", + "x-foo": "foo", + "kat-req-extauth-requested-header": "x-foo", + }, + expected=(500 if self.expected_protocol_version == "invalid" else 401), + ) # [3] - yield Query(self.url("target/"), headers={"kat-req-extauth-requested-status": "200", - "authorization": "foo-11111", - "foo" : "foo", - "kat-req-extauth-append": "foo=bar;baz=bar", - "kat-req-http-requested-header": "Authorization"}, - expected=(500 if self.expected_protocol_version == 'invalid' else 200)) + yield Query( + self.url("target/"), + headers={ + "kat-req-extauth-requested-status": "200", + "authorization": "foo-11111", + "foo": "foo", + "kat-req-extauth-append": "foo=bar;baz=bar", + "kat-req-http-requested-header": "Authorization", + }, + expected=(500 if self.expected_protocol_version == "invalid" else 200), + ) def check(self): - if self.expected_protocol_version == 'invalid': + if self.expected_protocol_version == "invalid": for i, result in enumerate(self.results): # Verify the basic structure of the HTTP 500's JSON body. assert result.json, f"self.results[{i}] does not have a JSON body" - assert result.json['status_code'] == 500, f"self.results[{i}] JSON body={repr(result.json)} does not have status_code=500" - assert result.json['request_id'], f"self.results[{i}] JSON body={repr(result.json)} does not have request_id" - assert self.path.k8s in result.json['message'], f"self.results[{i}] JSON body={repr(result.json)} does not have thing-containing-the-annotation-containing-the-AuthService name {repr(self.path.k8s)} in message" - assert 'AuthService' in result.json['message'], f"self.results[{i}] JSON body={repr(result.json)} does not have type 'AuthService' in message" + assert ( + result.json["status_code"] == 500 + ), f"self.results[{i}] JSON body={repr(result.json)} does not have status_code=500" + assert result.json[ + "request_id" + ], f"self.results[{i}] JSON body={repr(result.json)} does not have request_id" + assert ( + self.path.k8s in result.json["message"] + ), f"self.results[{i}] JSON body={repr(result.json)} does not have thing-containing-the-annotation-containing-the-AuthService name {repr(self.path.k8s)} in message" + assert ( + "AuthService" in result.json["message"] + ), f"self.results[{i}] JSON body={repr(result.json)} does not have type 'AuthService' in message" return # [0] Verifies all request headers sent to the authorization server. @@ -1018,44 +1287,64 @@ def check(self): assert self.results[0].backend.name == self.auth.path.k8s assert self.results[0].backend.request assert self.results[0].backend.request.url.path == "/target/" - assert self.results[0].backend.request.headers["x-forwarded-proto"]== ["http"] + assert self.results[0].backend.request.headers["x-forwarded-proto"] == ["http"] assert "user-agent" in self.results[0].backend.request.headers assert "baz" in self.results[0].backend.request.headers assert self.results[0].status == 401 assert self.results[0].headers["Server"] == ["envoy"] - assert self.results[0].headers['Kat-Resp-Extauth-Protocol-Version'] == [self.expected_protocol_version] + assert self.results[0].headers["Kat-Resp-Extauth-Protocol-Version"] == [ + self.expected_protocol_version + ] # [1] Verifies that Location header is returned from Envoy. assert self.results[1].backend assert self.results[1].backend.name == self.auth.path.k8s assert self.results[1].backend.request - assert self.results[1].backend.request.headers["kat-req-extauth-requested-status"] == ["302"] - assert self.results[1].backend.request.headers["kat-req-extauth-requested-location"] == ["foo"] + assert self.results[1].backend.request.headers["kat-req-extauth-requested-status"] == [ + "302" + ] + assert self.results[1].backend.request.headers["kat-req-extauth-requested-location"] == [ + "foo" + ] assert self.results[1].status == 302 assert self.results[1].headers["Location"] == ["foo"] - assert self.results[1].headers['Kat-Resp-Extauth-Protocol-Version'] == [self.expected_protocol_version] + assert self.results[1].headers["Kat-Resp-Extauth-Protocol-Version"] == [ + self.expected_protocol_version + ] # [2] Verifies Envoy returns whitelisted headers input by the user. assert self.results[2].backend assert self.results[2].backend.name == self.auth.path.k8s assert self.results[2].backend.request - assert self.results[2].backend.request.headers["kat-req-extauth-requested-status"] == ["401"] - assert self.results[2].backend.request.headers["kat-req-extauth-requested-header"] == ["x-foo"] + assert self.results[2].backend.request.headers["kat-req-extauth-requested-status"] == [ + "401" + ] + assert self.results[2].backend.request.headers["kat-req-extauth-requested-header"] == [ + "x-foo" + ] assert self.results[2].backend.request.headers["x-foo"] == ["foo"] assert self.results[2].status == 401 assert self.results[2].headers["Server"] == ["envoy"] assert self.results[2].headers["X-Foo"] == ["foo"] - assert self.results[2].headers['Kat-Resp-Extauth-Protocol-Version'] == [self.expected_protocol_version] + assert self.results[2].headers["Kat-Resp-Extauth-Protocol-Version"] == [ + self.expected_protocol_version + ] # [3] Verifies default whitelisted Authorization request header. assert self.results[3].backend assert self.results[3].backend.request - assert self.results[3].backend.request.headers["kat-req-extauth-requested-status"] == ["200"] - assert self.results[3].backend.request.headers["kat-req-http-requested-header"] == ["Authorization"] + assert self.results[3].backend.request.headers["kat-req-extauth-requested-status"] == [ + "200" + ] + assert self.results[3].backend.request.headers["kat-req-http-requested-header"] == [ + "Authorization" + ] assert self.results[3].backend.request.headers["authorization"] == ["foo-11111"] assert self.results[3].backend.request.headers["foo"] == ["foo,bar"] assert self.results[3].backend.request.headers["baz"] == ["bar"] assert self.results[3].status == 200 assert self.results[3].headers["Server"] == ["envoy"] assert self.results[3].headers["Authorization"] == ["foo-11111"] - assert self.results[3].backend.request.headers['kat-resp-extauth-protocol-version'] == [self.expected_protocol_version] + assert self.results[3].backend.request.headers["kat-resp-extauth-protocol-version"] == [ + self.expected_protocol_version + ] diff --git a/python/tests/kat/t_grpc.py b/python/tests/kat/t_grpc.py index 0a79c95cda..a5c93bab94 100644 --- a/python/tests/kat/t_grpc.py +++ b/python/tests/kat/t_grpc.py @@ -4,6 +4,7 @@ from abstract_tests import AmbassadorTest, ServiceType, EGRPC, Node + class AcceptanceGrpcTest(AmbassadorTest): target: ServiceType @@ -11,14 +12,15 @@ def init(self): self.target = EGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: -# yield self, self.format(""" -# --- -# apiVersion: getambassador.io/v3alpha1 -# kind: Module -# name: ambassador -# # """) - - yield self, self.format(""" + # yield self, self.format(""" + # --- + # apiVersion: getambassador.io/v3alpha1 + # kind: Module + # name: ambassador + # # """) + + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -28,35 +30,44 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: "" # This means to leave the prefix unaltered. name: {self.target.path.k8s} service: {self.target.path.k8s} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "0" }, - expected=200, - grpc_type="real") + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + expected=200, + grpc_type="real", + ) # [1] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "7" }, - expected=200, - grpc_type="real") + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "7"}, + expected=200, + grpc_type="real", + ) # [2] -- PHASE 2 yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), phase=2) def check(self): # [0] - assert self.results[0].headers["Grpc-Status"] == ["0"], f'0 expected ["0"], got {self.results[0].headers["Grpc-Status"]}' + assert self.results[0].headers["Grpc-Status"] == [ + "0" + ], f'0 expected ["0"], got {self.results[0].headers["Grpc-Status"]}' # [1] - assert self.results[1].headers["Grpc-Status"] == ["7"], f'0 expected ["0"], got {self.results[0].headers["Grpc-Status"]}' + assert self.results[1].headers["Grpc-Status"] == [ + "7" + ], f'0 expected ["0"], got {self.results[0].headers["Grpc-Status"]}' # [2] # XXX Ew. If self.results[2].json is empty, the harness won't convert it to a response. errors = self.results[2].json - assert(len(errors) == 0) + assert len(errors) == 0 class EndpointGrpcTest(AmbassadorTest): @@ -66,7 +77,9 @@ def init(self): self.target = EGRPC() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: KubernetesEndpointResolver @@ -89,32 +102,43 @@ def manifests(self) -> str: resolver: my-endpoint load_balancer: policy: round_robin -''') + super().manifests() +""" + ) + + super().manifests() + ) def queries(self): # [0] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "0" }, - expected=200, - grpc_type="real") + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + expected=200, + grpc_type="real", + ) # [1] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "7" }, - expected=200, - grpc_type="real") + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "7"}, + expected=200, + grpc_type="real", + ) # [2] -- PHASE 2 yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), phase=2) def check(self): # [0] - assert self.results[0].headers["Grpc-Status"] == ["0"], f'results[0]: expected ["0"], got {self.results[0].headers["Grpc-Status"]}' + assert self.results[0].headers["Grpc-Status"] == [ + "0" + ], f'results[0]: expected ["0"], got {self.results[0].headers["Grpc-Status"]}' # [1] - assert self.results[1].headers["Grpc-Status"] == ["7"], f'results[1]: expected ["7"], got {self.results[0].headers["Grpc-Status"]}' + assert self.results[1].headers["Grpc-Status"] == [ + "7" + ], f'results[1]: expected ["7"], got {self.results[0].headers["Grpc-Status"]}' # [2] # XXX Ew. If self.results[2].json is empty, the harness won't convert it to a response. errors = self.results[2].json - assert(len(errors) == 0) + assert len(errors) == 0 diff --git a/python/tests/kat/t_grpc_bridge.py b/python/tests/kat/t_grpc_bridge.py index 901d7185b8..d85f39874e 100644 --- a/python/tests/kat/t_grpc_bridge.py +++ b/python/tests/kat/t_grpc_bridge.py @@ -4,6 +4,7 @@ from abstract_tests import AmbassadorTest, ServiceType, EGRPC, Node + class AcceptanceGrpcBridgeTest(AmbassadorTest): target: ServiceType @@ -12,16 +13,19 @@ def init(self): self.target = EGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module name: ambassador config: enable_grpc_http11_bridge: True -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -31,20 +35,25 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /echo.EchoService/ name: {self.target.path.k8s} service: {self.target.path.k8s} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "0" }, - expected=200, - grpc_type="bridge") + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + expected=200, + grpc_type="bridge", + ) # [1] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "7" }, - expected=503, - grpc_type="bridge") + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "7"}, + expected=503, + grpc_type="bridge", + ) def check(self): # [0] diff --git a/python/tests/kat/t_grpc_stats.py b/python/tests/kat/t_grpc_stats.py index de2d93d3ae..04a29049fc 100644 --- a/python/tests/kat/t_grpc_stats.py +++ b/python/tests/kat/t_grpc_stats.py @@ -4,12 +4,14 @@ from abstract_tests import AmbassadorTest, EGRPC, Node + class AcceptanceGrpcStatsTest(AmbassadorTest): def init(self): self.target = EGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -18,9 +20,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: grpc_stats: all_methods: true upstream_stats: true -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -30,9 +34,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /echo.EchoService/ name: {self.target.path.k8s} service: {self.target.path.k8s} -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ apiVersion: getambassador.io/v3alpha1 kind: Mapping name: metrics @@ -40,63 +46,65 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /metrics rewrite: /metrics service: http://127.0.0.1:8877 -""") - +""" + ) def queries(self): # [0] for i in range(10): - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "0" }, - grpc_type="real", - phase=1) + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + grpc_type="real", + phase=1, + ) for i in range(10): - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "13" }, - grpc_type="real", - phase=1) + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "13"}, + grpc_type="real", + phase=1, + ) # [1] yield Query(self.url("metrics"), phase=2) - def check(self): # [0] stats = self.results[-1].text metrics = [ - 'envoy_cluster_grpc_Echo_0', - 'envoy_cluster_grpc_Echo_13', - 'envoy_cluster_grpc_Echo_request_message_count', - 'envoy_cluster_grpc_Echo_response_message_count', - 'envoy_cluster_grpc_Echo_success', - 'envoy_cluster_grpc_Echo_total', + "envoy_cluster_grpc_Echo_0", + "envoy_cluster_grpc_Echo_13", + "envoy_cluster_grpc_Echo_request_message_count", + "envoy_cluster_grpc_Echo_response_message_count", + "envoy_cluster_grpc_Echo_success", + "envoy_cluster_grpc_Echo_total", # present only when enable_upstream_stats is true - 'envoy_cluster_grpc_Echo_upstream_rq_time' + "envoy_cluster_grpc_Echo_upstream_rq_time", ] # these metrics SHOULD NOT be there based on the filter config absent_metrics = [ # since all_methods is true, we should not see the generic metrics - 'envoy_cluster_grpc_0' - 'envoy_cluster_grpc_13', - 'envoy_cluster_grpc_request_message_count', - 'envoy_cluster_grpc_response_message_count', - 'envoy_cluster_grpc_success', - 'envoy_cluster_grpc_total' + "envoy_cluster_grpc_0" "envoy_cluster_grpc_13", + "envoy_cluster_grpc_request_message_count", + "envoy_cluster_grpc_response_message_count", + "envoy_cluster_grpc_success", + "envoy_cluster_grpc_total", ] # check if the metrics are there for metric in metrics: - assert metric in stats, f'coult not find metric: {metric}' + assert metric in stats, f"coult not find metric: {metric}" for absent_metric in absent_metrics: - assert absent_metric not in stats, f'metric {metric} should not be present' + assert absent_metric not in stats, f"metric {metric} should not be present" # check if the metrics are there for metric in metrics: - assert metric in stats, f'coult not find metric: {metric}' + assert metric in stats, f"coult not find metric: {metric}" class GrpcStatsTestOnlySelectedServices(AmbassadorTest): @@ -104,7 +112,8 @@ def init(self): self.target = EGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -116,9 +125,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: services: - name: echo.EchoService method_names: [Echo] -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -128,9 +139,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /echo.EchoService/ name: {self.target.path.k8s} service: {self.target.path.k8s} -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ apiVersion: getambassador.io/v3alpha1 kind: Mapping name: metrics @@ -138,66 +151,70 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /metrics rewrite: /metrics service: http://127.0.0.1:8877 -""") - +""" + ) def queries(self): # [0] for i in range(10): - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "0" }, - grpc_type="real", - phase=1) + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + grpc_type="real", + phase=1, + ) for i in range(10): - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "13" }, - grpc_type="real", - phase=1) + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "13"}, + grpc_type="real", + phase=1, + ) # [1] yield Query(self.url("metrics"), phase=2) - def check(self): # [0] stats = self.results[-1].text metrics = [ - 'envoy_cluster_grpc_Echo_0', - 'envoy_cluster_grpc_Echo_13', - 'envoy_cluster_grpc_Echo_request_message_count', - 'envoy_cluster_grpc_Echo_response_message_count', - 'envoy_cluster_grpc_Echo_success', - 'envoy_cluster_grpc_Echo_total', + "envoy_cluster_grpc_Echo_0", + "envoy_cluster_grpc_Echo_13", + "envoy_cluster_grpc_Echo_request_message_count", + "envoy_cluster_grpc_Echo_response_message_count", + "envoy_cluster_grpc_Echo_success", + "envoy_cluster_grpc_Echo_total", ] # these metrics SHOULD NOT be there based on the filter config absent_metrics = [ # upstream stats is disabled - 'envoy_cluster_grpc_upstream_rq_time', + "envoy_cluster_grpc_upstream_rq_time", # the generic metrics shouldn't be present since all the methods being called are on the allowed list - 'envoy_cluster_grpc_0' - 'envoy_cluster_grpc_13', - 'envoy_cluster_grpc_request_message_count', - 'envoy_cluster_grpc_response_message_count', - 'envoy_cluster_grpc_success', - 'envoy_cluster_grpc_total' + "envoy_cluster_grpc_0" "envoy_cluster_grpc_13", + "envoy_cluster_grpc_request_message_count", + "envoy_cluster_grpc_response_message_count", + "envoy_cluster_grpc_success", + "envoy_cluster_grpc_total", ] # check if the metrics are there for metric in metrics: - assert metric in stats, f'coult not find metric: {metric}' + assert metric in stats, f"coult not find metric: {metric}" for absent_metric in absent_metrics: - assert absent_metric not in stats, f'metric {metric} should not be present' + assert absent_metric not in stats, f"metric {metric} should not be present" + class GrpcStatsTestNoUpstreamAllMethodsFalseInvalidKeys(AmbassadorTest): def init(self): self.target = EGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -207,9 +224,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: all_methods: false upstream_stats: false i_will_not_break_envoy: true -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -219,9 +238,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /echo.EchoService/ name: {self.target.path.k8s} service: {self.target.path.k8s} -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ apiVersion: getambassador.io/v3alpha1 kind: Mapping name: metrics @@ -229,50 +250,50 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /metrics rewrite: /metrics service: http://127.0.0.1:8877 -""") - +""" + ) def queries(self): # [0] for i in range(10): - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "0" }, - grpc_type="real", - phase=1) + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "0"}, + grpc_type="real", + phase=1, + ) for i in range(10): - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc", "kat-req-echo-requested-status": "13" }, - grpc_type="real", - phase=1) + yield Query( + self.url("echo.EchoService/Echo"), + headers={"content-type": "application/grpc", "kat-req-echo-requested-status": "13"}, + grpc_type="real", + phase=1, + ) # [1] yield Query(self.url("metrics"), phase=2) - def check(self): # [0] stats = self.results[-1].text # stat_all_methods is disabled and the list of services is empty, so we should only see generic metrics metrics = [ - 'envoy_cluster_grpc_0', - 'envoy_cluster_grpc_13', - 'envoy_cluster_grpc_request_message_count', - 'envoy_cluster_grpc_response_message_count', - 'envoy_cluster_grpc_success', - 'envoy_cluster_grpc_total', + "envoy_cluster_grpc_0", + "envoy_cluster_grpc_13", + "envoy_cluster_grpc_request_message_count", + "envoy_cluster_grpc_response_message_count", + "envoy_cluster_grpc_success", + "envoy_cluster_grpc_total", ] # these metrics SHOULD NOT be there based on the filter config - absent_metrics = [ - 'envoy_cluster_grpc_upstream_rq_time', - 'envoy_cluster_grpc_EchoService_0' - ] + absent_metrics = ["envoy_cluster_grpc_upstream_rq_time", "envoy_cluster_grpc_EchoService_0"] # check if the metrics are there for metric in metrics: - assert metric in stats, f'coult not find metric: {metric}' + assert metric in stats, f"coult not find metric: {metric}" for absent_metric in absent_metrics: - assert absent_metric not in stats, f'metric {metric} should not be present' + assert absent_metric not in stats, f"metric {metric} should not be present" diff --git a/python/tests/kat/t_grpc_web.py b/python/tests/kat/t_grpc_web.py index 2dcc2e737a..1fac9c0ea3 100644 --- a/python/tests/kat/t_grpc_web.py +++ b/python/tests/kat/t_grpc_web.py @@ -6,6 +6,7 @@ from abstract_tests import AmbassadorTest, ServiceType, EGRPC, Node + class AcceptanceGrpcWebTest(AmbassadorTest): target: ServiceType @@ -14,16 +15,19 @@ def init(self): self.target = EGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module name: ambassador config: enable_grpc_web: True -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -33,24 +37,33 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /echo.EchoService/ name: {self.target.path.k8s} service: {self.target.path.k8s} -""") +""" + ) def queries(self): # [0] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc-web-text", - "accept": "application/grpc-web-text", - "kat-req-echo-requested-status": "0" }, - expected=200, - grpc_type="web") + yield Query( + self.url("echo.EchoService/Echo"), + headers={ + "content-type": "application/grpc-web-text", + "accept": "application/grpc-web-text", + "kat-req-echo-requested-status": "0", + }, + expected=200, + grpc_type="web", + ) # [1] - yield Query(self.url("echo.EchoService/Echo"), - headers={ "content-type": "application/grpc-web-text", - "accept": "application/grpc-web-text", - "kat-req-echo-requested-status": "7" }, - expected=200, - grpc_type="web") + yield Query( + self.url("echo.EchoService/Echo"), + headers={ + "content-type": "application/grpc-web-text", + "accept": "application/grpc-web-text", + "kat-req-echo-requested-status": "7", + }, + expected=200, + grpc_type="web", + ) def check(self): # print('AcceptanceGrpcWebTest results:') @@ -67,7 +80,7 @@ def check(self): gstat = self.results[0].headers.get("Grpc-Status", "-none-") # print(f' grpc-status {gstat}') - if gstat == [ '0' ]: + if gstat == ["0"]: assert True else: assert False, f'0: got {gstat} instead of ["0"]' diff --git a/python/tests/kat/t_gzip.py b/python/tests/kat/t_gzip.py index d6dacadb9d..72f7b6294b 100644 --- a/python/tests/kat/t_gzip.py +++ b/python/tests/kat/t_gzip.py @@ -12,7 +12,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -20,8 +21,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: config: gzip: enabled: true -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -29,13 +32,14 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/"), headers={"Accept-Encoding": "gzip"}, expected=200) def check(self): - assert self.results[0].headers["Content-Encoding"] == [ "gzip" ] + assert self.results[0].headers["Content-Encoding"] == ["gzip"] class GzipTest(AmbassadorTest): @@ -45,7 +49,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -56,8 +61,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: window_bits: 15 content_type: - text/plain -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -65,13 +72,14 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/"), headers={"Accept-Encoding": "gzip"}, expected=200) def check(self): - assert self.results[0].headers["Content-Encoding"] == [ "gzip" ] + assert self.results[0].headers["Content-Encoding"] == ["gzip"] class GzipNotSupportedContentTypeTest(AmbassadorTest): @@ -81,7 +89,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -91,8 +100,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: min_content_length: 32 content_type: - application/json -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -100,7 +111,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/"), headers={"Accept-Encoding": "gzip"}, expected=200) diff --git a/python/tests/kat/t_headerrouting.py b/python/tests/kat/t_headerrouting.py index aeca2e7a55..8ca5decdb7 100644 --- a/python/tests/kat/t_headerrouting.py +++ b/python/tests/kat/t_headerrouting.py @@ -24,7 +24,8 @@ def init(self, target: ServiceType, target2: ServiceType): # type: ignore self.target2 = target2 def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -32,8 +33,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}/ service: http://{self.target.path.fqdn} -""") - yield self.target2, self.format(""" +""" + ) + yield self.target2, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -43,7 +46,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: http://{self.target2.path.fqdn} headers: X-Route: target2 -""") +""" + ) def queries(self): yield Query(self.parent.url(self.name + "/")) @@ -51,16 +55,23 @@ def queries(self): def check(self): assert self.results[0].backend - assert self.results[0].backend.name == self.target.path.k8s, f"r0 wanted {self.target.path.k8s} got {self.results[0].backend.name}" + assert ( + self.results[0].backend.name == self.target.path.k8s + ), f"r0 wanted {self.target.path.k8s} got {self.results[0].backend.name}" assert self.results[1].backend - assert self.results[1].backend.name == self.target2.path.k8s, f"r1 wanted {self.target2.path.k8s} got {self.results[1].backend.name}" + assert ( + self.results[1].backend.name == self.target2.path.k8s + ), f"r1 wanted {self.target2.path.k8s} got {self.results[1].backend.name}" + class HeaderRoutingAuth(ServiceType): skip_variant: ClassVar[bool] = True def __init__(self, *args, **kwargs) -> None: # Do this unconditionally, since that's part of the point of this class. - kwargs["service_manifests"] = """ + kwargs[ + "service_manifests" + ] = """ --- kind: Service apiVersion: v1 @@ -101,6 +112,7 @@ def __init__(self, *args, **kwargs) -> None: def requirements(self): yield ("url", Query("http://%s/ambassador/check/" % self.path.fqdn)) + class AuthenticationHeaderRouting(AmbassadorTest): target1: ServiceType target2: ServiceType @@ -120,7 +132,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # prefix ENDS WITH /nohdr/ -> 200, no X-Auth-Route -> we should hit target1 # anything else -> 403 -> we should see the 403 - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -133,8 +146,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: allowed_authorization_headers: - X-Auth-Route - Extauth -""") - yield self.target1, self.format(""" +""" + ) + yield self.target1, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -142,8 +157,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target1.path.fqdn} -""") - yield self.target2, self.format(""" +""" + ) + yield self.target2, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -153,7 +170,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: http://{self.target2.path.fqdn} headers: X-Auth-Route: Route -""") +""" + ) def queries(self): # [0] @@ -171,16 +189,24 @@ def queries(self): def check(self): # [0] should be a 403 from auth assert self.results[0].backend - assert self.results[0].backend.name == self.auth.path.k8s, f"r0 wanted {self.auth.path.k8s} got {self.results[0].backend.name}" + assert ( + self.results[0].backend.name == self.auth.path.k8s + ), f"r0 wanted {self.auth.path.k8s} got {self.results[0].backend.name}" # [1] should go to target2 assert self.results[1].backend - assert self.results[1].backend.name == self.target2.path.k8s, f"r1 wanted {self.target2.path.k8s} got {self.results[1].backend.name}" + assert ( + self.results[1].backend.name == self.target2.path.k8s + ), f"r1 wanted {self.target2.path.k8s} got {self.results[1].backend.name}" # [2] should go to target1 assert self.results[2].backend - assert self.results[2].backend.name == self.target1.path.k8s, f"r2 wanted {self.target1.path.k8s} got {self.results[2].backend.name}" + assert ( + self.results[2].backend.name == self.target1.path.k8s + ), f"r2 wanted {self.target1.path.k8s} got {self.results[2].backend.name}" # [3] should be a 403 from auth assert self.results[3].backend - assert self.results[3].backend.name == self.auth.path.k8s, f"r3 wanted {self.auth.path.k8s} got {self.results[3].backend.name}" + assert ( + self.results[3].backend.name == self.auth.path.k8s + ), f"r3 wanted {self.auth.path.k8s} got {self.results[3].backend.name}" diff --git a/python/tests/kat/t_headerswithunderscoresaction.py b/python/tests/kat/t_headerswithunderscoresaction.py index 70cdef1154..c45d2b1606 100644 --- a/python/tests/kat/t_headerswithunderscoresaction.py +++ b/python/tests/kat/t_headerswithunderscoresaction.py @@ -3,6 +3,7 @@ from kat.harness import Query from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node + class AllowHeadersWithUnderscoresTest(AmbassadorTest): target: ServiceType @@ -10,7 +11,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -19,14 +21,16 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): - yield Query(self.url("target/"), expected=200, headers={'t_underscore':'foo'}) + yield Query(self.url("target/"), expected=200, headers={"t_underscore": "foo"}) def check(self): assert self.results[0].status == 200 + class RejectHeadersWithUnderscoresTest(AmbassadorTest): target: ServiceType @@ -34,7 +38,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -42,8 +47,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: ambassador_id: [{self.ambassador_id}] config: headers_with_underscores_action: REJECT_REQUEST -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -52,10 +59,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): - yield Query(self.url("target/"), expected=400, headers={'t_underscore':'foo'}) + yield Query(self.url("target/"), expected=400, headers={"t_underscore": "foo"}) def check(self): assert self.results[0].status == 400 diff --git a/python/tests/kat/t_hosts.py b/python/tests/kat/t_hosts.py index 16a0a502df..602a5bfa8a 100644 --- a/python/tests/kat/t_hosts.py +++ b/python/tests/kat/t_hosts.py @@ -16,9 +16,13 @@ # Host where a TLSContext with the inferred name already exists bug_single_insecure_action = False # Do all Hosts have to have the same insecure.action? -bug_forced_star = True # Do we erroneously send replies in cleartext instead of TLS for unknown hosts? -bug_404_routes = True # Do we erroneously send 404 responses directly instead of redirect-to-tls first? -bug_clientcert_reset = True # Do we sometimes just close the connection instead of sending back tls certificate_required? +bug_forced_star = ( + True # Do we erroneously send replies in cleartext instead of TLS for unknown hosts? +) +bug_404_routes = ( + True # Do we erroneously send 404 responses directly instead of redirect-to-tls first? +) +bug_clientcert_reset = True # Do we sometimes just close the connection instead of sending back tls certificate_required? class HostCRDSingle(AmbassadorTest): @@ -26,6 +30,7 @@ class HostCRDSingle(AmbassadorTest): HostCRDSingle: a single Host with a manually-configured TLS. Since the Host is handling the TLSContext, we expect both OSS and Edge Stack to redirect cleartext from 8080 to 8443 here. """ + target: ServiceType def init(self): @@ -33,7 +38,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: v1 kind: Secret @@ -43,8 +50,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["localhost"].k8s_crt+''' - tls.key: '''+TLSCerts["localhost"].k8s_key+''' + tls.crt: """ + + TLSCerts["localhost"].k8s_crt + + """ + tls.key: """ + + TLSCerts["localhost"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -73,7 +84,10 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" @@ -88,6 +102,7 @@ class HostCRDNo8080(AmbassadorTest): HostCRDNo8080: a single Host with manually-configured TLS that explicitly turns off redirection from 8080. """ + target: ServiceType def init(self): @@ -97,7 +112,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: v1 kind: Secret @@ -107,8 +124,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["localhost"].k8s_crt+''' - tls.key: '''+TLSCerts["localhost"].k8s_key+''' + tls.crt: """ + + TLSCerts["localhost"].k8s_crt + + """ + tls.key: """ + + TLSCerts["localhost"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -155,14 +176,17 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): yield Query(self.url("target/"), insecure=True) - yield Query(self.url("target/", scheme="http"), error=[ "EOF", "connection refused" ]) + yield Query(self.url("target/", scheme="http"), error=["EOF", "connection refused"]) class HostCRDManualContext(AmbassadorTest): @@ -170,6 +194,7 @@ class HostCRDManualContext(AmbassadorTest): A single Host with a manually-specified TLS secret and a manually-specified TLSContext, too. """ + target: ServiceType def init(self): @@ -178,7 +203,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: v1 kind: Secret @@ -188,8 +215,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["localhost"].k8s_crt+''' - tls.key: '''+TLSCerts["localhost"].k8s_key+''' + tls.crt: """ + + TLSCerts["localhost"].k8s_crt + + """ + tls.key: """ + + TLSCerts["localhost"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -232,20 +263,28 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("target/tls-1.2-1.3"), insecure=True, - minTLSv="v1.2", maxTLSv="v1.3") - - yield Query(self.url("target/tls-1.0-1.0"), insecure=True, - minTLSv="v1.0", maxTLSv="v1.0", - error=["tls: server selected unsupported protocol version 303", - "tls: no supported versions satisfy MinVersion and MaxVersion", - "tls: protocol version not supported"]) + yield Query(self.url("target/tls-1.2-1.3"), insecure=True, minTLSv="v1.2", maxTLSv="v1.3") + + yield Query( + self.url("target/tls-1.0-1.0"), + insecure=True, + minTLSv="v1.0", + maxTLSv="v1.0", + error=[ + "tls: server selected unsupported protocol version 303", + "tls: no supported versions satisfy MinVersion and MaxVersion", + "tls: protocol version not supported", + ], + ) yield Query(self.url("target/cleartext", scheme="http"), expected=301) @@ -255,6 +294,7 @@ class HostCRDManualContextCRL(AmbassadorTest): A single Host with a manually-specified TLS secret, a manually-specified TLSContext and a manually specified mTLS config with CRL list too. """ + target: ServiceType def init(self): @@ -264,7 +304,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -289,8 +331,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["ambassador.example.com"].k8s_crt+''' - tls.key: '''+TLSCerts["ambassador.example.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["ambassador.example.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["ambassador.example.com"].k8s_key + + """ --- apiVersion: v1 kind: Secret @@ -300,7 +346,9 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["master.datawire.io"].k8s_crt+''' + tls.crt: """ + + TLSCerts["master.datawire.io"].k8s_crt + + """ tls.key: "" --- apiVersion: v1 @@ -311,7 +359,13 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: Opaque data: - crl.pem: '''+create_crl_pem_b64(TLSCerts["master.datawire.io"].pubcert, TLSCerts["master.datawire.io"].privkey, [TLSCerts["presto.example.com"].pubcert])+''' + crl.pem: """ + + create_crl_pem_b64( + TLSCerts["master.datawire.io"].pubcert, + TLSCerts["master.datawire.io"].privkey, + [TLSCerts["presto.example.com"].pubcert], + ) + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -356,26 +410,30 @@ def manifests(self) -> str: hostname: ambassador.example.com prefix: / service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): base = { - 'url': self.url(""), - 'ca_cert': TLSCerts["master.datawire.io"].pubcert, - 'headers': {"Host": "ambassador.example.com"}, - 'sni': True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + "url": self.url(""), + "ca_cert": TLSCerts["master.datawire.io"].pubcert, + "headers": {"Host": "ambassador.example.com"}, + "sni": True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI } - yield Query(**base, - error="tls: certificate required") + yield Query(**base, error="tls: certificate required") - yield Query(**base, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey, - error="tls: revoked certificate") + yield Query( + **base, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey, + error="tls: revoked certificate" + ) def requirements(self): yield ("pod", self.path.k8s) @@ -389,7 +447,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: v1 kind: Secret @@ -399,8 +459,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["localhost"].k8s_crt+''' - tls.key: '''+TLSCerts["localhost"].k8s_key+''' + tls.crt: """ + + TLSCerts["localhost"].k8s_crt + + """ + tls.key: """ + + TLSCerts["localhost"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -443,20 +507,28 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("target/"), insecure=True, - minTLSv="v1.2", maxTLSv="v1.3") + yield Query(self.url("target/"), insecure=True, minTLSv="v1.2", maxTLSv="v1.3") - yield Query(self.url("target/"), insecure=True, - minTLSv="v1.0", maxTLSv="v1.0", - error=["tls: server selected unsupported protocol version 303", - "tls: no supported versions satisfy MinVersion and MaxVersion", - "tls: protocol version not supported"]) + yield Query( + self.url("target/"), + insecure=True, + minTLSv="v1.0", + maxTLSv="v1.0", + error=[ + "tls: server selected unsupported protocol version 303", + "tls: no supported versions satisfy MinVersion and MaxVersion", + "tls: protocol version not supported", + ], + ) class HostCRDTLSConfig(AmbassadorTest): @@ -467,7 +539,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: v1 kind: Secret @@ -477,8 +551,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["localhost"].k8s_crt+''' - tls.key: '''+TLSCerts["localhost"].k8s_key+''' + tls.crt: """ + + TLSCerts["localhost"].k8s_crt + + """ + tls.key: """ + + TLSCerts["localhost"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -510,20 +588,28 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("target/"), insecure=True, - minTLSv="v1.2", maxTLSv="v1.3") + yield Query(self.url("target/"), insecure=True, minTLSv="v1.2", maxTLSv="v1.3") - yield Query(self.url("target/"), insecure=True, - minTLSv="v1.0", maxTLSv="v1.0", - error=["tls: server selected unsupported protocol version 303", - "tls: no supported versions satisfy MinVersion and MaxVersion", - "tls: protocol version not supported"]) + yield Query( + self.url("target/"), + insecure=True, + minTLSv="v1.0", + maxTLSv="v1.0", + error=[ + "tls: server selected unsupported protocol version 303", + "tls: no supported versions satisfy MinVersion and MaxVersion", + "tls: protocol version not supported", + ], + ) class HostCRDClearText(AmbassadorTest): @@ -531,6 +617,7 @@ class HostCRDClearText(AmbassadorTest): A single Host specifying cleartext only. Since it's just cleartext, no redirection comes into play. """ + target: ServiceType def init(self): @@ -543,7 +630,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -573,15 +662,17 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "http" def queries(self): yield Query(self.url("target/"), insecure=True) - yield Query(self.url("target/", scheme="https"), - error=[ "EOF", "connection refused" ]) + yield Query(self.url("target/", scheme="https"), error=["EOF", "connection refused"]) class HostCRDDouble(AmbassadorTest): @@ -597,6 +688,7 @@ class HostCRDDouble(AmbassadorTest): XXX In the future, the hostname matches should be unnecessary, as it should use metadata.labels.hostname. """ + target1: ServiceType target2: ServiceType target3: ServiceType @@ -610,7 +702,9 @@ def init(self): self.targetshared = HTTP(name="targetshared") def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -640,8 +734,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["tls-context-host-1"].k8s_crt+''' - tls.key: '''+TLSCerts["tls-context-host-1"].k8s_key+''' + tls.crt: """ + + TLSCerts["tls-context-host-1"].k8s_crt + + """ + tls.key: """ + + TLSCerts["tls-context-host-1"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -682,8 +780,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["tls-context-host-2"].k8s_crt+''' - tls.key: '''+TLSCerts["tls-context-host-2"].k8s_key+''' + tls.crt: """ + + TLSCerts["tls-context-host-2"].k8s_crt + + """ + tls.key: """ + + TLSCerts["tls-context-host-2"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -727,8 +829,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["ambassador.example.com"].k8s_crt+''' - tls.key: '''+TLSCerts["ambassador.example.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["ambassador.example.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["ambassador.example.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -770,86 +876,211 @@ def manifests(self) -> str: hostname: "*" prefix: /target-shared/ service: {self.targetshared.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): # 0: Get some info from diagd for self.check() to inspect - yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), - headers={"Host": "tls-context-host-1" }, - insecure=True, - sni=True) + yield Query( + self.url("ambassador/v0/diag/?json=true&filter=errors"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ) # 1-5: Host #1 - TLS - yield Query(self.url("target-1/", scheme="https"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, - expected=200) - yield Query(self.url("target-2/", scheme="https"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, - expected=404) - yield Query(self.url("target-3/", scheme="https"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, - expected=404) - yield Query(self.url("target-shared/", scheme="https"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, - expected=200) - yield Query(self.url(".well-known/acme-challenge/foo", scheme="https"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, - expected=404) + yield Query( + self.url("target-1/", scheme="https"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + expected=200, + ) + yield Query( + self.url("target-2/", scheme="https"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + expected=404, + ) + yield Query( + self.url("target-3/", scheme="https"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + expected=404, + ) + yield Query( + self.url("target-shared/", scheme="https"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + expected=200, + ) + yield Query( + self.url(".well-known/acme-challenge/foo", scheme="https"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + expected=404, + ) # 6-10: Host #1 - cleartext (action: Route) - yield Query(self.url("target-1/", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=200) - yield Query(self.url("target-2/", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=404) - yield Query(self.url("target-3/", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=404) - yield Query(self.url("target-shared/", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=200) - yield Query(self.url(".well-known/acme-challenge/foo", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=404) + yield Query( + self.url("target-1/", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=200, + ) + yield Query( + self.url("target-2/", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=404, + ) + yield Query( + self.url("target-3/", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=404, + ) + yield Query( + self.url("target-shared/", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=200, + ) + yield Query( + self.url(".well-known/acme-challenge/foo", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=404, + ) # 11-15: Host #2 - TLS - yield Query(self.url("target-1/", scheme="https"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True, - expected=404) - yield Query(self.url("target-2/", scheme="https"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True, - expected=200) - yield Query(self.url("target-3/", scheme="https"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True, - expected=404) - yield Query(self.url("target-shared/", scheme="https"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True, - expected=200) - yield Query(self.url(".well-known/acme-challenge/foo", scheme="https"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True, - expected=404) + yield Query( + self.url("target-1/", scheme="https"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + expected=404, + ) + yield Query( + self.url("target-2/", scheme="https"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + expected=200, + ) + yield Query( + self.url("target-3/", scheme="https"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + expected=404, + ) + yield Query( + self.url("target-shared/", scheme="https"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + expected=200, + ) + yield Query( + self.url(".well-known/acme-challenge/foo", scheme="https"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + expected=404, + ) # 16-20: Host #2 - cleartext (action: Redirect) - yield Query(self.url("target-1/", scheme="http"), headers={"Host": "tls-context-host-2"}, - expected=404) - yield Query(self.url("target-2/", scheme="http"), headers={"Host": "tls-context-host-2"}, - expected=301) - yield Query(self.url("target-3/", scheme="http"), headers={"Host": "tls-context-host-2"}, - expected=404) - yield Query(self.url("target-shared/", scheme="http"), headers={"Host": "tls-context-host-2"}, - expected=301) - yield Query(self.url(".well-known/acme-challenge/foo", scheme="http"), headers={"Host": "tls-context-host-2"}, - expected=404) + yield Query( + self.url("target-1/", scheme="http"), + headers={"Host": "tls-context-host-2"}, + expected=404, + ) + yield Query( + self.url("target-2/", scheme="http"), + headers={"Host": "tls-context-host-2"}, + expected=301, + ) + yield Query( + self.url("target-3/", scheme="http"), + headers={"Host": "tls-context-host-2"}, + expected=404, + ) + yield Query( + self.url("target-shared/", scheme="http"), + headers={"Host": "tls-context-host-2"}, + expected=301, + ) + yield Query( + self.url(".well-known/acme-challenge/foo", scheme="http"), + headers={"Host": "tls-context-host-2"}, + expected=404, + ) # 21-25: Host #3 - TLS - yield Query(self.url("target-1/", scheme="https"), headers={"Host": "ambassador.example.com"}, insecure=True, sni=True, - expected=404) - yield Query(self.url("target-2/", scheme="https"), headers={"Host": "ambassador.example.com"}, insecure=True, sni=True, - expected=404) - yield Query(self.url("target-3/", scheme="https"), headers={"Host": "ambassador.example.com"}, insecure=True, sni=True, - expected=200) - yield Query(self.url("target-shared/", scheme="https"), headers={"Host": "ambassador.example.com"}, insecure=True, sni=True, - expected=200) - yield Query(self.url(".well-known/acme-challenge/foo", scheme="https"), headers={"Host": "ambassador.example.com"}, insecure=True, sni=True, - expected=200) + yield Query( + self.url("target-1/", scheme="https"), + headers={"Host": "ambassador.example.com"}, + insecure=True, + sni=True, + expected=404, + ) + yield Query( + self.url("target-2/", scheme="https"), + headers={"Host": "ambassador.example.com"}, + insecure=True, + sni=True, + expected=404, + ) + yield Query( + self.url("target-3/", scheme="https"), + headers={"Host": "ambassador.example.com"}, + insecure=True, + sni=True, + expected=200, + ) + yield Query( + self.url("target-shared/", scheme="https"), + headers={"Host": "ambassador.example.com"}, + insecure=True, + sni=True, + expected=200, + ) + yield Query( + self.url(".well-known/acme-challenge/foo", scheme="https"), + headers={"Host": "ambassador.example.com"}, + insecure=True, + sni=True, + expected=200, + ) # 26-30: Host #3 - cleartext (action: Reject) - yield Query(self.url("target-1/", scheme="http"), headers={"Host": "ambassador.example.com"}, - expected=404) - yield Query(self.url("target-2/", scheme="http"), headers={"Host": "ambassador.example.com"}, - expected=404) - yield Query(self.url("target-3/", scheme="http"), headers={"Host": "ambassador.example.com"}, - expected=404) - yield Query(self.url("target-shared/", scheme="http"), headers={"Host": "ambassador.example.com"}, - expected=404) - yield Query(self.url(".well-known/acme-challenge/foo", scheme="http"), headers={"Host": "ambassador.example.com"}, - expected=200) + yield Query( + self.url("target-1/", scheme="http"), + headers={"Host": "ambassador.example.com"}, + expected=404, + ) + yield Query( + self.url("target-2/", scheme="http"), + headers={"Host": "ambassador.example.com"}, + expected=404, + ) + yield Query( + self.url("target-3/", scheme="http"), + headers={"Host": "ambassador.example.com"}, + expected=404, + ) + yield Query( + self.url("target-shared/", scheme="http"), + headers={"Host": "ambassador.example.com"}, + expected=404, + ) + yield Query( + self.url(".well-known/acme-challenge/foo", scheme="http"), + headers={"Host": "ambassador.example.com"}, + expected=200, + ) def check(self): # XXX Ew. If self.results[0].json is empty, the harness won't convert it to a response. @@ -861,19 +1092,55 @@ def check(self): for result in self.results: if result.status == 200 and result.query.headers and result.tls: - host_header = result.query.headers['Host'] - tls_common_name = result.tls[0]['Subject']['CommonName'] + host_header = result.query.headers["Host"] + tls_common_name = result.tls[0]["Subject"]["CommonName"] - assert host_header == tls_common_name, "test %d wanted CN %s, but got %s" % (idx, host_header, tls_common_name) + assert host_header == tls_common_name, "test %d wanted CN %s, but got %s" % ( + idx, + host_header, + tls_common_name, + ) idx += 1 def requirements(self): # We're replacing super()'s requirements deliberately here. Without a Host header they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) class HostCRDWildcards(AmbassadorTest): @@ -882,13 +1149,16 @@ class HostCRDWildcards(AmbassadorTest): prefix-match host-globs. But this is a solid start. """ + target: ServiceType def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -943,8 +1213,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["a.domain.com"].k8s_crt+''' - tls.key: '''+TLSCerts["a.domain.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["a.domain.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["a.domain.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -957,28 +1231,39 @@ def manifests(self) -> str: hostname: "*" prefix: /foo/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def insecure(self, suffix): return { - 'url': self.url('foo/%s' % suffix, scheme='http'), + "url": self.url("foo/%s" % suffix, scheme="http"), } def secure(self, suffix): return { - 'url': self.url('foo/%s' % suffix, scheme='https'), - 'ca_cert': TLSCerts["*.domain.com"].pubcert, - 'sni': True, + "url": self.url("foo/%s" % suffix, scheme="https"), + "ca_cert": TLSCerts["*.domain.com"].pubcert, + "sni": True, } def queries(self): - yield Query(**self.secure("0-200"), headers={'Host': 'a.domain.com'}, expected=200) # Host=a.domain.com - yield Query(**self.secure("1-200"), headers={'Host': 'wc.domain.com'}, expected=200) # Host=*.domain.com - yield Query(**self.secure("2-200"), headers={'Host': '127.0.0.1'}, expected=200) # Host=* - - yield Query(**self.insecure("3-301"), headers={'Host': 'a.domain.com'}, expected=301) # Host=a.domain.com - yield Query(**self.insecure("4-200"), headers={'Host': 'wc.domain.com'}, expected=200) # Host=*.domain.com - yield Query(**self.insecure("5-301"), headers={'Host': '127.0.0.1'}, expected=301) # Host=* + yield Query( + **self.secure("0-200"), headers={"Host": "a.domain.com"}, expected=200 + ) # Host=a.domain.com + yield Query( + **self.secure("1-200"), headers={"Host": "wc.domain.com"}, expected=200 + ) # Host=*.domain.com + yield Query(**self.secure("2-200"), headers={"Host": "127.0.0.1"}, expected=200) # Host=* + + yield Query( + **self.insecure("3-301"), headers={"Host": "a.domain.com"}, expected=301 + ) # Host=a.domain.com + yield Query( + **self.insecure("4-200"), headers={"Host": "wc.domain.com"}, expected=200 + ) # Host=*.domain.com + yield Query(**self.insecure("5-301"), headers={"Host": "127.0.0.1"}, expected=301) # Host=* def scheme(self) -> str: return "https" @@ -986,8 +1271,10 @@ def scheme(self) -> str: def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "127.0.0.1"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "127.0.0.1"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["*.domain.com"].pubcert yield (r[0], query) @@ -1007,7 +1294,10 @@ def manifests(self) -> str: # tls.ca_secret. And for ca_secret we still put the '.' in # the name so that we check that it's choosing the correct '.' # as the separator. - return namespace_manifest("alt-namespace") + self.format(''' + return ( + namespace_manifest("alt-namespace") + + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -1036,7 +1326,9 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["master.datawire.io"].k8s_crt+''' + tls.crt: """ + + TLSCerts["master.datawire.io"].k8s_crt + + """ tls.key: "" --- apiVersion: v1 @@ -1047,8 +1339,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["ambassador.example.com"].k8s_crt+''' - tls.key: '''+TLSCerts["ambassador.example.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["ambassador.example.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["ambassador.example.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1061,22 +1357,27 @@ def manifests(self) -> str: hostname: "*" prefix: / service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): base = { - 'url': self.url(""), - 'ca_cert': TLSCerts["master.datawire.io"].pubcert, - 'headers': {"Host": "ambassador.example.com"}, - 'sni': True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + "url": self.url(""), + "ca_cert": TLSCerts["master.datawire.io"].pubcert, + "headers": {"Host": "ambassador.example.com"}, + "sni": True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI } - yield Query(**base, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey) + yield Query( + **base, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey + ) # Check that it requires the client cert. # @@ -1084,24 +1385,41 @@ def queries(self): # so we get a generic alert=40 ("handshake_failure"). yield Query(**base, maxTLSv="v1.2", error="tls: handshake failure") # TLS 1.3 added a dedicated alert=116 ("certificate_required") for that scenario. - yield Query(**base, minTLSv="v1.3", error=(["tls: certificate required"] + (["write: connection reset by peer", "write: broken pipe"] if bug_clientcert_reset else []))) + yield Query( + **base, + minTLSv="v1.3", + error=( + ["tls: certificate required"] + + ( + ["write: connection reset by peer", "write: broken pipe"] + if bug_clientcert_reset + else [] + ) + ) + ) # Check that it's validating the client cert against the CA cert. - yield Query(**base, - client_crt=TLSCerts["localhost"].pubcert, - client_key=TLSCerts["localhost"].privkey, - maxTLSv="v1.2", error="tls: handshake failure") + yield Query( + **base, + client_crt=TLSCerts["localhost"].pubcert, + client_key=TLSCerts["localhost"].privkey, + maxTLSv="v1.2", + error="tls: handshake failure" + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "ambassador.example.com"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "ambassador.example.com"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["master.datawire.io"].pubcert query.client_cert = TLSCerts["presto.example.com"].pubcert query.client_key = TLSCerts["presto.example.com"].privkey yield (r[0], query) + class HostCRDClientCertSameNamespace(AmbassadorTest): target: ServiceType @@ -1116,7 +1434,10 @@ def manifests(self) -> str: # (unlike HostCRDClientCertCrossNamespace) the ca_secret # doesn't, so that we can check that it chooses the correct # namespace when a ".{namespace}" suffix isn't specified. - return namespace_manifest("alt2-namespace") + self.format(''' + return ( + namespace_manifest("alt2-namespace") + + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -1162,7 +1483,9 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["master.datawire.io"].k8s_crt+''' + tls.crt: """ + + TLSCerts["master.datawire.io"].k8s_crt + + """ tls.key: "" --- apiVersion: v1 @@ -1174,8 +1497,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["ambassador.example.com"].k8s_crt+''' - tls.key: '''+TLSCerts["ambassador.example.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["ambassador.example.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["ambassador.example.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1188,22 +1515,27 @@ def manifests(self) -> str: hostname: "*" prefix: / service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): base = { - 'url': self.url(""), - 'ca_cert': TLSCerts["master.datawire.io"].pubcert, - 'headers': {"Host": "ambassador.example.com"}, - 'sni': True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + "url": self.url(""), + "ca_cert": TLSCerts["master.datawire.io"].pubcert, + "headers": {"Host": "ambassador.example.com"}, + "sni": True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI } - yield Query(**base, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey) + yield Query( + **base, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey + ) # Check that it requires the client cert. # @@ -1211,19 +1543,35 @@ def queries(self): # so we get a generic alert=40 ("handshake_failure"). yield Query(**base, maxTLSv="v1.2", error="tls: handshake failure") # TLS 1.3 added a dedicated alert=116 ("certificate_required") for that scenario. - yield Query(**base, minTLSv="v1.3", error=(["tls: certificate required"] + (["write: connection reset by peer", "write: broken pipe"] if bug_clientcert_reset else []))) + yield Query( + **base, + minTLSv="v1.3", + error=( + ["tls: certificate required"] + + ( + ["write: connection reset by peer", "write: broken pipe"] + if bug_clientcert_reset + else [] + ) + ) + ) # Check that it's validating the client cert against the CA cert. - yield Query(**base, - client_crt=TLSCerts["localhost"].pubcert, - client_key=TLSCerts["localhost"].privkey, - maxTLSv="v1.2", error="tls: handshake failure") + yield Query( + **base, + client_crt=TLSCerts["localhost"].pubcert, + client_key=TLSCerts["localhost"].privkey, + maxTLSv="v1.2", + error="tls: handshake failure" + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "ambassador.example.com"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "ambassador.example.com"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["master.datawire.io"].pubcert query.client_cert = TLSCerts["presto.example.com"].pubcert query.client_key = TLSCerts["presto.example.com"].privkey @@ -1241,7 +1589,10 @@ def init(self): def manifests(self) -> str: # Similar to HostCRDClientCertSameNamespace, except we also # include a Certificate Revocation List in the TLS config - return namespace_manifest("alt3-namespace") + self.format(''' + return ( + namespace_manifest("alt3-namespace") + + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -1287,7 +1638,9 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["master.datawire.io"].k8s_crt+''' + tls.crt: """ + + TLSCerts["master.datawire.io"].k8s_crt + + """ tls.key: "" --- apiVersion: v1 @@ -1299,7 +1652,13 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: Opaque data: - crl.pem: '''+create_crl_pem_b64(TLSCerts["master.datawire.io"].pubcert, TLSCerts["master.datawire.io"].privkey, [])+''' + crl.pem: """ + + create_crl_pem_b64( + TLSCerts["master.datawire.io"].pubcert, + TLSCerts["master.datawire.io"].privkey, + [], + ) + + """ --- apiVersion: v1 kind: Secret @@ -1310,8 +1669,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["ambassador.example.com"].k8s_crt+''' - tls.key: '''+TLSCerts["ambassador.example.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["ambassador.example.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["ambassador.example.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1324,25 +1687,29 @@ def manifests(self) -> str: hostname: "*" prefix: / service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): base = { - 'url': self.url(""), - 'ca_cert': TLSCerts["master.datawire.io"].pubcert, - 'headers': {"Host": "ambassador.example.com"}, - 'sni': True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + "url": self.url(""), + "ca_cert": TLSCerts["master.datawire.io"].pubcert, + "headers": {"Host": "ambassador.example.com"}, + "sni": True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI } - yield Query(**base, - error="tls: certificate required") + yield Query(**base, error="tls: certificate required") - yield Query(**base, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey) + yield Query( + **base, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey + ) def requirements(self): yield ("pod", self.path.k8s) @@ -1359,7 +1726,10 @@ def init(self): def manifests(self) -> str: # Similar to HostCRDClientCertSameNamespace, except we also # include a Certificate Revocation List in the TLS config - return namespace_manifest("alt4-namespace") + self.format(''' + return ( + namespace_manifest("alt4-namespace") + + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -1405,7 +1775,9 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["master.datawire.io"].k8s_crt+''' + tls.crt: """ + + TLSCerts["master.datawire.io"].k8s_crt + + """ tls.key: "" --- apiVersion: v1 @@ -1417,7 +1789,13 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: Opaque data: - crl.pem: '''+create_crl_pem_b64(TLSCerts["master.datawire.io"].pubcert, TLSCerts["master.datawire.io"].privkey, [TLSCerts["presto.example.com"].pubcert])+''' + crl.pem: """ + + create_crl_pem_b64( + TLSCerts["master.datawire.io"].pubcert, + TLSCerts["master.datawire.io"].privkey, + [TLSCerts["presto.example.com"].pubcert], + ) + + """ --- apiVersion: v1 kind: Secret @@ -1428,8 +1806,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["ambassador.example.com"].k8s_crt+''' - tls.key: '''+TLSCerts["ambassador.example.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["ambassador.example.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["ambassador.example.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1442,35 +1824,40 @@ def manifests(self) -> str: hostname: "*" prefix: / service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): base = { - 'url': self.url(""), - 'ca_cert': TLSCerts["master.datawire.io"].pubcert, - 'headers': {"Host": "ambassador.example.com"}, - 'sni': True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + "url": self.url(""), + "ca_cert": TLSCerts["master.datawire.io"].pubcert, + "headers": {"Host": "ambassador.example.com"}, + "sni": True, # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI } - yield Query(**base, - error="tls: certificate required") + yield Query(**base, error="tls: certificate required") - yield Query(**base, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey, - error="tls: revoked certificate") + yield Query( + **base, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey, + error="tls: revoked certificate" + ) def requirements(self): yield ("pod", self.path.k8s) class HostCRDRootRedirectCongratulations(AmbassadorTest): - def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -1500,29 +1887,54 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["tls-context-host-1"].k8s_crt+''' - tls.key: '''+TLSCerts["tls-context-host-1"].k8s_key+''' -''') + super().manifests() + tls.crt: """ + + TLSCerts["tls-context-host-1"].k8s_crt + + """ + tls.key: """ + + TLSCerts["tls-context-host-1"].k8s_key + + """ +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=(404 if (EDGE_STACK or bug_404_routes) else 301)) - yield Query(self.url("other", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=(404 if bug_404_routes else 301)) - - yield Query(self.url("", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=404) - yield Query(self.url("other", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=404) + yield Query( + self.url("", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=(404 if (EDGE_STACK or bug_404_routes) else 301), + ) + yield Query( + self.url("other", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=(404 if bug_404_routes else 301), + ) + + yield Query( + self.url("", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=404, + ) + yield Query( + self.url("other", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=404, + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "tls-context-host-1"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "tls-context-host-1"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["tls-context-host-1"].pubcert yield (r[0], query) @@ -1534,7 +1946,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -1564,8 +1978,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["tls-context-host-1"].k8s_crt+''' - tls.key: '''+TLSCerts["tls-context-host-1"].k8s_key+''' + tls.crt: """ + + TLSCerts["tls-context-host-1"].k8s_crt + + """ + tls.key: """ + + TLSCerts["tls-context-host-1"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1577,27 +1995,44 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: / service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=301) - yield Query(self.url("other", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=301) - - yield Query(self.url("", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=200) - yield Query(self.url("other", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=200) + yield Query( + self.url("", scheme="http"), headers={"Host": "tls-context-host-1"}, expected=301 + ) + yield Query( + self.url("other", scheme="http"), headers={"Host": "tls-context-host-1"}, expected=301 + ) + + yield Query( + self.url("", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=200, + ) + yield Query( + self.url("other", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=200, + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "tls-context-host-1"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "tls-context-host-1"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["tls-context-host-1"].pubcert yield (r[0], query) @@ -1609,7 +2044,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -1639,8 +2076,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["tls-context-host-1"].k8s_crt+''' - tls.key: '''+TLSCerts["tls-context-host-1"].k8s_key+''' + tls.crt: """ + + TLSCerts["tls-context-host-1"].k8s_crt + + """ + tls.key: """ + + TLSCerts["tls-context-host-1"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1653,31 +2094,56 @@ def manifests(self) -> str: prefix: "/[[:word:]]*" # :word: is in RE2 but not ECMAScript RegExp or Python 're' prefix_regex: true service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=301) - yield Query(self.url("other", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=301) - yield Query(self.url("-other", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=(404 if bug_404_routes else 301)) - - yield Query(self.url("", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=200) - yield Query(self.url("other", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=200) - yield Query(self.url("-other", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=404) + yield Query( + self.url("", scheme="http"), headers={"Host": "tls-context-host-1"}, expected=301 + ) + yield Query( + self.url("other", scheme="http"), headers={"Host": "tls-context-host-1"}, expected=301 + ) + yield Query( + self.url("-other", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=(404 if bug_404_routes else 301), + ) + + yield Query( + self.url("", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=200, + ) + yield Query( + self.url("other", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=200, + ) + yield Query( + self.url("-other", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=404, + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "tls-context-host-1"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "tls-context-host-1"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["tls-context-host-1"].pubcert yield (r[0], query) @@ -1686,6 +2152,7 @@ class HostCRDForcedStar(AmbassadorTest): """This test verifies that Ambassador responds properly if we try talking to it on a hostname that it doesn't recognize. """ + target: ServiceType def init(self): @@ -1700,7 +2167,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Host @@ -1742,8 +2211,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["tls-context-host-1"].k8s_crt+''' - tls.key: '''+TLSCerts["tls-context-host-1"].k8s_key+''' + tls.crt: """ + + TLSCerts["tls-context-host-1"].k8s_crt + + """ + tls.key: """ + + TLSCerts["tls-context-host-1"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1754,7 +2227,10 @@ def manifests(self) -> str: hostname: "*" prefix: /foo/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" @@ -1765,33 +2241,71 @@ def queries(self): # ("nonmatching-host") to make sure that it is handled the same way. # 0-1: cleartext 200/301 - yield Query(self.url("foo/0-200", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=200) - yield Query(self.url("foo/1-301", scheme="http"), headers={"Host": "nonmatching-hostname"}, - expected=301) + yield Query( + self.url("foo/0-200", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=200, + ) + yield Query( + self.url("foo/1-301", scheme="http"), + headers={"Host": "nonmatching-hostname"}, + expected=301, + ) # 2-3: cleartext 404 - yield Query(self.url("bar/2-404", scheme="http"), headers={"Host": "tls-context-host-1"}, - expected=404) - yield Query(self.url("bar/3-301-or-404", scheme="http"), headers={"Host": "nonmatching-hostname"}, - expected=404 if bug_404_routes else 301) + yield Query( + self.url("bar/2-404", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=404, + ) + yield Query( + self.url("bar/3-301-or-404", scheme="http"), + headers={"Host": "nonmatching-hostname"}, + expected=404 if bug_404_routes else 301, + ) # 4-5: TLS 200 - yield Query(self.url("foo/4-200", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=200) - yield Query(self.url("foo/5-200", scheme="https"), headers={"Host": "nonmatching-hostname"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, insecure=True, - expected=200, error=("http: server gave HTTP response to HTTPS client" if bug_forced_star else None)) + yield Query( + self.url("foo/4-200", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=200, + ) + yield Query( + self.url("foo/5-200", scheme="https"), + headers={"Host": "nonmatching-hostname"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + insecure=True, + expected=200, + error=("http: server gave HTTP response to HTTPS client" if bug_forced_star else None), + ) # 6-7: TLS 404 - yield Query(self.url("bar/6-404", scheme="https"), headers={"Host": "tls-context-host-1"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, - expected=404) - yield Query(self.url("bar/7-404", scheme="https"), headers={"Host": "nonmatching-hostname"}, ca_cert=TLSCerts["tls-context-host-1"].pubcert, sni=True, insecure=True, - expected=404, error=("http: server gave HTTP response to HTTPS client" if bug_forced_star else None)) + yield Query( + self.url("bar/6-404", scheme="https"), + headers={"Host": "tls-context-host-1"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + expected=404, + ) + yield Query( + self.url("bar/7-404", scheme="https"), + headers={"Host": "nonmatching-hostname"}, + ca_cert=TLSCerts["tls-context-host-1"].pubcert, + sni=True, + insecure=True, + expected=404, + error=("http: server gave HTTP response to HTTPS client" if bug_forced_star else None), + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "tls-context-host-1"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "tls-context-host-1"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["tls-context-host-1"].pubcert yield (r[0], query) diff --git a/python/tests/kat/t_ingress.py b/python/tests/kat/t_ingress.py index 82f79c3436..778217e626 100644 --- a/python/tests/kat/t_ingress.py +++ b/python/tests/kat/t_ingress.py @@ -12,20 +12,16 @@ from tests.integration.utils import KUBESTATUS_PATH from ambassador.utils import parse_bool + class IngressStatusTest1(AmbassadorTest): - status_update = { - "loadBalancer": { - "ingress": [{ - "ip": "42.42.42.42" - }] - } - } + status_update = {"loadBalancer": {"ingress": [{"ip": "42.42.42.42"}]}} def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -45,57 +41,77 @@ def manifests(self) -> str: number: 80 path: /{self.name}/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def queries(self): - if True or sys.platform != 'darwin': + if True or sys.platform != "darwin": text = json.dumps(self.status_update) - update_cmd = [KUBESTATUS_PATH, 'Service', '-n', 'default', '-f', f'metadata.name={self.name.k8s}', '-u', '/dev/fd/0'] - subprocess.run(update_cmd, input=text.encode('utf-8'), timeout=10) + update_cmd = [ + KUBESTATUS_PATH, + "Service", + "-n", + "default", + "-f", + f"metadata.name={self.name.k8s}", + "-u", + "/dev/fd/0", + ] + subprocess.run(update_cmd, input=text.encode("utf-8"), timeout=10) # If you run these tests individually, the time between running kubestatus # and the ingress resource actually getting updated is longer than the # time spent waiting for resources to be ready, so this test will fail (most of the time) time.sleep(1) yield Query(self.url(self.name + "/")) - yield Query(self.url(f'need-normalization/../{self.name}/')) + yield Query(self.url(f"need-normalization/../{self.name}/")) def check(self): if not parse_bool(os.environ.get("AMBASSADOR_PYTEST_INGRESS_TEST", "false")): - pytest.xfail('AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...') + pytest.xfail("AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...") - if False and sys.platform == 'darwin': - pytest.xfail('not supported on Darwin') + if False and sys.platform == "darwin": + pytest.xfail("not supported on Darwin") for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" # check for Ingress IP here - ingress_cmd = ["tools/bin/kubectl", "get", "-n", "default", "-o", "json", "ingress", self.path.k8s] + ingress_cmd = [ + "tools/bin/kubectl", + "get", + "-n", + "default", + "-o", + "json", + "ingress", + self.path.k8s, + ] ingress_run = subprocess.Popen(ingress_cmd, stdout=subprocess.PIPE) ingress_out, _ = ingress_run.communicate() ingress_json = json.loads(ingress_out) - assert ingress_json['status'] == self.status_update, f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" + assert ( + ingress_json["status"] == self.status_update + ), f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" class IngressStatusTest2(AmbassadorTest): - status_update = { - "loadBalancer": { - "ingress": [{ - "ip": "84.84.84.84" - }] - } - } + status_update = {"loadBalancer": {"ingress": [{"ip": "84.84.84.84"}]}} def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -115,57 +131,78 @@ def manifests(self) -> str: number: 80 path: /{self.name}/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def queries(self): - if True or sys.platform != 'darwin': + if True or sys.platform != "darwin": text = json.dumps(self.status_update) - update_cmd = [KUBESTATUS_PATH, 'Service', '-n', 'default', '-f', f'metadata.name={self.name.k8s}', '-u', '/dev/fd/0'] - subprocess.run(update_cmd, input=text.encode('utf-8'), timeout=10) + update_cmd = [ + KUBESTATUS_PATH, + "Service", + "-n", + "default", + "-f", + f"metadata.name={self.name.k8s}", + "-u", + "/dev/fd/0", + ] + subprocess.run(update_cmd, input=text.encode("utf-8"), timeout=10) # If you run these tests individually, the time between running kubestatus # and the ingress resource actually getting updated is longer than the # time spent waiting for resources to be ready, so this test will fail (most of the time) time.sleep(1) yield Query(self.url(self.name + "/")) - yield Query(self.url(f'need-normalization/../{self.name}/')) + yield Query(self.url(f"need-normalization/../{self.name}/")) def check(self): if not parse_bool(os.environ.get("AMBASSADOR_PYTEST_INGRESS_TEST", "false")): - pytest.xfail('AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...') + pytest.xfail("AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...") - if False and sys.platform == 'darwin': - pytest.xfail('not supported on Darwin') + if False and sys.platform == "darwin": + pytest.xfail("not supported on Darwin") for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" # check for Ingress IP here - ingress_cmd = ["tools/bin/kubectl", "get", "-n", "default", "-o", "json", "ingress", self.path.k8s] + ingress_cmd = [ + "tools/bin/kubectl", + "get", + "-n", + "default", + "-o", + "json", + "ingress", + self.path.k8s, + ] ingress_run = subprocess.Popen(ingress_cmd, stdout=subprocess.PIPE) ingress_out, _ = ingress_run.communicate() ingress_json = json.loads(ingress_out) - assert ingress_json['status'] == self.status_update, f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" + assert ( + ingress_json["status"] == self.status_update + ), f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" class IngressStatusTestAcrossNamespaces(AmbassadorTest): - status_update = { - "loadBalancer": { - "ingress": [{ - "ip": "168.168.168.168" - }] - } - } + status_update = {"loadBalancer": {"ingress": [{"ip": "168.168.168.168"}]}} def init(self): self.target = HTTP(namespace="alt-namespace") def manifests(self) -> str: - return namespace_manifest("alt-namespace") + """ + return ( + namespace_manifest("alt-namespace") + + """ --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -186,57 +223,77 @@ def manifests(self) -> str: number: 80 path: /{self.name}/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def queries(self): - if True or sys.platform != 'darwin': + if True or sys.platform != "darwin": text = json.dumps(self.status_update) - update_cmd = [KUBESTATUS_PATH, 'Service', '-n', 'default', '-f', f'metadata.name={self.name.k8s}', '-u', '/dev/fd/0'] - subprocess.run(update_cmd, input=text.encode('utf-8'), timeout=10) + update_cmd = [ + KUBESTATUS_PATH, + "Service", + "-n", + "default", + "-f", + f"metadata.name={self.name.k8s}", + "-u", + "/dev/fd/0", + ] + subprocess.run(update_cmd, input=text.encode("utf-8"), timeout=10) # If you run these tests individually, the time between running kubestatus # and the ingress resource actually getting updated is longer than the # time spent waiting for resources to be ready, so this test will fail (most of the time) time.sleep(1) yield Query(self.url(self.name + "/")) - yield Query(self.url(f'need-normalization/../{self.name}/')) + yield Query(self.url(f"need-normalization/../{self.name}/")) def check(self): if not parse_bool(os.environ.get("AMBASSADOR_PYTEST_INGRESS_TEST", "false")): - pytest.xfail('AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...') + pytest.xfail("AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...") - if False and sys.platform == 'darwin': - pytest.xfail('not supported on Darwin') + if False and sys.platform == "darwin": + pytest.xfail("not supported on Darwin") for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" # check for Ingress IP here - ingress_cmd = ["tools/bin/kubectl", "get", "-o", "json", "ingress", self.path.k8s, "-n", "alt-namespace"] + ingress_cmd = [ + "tools/bin/kubectl", + "get", + "-o", + "json", + "ingress", + self.path.k8s, + "-n", + "alt-namespace", + ] ingress_run = subprocess.Popen(ingress_cmd, stdout=subprocess.PIPE) ingress_out, _ = ingress_run.communicate() ingress_json = json.loads(ingress_out) - assert ingress_json['status'] == self.status_update, f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" + assert ( + ingress_json["status"] == self.status_update + ), f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" class IngressStatusTestWithAnnotations(AmbassadorTest): - status_update = { - "loadBalancer": { - "ingress": [{ - "ip": "200.200.200.200" - }] - } - } + status_update = {"loadBalancer": {"ingress": [{"ip": "200.200.200.200"}]}} def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -265,13 +322,24 @@ def manifests(self) -> str: number: 80 path: /{self.name}/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def queries(self): text = json.dumps(self.status_update) - update_cmd = [KUBESTATUS_PATH, 'Service', '-n', 'default', '-f', f'metadata.name={self.name.k8s}', '-u', '/dev/fd/0'] - subprocess.run(update_cmd, input=text.encode('utf-8'), timeout=10) + update_cmd = [ + KUBESTATUS_PATH, + "Service", + "-n", + "default", + "-f", + f"metadata.name={self.name.k8s}", + "-u", + "/dev/fd/0", + ] + subprocess.run(update_cmd, input=text.encode("utf-8"), timeout=10) # If you run these tests individually, the time between running kubestatus # and the ingress resource actually getting updated is longer than the # time spent waiting for resources to be ready, so this test will fail (most of the time) @@ -279,28 +347,33 @@ def queries(self): yield Query(self.url(self.name + "/")) yield Query(self.url(self.name + "-nested/")) - yield Query(self.url(f'need-normalization/../{self.name}/')) + yield Query(self.url(f"need-normalization/../{self.name}/")) def check(self): if not parse_bool(os.environ.get("AMBASSADOR_PYTEST_INGRESS_TEST", "false")): - pytest.xfail('AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...') + pytest.xfail("AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...") # check for Ingress IP here - ingress_cmd = ["tools/bin/kubectl", "get", "-n", "default", "-o", "json", "ingress", self.path.k8s] + ingress_cmd = [ + "tools/bin/kubectl", + "get", + "-n", + "default", + "-o", + "json", + "ingress", + self.path.k8s, + ] ingress_run = subprocess.Popen(ingress_cmd, stdout=subprocess.PIPE) ingress_out, _ = ingress_run.communicate() ingress_json = json.loads(ingress_out) - assert ingress_json['status'] == self.status_update, f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" + assert ( + ingress_json["status"] == self.status_update + ), f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" class SameIngressMultipleNamespaces(AmbassadorTest): - status_update = { - "loadBalancer": { - "ingress": [{ - "ip": "210.210.210.210" - }] - } - } + status_update = {"loadBalancer": {"ingress": [{"ip": "210.210.210.210"}]}} def init(self): self.target = HTTP() @@ -308,7 +381,9 @@ def init(self): self.target2 = HTTP(name="target2", namespace="same-ingress-2") def manifests(self) -> str: - return namespace_manifest("same-ingress-1") + """ + return ( + namespace_manifest("same-ingress-1") + + """ --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -329,7 +404,9 @@ def manifests(self) -> str: number: 80 path: /{self.name}-target1/ pathType: Prefix -""" + namespace_manifest("same-ingress-2") + """ +""" + + namespace_manifest("same-ingress-2") + + """ --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -350,14 +427,25 @@ def manifests(self) -> str: number: 80 path: /{self.name}-target2/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def queries(self): - if True or sys.platform != 'darwin': + if True or sys.platform != "darwin": text = json.dumps(self.status_update) - update_cmd = [KUBESTATUS_PATH, 'Service', '-n', 'default', '-f', f'metadata.name={self.name.k8s}', '-u', '/dev/fd/0'] - subprocess.run(update_cmd, input=text.encode('utf-8'), timeout=10) + update_cmd = [ + KUBESTATUS_PATH, + "Service", + "-n", + "default", + "-f", + f"metadata.name={self.name.k8s}", + "-u", + "/dev/fd/0", + ] + subprocess.run(update_cmd, input=text.encode("utf-8"), timeout=10) # If you run these tests individually, the time between running kubestatus # and the ingress resource actually getting updated is longer than the # time spent waiting for resources to be ready, so this test will fail (most of the time) @@ -368,37 +456,45 @@ def queries(self): def check(self): if not parse_bool(os.environ.get("AMBASSADOR_PYTEST_INGRESS_TEST", "false")): - pytest.xfail('AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...') + pytest.xfail("AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...") - if False and sys.platform == 'darwin': - pytest.xfail('not supported on Darwin') + if False and sys.platform == "darwin": + pytest.xfail("not supported on Darwin") - for namespace in ['same-ingress-1', 'same-ingress-2']: + for namespace in ["same-ingress-1", "same-ingress-2"]: # check for Ingress IP here - ingress_cmd = ["tools/bin/kubectl", "get", "-n", "default", "-o", "json", "ingress", self.path.k8s, "-n", namespace] + ingress_cmd = [ + "tools/bin/kubectl", + "get", + "-n", + "default", + "-o", + "json", + "ingress", + self.path.k8s, + "-n", + namespace, + ] ingress_run = subprocess.Popen(ingress_cmd, stdout=subprocess.PIPE) ingress_out, _ = ingress_run.communicate() ingress_json = json.loads(ingress_out) - assert ingress_json['status'] == self.status_update, f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" + assert ( + ingress_json["status"] == self.status_update + ), f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" class IngressStatusTestWithIngressClass(AmbassadorTest): - status_update = { - "loadBalancer": { - "ingress": [{ - "ip": "42.42.42.42" - }] - } - } + status_update = {"loadBalancer": {"ingress": [{"ip": "42.42.42.42"}]}} def init(self): self.target = HTTP() if not is_ingress_class_compatible(): - self.xfail = 'IngressClass is not supported in this cluster' + self.xfail = "IngressClass is not supported in this cluster" def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -449,38 +545,63 @@ def manifests(self) -> str: number: 80 path: /{self.name}/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def queries(self): - if True or sys.platform != 'darwin': + if True or sys.platform != "darwin": text = json.dumps(self.status_update) - update_cmd = [KUBESTATUS_PATH, 'Service', '-n', 'default', '-f', f'metadata.name={self.name.k8s}', '-u', '/dev/fd/0'] - subprocess.run(update_cmd, input=text.encode('utf-8'), timeout=10) + update_cmd = [ + KUBESTATUS_PATH, + "Service", + "-n", + "default", + "-f", + f"metadata.name={self.name.k8s}", + "-u", + "/dev/fd/0", + ] + subprocess.run(update_cmd, input=text.encode("utf-8"), timeout=10) # If you run these tests individually, the time between running kubestatus # and the ingress resource actually getting updated is longer than the # time spent waiting for resources to be ready, so this test will fail (most of the time) time.sleep(1) yield Query(self.url(self.name + "/")) - yield Query(self.url(f'need-normalization/../{self.name}/')) + yield Query(self.url(f"need-normalization/../{self.name}/")) def check(self): if not parse_bool(os.environ.get("AMBASSADOR_PYTEST_INGRESS_TEST", "false")): - pytest.xfail('AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...') + pytest.xfail("AMBASSADOR_PYTEST_INGRESS_TEST not set, xfailing...") - if False and sys.platform == 'darwin': - pytest.xfail('not supported on Darwin') + if False and sys.platform == "darwin": + pytest.xfail("not supported on Darwin") for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" # check for Ingress IP here - ingress_cmd = ["tools/bin/kubectl", "get", "-n", "default", "-o", "json", "ingress", self.path.k8s] + ingress_cmd = [ + "tools/bin/kubectl", + "get", + "-n", + "default", + "-o", + "json", + "ingress", + self.path.k8s, + ] ingress_run = subprocess.Popen(ingress_cmd, stdout=subprocess.PIPE) ingress_out, _ = ingress_run.communicate() ingress_json = json.loads(ingress_out) - assert ingress_json['status'] == self.status_update, f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" + assert ( + ingress_json["status"] == self.status_update + ), f"Expected Ingress status to be {self.status_update}, got {ingress_json['status']} instead" diff --git a/python/tests/kat/t_ip_allow_deny.py b/python/tests/kat/t_ip_allow_deny.py index 1a911c041f..2ad9c4054d 100644 --- a/python/tests/kat/t_ip_allow_deny.py +++ b/python/tests/kat/t_ip_allow_deny.py @@ -43,7 +43,9 @@ def init(self): self.add_default_https_listener = False def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -82,10 +84,14 @@ def manifests(self) -> str: prefix: /localhost/ rewrite: /target/ # See NOTE above service: 127.0.0.1:8080 # See NOTE above -''') + super().manifests() +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(''' + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -95,7 +101,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: ip_allow: - peer: 127.0.0.1 # peer address must be localhost - remote: 99.99.0.0/16 # honors PROXY and XFF -''') +""" + ) def queries(self): # 0. Straightforward: hit /target/ and /localhost/ with nothing special, get 403s. @@ -108,15 +115,23 @@ def queries(self): # 2. Hit /target/ and /localhost/ with X-Forwarded-For specifying something bad, get a 403. yield Query(self.url("target/20"), headers={"X-Forwarded-For": "99.98.0.1"}, expected=403) - yield Query(self.url("localhost/21"), headers={"X-Forwarded-For": "99.98.0.1"}, expected=403) + yield Query( + self.url("localhost/21"), headers={"X-Forwarded-For": "99.98.0.1"}, expected=403 + ) # Done. Note that the /localhost/ endpoint is wrapping around to make a localhost call back # to Ambassador to check the peer: principal -- see the NOTE above. def requirements(self): # We're replacing super()'s requirements deliberately here. Without X-Forwarded-For they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"X-Forwarded-For": "99.99.0.1"})) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"X-Forwarded-For": "99.99.0.1"})) + yield ( + "url", + Query(self.url("ambassador/v0/check_ready"), headers={"X-Forwarded-For": "99.99.0.1"}), + ) + yield ( + "url", + Query(self.url("ambassador/v0/check_alive"), headers={"X-Forwarded-For": "99.99.0.1"}), + ) class IPDeny(AmbassadorTest): @@ -128,7 +143,9 @@ def init(self): self.add_default_https_listener = False def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -167,10 +184,14 @@ def manifests(self) -> str: prefix: /localhost/ rewrite: /target/ # See NOTE above service: 127.0.0.1:8080 # See NOTE above -''') + super().manifests() +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(''' + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -180,26 +201,37 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: ip_deny: - peer: 127.0.0.1 # peer address cannot be localhost (weird, huh?) - remote: 99.98.0.0/16 # honors PROXY and XFF -''') +""" + ) def queries(self): # 0. Straightforward: hit /target/ and /localhost/ with nothing special, get 403s. yield Query(self.url("target/00"), expected=200) - yield Query(self.url("localhost/01"), expected=403) # This should _never_ work. + yield Query(self.url("localhost/01"), expected=403) # This should _never_ work. # 1. Hit /target/ and /localhost/ with X-Forwarded-For specifying something bad, get 403s. yield Query(self.url("target/10"), headers={"X-Forwarded-For": "99.98.0.1"}, expected=403) - yield Query(self.url("localhost/11"), headers={"X-Forwarded-For": "99.98.0.1"}, expected=403) + yield Query( + self.url("localhost/11"), headers={"X-Forwarded-For": "99.98.0.1"}, expected=403 + ) # 2. Hit /target/ with X-Forwarded-For specifying something not so bad, get a 200. /localhost/ # will _still_ get a 403 though. yield Query(self.url("target/20"), headers={"X-Forwarded-For": "99.99.0.1"}, expected=200) - yield Query(self.url("localhost/21"), headers={"X-Forwarded-For": "99.99.0.1"}, expected=403) + yield Query( + self.url("localhost/21"), headers={"X-Forwarded-For": "99.99.0.1"}, expected=403 + ) # Done. Note that the /localhost/ endpoint is wrapping around to make a localhost call back # to Ambassador to check the peer: principal -- see the NOTE above. def requirements(self): # We're replacing super()'s requirements deliberately here. Without X-Forwarded-For they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"X-Forwarded-For": "99.99.0.1"})) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"X-Forwarded-For": "99.99.0.1"})) + yield ( + "url", + Query(self.url("ambassador/v0/check_ready"), headers={"X-Forwarded-For": "99.99.0.1"}), + ) + yield ( + "url", + Query(self.url("ambassador/v0/check_alive"), headers={"X-Forwarded-For": "99.99.0.1"}), + ) diff --git a/python/tests/kat/t_listeneridletimeout.py b/python/tests/kat/t_listeneridletimeout.py index 776ed39253..8aaa4ad247 100644 --- a/python/tests/kat/t_listeneridletimeout.py +++ b/python/tests/kat/t_listeneridletimeout.py @@ -4,6 +4,7 @@ from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node import json + class ListenerIdleTimeout(AmbassadorTest): target: ServiceType @@ -11,7 +12,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -19,8 +21,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: ambassador_id: [{self.ambassador_id}] config: listener_idle_timeout_ms: 30000 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -29,35 +33,51 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /config_dump rewrite: /config_dump service: http://127.0.0.1:8001 -""") +""" + ) def queries(self): yield Query(self.url("config_dump"), phase=2) def check(self): - expected_val = '30s' - actual_val = '' + expected_val = "30s" + actual_val = "" assert self.results[0].body body = json.loads(self.results[0].body) - for config_obj in body.get('configs'): - if config_obj.get('@type') == 'type.googleapis.com/envoy.admin.v3.ListenersConfigDump': - listeners = config_obj.get('dynamic_listeners') - found_idle_timeout = False - for listener_obj in listeners: - listener = listener_obj.get('active_state').get('listener') - filter_chains = listener.get('filter_chains') - for filters in filter_chains: - for filter in filters.get('filters'): - if filter.get('name') == 'envoy.filters.network.http_connection_manager': - filter_config = filter.get('typed_config') - common_http_protocol_options = filter_config.get('common_http_protocol_options') - if common_http_protocol_options: - actual_val = common_http_protocol_options.get('idle_timeout', '') - if actual_val != '': - if actual_val == expected_val: - found_idle_timeout = True - else: - assert False, "Expected to find common_http_protocol_options.idle_timeout property on listener" - else: - assert False, "Expected to find common_http_protocol_options property on listener" - assert found_idle_timeout, "Expected common_http_protocol_options.idle_timeout = {}, Got common_http_protocol_options.idle_timeout = {}".format(expected_val, actual_val) + for config_obj in body.get("configs"): + if config_obj.get("@type") == "type.googleapis.com/envoy.admin.v3.ListenersConfigDump": + listeners = config_obj.get("dynamic_listeners") + found_idle_timeout = False + for listener_obj in listeners: + listener = listener_obj.get("active_state").get("listener") + filter_chains = listener.get("filter_chains") + for filters in filter_chains: + for filter in filters.get("filters"): + if ( + filter.get("name") + == "envoy.filters.network.http_connection_manager" + ): + filter_config = filter.get("typed_config") + common_http_protocol_options = filter_config.get( + "common_http_protocol_options" + ) + if common_http_protocol_options: + actual_val = common_http_protocol_options.get( + "idle_timeout", "" + ) + if actual_val != "": + if actual_val == expected_val: + found_idle_timeout = True + else: + assert ( + False + ), "Expected to find common_http_protocol_options.idle_timeout property on listener" + else: + assert ( + False + ), "Expected to find common_http_protocol_options property on listener" + assert ( + found_idle_timeout + ), "Expected common_http_protocol_options.idle_timeout = {}, Got common_http_protocol_options.idle_timeout = {}".format( + expected_val, actual_val + ) diff --git a/python/tests/kat/t_loadbalancer.py b/python/tests/kat/t_loadbalancer.py index 1bb4376104..55eb3e29c1 100644 --- a/python/tests/kat/t_loadbalancer.py +++ b/python/tests/kat/t_loadbalancer.py @@ -29,6 +29,7 @@ value: {backend_env} """ + class LoadBalancerTest(AmbassadorTest): target: ServiceType @@ -36,7 +37,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -146,7 +148,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: policy: least_request cookie: name: test-cookie -""") +""" + ) def queries(self): yield Query(self.url(self.name + "-0/")) @@ -168,20 +171,27 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - backend = self.name.lower() + '-backend' - return \ - integration_manifests.format(LOADBALANCER_POD, - name='{}-1'.format(self.path.k8s), - backend=backend, - backend_env='{}-1'.format(self.path.k8s)) + \ - integration_manifests.format(LOADBALANCER_POD, - name='{}-2'.format(self.path.k8s), - backend=backend, - backend_env='{}-2'.format(self.path.k8s)) + \ - integration_manifests.format(LOADBALANCER_POD, - name='{}-3'.format(self.path.k8s), - backend=backend, - backend_env='{}-3'.format(self.path.k8s)) + """ + backend = self.name.lower() + "-backend" + return ( + integration_manifests.format( + LOADBALANCER_POD, + name="{}-1".format(self.path.k8s), + backend=backend, + backend_env="{}-1".format(self.path.k8s), + ) + + integration_manifests.format( + LOADBALANCER_POD, + name="{}-2".format(self.path.k8s), + backend=backend, + backend_env="{}-2".format(self.path.k8s), + ) + + integration_manifests.format( + LOADBALANCER_POD, + name="{}-3".format(self.path.k8s), + backend=backend, + backend_env="{}-3".format(self.path.k8s), + ) + + """ --- apiVersion: v1 kind: Service @@ -196,11 +206,15 @@ def manifests(self) -> str: targetPort: 8080 selector: backend: {backend} -""".format(backend=backend) + \ - super().manifests() +""".format( + backend=backend + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ apiVersion: getambassador.io/v3alpha1 kind: Module name: ambassador @@ -227,42 +241,37 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}-generic/ service: globalloadbalancing-service -""") +""" + ) def queries(self): # generic header queries for i in range(50): - yield Query(self.url(self.name) + '-header/') + yield Query(self.url(self.name) + "-header/") # header queries for i in range(50): - yield Query(self.url(self.name) + '-header/', headers={"LB-HEADER": "yes"}) + yield Query(self.url(self.name) + "-header/", headers={"LB-HEADER": "yes"}) # cookie queries for i in range(50): - yield Query(self.url(self.name) + '-header/', cookies=[ - { - 'name': 'lb-cookie', - 'value': 'yes' - } - ]) + yield Query( + self.url(self.name) + "-header/", cookies=[{"name": "lb-cookie", "value": "yes"}] + ) # generic - generic queries for i in range(50): - yield Query(self.url(self.name) + '-generic/') + yield Query(self.url(self.name) + "-generic/") # generic - header queries for i in range(50): - yield Query(self.url(self.name) + '-generic/', headers={"LB-HEADER": "yes"}) + yield Query(self.url(self.name) + "-generic/", headers={"LB-HEADER": "yes"}) # generic - cookie queries for i in range(50): - yield Query(self.url(self.name) + '-generic/', cookies=[ - { - 'name': 'lb-cookie', - 'value': 'yes' - } - ]) + yield Query( + self.url(self.name) + "-generic/", cookies=[{"name": "lb-cookie", "value": "yes"}] + ) def check(self): assert len(self.results) == 300 @@ -279,48 +288,60 @@ def check(self): generic_dict: Dict[str, int] = {} for result in generic_queries: assert result.backend - generic_dict[result.backend.name] = \ + generic_dict[result.backend.name] = ( generic_dict[result.backend.name] + 1 if result.backend.name in generic_dict else 1 + ) assert len(generic_dict) == 3 # header queries - no cookie - no sticky expected header_dict: Dict[str, int] = {} for result in header_queries: assert result.backend - header_dict[result.backend.name] = \ + header_dict[result.backend.name] = ( header_dict[result.backend.name] + 1 if result.backend.name in header_dict else 1 + ) assert len(header_dict) == 3 # cookie queries - no headers - sticky expected cookie_dict: Dict[str, int] = {} for result in cookie_queries: assert result.backend - cookie_dict[result.backend.name] = \ + cookie_dict[result.backend.name] = ( cookie_dict[result.backend.name] + 1 if result.backend.name in cookie_dict else 1 + ) assert len(cookie_dict) == 1 # generic header queries - no cookie, no header generic_generic_dict: Dict[str, int] = {} for result in generic_generic_queries: assert result.backend - generic_generic_dict[result.backend.name] = \ - generic_generic_dict[result.backend.name] + 1 if result.backend.name in generic_generic_dict else 1 + generic_generic_dict[result.backend.name] = ( + generic_generic_dict[result.backend.name] + 1 + if result.backend.name in generic_generic_dict + else 1 + ) assert len(generic_generic_dict) == 3 # header queries - no cookie - sticky expected generic_header_dict: Dict[str, int] = {} for result in generic_header_queries: assert result.backend - generic_header_dict[result.backend.name] = \ - generic_header_dict[result.backend.name] + 1 if result.backend.name in generic_header_dict else 1 + generic_header_dict[result.backend.name] = ( + generic_header_dict[result.backend.name] + 1 + if result.backend.name in generic_header_dict + else 1 + ) assert len(generic_header_dict) == 1 # cookie queries - no headers - no sticky expected generic_cookie_dict: Dict[str, int] = {} for result in generic_cookie_queries: assert result.backend - generic_cookie_dict[result.backend.name] = \ - generic_cookie_dict[result.backend.name] + 1 if result.backend.name in generic_cookie_dict else 1 + generic_cookie_dict[result.backend.name] = ( + generic_cookie_dict[result.backend.name] + 1 + if result.backend.name in generic_cookie_dict + else 1 + ) assert len(generic_cookie_dict) == 3 @@ -332,20 +353,27 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - backend = self.name.lower() + '-backend' - return \ - integration_manifests.format(LOADBALANCER_POD, - name='{}-1'.format(self.path.k8s), - backend=backend, - backend_env='{}-1'.format(self.path.k8s)) + \ - integration_manifests.format(LOADBALANCER_POD, - name='{}-2'.format(self.path.k8s), - backend=backend, - backend_env='{}-2'.format(self.path.k8s)) + \ - integration_manifests.format(LOADBALANCER_POD, - name='{}-3'.format(self.path.k8s), - backend=backend, - backend_env='{}-3'.format(self.path.k8s)) + """ + backend = self.name.lower() + "-backend" + return ( + integration_manifests.format( + LOADBALANCER_POD, + name="{}-1".format(self.path.k8s), + backend=backend, + backend_env="{}-1".format(self.path.k8s), + ) + + integration_manifests.format( + LOADBALANCER_POD, + name="{}-2".format(self.path.k8s), + backend=backend, + backend_env="{}-2".format(self.path.k8s), + ) + + integration_manifests.format( + LOADBALANCER_POD, + name="{}-3".format(self.path.k8s), + backend=backend, + backend_env="{}-3".format(self.path.k8s), + ) + + """ --- apiVersion: v1 kind: Service @@ -360,13 +388,17 @@ def manifests(self) -> str: targetPort: 8080 selector: backend: {backend} -""".format(backend=backend) + \ - super().manifests() +""".format( + backend=backend + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - for policy in ['ring_hash', 'maglev']: + for policy in ["ring_hash", "maglev"]: self.policy = policy - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -415,110 +447,127 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: policy: {self.policy} cookie: name: lb-cookie -""") +""" + ) def queries(self): - for policy in ['ring_hash', 'maglev']: + for policy in ["ring_hash", "maglev"]: # generic header queries for i in range(50): - yield Query(self.url(self.name) + '-header-{}/'.format(policy)) + yield Query(self.url(self.name) + "-header-{}/".format(policy)) # header queries for i in range(50): - yield Query(self.url(self.name) + '-header-{}/'.format(policy), headers={"LB-HEADER": "yes"}) + yield Query( + self.url(self.name) + "-header-{}/".format(policy), headers={"LB-HEADER": "yes"} + ) # source IP queries for i in range(50): - yield Query(self.url(self.name) + '-sourceip-{}/'.format(policy)) + yield Query(self.url(self.name) + "-sourceip-{}/".format(policy)) # generic cookie queries for i in range(50): - yield Query(self.url(self.name) + '-cookie-{}/'.format(policy)) + yield Query(self.url(self.name) + "-cookie-{}/".format(policy)) # cookie queries for i in range(50): - yield Query(self.url(self.name) + '-cookie-{}/'.format(policy), cookies=[ - { - 'name': 'lb-cookie', - 'value': 'yes' - } - ]) + yield Query( + self.url(self.name) + "-cookie-{}/".format(policy), + cookies=[{"name": "lb-cookie", "value": "yes"}], + ) # cookie no TTL queries for i in range(50): - yield Query(self.url(self.name) + '-cookie-no-ttl-{}/'.format(policy), cookies=[ - { - 'name': 'lb-cookie', - 'value': 'yes' - } - ]) + yield Query( + self.url(self.name) + "-cookie-no-ttl-{}/".format(policy), + cookies=[{"name": "lb-cookie", "value": "yes"}], + ) def check(self): assert len(self.results) == 600 for i in [0, 300]: - generic_header_queries = self.results[0+i:50+i] - header_queries = self.results[50+i:100+i] - source_ip_queries = self.results[100+i:150+i] - generic_cookie_queries = self.results[150+i:200+i] - cookie_queries = self.results[200+i:250+i] - cookie_no_ttl_queries = self.results[250+i:300+i] + generic_header_queries = self.results[0 + i : 50 + i] + header_queries = self.results[50 + i : 100 + i] + source_ip_queries = self.results[100 + i : 150 + i] + generic_cookie_queries = self.results[150 + i : 200 + i] + cookie_queries = self.results[200 + i : 250 + i] + cookie_no_ttl_queries = self.results[250 + i : 300 + i] # generic header queries generic_header_dict: Dict[str, int] = {} for result in generic_header_queries: assert result.backend - generic_header_dict[result.backend.name] =\ - generic_header_dict[result.backend.name] + 1 if result.backend.name in generic_header_dict else 1 + generic_header_dict[result.backend.name] = ( + generic_header_dict[result.backend.name] + 1 + if result.backend.name in generic_header_dict + else 1 + ) assert len(generic_header_dict) == 3 # header queries header_dict: Dict[str, int] = {} for result in header_queries: assert result.backend - header_dict[result.backend.name] = \ - header_dict[result.backend.name] + 1 if result.backend.name in header_dict else 1 + header_dict[result.backend.name] = ( + header_dict[result.backend.name] + 1 + if result.backend.name in header_dict + else 1 + ) assert len(header_dict) == 1 # source IP queries source_ip_dict: Dict[str, int] = {} for result in source_ip_queries: assert result.backend - source_ip_dict[result.backend.name] = \ - source_ip_dict[result.backend.name] + 1 if result.backend.name in source_ip_dict else 1 + source_ip_dict[result.backend.name] = ( + source_ip_dict[result.backend.name] + 1 + if result.backend.name in source_ip_dict + else 1 + ) assert len(source_ip_dict) == 1 assert list(source_ip_dict.values())[0] == 50 # generic cookie queries - results must include Set-Cookie header generic_cookie_dict: Dict[str, int] = {} for result in generic_cookie_queries: - assert 'Set-Cookie' in result.headers - assert len(result.headers['Set-Cookie']) == 1 - assert 'lb-cookie=' in result.headers['Set-Cookie'][0] - assert 'Max-Age=125' in result.headers['Set-Cookie'][0] - assert 'Path=/foo' in result.headers['Set-Cookie'][0] + assert "Set-Cookie" in result.headers + assert len(result.headers["Set-Cookie"]) == 1 + assert "lb-cookie=" in result.headers["Set-Cookie"][0] + assert "Max-Age=125" in result.headers["Set-Cookie"][0] + assert "Path=/foo" in result.headers["Set-Cookie"][0] assert result.backend - generic_cookie_dict[result.backend.name] = \ - generic_cookie_dict[result.backend.name] + 1 if result.backend.name in generic_cookie_dict else 1 + generic_cookie_dict[result.backend.name] = ( + generic_cookie_dict[result.backend.name] + 1 + if result.backend.name in generic_cookie_dict + else 1 + ) assert len(generic_cookie_dict) == 3 # cookie queries cookie_dict: Dict[str, int] = {} for result in cookie_queries: - assert 'Set-Cookie' not in result.headers + assert "Set-Cookie" not in result.headers assert result.backend - cookie_dict[result.backend.name] = \ - cookie_dict[result.backend.name] + 1 if result.backend.name in cookie_dict else 1 + cookie_dict[result.backend.name] = ( + cookie_dict[result.backend.name] + 1 + if result.backend.name in cookie_dict + else 1 + ) assert len(cookie_dict) == 1 # cookie no TTL queries cookie_no_ttl_dict: Dict[str, int] = {} for result in cookie_no_ttl_queries: - assert 'Set-Cookie' not in result.headers + assert "Set-Cookie" not in result.headers assert result.backend - cookie_no_ttl_dict[result.backend.name] = \ - cookie_no_ttl_dict[result.backend.name] + 1 if result.backend.name in cookie_no_ttl_dict else 1 + cookie_no_ttl_dict[result.backend.name] = ( + cookie_no_ttl_dict[result.backend.name] + 1 + if result.backend.name in cookie_no_ttl_dict + else 1 + ) assert len(cookie_no_ttl_dict) == 1 diff --git a/python/tests/kat/t_logservice.py b/python/tests/kat/t_logservice.py index 1d1a423ed4..b2052c92f8 100644 --- a/python/tests/kat/t_logservice.py +++ b/python/tests/kat/t_logservice.py @@ -11,23 +11,26 @@ class LogServiceTest(AmbassadorTest): target: ServiceType - specified_protocol_version: Literal['v2', 'v3', 'default'] - expected_protocol_version: Literal['v3', 'invalid'] + specified_protocol_version: Literal["v2", "v3", "default"] + expected_protocol_version: Literal["v3", "invalid"] als: ServiceType @classmethod def variants(cls) -> Generator[Node, None, None]: - for protocol_version in ['v2', 'v3', 'default']: + for protocol_version in ["v2", "v3", "default"]: yield cls(protocol_version, name="{self.specified_protocol_version}") - def init(self, protocol_version: Literal['v3', 'default']): + def init(self, protocol_version: Literal["v3", "default"]): self.target = HTTP() self.specified_protocol_version = protocol_version - self.expected_protocol_version = cast(Literal['v3', 'invalid'], protocol_version if protocol_version in ['v3'] else 'invalid') + self.expected_protocol_version = cast( + Literal["v3", "invalid"], protocol_version if protocol_version in ["v3"] else "invalid" + ) self.als = ALSGRPC() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -49,8 +52,14 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: during_request: false flush_interval_time: 1 flush_interval_byte_size: 1 -""") + ("" if self.specified_protocol_version == "default" else f"protocol_version: '{self.specified_protocol_version}'") - yield self, self.format(""" +""" + ) + ( + "" + if self.specified_protocol_version == "default" + else f"protocol_version: '{self.specified_protocol_version}'" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -58,10 +67,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): - yield Query(f"http://{self.als.path.fqdn}/logs", method='DELETE', phase=1) + yield Query(f"http://{self.als.path.fqdn}/logs", method="DELETE", phase=1) yield Query(self.url("target/foo"), phase=2) yield Query(self.url("target/bar"), phase=3) yield Query(f"http://{self.als.path.fqdn}/logs", phase=4) @@ -69,19 +79,19 @@ def queries(self): def check(self): logs = self.results[3].json expkey = f"als{self.expected_protocol_version}-http" - for key in ['alsv2-http', 'alsv2-tcp', 'alsv3-http', 'alsv3-tcp']: + for key in ["alsv2-http", "alsv2-tcp", "alsv3-http", "alsv3-tcp"]: if key == expkey: continue assert not logs[key] - if self.expected_protocol_version == 'invalid': + if self.expected_protocol_version == "invalid": assert expkey not in logs return assert logs[expkey] assert len(logs[expkey]) == 2 - assert logs[expkey][0]['request']['original_path'] == '/target/foo' - assert logs[expkey][1]['request']['original_path'] == '/target/bar' + assert logs[expkey][0]["request"]["original_path"] == "/target/foo" + assert logs[expkey][1]["request"]["original_path"] == "/target/bar" class LogServiceLongServiceNameTest(AmbassadorTest): @@ -93,7 +103,9 @@ def init(self): self.als = ALSGRPC() def manifests(self) -> str: - return self.format(""" + return ( + self.format( + """ --- kind: Service apiVersion: v1 @@ -111,10 +123,14 @@ def manifests(self) -> str: protocol: TCP port: 443 targetPort: 8443 -""") + super().manifests() +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -137,8 +153,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: during_request: false flush_interval_time: 1 flush_interval_byte_size: 1 - """) - yield self, self.format(""" + """ + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -146,22 +164,22 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): - yield Query(f"http://{self.als.path.fqdn}/logs", method='DELETE', phase=1) + yield Query(f"http://{self.als.path.fqdn}/logs", method="DELETE", phase=1) yield Query(self.url("target/foo"), phase=2) yield Query(self.url("target/bar"), phase=3) yield Query(f"http://{self.als.path.fqdn}/logs", phase=4) def check(self): logs = self.results[3].json - assert not logs['alsv2-http'] - assert not logs['alsv2-tcp'] - assert logs['alsv3-http'] - assert not logs['alsv3-tcp'] - - - assert len(logs['alsv3-http']) == 2 - assert logs['alsv3-http'][0]['request']['original_path'] == '/target/foo' - assert logs['alsv3-http'][1]['request']['original_path'] == '/target/bar' + assert not logs["alsv2-http"] + assert not logs["alsv2-tcp"] + assert logs["alsv3-http"] + assert not logs["alsv3-tcp"] + + assert len(logs["alsv3-http"]) == 2 + assert logs["alsv3-http"][0]["request"]["original_path"] == "/target/foo" + assert logs["alsv3-http"][1]["request"]["original_path"] == "/target/bar" diff --git a/python/tests/kat/t_lua_scripts.py b/python/tests/kat/t_lua_scripts.py index b1897bb5ec..f3557a9ad6 100644 --- a/python/tests/kat/t_lua_scripts.py +++ b/python/tests/kat/t_lua_scripts.py @@ -14,7 +14,9 @@ def init(self): """ def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -37,11 +39,14 @@ def manifests(self) -> str: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def queries(self): yield Query(self.url("target/")) def check(self): for r in self.results: - assert r.headers.get('Lua-Scripts-Enabled', None) == ['Processed'] + assert r.headers.get("Lua-Scripts-Enabled", None) == ["Processed"] diff --git a/python/tests/kat/t_mappingtests_default.py b/python/tests/kat/t_mappingtests_default.py index 045d06e425..e3711f63dd 100644 --- a/python/tests/kat/t_mappingtests_default.py +++ b/python/tests/kat/t_mappingtests_default.py @@ -20,7 +20,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -34,7 +35,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: http://{self.target.path.fqdn} host: myhostname.com -""") +""" + ) def queries(self): # Sanity test that a missing or incorrect hostname does not route, and it does route with a correct hostname. @@ -43,10 +45,15 @@ def queries(self): yield Query(self.url(self.name + "/"), headers={"Host": "myhostname.com"}) # Test that a host header with a port value that does match the listener's configured port is correctly # stripped for the purpose of routing, and matches the mapping. - yield Query(self.url(self.name + "/"), headers={"Host": "myhostname.com:" + str(Constants.SERVICE_PORT_HTTP)}) + yield Query( + self.url(self.name + "/"), + headers={"Host": "myhostname.com:" + str(Constants.SERVICE_PORT_HTTP)}, + ) # Test that a host header with a port value that does _not_ match the listener's configured does not have its # port value stripped for the purpose of routing, so it does not match the mapping. - yield Query(self.url(self.name + "/"), headers={"Host": "myhostname.com:11875"}, expected=404) + yield Query( + self.url(self.name + "/"), headers={"Host": "myhostname.com:11875"}, expected=404 + ) # This has to be an `AmbassadorTest` because we're going to set up a Module that @@ -59,7 +66,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -68,7 +76,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/status/ rewrite: /status/ service: httpbin.default -""") +""" + ) def queries(self): yield Query(self.url(self.name + "/status/200")) @@ -89,7 +98,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -104,7 +114,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/status/ rewrite: /status/ service: httpbin.default -""") +""" + ) def queries(self): yield Query(self.url(self.name + "/status/200")) @@ -114,6 +125,7 @@ def queries(self): yield Query(self.url("/" + self.name + "//status/200")) yield Query(self.url(self.name + "//status/200")) + # This has to be an `AmbassadorTest` because we're going to set up a Module that # needs to apply to just this test. If this were a MappingTest, then the Module # would apply to all other MappingTest's and we don't want that. @@ -124,7 +136,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -133,7 +146,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/status/ rewrite: /status/ service: httpbin.default -""") +""" + ) def queries(self): # Sanity test that escaped slashes are not rejected by default. The upstream @@ -143,8 +157,8 @@ def queries(self): def check(self): # We should have observed this 404 upstream from httpbin. The presence of this header verifies that. - print ("headers=%s", repr(self.results[0].headers)) - assert 'X-Envoy-Upstream-Service-Time' in self.results[0].headers + print("headers=%s", repr(self.results[0].headers)) + assert "X-Envoy-Upstream-Service-Time" in self.results[0].headers # This has to be an `AmbassadorTest` because we're going to set up a Module that @@ -157,7 +171,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -172,7 +187,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/status/ rewrite: /status/ service: httpbin -""") +""" + ) def queries(self): # Expect that requests with escaped slashes are rejected by Envoy. We know this is rejected @@ -183,7 +199,8 @@ def queries(self): def check(self): # We should have not have observed this 400 upstream from httpbin. The absence of this header # suggests that (though does not prove, in theory). - assert 'X-Envoy-Upstream-Service-Time' not in self.results[0].headers + assert "X-Envoy-Upstream-Service-Time" not in self.results[0].headers + class LinkerdHeaderMapping(AmbassadorTest): target: ServiceType @@ -194,7 +211,8 @@ def init(self): self.target_add_linkerd_header_only = HTTP(name="addlinkerdonly") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -239,49 +257,66 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: value: banana remove_request_headers: - x-evilness -""") +""" + ) def queries(self): # [0] expect Linkerd headers set through mapping - yield Query(self.url("target/"), headers={ "x-evil-header": "evilness", "x-evilness": "more evilness" }, expected=200) + yield Query( + self.url("target/"), + headers={"x-evil-header": "evilness", "x-evilness": "more evilness"}, + expected=200, + ) # [1] expect no Linkerd headers - yield Query(self.url("target_no_header/"), headers={ "x-evil-header": "evilness", "x-evilness": "more evilness" }, expected=200) + yield Query( + self.url("target_no_header/"), + headers={"x-evil-header": "evilness", "x-evilness": "more evilness"}, + expected=200, + ) # [2] expect Linkerd headers only - yield Query(self.url("target_add_linkerd_header_only/"), headers={ "x-evil-header": "evilness", "x-evilness": "more evilness" }, expected=200) + yield Query( + self.url("target_add_linkerd_header_only/"), + headers={"x-evil-header": "evilness", "x-evilness": "more evilness"}, + expected=200, + ) def check(self): # [0] assert self.results[0].backend assert self.results[0].backend.request - assert len(self.results[0].backend.request.headers['l5d-dst-override']) > 0 - assert self.results[0].backend.request.headers['l5d-dst-override'] == ["{}:80".format(self.target.path.fqdn)] - assert len(self.results[0].backend.request.headers['fruit']) > 0 - assert self.results[0].backend.request.headers['fruit'] == [ 'banana'] - assert len(self.results[0].backend.request.headers['x-evil-header']) > 0 - assert self.results[0].backend.request.headers['x-evil-header'] == [ 'evilness' ] - assert 'x-evilness' not in self.results[0].backend.request.headers + assert len(self.results[0].backend.request.headers["l5d-dst-override"]) > 0 + assert self.results[0].backend.request.headers["l5d-dst-override"] == [ + "{}:80".format(self.target.path.fqdn) + ] + assert len(self.results[0].backend.request.headers["fruit"]) > 0 + assert self.results[0].backend.request.headers["fruit"] == ["banana"] + assert len(self.results[0].backend.request.headers["x-evil-header"]) > 0 + assert self.results[0].backend.request.headers["x-evil-header"] == ["evilness"] + assert "x-evilness" not in self.results[0].backend.request.headers # [1] assert self.results[1].backend assert self.results[1].backend.request - assert 'l5d-dst-override' not in self.results[1].backend.request.headers - assert len(self.results[1].backend.request.headers['fruit']) > 0 - assert self.results[1].backend.request.headers['fruit'] == [ 'orange'] - assert 'x-evil-header' not in self.results[1].backend.request.headers - assert len(self.results[1].backend.request.headers['x-evilness']) > 0 - assert self.results[1].backend.request.headers['x-evilness'] == [ 'more evilness' ] + assert "l5d-dst-override" not in self.results[1].backend.request.headers + assert len(self.results[1].backend.request.headers["fruit"]) > 0 + assert self.results[1].backend.request.headers["fruit"] == ["orange"] + assert "x-evil-header" not in self.results[1].backend.request.headers + assert len(self.results[1].backend.request.headers["x-evilness"]) > 0 + assert self.results[1].backend.request.headers["x-evilness"] == ["more evilness"] # [2] assert self.results[2].backend assert self.results[2].backend.request - assert len(self.results[2].backend.request.headers['l5d-dst-override']) > 0 - assert self.results[2].backend.request.headers['l5d-dst-override'] == ["{}:80".format(self.target_add_linkerd_header_only.path.fqdn)] - assert len(self.results[2].backend.request.headers['x-evil-header']) > 0 - assert self.results[2].backend.request.headers['x-evil-header'] == [ 'evilness' ] - assert len(self.results[2].backend.request.headers['x-evilness']) > 0 - assert self.results[2].backend.request.headers['x-evilness'] == [ 'more evilness' ] + assert len(self.results[2].backend.request.headers["l5d-dst-override"]) > 0 + assert self.results[2].backend.request.headers["l5d-dst-override"] == [ + "{}:80".format(self.target_add_linkerd_header_only.path.fqdn) + ] + assert len(self.results[2].backend.request.headers["x-evil-header"]) > 0 + assert self.results[2].backend.request.headers["x-evil-header"] == ["evilness"] + assert len(self.results[2].backend.request.headers["x-evilness"]) > 0 + assert self.results[2].backend.request.headers["x-evilness"] == ["more evilness"] class SameMappingDifferentNamespaces(AmbassadorTest): @@ -291,9 +326,11 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return namespace_manifest('same-mapping-1') + \ - namespace_manifest('same-mapping-2') + \ - self.format(''' + return ( + namespace_manifest("same-mapping-1") + + namespace_manifest("same-mapping-2") + + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -316,7 +353,10 @@ def manifests(self) -> str: hostname: "*" prefix: /{self.name}-2/ service: {self.target.path.fqdn}.default -''') + super().manifests() +""" + ) + + super().manifests() + ) def queries(self): yield Query(self.url(self.name + "-1/")) @@ -330,7 +370,9 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: v1 kind: Service @@ -349,7 +391,10 @@ def manifests(self) -> str: hostname: "*" prefix: /{self.name}-1/ service: thisisaverylongservicenameoverwithsixythreecharacters123456789 -''') + super().manifests() +""" + ) + + super().manifests() + ) def queries(self): yield Query(self.url(self.name + "-1/")) diff --git a/python/tests/kat/t_mappingtests_plain.py b/python/tests/kat/t_mappingtests_plain.py index 490c82e290..538623d301 100644 --- a/python/tests/kat/t_mappingtests_plain.py +++ b/python/tests/kat/t_mappingtests_plain.py @@ -33,11 +33,15 @@ def variants(cls) -> Generator[Node, None, None]: for mot in variants(OptionTest): yield cls(st, (mot,), name="{self.target.name}-{self.options[0].name}") - yield cls(st, unique(v for v in variants(OptionTest) - if not getattr(v, "isolated", False)), name="{self.target.name}-all") + yield cls( + st, + unique(v for v in variants(OptionTest) if not getattr(v, "isolated", False)), + name="{self.target.name}-all", + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -45,18 +49,22 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.parent.url(self.name + "/")) - yield Query(self.parent.url(f'need-normalization/../{self.name}/')) + yield Query(self.parent.url(f"need-normalization/../{self.name}/")) def check(self): for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" class SimpleMappingIngress(MappingTest): @@ -92,15 +100,21 @@ def manifests(self) -> str: """ def queries(self): - yield Query(self.parent.url(self.name + "/")) # , xfail="IHA hostglob") - yield Query(self.parent.url(f'need-normalization/../{self.name}/')) # , xfail="IHA hostglob") + yield Query(self.parent.url(self.name + "/")) # , xfail="IHA hostglob") + yield Query( + self.parent.url(f"need-normalization/../{self.name}/") + ) # , xfail="IHA hostglob") def check(self): for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] == f'/{self.name}/' + assert r.backend.request.headers["x-envoy-original-path"][0] == f"/{self.name}/" + # Disabled SimpleMappingIngressDefaultBackend since adding a default fallback mapping would break other # assertions, expecting to 404 if mappings don't match in Plain. @@ -183,17 +197,27 @@ def manifests(self) -> str: """ def queries(self): - yield Query(self.parent.url(self.name + "/")) # , xfail="IHA hostglob") - yield Query(self.parent.url(f'need-normalization/../{self.name}/')) # , xfail="IHA hostglob") - yield Query(self.parent.url(self.name + "-nested/")) # , xfail="IHA hostglob") - yield Query(self.parent.url(self.name + "-non-existent/"), expected=404) # , xfail="IHA hostglob") + yield Query(self.parent.url(self.name + "/")) # , xfail="IHA hostglob") + yield Query( + self.parent.url(f"need-normalization/../{self.name}/") + ) # , xfail="IHA hostglob") + yield Query(self.parent.url(self.name + "-nested/")) # , xfail="IHA hostglob") + yield Query( + self.parent.url(self.name + "-non-existent/"), expected=404 + ) # , xfail="IHA hostglob") def check(self): for r in self.results: if r.backend: - assert r.backend.name == self.target.path.k8s, (r.backend.name, self.target.path.k8s) + assert r.backend.name == self.target.path.k8s, ( + r.backend.name, + self.target.path.k8s, + ) assert r.backend.request - assert r.backend.request.headers['x-envoy-original-path'][0] in (f'/{self.name}/', f'/{self.name}-nested/') + assert r.backend.request.headers["x-envoy-original-path"][0] in ( + f"/{self.name}/", + f"/{self.name}-nested/", + ) class HostHeaderMappingIngress(MappingTest): @@ -230,7 +254,9 @@ def manifests(self) -> str: def queries(self): yield Query(self.parent.url(self.name + "/"), expected=404) - yield Query(self.parent.url(self.name + "/"), headers={"Host": "inspector.internal"}, expected=404) + yield Query( + self.parent.url(self.name + "/"), headers={"Host": "inspector.internal"}, expected=404 + ) yield Query(self.parent.url(self.name + "/"), headers={"Host": "inspector.external"}) @@ -244,7 +270,8 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(st, name="{self.target.name}") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -252,16 +279,24 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: http://{self.target.path.fqdn} host: inspector.external -""") +""" + ) def queries(self): yield Query(self.parent.url(self.name + "/"), expected=404) - yield Query(self.parent.url(self.name + "/"), headers={"Host": "inspector.internal"}, expected=404) + yield Query( + self.parent.url(self.name + "/"), headers={"Host": "inspector.internal"}, expected=404 + ) yield Query(self.parent.url(self.name + "/"), headers={"Host": "inspector.external"}) # Test that a host header with a port value that does match the listener's configured port is not # stripped for the purpose of routing, so it does not match the Mapping. This is the default behavior, # and can be overridden using `strip_matching_host_port`, tested below. - yield Query(self.parent.url(self.name + "/"), headers={"Host": "inspector.external:" + str(Constants.SERVICE_PORT_HTTP)}, expected=404) + yield Query( + self.parent.url(self.name + "/"), + headers={"Host": "inspector.external:" + str(Constants.SERVICE_PORT_HTTP)}, + expected=404, + ) + class InvalidPortMapping(MappingTest): @@ -273,7 +308,8 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(st, name="{self.target.name}") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -281,13 +317,14 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}/ service: http://{self.target.path.fqdn}:80.invalid -""") +""" + ) def queries(self): yield Query(self.parent.url("ambassador/v0/diag/?json=true&filter=errors")) def check(self): - error_string = 'found invalid port for service' + error_string = "found invalid port for service" found_error = False for error_list in self.results[0].json: for error in error_list: @@ -306,7 +343,8 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(st, name="{self.target.name}") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -315,7 +353,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: websocket-echo-server.plain-namespace use_websocket: true -""") +""" + ) def queries(self): yield Query(self.parent.url(self.name + "/"), expected=404) @@ -323,7 +362,9 @@ def queries(self): yield Query(self.parent.url(self.name + "/", scheme="ws"), messages=["one", "two", "three"]) def check(self): - assert self.results[-1].messages == ["one", "two", "three"], "invalid messages: %s" % repr(self.results[-1].messages) + assert self.results[-1].messages == ["one", "two", "three"], "invalid messages: %s" % repr( + self.results[-1].messages + ) class TLSOrigination(MappingTest): @@ -350,6 +391,7 @@ class TLSOrigination(MappingTest): prefix: /{self.name}/ service: https://{self.target.path.fqdn} """ + @classmethod def variants(cls) -> Generator[Node, None, None]: for v in variants(ServiceType): @@ -381,7 +423,8 @@ def init(self): MappingTest.init(self, HTTP()) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -432,7 +475,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: pattern: /{self.name}-5/assets/([a-f0-9]{{12}})/images substitution: /images/\\1 redirect_response_code: 308 -""") +""" + ) def queries(self): # [0] @@ -457,39 +501,48 @@ def queries(self): yield Query(self.parent.url(self.name + "-5/assets/abcd0000f123/images"), expected=308) # [7] - yield Query(self.parent.url(self.name + "-5/assets/abcd0000f123/images?itworked=true"), expected=308) + yield Query( + self.parent.url(self.name + "-5/assets/abcd0000f123/images?itworked=true"), expected=308 + ) def check(self): # [0] - assert self.results[0].headers['Location'] == [self.format("http://foobar.com/{self.name}/anything?itworked=true")], \ - f"Unexpected Location {self.results[0].headers['Location']}" + assert self.results[0].headers["Location"] == [ + self.format("http://foobar.com/{self.name}/anything?itworked=true") + ], f"Unexpected Location {self.results[0].headers['Location']}" # [1] assert self.results[1].status == 404 # [2] - assert self.results[2].headers['Location'] == [self.format("http://foobar.com/{self.name}-2/anything?itworked=true")], \ - f"Unexpected Location {self.results[2].headers['Location']}" + assert self.results[2].headers["Location"] == [ + self.format("http://foobar.com/{self.name}-2/anything?itworked=true") + ], f"Unexpected Location {self.results[2].headers['Location']}" # [3] - assert self.results[3].headers['Location'] == [self.format("http://foobar.com/" + self.name.upper() + "-2/anything?itworked=true")], \ - f"Unexpected Location {self.results[3].headers['Location']}" + assert self.results[3].headers["Location"] == [ + self.format("http://foobar.com/" + self.name.upper() + "-2/anything?itworked=true") + ], f"Unexpected Location {self.results[3].headers['Location']}" # [4] - assert self.results[4].headers['Location'] == [self.format("http://foobar.com/redirect/")], \ - f"Unexpected Location {self.results[4].headers['Location']}" + assert self.results[4].headers["Location"] == [ + self.format("http://foobar.com/redirect/") + ], f"Unexpected Location {self.results[4].headers['Location']}" # [5] - assert self.results[5].headers['Location'] == [self.format("http://foobar.com/foobar/baz/anything")], \ - f"Unexpected Location {self.results[5].headers['Location']}" + assert self.results[5].headers["Location"] == [ + self.format("http://foobar.com/foobar/baz/anything") + ], f"Unexpected Location {self.results[5].headers['Location']}" # [6] - assert self.results[6].headers['Location'] == [self.format("http://foobar.com/images/abcd0000f123")], \ - f"Unexpected Location {self.results[6].headers['Location']}" + assert self.results[6].headers["Location"] == [ + self.format("http://foobar.com/images/abcd0000f123") + ], f"Unexpected Location {self.results[6].headers['Location']}" # [7] - assert self.results[7].headers['Location'] == [self.format("http://foobar.com/images/abcd0000f123?itworked=true")], \ - f"Unexpected Location {self.results[7].headers['Location']}" + assert self.results[7].headers["Location"] == [ + self.format("http://foobar.com/images/abcd0000f123?itworked=true") + ], f"Unexpected Location {self.results[7].headers['Location']}" class CanaryMapping(MappingTest): @@ -509,13 +562,14 @@ def variants(cls) -> Generator[Node, None, None]: # parent's init to have a different signature... but it's also intimately # (nay, incestuously) related to the variant()'s yield() above, and I really # don't want to deal with that right now. So. We'll deal with it later. - def init(self, target: ServiceType, canary: ServiceType, weight): # type: ignore + def init(self, target: ServiceType, canary: ServiceType, weight): # type: ignore MappingTest.init(self, target) self.canary = canary self.weight = weight def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -523,8 +577,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}/ service: http://{self.target.path.fqdn} -""") - yield self.canary, self.format(""" +""" + ) + yield self.canary, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -533,7 +589,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: http://{self.canary.path.fqdn} weight: {self.weight} -""") +""" + ) def queries(self): for i in range(100): @@ -553,11 +610,15 @@ def check(self): assert hist.get(self.canary.path.k8s, 0) == 100 assert hist.get(self.target.path.k8s, 0) == 0 else: - canary = 100*hist.get(self.canary.path.k8s, 0)/len(self.results) - main = 100*hist.get(self.target.path.k8s, 0)/len(self.results) + canary = 100 * hist.get(self.canary.path.k8s, 0) / len(self.results) + main = 100 * hist.get(self.target.path.k8s, 0) / len(self.results) - assert abs(self.weight - canary) < 25, f'weight {self.weight} routed {canary}% to canary' - assert abs(100 - (canary + main)) < 2, f'weight {self.weight} routed only {canary + main}% at all?' + assert ( + abs(self.weight - canary) < 25 + ), f"weight {self.weight} routed {canary}% to canary" + assert ( + abs(100 - (canary + main)) < 2 + ), f"weight {self.weight} routed only {canary + main}% at all?" class CanaryDiffMapping(MappingTest): @@ -577,13 +638,14 @@ def variants(cls) -> Generator[Node, None, None]: # parent's init to have a different signature... but it's also intimately # (nay, incestuously) related to the variant()'s yield() above, and I really # don't want to deal with that right now. So. We'll deal with it later. - def init(self, target: ServiceType, canary: ServiceType, weight): # type: ignore + def init(self, target: ServiceType, canary: ServiceType, weight): # type: ignore MappingTest.init(self, target) self.canary = canary self.weight = weight def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -592,8 +654,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: http://{self.target.path.fqdn} host_rewrite: canary.1.example.com -""") - yield self.canary, self.format(""" +""" + ) + yield self.canary, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -603,14 +667,15 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: http://{self.canary.path.fqdn} host_rewrite: canary.2.example.com weight: {self.weight} -""") +""" + ) def queries(self): for i in range(100): yield Query(self.parent.url(self.name + "/")) def check(self): - request_hosts = ['canary.1.example.com', 'canary.2.example.com'] + request_hosts = ["canary.1.example.com", "canary.2.example.com"] hist: Dict[str, int] = {} @@ -618,7 +683,9 @@ def check(self): assert r.backend hist[r.backend.name] = hist.get(r.backend.name, 0) + 1 assert r.backend.request - assert r.backend.request.host in request_hosts, f'Expected host {request_hosts}, got {r.backend.request.host}' + assert ( + r.backend.request.host in request_hosts + ), f"Expected host {request_hosts}, got {r.backend.request.host}" if self.weight == 0: assert hist.get(self.canary.path.k8s, 0) == 0 @@ -630,8 +697,12 @@ def check(self): canary = 100 * hist.get(self.canary.path.k8s, 0) / len(self.results) main = 100 * hist.get(self.target.path.k8s, 0) / len(self.results) - assert abs(self.weight - canary) < 25, f'weight {self.weight} routed {canary}% to canary' - assert abs(100 - (canary + main)) < 2, f'weight {self.weight} routed only {canary + main}% at all?' + assert ( + abs(self.weight - canary) < 25 + ), f"weight {self.weight} routed {canary}% to canary" + assert ( + abs(100 - (canary + main)) < 2 + ), f"weight {self.weight} routed only {canary + main}% at all?" class AddRespHeadersMapping(MappingTest): @@ -644,7 +715,8 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(st, name="{self.target.name}") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -663,19 +735,21 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: value: boo foo: value: Foo -""") +""" + ) def queries(self): - yield Query(self.parent.url(self.name)+"/response-headers?zoo=Zoo&test=Test&koo=Koot") + yield Query(self.parent.url(self.name) + "/response-headers?zoo=Zoo&test=Test&koo=Koot") def check(self): for r in self.results: if r.headers: # print(r.headers) - assert r.headers['Koo'] == ['KooK'] - assert r.headers['Zoo'] == ['Zoo', 'ZooZ'] - assert r.headers['Test'] == ['Test', 'boo'] - assert r.headers['Foo'] == ['Foo'] + assert r.headers["Koo"] == ["KooK"] + assert r.headers["Zoo"] == ["Zoo", "ZooZ"] + assert r.headers["Test"] == ["Test", "boo"] + assert r.headers["Foo"] == ["Foo"] + # To make sure queries to Edge stack related paths adds X-Content-Type-Options = nosniff in the response header # and not to any other mappings/routes @@ -690,7 +764,8 @@ def init(self): self.skip_node = True def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -698,7 +773,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.parent.url("edge_stack/admin/"), expected=404) @@ -708,6 +784,7 @@ def check(self): # assert self.results[0].headers['X-Content-Type-Options'] == ['nosniff'] assert "X-Content-Type-Options" not in self.results[1].headers + class RemoveReqHeadersMapping(MappingTest): parent: AmbassadorTest target: ServiceType @@ -718,7 +795,8 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(st, name="{self.target.name}") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -729,22 +807,23 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: remove_request_headers: - zoo - aoo -""") +""" + ) def queries(self): - yield Query(self.parent.url(self.name + "/headers"), headers={ - "zoo": "ZooZ", - "aoo": "AooA", - "foo": "FooF" - }) + yield Query( + self.parent.url(self.name + "/headers"), + headers={"zoo": "ZooZ", "aoo": "AooA", "foo": "FooF"}, + ) def check(self): for r in self.results: # print(r.json) - if 'headers' in r.json: - assert r.json['headers']['Foo'] == 'FooF' - assert 'Zoo' not in r.json['headers'] - assert 'Aoo' not in r.json['headers'] + if "headers" in r.json: + assert r.json["headers"]["Foo"] == "FooF" + assert "Zoo" not in r.json["headers"] + assert "Aoo" not in r.json["headers"] + class AddReqHeadersMapping(MappingTest): parent: AmbassadorTest @@ -756,7 +835,8 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(st, name="{self.target.name}") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -775,21 +855,20 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: value: boo foo: value: Foo -""") +""" + ) def queries(self): - yield Query(self.parent.url(self.name + "/"), headers={ - "zoo": "ZooZ", - "aoo": "AooA", - "boo": "BooB", - "foo": "FooF" - }) + yield Query( + self.parent.url(self.name + "/"), + headers={"zoo": "ZooZ", "aoo": "AooA", "boo": "BooB", "foo": "FooF"}, + ) def check(self): for r in self.results: if r.backend: assert r.backend.request - assert r.backend.request.headers['zoo'] == ['Zoo'] - assert r.backend.request.headers['aoo'] == ['AooA','aoo'] - assert r.backend.request.headers['boo'] == ['BooB','boo'] - assert r.backend.request.headers['foo'] == ['FooF','Foo'] + assert r.backend.request.headers["zoo"] == ["Zoo"] + assert r.backend.request.headers["aoo"] == ["AooA", "aoo"] + assert r.backend.request.headers["boo"] == ["BooB", "boo"] + assert r.backend.request.headers["foo"] == ["FooF", "Foo"] diff --git a/python/tests/kat/t_max_req_header_kb.py b/python/tests/kat/t_max_req_header_kb.py index 85a1b3951d..6f9ba34032 100644 --- a/python/tests/kat/t_max_req_header_kb.py +++ b/python/tests/kat/t_max_req_header_kb.py @@ -3,6 +3,7 @@ from kat.harness import Query from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node + class MaxRequestHeaderKBTest(AmbassadorTest): target: ServiceType @@ -10,7 +11,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -18,8 +20,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: ambassador_id: [{self.ambassador_id}] config: max_request_headers_kb: 30 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -27,20 +31,20 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): - h1 = 'i' * (31 * 1024) - yield Query(self.url("target/"), expected=431, - headers={'big':h1}) - h2 = 'i' * (29 * 1024) - yield Query(self.url("target/"), expected=200, - headers={'small':h2}) + h1 = "i" * (31 * 1024) + yield Query(self.url("target/"), expected=431, headers={"big": h1}) + h2 = "i" * (29 * 1024) + yield Query(self.url("target/"), expected=200, headers={"small": h2}) def check(self): # We're just testing the status codes above, so nothing to check here assert True + class MaxRequestHeaderKBMaxTest(AmbassadorTest): target: ServiceType @@ -48,7 +52,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -56,8 +61,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: ambassador_id: [{self.ambassador_id}] config: max_request_headers_kb: 96 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -65,16 +72,19 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): # without the override the response headers will cause envoy to respond with a 503 - h1 = 'i' * (97 * 1024) - yield Query(self.url("target/?override_extauth_header=1"), expected=431, - headers={'big':h1}) - h2 = 'i' * (95 * 1024) - yield Query(self.url("target/?override_extauth_header=1"), expected=200, - headers={'small':h2}) + h1 = "i" * (97 * 1024) + yield Query( + self.url("target/?override_extauth_header=1"), expected=431, headers={"big": h1} + ) + h2 = "i" * (95 * 1024) + yield Query( + self.url("target/?override_extauth_header=1"), expected=200, headers={"small": h2} + ) def check(self): # We're just testing the status codes above, so nothing to check here diff --git a/python/tests/kat/t_no_ui.py b/python/tests/kat/t_no_ui.py index 1e0fb24b39..edd6b79c9c 100644 --- a/python/tests/kat/t_no_ui.py +++ b/python/tests/kat/t_no_ui.py @@ -3,7 +3,7 @@ from abstract_tests import AmbassadorTest, ServiceType, HTTP -class NoUITest (AmbassadorTest): +class NoUITest(AmbassadorTest): # Don't use single_namespace -- we want CRDs, so we want # the cluster-scope RBAC instead of the namespace-scope # RBAC. Our ambassador_id filters out the stuff we want. @@ -11,7 +11,9 @@ class NoUITest (AmbassadorTest): extra_ports = [8877] def manifests(self) -> str: - return self.format(""" + return ( + self.format( + """ --- apiVersion: v1 kind: Namespace @@ -30,9 +32,12 @@ def manifests(self) -> str: config: diagnostics: enabled: false -""") + super().manifests() +""" + ) + + super().manifests() + ) def queries(self): - yield(Query(self.url("ambassador/v0/diag/"), expected=404)) - yield(Query(self.url("edge_stack/admin/"), expected=404)) + yield (Query(self.url("ambassador/v0/diag/"), expected=404)) + yield (Query(self.url("edge_stack/admin/"), expected=404)) yield Query(self.url("ambassador/v0/diag/", scheme="http", port=8877), expected=404) diff --git a/python/tests/kat/t_no_ui_allow_non_local.py b/python/tests/kat/t_no_ui_allow_non_local.py index 9660df89c8..15d7c215a0 100644 --- a/python/tests/kat/t_no_ui_allow_non_local.py +++ b/python/tests/kat/t_no_ui_allow_non_local.py @@ -3,7 +3,7 @@ from abstract_tests import AmbassadorTest, ServiceType, HTTP -class NoUITestAllowNoLocal (AmbassadorTest): +class NoUITestAllowNoLocal(AmbassadorTest): # Don't use single_namespace -- we want CRDs, so we want # the cluster-scope RBAC instead of the namespace-scope # RBAC. Our ambassador_id filters out the stuff we want. @@ -11,7 +11,9 @@ class NoUITestAllowNoLocal (AmbassadorTest): extra_ports = [8877] def manifests(self) -> str: - return self.format(""" + return ( + self.format( + """ --- apiVersion: v1 kind: Namespace @@ -31,9 +33,12 @@ def manifests(self) -> str: diagnostics: enabled: false allow_non_local: true -""") + super().manifests() +""" + ) + + super().manifests() + ) def queries(self): - yield(Query(self.url("ambassador/v0/diag/"), expected=404)) - yield(Query(self.url("edge_stack/admin/"), expected=404)) + yield (Query(self.url("ambassador/v0/diag/"), expected=404)) + yield (Query(self.url("edge_stack/admin/"), expected=404)) yield Query(self.url("ambassador/v0/diag/", scheme="http", port=8877), expected=200) diff --git a/python/tests/kat/t_optiontests.py b/python/tests/kat/t_optiontests.py index f90dce3994..1163dafec1 100644 --- a/python/tests/kat/t_optiontests.py +++ b/python/tests/kat/t_optiontests.py @@ -15,19 +15,11 @@ class AddRequestHeaders(OptionTest): parent: Test VALUES: ClassVar[Sequence[Dict[str, Dict[str, Union[str, bool]]]]] = [ - { "foo": { "value": "bar" } }, - { "moo": { "value": "arf" } }, - { "zoo": { - "append": True, - "value": "bar" - }}, - { "xoo": { - "append": False, - "value": "dwe" - }}, - { "aoo": { - "value": "tyu" - }} + {"foo": {"value": "bar"}}, + {"moo": {"value": "arf"}}, + {"zoo": {"append": True, "value": "bar"}}, + {"xoo": {"append": False, "value": "dwe"}}, + {"aoo": {"value": "tyu"}}, ] def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: @@ -39,7 +31,7 @@ def check(self): assert r.backend assert r.backend.request actual = r.backend.request.headers.get(k.lower()) - if isinstance(v,dict): + if isinstance(v, dict): assert actual == [v["value"]], (actual, [v["value"]]) else: assert actual == [v], (actual, [v]) @@ -50,19 +42,11 @@ class AddResponseHeaders(OptionTest): parent: Test VALUES: ClassVar[Sequence[Dict[str, Dict[str, Union[str, bool]]]]] = [ - { "foo": { "value": "bar" } }, - { "moo": { "value": "arf" } }, - { "zoo": { - "append": True, - "value": "bar" - }}, - { "xoo": { - "append": False, - "value": "dwe" - }}, - { "aoo": { - "value": "tyu" - }} + {"foo": {"value": "bar"}}, + {"moo": {"value": "arf"}}, + {"zoo": {"append": True, "value": "bar"}}, + {"xoo": {"append": False, "value": "dwe"}}, + {"aoo": {"value": "tyu"}}, ] def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: @@ -71,12 +55,16 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: def check(self): for r in self.parent.results: # Why do we end up with capitalized headers anyway?? - lowercased_headers = { k.lower(): v for k, v in r.headers.items() } + lowercased_headers = {k.lower(): v for k, v in r.headers.items()} for k, v in self.value.items(): actual = lowercased_headers.get(k.lower()) - if isinstance(v,dict): - assert actual == [v["value"]], "expected %s: %s but got %s" % (k, v["value"], lowercased_headers) + if isinstance(v, dict): + assert actual == [v["value"]], "expected %s: %s but got %s" % ( + k, + v["value"], + lowercased_headers, + ) else: assert actual == [v], "expected %s: %s but got %s" % (k, v, lowercased_headers) @@ -85,7 +73,7 @@ class UseWebsocket(OptionTest): # TODO: add a check with a websocket client as soon as we have backend support for it def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield 'use_websocket: true' + yield "use_websocket: true" class CORS(OptionTest): @@ -102,7 +90,7 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: def queries(self): for q in self.parent.queries(): yield Query(q.url) # redundant with parent - yield Query(q.url, headers={ "Origin": "https://www.test-cors.org" }) + yield Query(q.url, headers={"Origin": "https://www.test-cors.org"}) def check(self): # can assert about self.parent.results too @@ -114,7 +102,9 @@ def check(self): assert self.results[1].backend assert self.results[1].backend.name == self.parent.target.path.k8s # Uh. Is it OK that this is case-sensitive? - assert self.results[1].headers["Access-Control-Allow-Origin"] == [ "https://www.test-cors.org" ] + assert self.results[1].headers["Access-Control-Allow-Origin"] == [ + "https://www.test-cors.org" + ] class CaseSensitive(OptionTest): @@ -146,7 +136,9 @@ def check(self): request_host = r.backend.request.host response_host = self.parent.get_fqdn(r.backend.name) - assert response_host == request_host, f'backend {response_host} != request host {request_host}' + assert ( + response_host == request_host + ), f"backend {response_host} != request host {request_host}" class Rewrite(OptionTest): @@ -184,4 +176,6 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: def check(self): for r in self.parent.results: - assert r.headers.get("x-envoy-upstream-service-time", None) == None, "x-envoy-upstream-service-time header was meant to be dropped but wasn't" + assert ( + r.headers.get("x-envoy-upstream-service-time", None) == None + ), "x-envoy-upstream-service-time header was meant to be dropped but wasn't" diff --git a/python/tests/kat/t_plain.py b/python/tests/kat/t_plain.py index d88f88b8db..5c8582b652 100644 --- a/python/tests/kat/t_plain.py +++ b/python/tests/kat/t_plain.py @@ -20,7 +20,10 @@ def variants(cls) -> Generator[Node, None, None]: yield cls(variants(MappingTest)) def manifests(self) -> str: - m = namespace_manifest("plain-namespace") + namespace_manifest("evil-namespace") + """ + m = ( + namespace_manifest("plain-namespace") + + namespace_manifest("evil-namespace") + + """ --- kind: Service apiVersion: v1 @@ -67,6 +70,7 @@ def manifests(self) -> str: port: 443 targetPort: 8443 """ + ) if EDGE_STACK: m += """ @@ -131,11 +135,11 @@ def check(self): # We shouldn't have any missing-CRD-types errors any more. for source, error in errors: - if (('could not find' in error) and ('CRD definitions' in error)): - assert False, f"Missing CRDs: {error}" + if ("could not find" in error) and ("CRD definitions" in error): + assert False, f"Missing CRDs: {error}" - if 'Ingress resources' in error: - assert False, f"Ingress resource error: {error}" + if "Ingress resources" in error: + assert False, f"Ingress resource error: {error}" # The default errors assume that we have missing CRDs, and that's not correct any more, # so don't try to use assert_default_errors here. diff --git a/python/tests/kat/t_queryparameter_routing.py b/python/tests/kat/t_queryparameter_routing.py index a47cdc4f29..109cb34c2e 100644 --- a/python/tests/kat/t_queryparameter_routing.py +++ b/python/tests/kat/t_queryparameter_routing.py @@ -3,6 +3,7 @@ from kat.harness import Query from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node + class QueryParameterRoutingTest(AmbassadorTest): target1: ServiceType target2: ServiceType @@ -12,7 +13,8 @@ def init(self): self.target2 = HTTP(name="target2") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target1, self.format(""" + yield self.target1, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -20,8 +22,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target1.path.fqdn} -""") - yield self.target2, self.format(""" +""" + ) + yield self.target2, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -31,7 +35,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: http://{self.target2.path.fqdn} query_parameters: test_param: target2 -""") +""" + ) def queries(self): yield Query(self.url("target/"), expected=200) @@ -39,9 +44,14 @@ def queries(self): def check(self): assert self.results[0].backend - assert self.results[0].backend.name == self.target1.path.k8s, f"r0 wanted {self.target1.path.k8s} got {self.results[0].backend.name}" + assert ( + self.results[0].backend.name == self.target1.path.k8s + ), f"r0 wanted {self.target1.path.k8s} got {self.results[0].backend.name}" assert self.results[1].backend - assert self.results[1].backend.name == self.target2.path.k8s, f"r1 wanted {self.target2.path.k8s} got {self.results[1].backend.name}" + assert ( + self.results[1].backend.name == self.target2.path.k8s + ), f"r1 wanted {self.target2.path.k8s} got {self.results[1].backend.name}" + class QueryParameterRoutingWithRegexTest(AmbassadorTest): target: ServiceType @@ -50,7 +60,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -60,7 +71,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: http://{self.target.path.fqdn} regex_query_parameters: test_param: "^[a-z].*" -""") +""" + ) def queries(self): yield Query(self.url("target/?test_param=hello"), expected=200) @@ -71,7 +83,10 @@ def queries(self): def check(self): assert self.results[0].backend - assert self.results[0].backend.name == self.target.path.k8s, f"r0 wanted {self.target.path.k8s} got {self.results[0].backend.name}" + assert ( + self.results[0].backend.name == self.target.path.k8s + ), f"r0 wanted {self.target.path.k8s} got {self.results[0].backend.name}" + class QueryParameterPresentRoutingTest(AmbassadorTest): target: ServiceType @@ -80,7 +95,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -90,7 +106,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: http://{self.target.path.fqdn} regex_query_parameters: test_param: ".*" -""") +""" + ) def queries(self): yield Query(self.url("target/?test_param=true"), expected=200) @@ -98,4 +115,6 @@ def queries(self): def check(self): assert self.results[0].backend - assert self.results[0].backend.name == self.target.path.k8s, f"r0 wanted {self.target.path.k8s} got {self.results[0].backend.name}" + assert ( + self.results[0].backend.name == self.target.path.k8s + ), f"r0 wanted {self.target.path.k8s} got {self.results[0].backend.name}" diff --git a/python/tests/kat/t_ratelimit.py b/python/tests/kat/t_ratelimit.py index 74297ff3d5..ae9465d3a3 100644 --- a/python/tests/kat/t_ratelimit.py +++ b/python/tests/kat/t_ratelimit.py @@ -21,7 +21,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. # ambassador_id: [ {self.with_tracing.ambassador_id}, {self.no_tracing.ambassador_id} ] - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -64,10 +65,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: key: custom-label header_name: "x-omg" default: "OMFG!" -""") +""" + ) # For self.with_tracing, we want to configure the TracingService. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: RateLimitService @@ -75,7 +78,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: "{self.rls.path.fqdn}" timeout_ms: 500 protocol_version: "v3" -""") +""" + ) def queries(self): # Speak through each Ambassador to the traced service... @@ -88,28 +92,37 @@ def queries(self): # [1] # Header instructing dummy ratelimit-service to allow request - yield Query(self.url("target/"), expected=200, headers={ - 'kat-req-rls-allow': 'true', - 'kat-req-rls-headers-append': 'no header', - }) + yield Query( + self.url("target/"), + expected=200, + headers={ + "kat-req-rls-allow": "true", + "kat-req-rls-headers-append": "no header", + }, + ) # [2] # Header instructing dummy ratelimit-service to reject request with # a custom response body - yield Query(self.url("target/"), expected=429, headers={ - 'kat-req-rls-allow': 'over my dead body', - 'kat-req-rls-headers-append': 'Hello=Foo; Hi=Baz', - }) + yield Query( + self.url("target/"), + expected=429, + headers={ + "kat-req-rls-allow": "over my dead body", + "kat-req-rls-headers-append": "Hello=Foo; Hi=Baz", + }, + ) def check(self): # [2] Verifies the 429 response and the proper content-type. # The kat-server gRPC ratelimit implementation explicitly overrides # the content-type to json, because the response is in fact json # and we need to verify that this override is possible/correct. - assert self.results[2].headers["Hello"] == [ "Foo" ] - assert self.results[2].headers["Hi"] == [ "Baz" ] - assert self.results[2].headers["Content-Type"] == [ "application/json" ] - assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == [ "v3" ] + assert self.results[2].headers["Hello"] == ["Foo"] + assert self.results[2].headers["Hi"] == ["Baz"] + assert self.results[2].headers["Content-Type"] == ["application/json"] + assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == ["v3"] + class RateLimitV1Test(AmbassadorTest): # debug = True @@ -122,7 +135,8 @@ def init(self): def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -141,9 +155,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: key: kat-req-rls-headers-append header_name: "kat-req-rls-headers-append" omit_if_not_present: true -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: RateLimitService @@ -151,7 +167,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: service: "{self.rls.path.fqdn}" timeout_ms: 500 protocol_version: "v3" -""") +""" + ) def queries(self): # [0] @@ -160,27 +177,36 @@ def queries(self): # [1] # Header instructing dummy ratelimit-service to allow request - yield Query(self.url("target/"), expected=200, headers={ - 'kat-req-rls-allow': 'true', - 'kat-req-rls-headers-append': 'no header', - }) + yield Query( + self.url("target/"), + expected=200, + headers={ + "kat-req-rls-allow": "true", + "kat-req-rls-headers-append": "no header", + }, + ) # [2] # Header instructing dummy ratelimit-service to reject request - yield Query(self.url("target/"), expected=429, headers={ - 'kat-req-rls-allow': 'over my dead body', - 'kat-req-rls-headers-append': 'Hello=Foo; Hi=Baz', - }) + yield Query( + self.url("target/"), + expected=429, + headers={ + "kat-req-rls-allow": "over my dead body", + "kat-req-rls-headers-append": "Hello=Foo; Hi=Baz", + }, + ) def check(self): # [2] Verifies the 429 response and the proper content-type. # The kat-server gRPC ratelimit implementation explicitly overrides # the content-type to json, because the response is in fact json # and we need to verify that this override is possible/correct. - assert self.results[2].headers["Hello"] == [ "Foo" ] - assert self.results[2].headers["Hi"] == [ "Baz" ] - assert self.results[2].headers["Content-Type"] == [ "application/json" ] - assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == [ "v3" ] + assert self.results[2].headers["Hello"] == ["Foo"] + assert self.results[2].headers["Hi"] == ["Baz"] + assert self.results[2].headers["Content-Type"] == ["application/json"] + assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == ["v3"] + class RateLimitV1WithTLSTest(AmbassadorTest): # debug = True @@ -191,7 +217,8 @@ def init(self): self.rls = RLSGRPC() def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -201,12 +228,15 @@ def manifests(self) -> str: metadata: name: ratelimit-tls-secret type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -231,9 +261,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: key: kat-req-rls-headers-append header_name: "kat-req-rls-headers-append" omit_if_not_present: true -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: RateLimitService @@ -242,56 +274,68 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: timeout_ms: 500 tls: ratelimit-tls-context protocol_version: "v3" -""") +""" + ) def queries(self): # No matching headers, won't even go through ratelimit-service filter yield Query(self.url("target/")) # Header instructing dummy ratelimit-service to allow request - yield Query(self.url("target/"), expected=200, headers={ - 'kat-req-rls-allow': 'true' - }) + yield Query(self.url("target/"), expected=200, headers={"kat-req-rls-allow": "true"}) # Header instructing dummy ratelimit-service to reject request - yield Query(self.url("target/"), expected=429, headers={ - 'kat-req-rls-allow': 'nope', - 'kat-req-rls-headers-append': 'Hello=Foo; Hi=Baz' - }) + yield Query( + self.url("target/"), + expected=429, + headers={ + "kat-req-rls-allow": "nope", + "kat-req-rls-headers-append": "Hello=Foo; Hi=Baz", + }, + ) def check(self): # [2] Verifies the 429 response and the proper content-type. # The kat-server gRPC ratelimit implementation explicitly overrides # the content-type to json, because the response is in fact json # and we need to verify that this override is possible/correct. - assert self.results[2].headers["Hello"] == [ "Foo" ] - assert self.results[2].headers["Hi"] == [ "Baz" ] - assert self.results[2].headers["Content-Type"] == [ "application/json" ] - assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == [ "v3" ] + assert self.results[2].headers["Hello"] == ["Foo"] + assert self.results[2].headers["Hi"] == ["Baz"] + assert self.results[2].headers["Content-Type"] == ["application/json"] + assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == ["v3"] class RateLimitVerTest(AmbassadorTest): # debug = True target: ServiceType - specified_protocol_version: Literal['v2', 'v3', 'default'] - expected_protocol_version: Literal['v3', 'invalid'] + specified_protocol_version: Literal["v2", "v3", "default"] + expected_protocol_version: Literal["v3", "invalid"] rls: ServiceType @classmethod def variants(cls) -> Generator[Node, None, None]: - for protocol_version in ['v2', 'v3', 'default']: + for protocol_version in ["v2", "v3", "default"]: yield cls(protocol_version, name="{self.specified_protocol_version}") - def init(self, protocol_version: Literal['v2', 'v3', 'default']): + def init(self, protocol_version: Literal["v2", "v3", "default"]): self.target = HTTP() self.specified_protocol_version = protocol_version - self.expected_protocol_version = cast(Literal['v3', 'invalid'], protocol_version if protocol_version in ['v3'] else 'invalid') - self.rls = RLSGRPC(protocol_version=(self.expected_protocol_version if self.expected_protocol_version != 'invalid' else 'v3')) + self.expected_protocol_version = cast( + Literal["v3", "invalid"], protocol_version if protocol_version in ["v3"] else "invalid" + ) + self.rls = RLSGRPC( + protocol_version=( + self.expected_protocol_version + if self.expected_protocol_version != "invalid" + else "v3" + ) + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -310,16 +354,23 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: key: kat-req-rls-headers-append header_name: "kat-req-rls-headers-append" omit_if_not_present: true -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: RateLimitService name: {self.rls.path.k8s} service: "{self.rls.path.fqdn}" timeout_ms: 500 -""") + ("" if self.specified_protocol_version == "default" else f"protocol_version: '{self.specified_protocol_version}'") +""" + ) + ( + "" + if self.specified_protocol_version == "default" + else f"protocol_version: '{self.specified_protocol_version}'" + ) def queries(self): # [0] @@ -328,20 +379,28 @@ def queries(self): # [1] # Header instructing dummy ratelimit-service to allow request - yield Query(self.url("target/"), expected=200, headers={ - 'kat-req-rls-allow': 'true', - 'kat-req-rls-headers-append': 'no header', - }) + yield Query( + self.url("target/"), + expected=200, + headers={ + "kat-req-rls-allow": "true", + "kat-req-rls-headers-append": "no header", + }, + ) # [2] # Header instructing dummy ratelimit-service to reject request - yield Query(self.url("target/"), expected=(429 if self.expected_protocol_version != 'invalid' else 200), headers={ - 'kat-req-rls-allow': 'over my dead body', - 'kat-req-rls-headers-append': 'Hello=Foo; Hi=Baz', - }) + yield Query( + self.url("target/"), + expected=(429 if self.expected_protocol_version != "invalid" else 200), + headers={ + "kat-req-rls-allow": "over my dead body", + "kat-req-rls-headers-append": "Hello=Foo; Hi=Baz", + }, + ) def check(self): - if self.expected_protocol_version == 'invalid': + if self.expected_protocol_version == "invalid": # all queries should succeed because the rate-limit filter was dropped, due to bad protocol assert "Hello" not in self.results[2].headers assert "Hi" not in self.results[2].headers @@ -352,7 +411,9 @@ def check(self): # The kat-server gRPC ratelimit implementation explicitly overrides # the content-type to json, because the response is in fact json # and we need to verify that this override is possible/correct. - assert self.results[2].headers["Hello"] == [ "Foo" ] - assert self.results[2].headers["Hi"] == [ "Baz" ] - assert self.results[2].headers["Content-Type"] == [ "application/json" ] - assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == [ self.expected_protocol_version ] + assert self.results[2].headers["Hello"] == ["Foo"] + assert self.results[2].headers["Hi"] == ["Baz"] + assert self.results[2].headers["Content-Type"] == ["application/json"] + assert self.results[2].headers["Kat-Resp-Rls-Protocol-Version"] == [ + self.expected_protocol_version + ] diff --git a/python/tests/kat/t_redirect.py b/python/tests/kat/t_redirect.py index 4680eea7ea..1dcf9b40a5 100644 --- a/python/tests/kat/t_redirect.py +++ b/python/tests/kat/t_redirect.py @@ -17,6 +17,7 @@ # no way to subclass an AmbassadorTest without having your base class be run separately, which isn't # what I wanted here. Sigh. + class RedirectTests(AmbassadorTest): target: ServiceType edge_stack_cleartext_host = False @@ -31,10 +32,14 @@ def init(self): def requirements(self): # only check https urls since test readiness will only end up barfing on redirect - yield from (r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("https")) + yield from ( + r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("https") + ) def manifests(self): - return namespace_manifest("redirect-namespace") + f""" + return ( + namespace_manifest("redirect-namespace") + + f""" --- apiVersion: v1 kind: Secret @@ -54,12 +59,15 @@ def manifests(self): data: tls.crt: {TLSCerts["localhost"].k8s_crt} tls.key: {TLSCerts["localhost"].k8s_key} -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self here, not self.target, because we want the TLS module to # be annotated on the Ambassador itself. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -70,9 +78,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: enabled: True secret: redirect-cert redirect_cleartext_from: 8080 -""") +""" + ) - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -80,27 +90,30 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /tls-target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] yield Query(self.url("tls-target/", scheme="http"), expected=301) # [1] -- PHASE 2 - yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors", - scheme="https"), - insecure=True, - phase=2) + yield Query( + self.url("ambassador/v0/diag/?json=true&filter=errors", scheme="https"), + insecure=True, + phase=2, + ) def check(self): # For query 0, check the redirection target. - assert len(self.results[0].headers['Location']) > 0 - assert self.results[0].headers['Location'][0].find('/tls-target/') > 0 + assert len(self.results[0].headers["Location"]) > 0 + assert self.results[0].headers["Location"][0].find("/tls-target/") > 0 # For query 1, we require no errors. # XXX Ew. If self.results[1].json is empty, the harness won't convert it to a response. errors = self.results[1].json - assert(len(errors) == 0) + assert len(errors) == 0 + class RedirectTestsWithProxyProto(AmbassadorTest): @@ -112,10 +125,13 @@ def init(self): def requirements(self): # only check https urls since test readiness will only end up barfing on redirect - yield from (r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("https")) + yield from ( + r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("https") + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -123,9 +139,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: config: use_proxy_proto: true enable_ipv6: true -""") +""" + ) - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -133,7 +151,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /tls-target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # TODO (concaf): FWIW, this query only covers one side of the story. This tests that this is the correct @@ -141,7 +160,7 @@ def queries(self): # This is because net/http does not yet support adding proxy proto to HTTP requests, and hence it's difficult # to test with kat. We will need to open a raw TCP connection (e.g. telnet/nc) and send the entire HTTP Request # in plaintext to test this behavior (or use curl with --haproxy-protocol). - yield Query(self.url("tls-target/"), error=[ "EOF", "connection reset by peer" ]) + yield Query(self.url("tls-target/"), error=["EOF", "connection reset by peer"]) # We can't do the error check until we have the PROXY client mentioned above. # # [1] -- PHASE 2 @@ -174,10 +193,13 @@ def init(self): def requirements(self): # only check https urls since test readiness will only end up barfing on redirect - yield from (r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("https")) + yield from ( + r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("https") + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -188,9 +210,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: enabled: True secret: does-not-exist-secret redirect_cleartext_from: 8080 -""") +""" + ) - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -198,7 +222,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /tls-target/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] @@ -232,7 +257,9 @@ def init(self): self.add_default_https_listener = False def manifests(self): - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -257,11 +284,14 @@ def manifests(self): requestPolicy: insecure: action: Redirect -''') + super().manifests() - +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ apiVersion: getambassador.io/v3alpha1 kind: Module name: ambassador @@ -274,25 +304,35 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /{self.name}/ service: {self.target.path.fqdn} -""") +""" + ) def queries(self): # [0] - yield Query(self.url(self.name + "/target/"), headers={ "X-Forwarded-Proto": "http" }, expected=301) + yield Query( + self.url(self.name + "/target/"), headers={"X-Forwarded-Proto": "http"}, expected=301 + ) # [1] - yield Query(self.url(self.name + "/target/"), headers={ "X-Forwarded-Proto": "https" }, expected=200) + yield Query( + self.url(self.name + "/target/"), headers={"X-Forwarded-Proto": "https"}, expected=200 + ) # [2] -- PHASE 2 - yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), headers={ "X-Forwarded-Proto": "https" }, phase=2) + yield Query( + self.url("ambassador/v0/diag/?json=true&filter=errors"), + headers={"X-Forwarded-Proto": "https"}, + phase=2, + ) def check(self): # For query 0, check the redirection target. expected_location = ["https://" + self.path.fqdn + "/" + self.name + "/target/"] - actual_location = self.results[0].headers['Location'] - assert actual_location == expected_location, "Expected redirect location to be {}, got {} instead".format( - expected_location, - actual_location + actual_location = self.results[0].headers["Location"] + assert ( + actual_location == expected_location + ), "Expected redirect location to be {}, got {} instead".format( + expected_location, actual_location ) # For query 1, we don't have to check anything, the "expected" clause is enough. @@ -300,9 +340,15 @@ def check(self): # For query 2, we require no errors. # XXX Ew. If self.results[2].json is empty, the harness won't convert it to a response. errors = self.results[2].json - assert(len(errors) == 0) + assert len(errors) == 0 def requirements(self): # We're replacing super()'s requirements deliberately here: we need the XFP header or they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"X-Forwarded-Proto": "https"})) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"X-Forwarded-Proto": "https"})) + yield ( + "url", + Query(self.url("ambassador/v0/check_ready"), headers={"X-Forwarded-Proto": "https"}), + ) + yield ( + "url", + Query(self.url("ambassador/v0/check_alive"), headers={"X-Forwarded-Proto": "https"}), + ) diff --git a/python/tests/kat/t_regexrewrite_forwarding.py b/python/tests/kat/t_regexrewrite_forwarding.py index e98191a4b0..58766050fd 100644 --- a/python/tests/kat/t_regexrewrite_forwarding.py +++ b/python/tests/kat/t_regexrewrite_forwarding.py @@ -3,6 +3,7 @@ from kat.harness import variants, Query from abstract_tests import AmbassadorTest, ServiceType, HTTP, Node + class RegexRewriteForwardingTest(AmbassadorTest): target: ServiceType @@ -10,7 +11,8 @@ def init(self): self.target = HTTP(name="foo") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(r""" + yield self.target, self.format( + r""" --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -21,7 +23,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: regex_rewrite: pattern: "/foo/baz" substitution: "/baz/foo" -""") +""" + ) def queries(self): yield Query(self.url("foo/bar"), expected=200) @@ -31,13 +34,14 @@ def queries(self): def check(self): assert self.results[0].backend assert self.results[0].backend.request - assert self.results[0].backend.request.headers['x-envoy-original-path'][0] == f'/foo/bar' + assert self.results[0].backend.request.headers["x-envoy-original-path"][0] == f"/foo/bar" assert self.results[0].backend.request.url.path == "/foo/bar" assert self.results[1].backend assert self.results[1].backend.request - assert self.results[1].backend.request.headers['x-envoy-original-path'][0] == f'/foo/baz' + assert self.results[1].backend.request.headers["x-envoy-original-path"][0] == f"/foo/baz" assert self.results[1].backend.request.url.path == "/baz/foo" + class RegexRewriteForwardingWithExtractAndSubstituteTest(AmbassadorTest): target: ServiceType @@ -45,7 +49,8 @@ def init(self): self.target = HTTP(name="foo") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(r""" + yield self.target, self.format( + r""" --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -56,7 +61,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: regex_rewrite: pattern: "/foo/([0-9]*)/list" substitution: "/bar/\\1" -""") +""" + ) def queries(self): yield Query(self.url("foo/123456789/list"), expected=200) @@ -67,9 +73,15 @@ def queries(self): def check(self): assert self.results[0].backend assert self.results[0].backend.request - assert self.results[0].backend.request.headers['x-envoy-original-path'][0] == f'/foo/123456789/list' + assert ( + self.results[0].backend.request.headers["x-envoy-original-path"][0] + == f"/foo/123456789/list" + ) assert self.results[0].backend.request.url.path == "/bar/123456789" assert self.results[1].backend assert self.results[1].backend.request - assert self.results[1].backend.request.headers['x-envoy-original-path'][0] == f'/foo/987654321/list' + assert ( + self.results[1].backend.request.headers["x-envoy-original-path"][0] + == f"/foo/987654321/list" + ) assert self.results[1].backend.request.url.path == "/bar/987654321" diff --git a/python/tests/kat/t_request_header.py b/python/tests/kat/t_request_header.py index 8f8e54f239..4a3d03e014 100644 --- a/python/tests/kat/t_request_header.py +++ b/python/tests/kat/t_request_header.py @@ -11,7 +11,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -25,7 +26,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/"), headers={"x-request-id": "hello"}) @@ -33,7 +35,8 @@ def queries(self): def check(self): assert self.results[0].backend assert self.results[0].backend.request - assert self.results[0].backend.request.headers['x-request-id'] == ['hello'] + assert self.results[0].backend.request.headers["x-request-id"] == ["hello"] + class XRequestIdHeaderDefaultTest(AmbassadorTest): target: ServiceType @@ -43,7 +46,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -56,7 +60,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/"), headers={"X-Request-Id": "hello"}) @@ -64,7 +69,7 @@ def queries(self): def check(self): assert self.results[0].backend assert self.results[0].backend.request - assert self.results[0].backend.request.headers['x-request-id'] != ['hello'] + assert self.results[0].backend.request.headers["x-request-id"] != ["hello"] # Sanity test that Envoy headers are present if we do not suppress them @@ -75,7 +80,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -85,7 +91,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /rewrite/ timeout_ms: 5001 service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/")) @@ -98,8 +105,9 @@ def check(self): # All known Envoy headers should be set. The original path header is # include here because we made sure to include a rewrite in the Mapping. - assert headers['x-envoy-expected-rq-timeout-ms'] == ['5001'] - assert headers['x-envoy-original-path'] == ['/target/'] + assert headers["x-envoy-expected-rq-timeout-ms"] == ["5001"] + assert headers["x-envoy-original-path"] == ["/target/"] + # Sanity test that we can suppress Envoy headers when configured class SuppressEnvoyHeadersTest(AmbassadorTest): @@ -109,7 +117,8 @@ def init(self): self.target = HTTP(name="target") def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -125,7 +134,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: rewrite: /rewrite/ timeout_ms: 5001 service: http://{self.target.path.fqdn} -""") +""" + ) def queries(self): yield Query(self.url("target/")) @@ -137,5 +147,5 @@ def check(self): headers = self.results[0].backend.request.headers # No Envoy headers should be set - assert 'x-envoy-expected-rq-timeout-ms' not in headers - assert 'x-envoy-original-path' not in headers + assert "x-envoy-expected-rq-timeout-ms" not in headers + assert "x-envoy-original-path" not in headers diff --git a/python/tests/kat/t_retrypolicy.py b/python/tests/kat/t_retrypolicy.py index 59627f3372..a48f008fa8 100644 --- a/python/tests/kat/t_retrypolicy.py +++ b/python/tests/kat/t_retrypolicy.py @@ -15,7 +15,8 @@ def init(self) -> None: self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -24,9 +25,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}-normal/ service: {self.target.path.fqdn} timeout_ms: 3000 -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -38,9 +41,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: retry_policy: retry_on: "5xx" num_retries: 4 -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -49,26 +54,49 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: retry_policy: retry_on: "retriable-4xx" num_retries: 4 -""") +""" + ) def queries(self): - yield Query(self.url(self.name + '-normal/'), headers={"Kat-Req-Http-Requested-Backend-Delay": "0"}, expected=200) - yield Query(self.url(self.name + '-normal/'), headers={"Kat-Req-Http-Requested-Status": "500"}, expected=500) - yield Query(self.url(self.name + '-retry/'), headers={"Kat-Req-Http-Requested-Status": "500", "Kat-Req-Http-Requested-Backend-Delay": "2000"}, expected=504) - yield Query(self.url(self.name + '-normal/'), headers={"Kat-Req-Http-Requested-Status": "409", "Kat-Req-Http-Requested-Backend-Delay": "2000"}, expected=504) + yield Query( + self.url(self.name + "-normal/"), + headers={"Kat-Req-Http-Requested-Backend-Delay": "0"}, + expected=200, + ) + yield Query( + self.url(self.name + "-normal/"), + headers={"Kat-Req-Http-Requested-Status": "500"}, + expected=500, + ) + yield Query( + self.url(self.name + "-retry/"), + headers={ + "Kat-Req-Http-Requested-Status": "500", + "Kat-Req-Http-Requested-Backend-Delay": "2000", + }, + expected=504, + ) + yield Query( + self.url(self.name + "-normal/"), + headers={ + "Kat-Req-Http-Requested-Status": "409", + "Kat-Req-Http-Requested-Backend-Delay": "2000", + }, + expected=504, + ) def get_timestamp(self, hdr): - m = re.match(r'^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{1,6})', hdr) + m = re.match(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{1,6})", hdr) if m: - return datetime.strptime(m.group(1), '%Y-%m-%dT%H:%M:%S.%f').timestamp() + return datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S.%f").timestamp() else: assert False, f'header timestamp "{hdr}" is not parseable' return None def get_duration(self, result): - start_time = self.get_timestamp(result.headers['Client-Start-Date'][0]) - end_time = self.get_timestamp(result.headers['Client-End-Date'][0]) + start_time = self.get_timestamp(result.headers["Client-Start-Date"][0]) + end_time = self.get_timestamp(result.headers["Client-End-Date"][0]) return end_time - start_time @@ -84,16 +112,24 @@ def check(self): conflict_duration = self.get_duration(conflict_result) assert retry_duration >= 2, f"retry time {retry_duration} must be at least 2 seconds" - assert conflict_duration >= 2, f"conflict time {conflict_duration} must be at least 2 seconds" + assert ( + conflict_duration >= 2 + ), f"conflict time {conflict_duration} must be at least 2 seconds" ok_vs_normal = abs(ok_duration - normal_duration) - assert ok_vs_normal <= 1, f"time to 200 OK {ok_duration} is more than 1 second different from time to 500 {normal_duration}" + assert ( + ok_vs_normal <= 1 + ), f"time to 200 OK {ok_duration} is more than 1 second different from time to 500 {normal_duration}" retry_vs_normal = retry_duration - normal_duration - assert retry_vs_normal >= 2, f"retry time {retry_duration} is not at least 2 seconds slower than normal time {normal_duration}" + assert ( + retry_vs_normal >= 2 + ), f"retry time {retry_duration} is not at least 2 seconds slower than normal time {normal_duration}" conflict_vs_ok = conflict_duration - ok_duration - assert conflict_vs_ok >= 2, f"conflict time {conflict_duration} is not at least 2 seconds slower than ok time {ok_duration}" + assert ( + conflict_vs_ok >= 2 + ), f"conflict time {conflict_duration} is not at least 2 seconds slower than ok time {ok_duration}" diff --git a/python/tests/kat/t_shadow.py b/python/tests/kat/t_shadow.py index 6745f11d00..a3efcc7e57 100644 --- a/python/tests/kat/t_shadow.py +++ b/python/tests/kat/t_shadow.py @@ -14,12 +14,13 @@ class ShadowTestCANFLAKE(MappingTest): # parent's init to have a different signature... but it's also intimately # (nay, incestuously) related to the variant()'s yield() above, and I really # don't want to deal with that right now. So. We'll deal with it later. - def init(self) -> None: # type: ignore + def init(self) -> None: # type: ignore self.target = HTTP(name="target") self.options = [] def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -56,10 +57,13 @@ def manifests(self) -> str: ports: - name: http containerPort: 3000 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -103,7 +107,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/check/ rewrite: /check/ service: shadow.plain-namespace -""") +""" + ) def requirements(self): yield from super().requirements() @@ -119,7 +124,7 @@ def queries(self): # to our shadow service that's tallying calls by bucket. So, basically, each # shadow bucket 0-9 should end up with 10 call.s bucket = i % 10 - yield Query(self.parent.url(f'{self.name}/mark/{bucket}')) + yield Query(self.parent.url(f"{self.name}/mark/{bucket}")) for i in range(500): # We also do a call to weighted-mark, which is exactly the same _but_ the @@ -130,7 +135,7 @@ def queries(self): # shadow. bucket = (i % 10) + 100 - yield Query(self.parent.url(f'{self.name}/weighted-mark/{bucket}')) + yield Query(self.parent.url(f"{self.name}/weighted-mark/{bucket}")) # Finally, in phase 2, grab the bucket counts. yield Query(self.parent.url("%s/check/" % self.name), phase=2) @@ -141,18 +146,18 @@ def check(self): # We shouldn't have any missing-CRD-types errors any more. for source, error in errors: - if (('could not find' in error) and ('CRD definitions' in error)): - assert False, f"Missing CRDs: {error}" + if ("could not find" in error) and ("CRD definitions" in error): + assert False, f"Missing CRDs: {error}" - if 'Ingress resources' in error: - assert False, f"Ingress resource error: {error}" + if "Ingress resources" in error: + assert False, f"Ingress resource error: {error}" # The default errors assume that we have missing CRDs, and that's not correct any more, # so don't try to use assert_default_errors here. for result in self.results: if "mark" in result.query.url: - assert not result.headers.get('X-Shadowed', False) + assert not result.headers.get("X-Shadowed", False) elif "check" in result.query.url: data = result.json weighted_total = 0 @@ -163,7 +168,7 @@ def check(self): value = data.get(str(i), -1) error = abs(value - 10) - assert error <= 2, f'bucket {i} should have 10 calls, got {value}' + assert error <= 2, f"bucket {i} should have 10 calls, got {value}" # Buckets 100-109 should also have 10 per bucket... but honestly, this is # a pretty small sample size, and Envoy's randomization seems to kinda suck @@ -183,4 +188,6 @@ def check(self): # See above for why we're just doing a >0 check here. # assert abs(weighted_total - 50) <= 10, f'weighted buckets should have 50 total calls, got {weighted_total}' - assert weighted_total > 0, f'weighted buckets should have 50 total calls but got zero' + assert ( + weighted_total > 0 + ), f"weighted buckets should have 50 total calls but got zero" diff --git a/python/tests/kat/t_stats.py b/python/tests/kat/t_stats.py index 90e2399a7e..9f8dd9b19b 100644 --- a/python/tests/kat/t_stats.py +++ b/python/tests/kat/t_stats.py @@ -113,17 +113,21 @@ def manifests(self) -> str: value: 'true' """ - return self.format(integration_manifests.load("rbac_cluster_scope") + integration_manifests.load("ambassador"), - envs=envs, - extra_ports="", - capabilities_block="") + \ - GRAPHITE_CONFIG.format( - 'statsd-sink', - integration_manifests.get_images()['test-stats'], - f"{STATSD_TEST_CLUSTER}:{ALT_STATSD_TEST_CLUSTER}") + return self.format( + integration_manifests.load("rbac_cluster_scope") + + integration_manifests.load("ambassador"), + envs=envs, + extra_ports="", + capabilities_block="", + ) + GRAPHITE_CONFIG.format( + "statsd-sink", + integration_manifests.get_images()["test-stats"], + f"{STATSD_TEST_CLUSTER}:{ALT_STATSD_TEST_CLUSTER}", + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -156,7 +160,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /metrics rewrite: /metrics service: http://127.0.0.1:8877 -""") +""" + ) def requirements(self): yield from super().requirements() @@ -175,25 +180,29 @@ def check(self): stats = self.results[-2].json or {} cluster_stats = stats.get(STATSD_TEST_CLUSTER, {}) - rq_total = cluster_stats.get('upstream_rq_total', -1) - rq_200 = cluster_stats.get('upstream_rq_200', -1) + rq_total = cluster_stats.get("upstream_rq_total", -1) + rq_200 = cluster_stats.get("upstream_rq_200", -1) - assert rq_total == 1000, f'{STATSD_TEST_CLUSTER}: expected 1000 total calls, got {rq_total}' - assert rq_200 > 990, f'{STATSD_TEST_CLUSTER}: expected 1000 successful calls, got {rq_200}' + assert rq_total == 1000, f"{STATSD_TEST_CLUSTER}: expected 1000 total calls, got {rq_total}" + assert rq_200 > 990, f"{STATSD_TEST_CLUSTER}: expected 1000 successful calls, got {rq_200}" cluster_stats = stats.get(ALT_STATSD_TEST_CLUSTER, {}) - rq_total = cluster_stats.get('upstream_rq_total', -1) - rq_200 = cluster_stats.get('upstream_rq_200', -1) + rq_total = cluster_stats.get("upstream_rq_total", -1) + rq_200 = cluster_stats.get("upstream_rq_200", -1) - assert rq_total == 1000, f'{ALT_STATSD_TEST_CLUSTER}: expected 1000 total calls, got {rq_total}' - assert rq_200 > 990, f'{ALT_STATSD_TEST_CLUSTER}: expected 1000 successful calls, got {rq_200}' + assert ( + rq_total == 1000 + ), f"{ALT_STATSD_TEST_CLUSTER}: expected 1000 total calls, got {rq_total}" + assert ( + rq_200 > 990 + ), f"{ALT_STATSD_TEST_CLUSTER}: expected 1000 successful calls, got {rq_200}" # self.results[-1] is the text dump from Envoy's '/metrics' endpoint. metrics = self.results[-1].text # Somewhere in here, we want to see a metric explicitly for both our "real" # cluster and our alt cluster, returning a 200. Are they there? - wanted_metric = 'envoy_cluster_internal_upstream_rq' + wanted_metric = "envoy_cluster_internal_upstream_rq" wanted_status = 'envoy_response_code="200"' wanted_cluster_name = f'envoy_cluster_name="{STATSD_TEST_CLUSTER}"' alt_wanted_cluster_name = f'envoy_cluster_name="{ALT_STATSD_TEST_CLUSTER}"' @@ -210,8 +219,12 @@ def check(self): print(f"line '{line}'") found_alt = True - assert found_normal, f"wanted {STATSD_TEST_CLUSTER} in Prometheus metrics, but didn't find it" - assert found_alt, f"wanted {ALT_STATSD_TEST_CLUSTER} in Prometheus metrics, but didn't find it" + assert ( + found_normal + ), f"wanted {STATSD_TEST_CLUSTER} in Prometheus metrics, but didn't find it" + assert ( + found_alt + ), f"wanted {ALT_STATSD_TEST_CLUSTER} in Prometheus metrics, but didn't find it" class DogstatsdTest(AmbassadorTest): @@ -230,17 +243,21 @@ def manifests(self) -> str: value: 'true' """ - return self.format(integration_manifests.load("rbac_cluster_scope") + integration_manifests.load('ambassador'), - envs=envs, - extra_ports="", - capabilities_block="") + \ - DOGSTATSD_CONFIG.format( - 'dogstatsd-sink', - integration_manifests.get_images()['test-stats'], - DOGSTATSD_TEST_CLUSTER) + return self.format( + integration_manifests.load("rbac_cluster_scope") + + integration_manifests.load("ambassador"), + envs=envs, + extra_ports="", + capabilities_block="", + ) + DOGSTATSD_CONFIG.format( + "dogstatsd-sink", + integration_manifests.get_images()["test-stats"], + DOGSTATSD_TEST_CLUSTER, + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -257,7 +274,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /reset/ rewrite: /RESET/ service: dogstatsd-sink -""") +""" + ) def requirements(self): yield from super().requirements() @@ -273,8 +291,8 @@ def check(self): stats = self.results[-1].json or {} cluster_stats = stats.get(DOGSTATSD_TEST_CLUSTER, {}) - rq_total = cluster_stats.get('upstream_rq_total', -1) - rq_200 = cluster_stats.get('upstream_rq_200', -1) + rq_total = cluster_stats.get("upstream_rq_total", -1) + rq_200 = cluster_stats.get("upstream_rq_200", -1) - assert rq_total == 1000, f'expected 1000 total calls, got {rq_total}' - assert rq_200 > 990, f'expected 1000 successful calls, got {rq_200}' + assert rq_total == 1000, f"expected 1000 total calls, got {rq_total}" + assert rq_200 > 990, f"expected 1000 successful calls, got {rq_200}" diff --git a/python/tests/kat/t_tcpmapping.py b/python/tests/kat/t_tcpmapping.py index 394472ae5f..cde31319a3 100644 --- a/python/tests/kat/t_tcpmapping.py +++ b/python/tests/kat/t_tcpmapping.py @@ -9,10 +9,11 @@ # An AmbassadorTest subclass will actually create a running Ambassador. # "self" in this class will refer to the Ambassador. + class TCPMappingTest(AmbassadorTest): # single_namespace = True namespace = "tcp-namespace" - extra_ports = [ 6789, 7654, 8765, 9876 ] + extra_ports = [6789, 7654, 8765, 9876] # This test is written assuming explicit control of which Hosts are present, # so don't let Edge Stack mess with that. @@ -46,7 +47,10 @@ def init(self): # Kubernetes cluster before running any tests. def manifests(self) -> str: - return namespace_manifest("tcp-namespace") + namespace_manifest("other-namespace") + f""" + return ( + namespace_manifest("tcp-namespace") + + namespace_manifest("other-namespace") + + f""" --- apiVersion: v1 kind: Secret @@ -126,13 +130,16 @@ def manifests(self) -> str: requestPolicy: insecure: action: Reject -""" + super().manifests() +""" + + super().manifests() + ) # config() must _yield_ tuples of Node, Ambassador-YAML where the # Ambassador-YAML will be annotated onto the Node. def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -142,9 +149,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - tls-context-host-2 - tls-context-host-3 secret: supersecret -""") +""" + ) - yield self.target1, self.format(""" + yield self.target1, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TCPMapping @@ -171,10 +180,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: port: 6789 host: tls-context-host-1 service: {self.target1.path.fqdn}:80 -""") +""" + ) # Host-differentiated. - yield self.target2, self.format(""" + yield self.target2, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TCPMapping @@ -183,10 +194,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: host: tls-context-host-2 service: {self.target2.path.fqdn} tls: {self.name}-tlscontext -""") +""" + ) # Host-differentiated. - yield self.target3, self.format(""" + yield self.target3, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TCPMapping @@ -194,14 +207,47 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: port: 6789 host: tls-context-host-3 service: https://{self.target3.path.fqdn} -""") +""" + ) def requirements(self): # We're replacing super()'s requirements deliberately here. Without a Host header they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) # scheme defaults to HTTP; if you need to use HTTPS, have it return # "https"... @@ -214,35 +260,41 @@ def scheme(self): def queries(self): # 0: should hit target1, and use TLS - yield Query(self.url(self.name + "/wtfo/", port=9876), - insecure=True) + yield Query(self.url(self.name + "/wtfo/", port=9876), insecure=True) # 1: should hit target2, and use TLS - yield Query(self.url(self.name + "/wtfo/", port=7654, scheme='http'), - insecure=True) + yield Query(self.url(self.name + "/wtfo/", port=7654, scheme="http"), insecure=True) # 2: should hit target1 via SNI, and use cleartext - yield Query(self.url(self.name + "/wtfo/", port=6789), - headers={"Host": "tls-context-host-1"}, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/wtfo/", port=6789), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ) # 3: should hit target2 via SNI, and use TLS - yield Query(self.url(self.name + "/wtfo/", port=6789), - headers={"Host": "tls-context-host-2"}, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/wtfo/", port=6789), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ) # 4: should hit target3 via SNI, and use TLS - yield Query(self.url(self.name + "/wtfo/", port=6789), - headers={"Host": "tls-context-host-3"}, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/wtfo/", port=6789), + headers={"Host": "tls-context-host-3"}, + insecure=True, + sni=True, + ) # 5: should error since port 8765 is bound only to localhost - yield Query(self.url(self.name + "/wtfo/", port=8765), - error=[ 'connection reset by peer', 'EOF', 'connection refused' ], - insecure=True) + yield Query( + self.url(self.name + "/wtfo/", port=8765), + error=["connection reset by peer", "EOF", "connection refused"], + insecure=True, + ) # Once in check(), self.results is an ordered list of results from your # Queries. (You can also look at self.parent.results if you really want @@ -250,11 +302,11 @@ def queries(self): def check(self): for idx, target, tls_wanted in [ - ( 0, self.target1, True ), - ( 1, self.target2, True ), - ( 2, self.target1, False ), - ( 3, self.target2, True ), - ( 4, self.target3, True ), + (0, self.target1, True), + (1, self.target2, True), + (2, self.target1, False), + (3, self.target2, True), + (4, self.target3, True), # ( 5, self.target1 ), ]: r = self.results[idx] @@ -264,5 +316,9 @@ def check(self): assert r.backend.request tls_enabled = r.backend.request.tls.enabled - assert backend_fqdn == wanted_fqdn, f'{idx}: backend {backend_fqdn} != expected {wanted_fqdn}' - assert tls_enabled == tls_wanted, f'{idx}: TLS status {tls_enabled} != wanted {tls_wanted}' + assert ( + backend_fqdn == wanted_fqdn + ), f"{idx}: backend {backend_fqdn} != expected {wanted_fqdn}" + assert ( + tls_enabled == tls_wanted + ), f"{idx}: TLS status {tls_enabled} != wanted {tls_wanted}" diff --git a/python/tests/kat/t_tls.py b/python/tests/kat/t_tls.py index 66a1219d94..e48697e57e 100644 --- a/python/tests/kat/t_tls.py +++ b/python/tests/kat/t_tls.py @@ -8,7 +8,9 @@ from tests.utils import create_crl_pem_b64 -bug_404_routes = True # Do we erroneously send 404 responses directly instead of redirect-to-tls first? +bug_404_routes = ( + True # Do we erroneously send 404 responses directly instead of redirect-to-tls first? +) class TLSContextsTest(AmbassadorTest): @@ -26,7 +28,8 @@ def init(self): self.xfail = "FIXME: IHA" def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 metadata: @@ -37,10 +40,13 @@ def manifests(self) -> str: tls.crt: {TLSCerts["master.datawire.io"].k8s_crt} kind: Secret type: Opaque -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -50,35 +56,43 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: upstream: enabled: True secret: test-tlscontexts-secret -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping name: {self.target.path.k8s} prefix: /{self.name}/ service: {self.target.path.fqdn} -""") +""" + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url(self.name + "/"), error=['connection refused', 'connection reset by peer', 'EOF', 'request canceled']) + yield Query( + self.url(self.name + "/"), + error=["connection refused", "connection reset by peer", "EOF", "request canceled"], + ) def requirements(self): - yield from (r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("http://")) + yield from ( + r for r in super().requirements() if r[0] == "url" and r[1].url.startswith("http://") + ) class ClientCertificateAuthentication(AmbassadorTest): - def init(self): self.xfail = "FIXME: IHA" self.target = HTTP() def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 metadata: @@ -100,10 +114,13 @@ def manifests(self) -> str: data: tls.crt: {TLSCerts["ambassador.example.com"].k8s_crt} tls.key: {TLSCerts["ambassador.example.com"].k8s_key} -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -125,9 +142,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: enabled: True secret: test-clientcert-client-secret cert_required: True -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -139,46 +158,74 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: x-cert-end: { value: "%DOWNSTREAM_PEER_CERT_V_END%" } x-cert-start-custom: { value: "%DOWNSTREAM_PEER_CERT_V_START(%b %e %H:%M:%S %Y %Z)%" } x-cert-end-custom: { value: "%DOWNSTREAM_PEER_CERT_V_END(%b %e %H:%M:%S %Y %Z)%" } -""") +""" + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url(self.name + "/"), insecure=True, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey, - client_cert_required=True, - ca_cert=TLSCerts["master.datawire.io"].pubcert) + yield Query( + self.url(self.name + "/"), + insecure=True, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey, + client_cert_required=True, + ca_cert=TLSCerts["master.datawire.io"].pubcert, + ) # In TLS < 1.3, there's not a dedicated alert code for "the client forgot to include a certificate", # so we get a generic alert=40 ("handshake_failure"). We also include "write: connection reset by peer" # because we've seen cases where Envoy and the client library don't play nicely, so the error report doesn't # get back before the connection closes. - yield Query(self.url(self.name + "/"), insecure=True, maxTLSv="v1.2", - error=[ "tls: handshake failure", "write: connection reset by peer" ]) + yield Query( + self.url(self.name + "/"), + insecure=True, + maxTLSv="v1.2", + error=["tls: handshake failure", "write: connection reset by peer"], + ) # TLS 1.3 added a dedicated alert=116 ("certificate_required") for that scenario. See above for why # "write: connection reset by peer " is also accepted. - yield Query(self.url(self.name + "/"), insecure=True, minTLSv="v1.3", - error=[ "tls: certificate required", "write: connection reset by peer" ]) + yield Query( + self.url(self.name + "/"), + insecure=True, + minTLSv="v1.3", + error=["tls: certificate required", "write: connection reset by peer"], + ) def check(self): assert self.results[0].backend assert self.results[0].backend.request - assert self.results[0].backend.request.headers["x-forwarded-client-cert"] == \ - ["Hash=c2d41a5977dcd28a3ba21f59ed5508cc6538defa810843d8a593e668306c8c4f;Subject=\"CN=presto.example.com,OU=Engineering,O=Presto,L=Bangalore,ST=KA,C=IN\""] - assert self.results[0].backend.request.headers["x-cert-start"] == ["2019-01-10T19:19:52.000Z"], \ - "unexpected x-cert-start value: %s" % self.results[0].backend.request.headers["x-cert-start"] - assert self.results[0].backend.request.headers["x-cert-end"] == ["2118-12-17T19:19:52.000Z"], \ - "unexpected x-cert-end value: %s" % self.results[0].backend.request.headers["x-cert-end"] + assert self.results[0].backend.request.headers["x-forwarded-client-cert"] == [ + 'Hash=c2d41a5977dcd28a3ba21f59ed5508cc6538defa810843d8a593e668306c8c4f;Subject="CN=presto.example.com,OU=Engineering,O=Presto,L=Bangalore,ST=KA,C=IN"' + ] + assert self.results[0].backend.request.headers["x-cert-start"] == [ + "2019-01-10T19:19:52.000Z" + ], ( + "unexpected x-cert-start value: %s" + % self.results[0].backend.request.headers["x-cert-start"] + ) + assert self.results[0].backend.request.headers["x-cert-end"] == [ + "2118-12-17T19:19:52.000Z" + ], ( + "unexpected x-cert-end value: %s" + % self.results[0].backend.request.headers["x-cert-end"] + ) assert self.results[1].backend assert self.results[1].backend.request - assert self.results[0].backend.request.headers["x-cert-start-custom"] == ["Jan 10 19:19:52 2019 UTC"], \ - "unexpected x-cert-start-custom value: %s" % self.results[1].backend.request.headers["x-cert-start-custom"] - assert self.results[0].backend.request.headers["x-cert-end-custom"] == ["Dec 17 19:19:52 2118 UTC"], \ - "unexpected x-cert-end-custom value: %s" % self.results[0].backend.request.headers["x-cert-end-custom"] - + assert self.results[0].backend.request.headers["x-cert-start-custom"] == [ + "Jan 10 19:19:52 2019 UTC" + ], ( + "unexpected x-cert-start-custom value: %s" + % self.results[1].backend.request.headers["x-cert-start-custom"] + ) + assert self.results[0].backend.request.headers["x-cert-end-custom"] == [ + "Dec 17 19:19:52 2118 UTC" + ], ( + "unexpected x-cert-end-custom value: %s" + % self.results[0].backend.request.headers["x-cert-end-custom"] + ) def requirements(self): for r in super().requirements(): @@ -192,13 +239,14 @@ def requirements(self): class ClientCertificateAuthenticationContext(AmbassadorTest): - def init(self): self.xfail = "FIXME: IHA" self.target = HTTP() def manifests(self) -> str: - return self.format(f""" + return ( + self.format( + f""" --- apiVersion: v1 metadata: @@ -233,39 +281,55 @@ def manifests(self) -> str: secret: ccauthctx-server-secret ca_secret: ccauthctx-client-secret cert_required: True -""") + super().manifests() +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping name: {self.target.path.k8s} prefix: /{self.name}/ service: {self.target.path.fqdn} -""") +""" + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url(self.name + "/"), insecure=True, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey, - client_cert_required=True, - ca_cert=TLSCerts["master.datawire.io"].pubcert) + yield Query( + self.url(self.name + "/"), + insecure=True, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey, + client_cert_required=True, + ca_cert=TLSCerts["master.datawire.io"].pubcert, + ) # In TLS < 1.3, there's not a dedicated alert code for "the client forgot to include a certificate", # so we get a generic alert=40 ("handshake_failure"). We also include "write: connection reset by peer" # because we've seen cases where Envoy and the client library don't play nicely, so the error report doesn't # get back before the connection closes. - yield Query(self.url(self.name + "/"), insecure=True, maxTLSv="v1.2", - error=[ "tls: handshake failure", "write: connection reset by peer" ]) + yield Query( + self.url(self.name + "/"), + insecure=True, + maxTLSv="v1.2", + error=["tls: handshake failure", "write: connection reset by peer"], + ) # TLS 1.3 added a dedicated alert=116 ("certificate_required") for that scenario. See above for why # "write: connection reset by peer" is also accepted. - yield Query(self.url(self.name + "/"), insecure=True, minTLSv="v1.3", - error=[ "tls: certificate required", "write: connection reset by peer" ]) + yield Query( + self.url(self.name + "/"), + insecure=True, + minTLSv="v1.3", + error=["tls: certificate required", "write: connection reset by peer"], + ) def requirements(self): for r in super().requirements(): @@ -279,13 +343,14 @@ def requirements(self): class ClientCertificateAuthenticationContextCRL(AmbassadorTest): - def init(self): self.xfail = "FIXME: IHA" # This test should cover TLSContext with a crl_secret self.target = HTTP() def manifests(self) -> str: - return self.format(f""" + return ( + self.format( + f""" --- apiVersion: v1 metadata: @@ -331,10 +396,14 @@ def manifests(self) -> str: ca_secret: ccauthctxcrl-client-secret crl_secret: ccauthctxcrl-crl-secret cert_required: True -""") + super().manifests() +""" + ) + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -342,31 +411,35 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: / service: {self.target.path.fqdn} hostname: "*" -""") +""" + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url(self.name + "/"), insecure=True, - client_crt=TLSCerts["presto.example.com"].pubcert, - client_key=TLSCerts["presto.example.com"].privkey, - client_cert_required=True, - ca_cert=TLSCerts["master.datawire.io"].pubcert, - error=[ "tls: revoked certificate" ]) + yield Query( + self.url(self.name + "/"), + insecure=True, + client_crt=TLSCerts["presto.example.com"].pubcert, + client_key=TLSCerts["presto.example.com"].privkey, + client_cert_required=True, + ca_cert=TLSCerts["master.datawire.io"].pubcert, + error=["tls: revoked certificate"], + ) def requirements(self): yield ("pod", self.path.k8s) class TLSOriginationSecret(AmbassadorTest): - def init(self): self.xfail = "FIXME: IHA" self.target = HTTP() def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 kind: Secret @@ -378,10 +451,13 @@ def manifests(self) -> str: data: tls.crt: {TLSCerts["localhost"].k8s_crt} tls.key: {TLSCerts["localhost"].k8s_key} -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -393,9 +469,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: upstream-files: cert_chain_file: /tmp/ambassador/snapshots/default/secrets-decoded/test-origination-secret/F94E4DCF30ABC50DEF240AA8024599B67CC03991.crt private_key_file: /tmp/ambassador/snapshots/default/secrets-decoded/test-origination-secret/F94E4DCF30ABC50DEF240AA8024599B67CC03991.key -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -403,9 +481,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}/ service: {self.target.path.fqdn} tls: upstream -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -413,7 +493,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /{self.name}-files/ service: {self.target.path.fqdn} tls: upstream-files -""") +""" + ) def queries(self): yield Query(self.url(self.name + "/")) @@ -435,7 +516,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 kind: Secret @@ -472,22 +554,24 @@ def manifests(self) -> str: requestPolicy: insecure: action: Reject -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: -# # Use self here, not self.target, because we want the TLS module to -# # be annotated on the Ambassador itself. -# yield self, self.format(""" -# --- -# apiVersion: getambassador.io/v3alpha1 -# kind: Module -# name: tls -# ambassador_id: [{self.ambassador_id}] -# config: -# server: -# enabled: True -# secret: test-tls-secret -# """) + # # Use self here, not self.target, because we want the TLS module to + # # be annotated on the Ambassador itself. + # yield self, self.format(""" + # --- + # apiVersion: getambassador.io/v3alpha1 + # kind: Module + # name: tls + # ambassador_id: [{self.ambassador_id}] + # config: + # server: + # enabled: True + # secret: test-tls-secret + # """) # Use self.target _here_, because we want the mapping to be annotated # on the service, not the Ambassador. Also, you don't need to include @@ -497,14 +581,16 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # If the test were more complex, we'd probably need to do some sort # of mangling for the mapping name and prefix. For this simple test, # it's not necessary. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping name: tls_target_mapping prefix: /tls-target/ service: {self.target.path.fqdn} -""") +""" + ) def scheme(self) -> str: return "https" @@ -522,7 +608,8 @@ def init(self): self.target = HTTP() def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -541,16 +628,19 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: enabled: True secret: test-certs-secret-invalid ca_secret: ambassador-certs -""") +""" + ) - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping name: tls_target_mapping prefix: /tls-target/ service: {self.target.path.fqdn} -""") +""" + ) def scheme(self) -> str: return "http" @@ -562,13 +652,15 @@ def check(self): assert self.results[0].backend errors = self.results[0].backend.response - expected = set({ - "TLSContext server found no certificate in secret test-certs-secret-invalid in namespace default, ignoring...", - "TLSContext bad-path-info found no cert_chain_file '/nonesuch'", - "TLSContext bad-path-info found no private_key_file '/nonesuch'", - "TLSContext validation-without-termination found no certificate in secret test-certs-secret-invalid in namespace default, ignoring...", - "TLSContext missing-secret-key: 'cert_chain_file' requires 'private_key_file' as well", - }) + expected = set( + { + "TLSContext server found no certificate in secret test-certs-secret-invalid in namespace default, ignoring...", + "TLSContext bad-path-info found no cert_chain_file '/nonesuch'", + "TLSContext bad-path-info found no private_key_file '/nonesuch'", + "TLSContext validation-without-termination found no certificate in secret test-certs-secret-invalid in namespace default, ignoring...", + "TLSContext missing-secret-key: 'cert_chain_file' requires 'private_key_file' as well", + } + ) current = set({}) for errsvc, errtext in errors: @@ -576,7 +668,7 @@ def check(self): diff = expected - current - assert len(diff) == 0, f'expected {len(expected)} errors, got {len(errors)}: Missing {diff}' + assert len(diff) == 0, f"expected {len(expected)} errors, got {len(errors)}: Missing {diff}" class TLSContextTest(AmbassadorTest): @@ -590,7 +682,9 @@ def init(self): self.xfail = "XFailing for now" def manifests(self) -> str: - return namespace_manifest("secret-namespace") + f""" + return ( + namespace_manifest("secret-namespace") + + f""" --- apiVersion: v1 data: @@ -625,10 +719,13 @@ def manifests(self) -> str: labels: kat-ambassador-id: tlscontexttest type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -636,8 +733,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /tls-context-same/ service: http://{self.target.path.fqdn} host: tls-context-host-1 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -648,8 +747,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: min_tls_version: v1.0 max_tls_version: v1.3 redirect_cleartext_from: 8080 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -657,8 +758,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /tls-context-same/ service: http://{self.target.path.fqdn} host: tls-context-host-2 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -668,8 +771,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: secret: test-tlscontext-secret-2 alpn_protocols: h2,http/1.1 redirect_cleartext_from: 8080 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -678,17 +783,21 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: server: enabled: True secret: test-tlscontext-secret-0 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping name: {self.name}-other-mapping prefix: /{self.name}/ service: https://{self.target.path.fqdn} -""") +""" + ) # Ambassador should not return an error when hostname is not present. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -696,9 +805,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: min_tls_version: v1.0 max_tls_version: v1.3 redirect_cleartext_from: 8080 -""") +""" + ) # Ambassador should return an error for this configuration. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -706,9 +817,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hosts: - tls-context-host-1 redirect_cleartext_from: 8080 -""") - # Ambassador should return an error for this configuration. - yield self, self.format(""" +""" + ) + # Ambassador should return an error for this configuration. + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -716,7 +829,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hosts: - tls-context-host-1 redirect_cleartext_from: 8081 -""") +""" + ) def scheme(self) -> str: return "https" @@ -731,71 +845,91 @@ def _go_close_connection_error(url): def queries(self): # 0 - yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), - headers={"Host": "tls-context-host-2"}, - insecure=True, - sni=True) + yield Query( + self.url("ambassador/v0/diag/?json=true&filter=errors"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ) # 1 - Correct host #1 - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + ) # 2 - Correct host #2 - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-2"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-2"}, + expected=200, + insecure=True, + sni=True, + ) # 3 - Incorrect host - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-3"}, - # error=self._go_close_connection_error(self.url("tls-context-same/")), - expected=404, - insecure=True) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-3"}, + # error=self._go_close_connection_error(self.url("tls-context-same/")), + expected=404, + insecure=True, + ) # 4 - Incorrect path, correct host - yield Query(self.url("tls-context-different/"), - headers={"Host": "tls-context-host-1"}, - expected=404, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-different/"), + headers={"Host": "tls-context-host-1"}, + expected=404, + insecure=True, + sni=True, + ) # Other mappings with no host will respond with the fallbock cert. # 5 - no Host header, fallback cert from the TLS module - yield Query(self.url(self.name + "/"), - # error=self._go_close_connection_error(self.url(self.name + "/")), - insecure=True) + yield Query( + self.url(self.name + "/"), + # error=self._go_close_connection_error(self.url(self.name + "/")), + insecure=True, + ) # 6 - explicit Host header, fallback cert - yield Query(self.url(self.name + "/"), - # error=self._go_close_connection_error(self.url(self.name + "/")), - # sni=True, - headers={"Host": "tls-context-host-3"}, - insecure=True) + yield Query( + self.url(self.name + "/"), + # error=self._go_close_connection_error(self.url(self.name + "/")), + # sni=True, + headers={"Host": "tls-context-host-3"}, + insecure=True, + ) # 7 - explicit Host header 1 wins, we'll get the SNI cert for this overlapping path - yield Query(self.url(self.name + "/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + ) # 8 - explicit Host header 2 wins, we'll get the SNI cert for this overlapping path - yield Query(self.url(self.name + "/"), - headers={"Host": "tls-context-host-2"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/"), + headers={"Host": "tls-context-host-2"}, + expected=200, + insecure=True, + sni=True, + ) # 9 - Redirect cleartext from actually redirects. - yield Query(self.url("tls-context-same/", scheme="http"), - headers={"Host": "tls-context-host-1"}, - expected=301, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-same/", scheme="http"), + headers={"Host": "tls-context-host-1"}, + expected=301, + insecure=True, + sni=True, + ) def check(self): # XXX Ew. If self.results[0].json is empty, the harness won't convert it to a response. @@ -804,11 +938,11 @@ def check(self): assert num_errors == 5, "expected 5 errors, got {} -\n{}".format(num_errors, errors) errors_that_should_be_found = { - 'TLSContext TLSContextTest-no-secret has no certificate information at all?': False, - 'TLSContext TLSContextTest-same-context-error has no certificate information at all?': False, - 'TLSContext TLSContextTest-same-context-error is missing cert_chain_file': False, - 'TLSContext TLSContextTest-same-context-error is missing private_key_file': False, - 'TLSContext: TLSContextTest-rcf-error; configured conflicting redirect_from port: 8081': False + "TLSContext TLSContextTest-no-secret has no certificate information at all?": False, + "TLSContext TLSContextTest-same-context-error has no certificate information at all?": False, + "TLSContext TLSContextTest-same-context-error is missing cert_chain_file": False, + "TLSContext TLSContextTest-same-context-error is missing private_key_file": False, + "TLSContext: TLSContextTest-rcf-error; configured conflicting redirect_from port: 8081": False, } unknown_errors: List[str] = [] @@ -830,8 +964,8 @@ def check(self): for result in self.results: if result.status == 200 and result.query.headers: - host_header = result.query.headers['Host'] - tls_common_name = result.tls[0]['Issuer']['CommonName'] + host_header = result.query.headers["Host"] + tls_common_name = result.tls[0]["Issuer"]["CommonName"] # XXX Weirdness with the fallback cert here! You see, if we use host # tls-context-host-3 (or, really, anything except -1 or -2), then the @@ -840,23 +974,58 @@ def check(self): # # Ew. - if host_header == 'tls-context-host-3': - host_header = 'localhost' + if host_header == "tls-context-host-3": + host_header = "localhost" - assert host_header == tls_common_name, "test %d wanted CN %s, but got %s" % (idx, host_header, tls_common_name) + assert host_header == tls_common_name, "test %d wanted CN %s, but got %s" % ( + idx, + host_header, + tls_common_name, + ) idx += 1 def requirements(self): # We're replacing super()'s requirements deliberately here. Without a Host header they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) class TLSIngressTest(AmbassadorTest): - def init(self): self.xfail = "FIXME: IHA" self.target = HTTP() @@ -867,7 +1036,9 @@ def manifests(self) -> str: value: "diagd" """ - return namespace_manifest("secret-namespace-ingress") + f""" + return ( + namespace_manifest("secret-namespace-ingress") + + f""" --- apiVersion: v1 data: @@ -950,10 +1121,13 @@ def manifests(self) -> str: number: 80 path: /tls-context-same/ pathType: Prefix -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -962,9 +1136,11 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: server: enabled: True secret: test-tlscontext-secret-ingress-0 -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -972,7 +1148,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: name: {self.name}-other-mapping prefix: /{self.name}/ service: https://{self.target.path.fqdn} -""") +""" + ) def scheme(self) -> str: return "https" @@ -987,64 +1164,82 @@ def _go_close_connection_error(url): def queries(self): # 0 - yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), - headers={"Host": "tls-context-host-2"}, - insecure=True, - sni=True) + yield Query( + self.url("ambassador/v0/diag/?json=true&filter=errors"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ) # 1 - Correct host #1 - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + ) # 2 - Correct host #2 - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-2"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-2"}, + expected=200, + insecure=True, + sni=True, + ) # 3 - Incorrect host - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-3"}, - # error=self._go_close_connection_error(self.url("tls-context-same/")), - expected=404, - insecure=True) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-3"}, + # error=self._go_close_connection_error(self.url("tls-context-same/")), + expected=404, + insecure=True, + ) # 4 - Incorrect path, correct host - yield Query(self.url("tls-context-different/"), - headers={"Host": "tls-context-host-1"}, - expected=404, - insecure=True, - sni=True) + yield Query( + self.url("tls-context-different/"), + headers={"Host": "tls-context-host-1"}, + expected=404, + insecure=True, + sni=True, + ) # Other mappings with no host will respond with the fallbock cert. # 5 - no Host header, fallback cert from the TLS module - yield Query(self.url(self.name + "/"), - # error=self._go_close_connection_error(self.url(self.name + "/")), - insecure=True) + yield Query( + self.url(self.name + "/"), + # error=self._go_close_connection_error(self.url(self.name + "/")), + insecure=True, + ) # 6 - explicit Host header, fallback cert - yield Query(self.url(self.name + "/"), - # error=self._go_close_connection_error(self.url(self.name + "/")), - # sni=True, - headers={"Host": "tls-context-host-3"}, - insecure=True) + yield Query( + self.url(self.name + "/"), + # error=self._go_close_connection_error(self.url(self.name + "/")), + # sni=True, + headers={"Host": "tls-context-host-3"}, + insecure=True, + ) # 7 - explicit Host header 1 wins, we'll get the SNI cert for this overlapping path - yield Query(self.url(self.name + "/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + ) # 7 - explicit Host header 2 wins, we'll get the SNI cert for this overlapping path - yield Query(self.url(self.name + "/"), - headers={"Host": "tls-context-host-2"}, - expected=200, - insecure=True, - sni=True) + yield Query( + self.url(self.name + "/"), + headers={"Host": "tls-context-host-2"}, + expected=200, + insecure=True, + sni=True, + ) def check(self): # XXX Ew. If self.results[0].json is empty, the harness won't convert it to a response. @@ -1056,8 +1251,8 @@ def check(self): for result in self.results: if result.status == 200 and result.query.headers: - host_header = result.query.headers['Host'] - tls_common_name = result.tls[0]['Issuer']['CommonName'] + host_header = result.query.headers["Host"] + tls_common_name = result.tls[0]["Issuer"]["CommonName"] # XXX Weirdness with the fallback cert here! You see, if we use host # tls-context-host-3 (or, really, anything except -1 or -2), then the @@ -1066,24 +1261,60 @@ def check(self): # # Ew. - if host_header == 'tls-context-host-3': - host_header = 'localhost' + if host_header == "tls-context-host-3": + host_header = "localhost" # Yep, that's expected. Since the TLS secret for 'tls-context-host-1' is # not namespaced it should only resolve to the Ingress' own # namespace, and can't use the 'secret.namespace' Ambassador syntax - if host_header == 'tls-context-host-1': - host_header = 'localhost' + if host_header == "tls-context-host-1": + host_header = "localhost" - assert host_header == tls_common_name, "test %d wanted CN %s, but got %s" % (idx, host_header, tls_common_name) + assert host_header == tls_common_name, "test %d wanted CN %s, but got %s" % ( + idx, + host_header, + tls_common_name, + ) idx += 1 def requirements(self): - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-2"}, insecure=True, sni=True)) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-2"}, + insecure=True, + sni=True, + ), + ) class TLSContextProtocolMaxVersion(AmbassadorTest): @@ -1104,7 +1335,8 @@ def init(self): self.xfail = "FIXME: IHA" def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -1116,10 +1348,13 @@ def manifests(self) -> str: labels: kat-ambassador-id: tlscontextprotocolmaxversion type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -1143,7 +1378,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: secret: secret.max-version min_tls_version: v1.1 max_tls_version: v1.2 -""") +""" + ) def scheme(self) -> str: return "https" @@ -1166,35 +1402,43 @@ def queries(self): # For now, we're checking for the None result, but, ew. # ---- - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - minTLSv="v1.2", - maxTLSv="v1.2") + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + minTLSv="v1.2", + maxTLSv="v1.2", + ) # This should give us TLS v1.1 - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - minTLSv="v1.0", - maxTLSv="v1.1") + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + minTLSv="v1.0", + maxTLSv="v1.1", + ) # This should be an error. - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - minTLSv="v1.3", - maxTLSv="v1.3", - error=[ "tls: server selected unsupported protocol version 303", - "tls: no supported versions satisfy MinVersion and MaxVersion", - "tls: protocol version not supported", - "read: connection reset by peer"]) # The TLS inspector just closes the connection. Wow. + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + minTLSv="v1.3", + maxTLSv="v1.3", + error=[ + "tls: server selected unsupported protocol version 303", + "tls: no supported versions satisfy MinVersion and MaxVersion", + "tls: protocol version not supported", + "read: connection reset by peer", + ], + ) # The TLS inspector just closes the connection. Wow. def check(self): assert self.results[0].backend @@ -1210,8 +1454,27 @@ def check(self): def requirements(self): # We're replacing super()'s requirements deliberately here. Without a Host header they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, minTLSv="v1.2")) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True, minTLSv="v1.2")) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + minTLSv="v1.2", + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + minTLSv="v1.2", + ), + ) + class TLSContextProtocolMinVersion(AmbassadorTest): # Here we're testing that the client can't drop below the minimum TLS version @@ -1227,7 +1490,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -1239,10 +1503,13 @@ def manifests(self) -> str: labels: kat-ambassador-id: tlscontextprotocolminversion type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1260,7 +1527,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: secret_namespacing: False min_tls_version: v1.2 max_tls_version: v1.3 -""") +""" + ) def scheme(self) -> str: return "https" @@ -1275,34 +1543,42 @@ def _go_close_connection_error(url): def queries(self): # This should give v1.3, but it currently seems to give 1.2. - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - minTLSv="v1.2", - maxTLSv="v1.3") + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + minTLSv="v1.2", + maxTLSv="v1.3", + ) # This should give v1.2 - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - minTLSv="v1.1", - maxTLSv="v1.2") + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + minTLSv="v1.1", + maxTLSv="v1.2", + ) # This should be an error. - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - minTLSv="v1.0", - maxTLSv="v1.0", - error=[ "tls: server selected unsupported protocol version 303", - "tls: no supported versions satisfy MinVersion and MaxVersion", - "tls: protocol version not supported" ]) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + minTLSv="v1.0", + maxTLSv="v1.0", + error=[ + "tls: server selected unsupported protocol version 303", + "tls: no supported versions satisfy MinVersion and MaxVersion", + "tls: protocol version not supported", + ], + ) def check(self): assert self.results[0].backend @@ -1319,8 +1595,25 @@ def check(self): def requirements(self): # We're replacing super()'s requirements deliberately here. Without a Host header they can't work. - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + class TLSContextCipherSuites(AmbassadorTest): # debug = True @@ -1330,7 +1623,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 data: @@ -1342,10 +1636,13 @@ def manifests(self) -> str: labels: kat-ambassador-id: tlscontextciphersuites type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1353,8 +1650,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /tls-context-same/ service: https://{self.target.path.fqdn} host: tls-context-host-1 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -1368,7 +1667,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - ECDHE-RSA-AES128-GCM-SHA256 ecdh_curves: - P-256 -""") +""" + ) def scheme(self) -> str: return "https" @@ -1382,32 +1682,38 @@ def _go_close_connection_error(url): return "Get {}: EOF".format(url) def queries(self): - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - cipherSuites=["TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"], - maxTLSv="v1.2") - - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - cipherSuites=["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"], - maxTLSv="v1.2", - error="tls: handshake failure",) - - yield Query(self.url("tls-context-same/"), - headers={"Host": "tls-context-host-1"}, - expected=200, - insecure=True, - sni=True, - cipherSuites=["TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"], - ecdhCurves=["X25519"], - maxTLSv="v1.2", - error="tls: handshake failure",) + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + cipherSuites=["TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"], + maxTLSv="v1.2", + ) + + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + cipherSuites=["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"], + maxTLSv="v1.2", + error="tls: handshake failure", + ) + + yield Query( + self.url("tls-context-same/"), + headers={"Host": "tls-context-host-1"}, + expected=200, + insecure=True, + sni=True, + cipherSuites=["TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"], + ecdhCurves=["X25519"], + maxTLSv="v1.2", + error="tls: handshake failure", + ) def check(self): assert self.results[0].backend @@ -1417,8 +1723,25 @@ def check(self): assert tls_0_version == "v1.2", f"requesting TLS v1.2 got TLS {tls_0_version}" def requirements(self): - yield ("url", Query(self.url("ambassador/v0/check_ready"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) - yield ("url", Query(self.url("ambassador/v0/check_alive"), headers={"Host": "tls-context-host-1"}, insecure=True, sni=True)) + yield ( + "url", + Query( + self.url("ambassador/v0/check_ready"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + yield ( + "url", + Query( + self.url("ambassador/v0/check_alive"), + headers={"Host": "tls-context-host-1"}, + insecure=True, + sni=True, + ), + ) + class TLSContextIstioSecretTest(AmbassadorTest): # debug = True @@ -1430,7 +1753,9 @@ def init(self): self.xfail = "XFailing for now" def manifests(self) -> str: - return namespace_manifest("secret-namespace") + """ + return ( + namespace_manifest("secret-namespace") + + """ --- apiVersion: v1 data: @@ -1444,10 +1769,13 @@ def manifests(self) -> str: labels: kat-ambassador-id: tlscontextistiosecret type: istio.io/key-and-cert -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1455,8 +1783,10 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: prefix: /tls-context-istio/ service: https://{self.target.path.fqdn} tls: {self.name}-istio-context-1 -""") - yield self, self.format(""" +""" + ) + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -1464,17 +1794,19 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: secret: istio.test-tlscontext-istio-secret-1 namespace: secret-namespace secret_namespacing: False -""") +""" + ) def queries(self): yield Query(self.url("ambassador/v0/diag/?json=true&filter=errors"), phase=2) def check(self): - assert self.results[0].backend is None, f'expected 0 errors, got {len(self.results[0].backend.response)}: received {self.results[0].backend.response}' + assert ( + self.results[0].backend is None + ), f"expected 0 errors, got {len(self.results[0].backend.response)}: received {self.results[0].backend.response}" class TLSCoalescing(AmbassadorTest): - def init(self): self.target = HTTP() @@ -1484,7 +1816,8 @@ def init(self): self.xfail = "FIXME: IHA" def manifests(self) -> str: - return f""" + return ( + f""" --- apiVersion: v1 metadata: @@ -1496,10 +1829,13 @@ def manifests(self) -> str: tls.key: {TLSCerts["*.domain.com"].k8s_key} kind: Secret type: kubernetes.io/tls -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self, self.format(""" + yield self, self.format( + """ apiVersion: getambassador.io/v3alpha1 kind: TLSContext name: tlscoalescing-context @@ -1509,7 +1845,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - domain.com - a.domain.com - b.domain.com -""") +""" + ) def scheme(self) -> str: return "https" @@ -1523,14 +1860,18 @@ def _go_close_connection_error(url): return "Get {}: EOF".format(url) def queries(self): - yield Query(self.url("ambassador/v0/diag/"), - headers={"Host": "a.domain.com"}, - insecure=True, - sni=True) - yield Query(self.url("ambassador/v0/diag/"), - headers={"Host": "b.domain.com"}, - insecure=True, - sni=True) + yield Query( + self.url("ambassador/v0/diag/"), + headers={"Host": "a.domain.com"}, + insecure=True, + sni=True, + ) + yield Query( + self.url("ambassador/v0/diag/"), + headers={"Host": "b.domain.com"}, + insecure=True, + sni=True, + ) def requirements(self): yield ("url", Query(self.url("ambassador/v0/check_ready"), insecure=True, sni=True)) @@ -1546,7 +1887,8 @@ def init(self): def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # These are annotations instead of resources because the name matters. - yield self, self.format(''' + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -1556,10 +1898,13 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: server: enabled: True redirect_cleartext_from: 8080 -''') +""" + ) def manifests(self) -> str: - return self.format(''' + return ( + self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TLSContext @@ -1580,8 +1925,12 @@ def manifests(self) -> str: kat-ambassador-id: {self.ambassador_id} type: kubernetes.io/tls data: - tls.crt: '''+TLSCerts["a.domain.com"].k8s_crt+''' - tls.key: '''+TLSCerts["a.domain.com"].k8s_key+''' + tls.crt: """ + + TLSCerts["a.domain.com"].k8s_crt + + """ + tls.key: """ + + TLSCerts["a.domain.com"].k8s_key + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -1591,25 +1940,42 @@ def manifests(self) -> str: ambassador_id: [ {self.ambassador_id} ] prefix: /foo service: {self.target.path.fqdn} -''') + super().manifests() +""" + ) + + super().manifests() + ) def scheme(self) -> str: return "https" def queries(self): - yield Query(self.url("foo", scheme="http"), headers={"Host": "a.domain.com"}, - expected=301) - yield Query(self.url("bar", scheme="http"), headers={"Host": "a.domain.com"}, - expected=(404 if bug_404_routes else 301)) - yield Query(self.url("foo", scheme="https"), headers={"Host": "a.domain.com"}, ca_cert=TLSCerts["a.domain.com"].pubcert, sni=True, - expected=200) - yield Query(self.url("bar", scheme="https"), headers={"Host": "a.domain.com"}, ca_cert=TLSCerts["a.domain.com"].pubcert, sni=True, - expected=404) + yield Query(self.url("foo", scheme="http"), headers={"Host": "a.domain.com"}, expected=301) + yield Query( + self.url("bar", scheme="http"), + headers={"Host": "a.domain.com"}, + expected=(404 if bug_404_routes else 301), + ) + yield Query( + self.url("foo", scheme="https"), + headers={"Host": "a.domain.com"}, + ca_cert=TLSCerts["a.domain.com"].pubcert, + sni=True, + expected=200, + ) + yield Query( + self.url("bar", scheme="https"), + headers={"Host": "a.domain.com"}, + ca_cert=TLSCerts["a.domain.com"].pubcert, + sni=True, + expected=404, + ) def requirements(self): for r in super().requirements(): query = r[1] - query.headers={"Host": "a.domain.com"} - query.sni = True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + query.headers = {"Host": "a.domain.com"} + query.sni = ( + True # Use query.headers["Host"] instead of urlparse(query.url).hostname for SNI + ) query.ca_cert = TLSCerts["a.domain.com"].pubcert yield (r[0], query) diff --git a/python/tests/kat/t_tracing.py b/python/tests/kat/t_tracing.py index bc7dd2ed47..546c4f375f 100644 --- a/python/tests/kat/t_tracing.py +++ b/python/tests/kat/t_tracing.py @@ -16,12 +16,14 @@ # 2, so the hope is that phase 3 reduces the likelihood of the test flaking again. check_phase = 3 + class TracingTest(AmbassadorTest): def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -58,13 +60,16 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -72,10 +77,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) # Configure the TracingService. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TracingService @@ -84,7 +91,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: driver: zipkin tag_headers: - "x-watsup" -""") +""" + ) def requirements(self): yield from super().requirements() @@ -94,14 +102,17 @@ def queries(self): # Speak through each Ambassador to the traced service... for i in range(100): - yield Query(self.url("target/"), headers={'x-watsup':'nothin'}, phase=1) - + yield Query(self.url("target/"), headers={"x-watsup": "nothin"}, phase=1) # ...then ask the Zipkin for services and spans. Including debug=True in these queries # is particularly helpful. yield Query("http://zipkin:9411/api/v2/services", phase=check_phase) - yield Query("http://zipkin:9411/api/v2/spans?serviceName=tracingtest-default", phase=check_phase) - yield Query("http://zipkin:9411/api/v2/traces?serviceName=tracingtest-default", phase=check_phase) + yield Query( + "http://zipkin:9411/api/v2/spans?serviceName=tracingtest-default", phase=check_phase + ) + yield Query( + "http://zipkin:9411/api/v2/traces?serviceName=tracingtest-default", phase=check_phase + ) # The diagnostics page should load properly yield Query(self.url("ambassador/v0/diag/"), phase=check_phase) @@ -113,17 +124,18 @@ def check(self): assert result.backend.name == self.target.path.k8s print(f"self.results[100] = {self.results[100]}") - assert self.results[100].backend is not None and self.results[100].backend.name == "raw", \ - f"unexpected self.results[100] = {self.results[100]}" + assert ( + self.results[100].backend is not None and self.results[100].backend.name == "raw" + ), f"unexpected self.results[100] = {self.results[100]}" assert len(self.results[100].backend.response) == 1 - assert self.results[100].backend.response[0] == 'tracingtest-default' + assert self.results[100].backend.response[0] == "tracingtest-default" assert self.results[101].backend assert self.results[101].backend.name == "raw" - tracelist = { x: True for x in self.results[101].backend.response } + tracelist = {x: True for x in self.results[101].backend.response} - assert 'router cluster_tracingtest_http_default egress' in tracelist + assert "router cluster_tracingtest_http_default egress" in tracelist # Look for the host that we actually queried, since that's what appears in the spans. assert self.results[0].backend @@ -132,12 +144,12 @@ def check(self): # Ensure we generate 128-bit traceids by default trace = self.results[102].json[0][0] - traceId = trace['traceId'] + traceId = trace["traceId"] assert len(traceId) == 32 for t in self.results[102].json[0]: - if t.get('tags', {}).get('node_id') == 'test-id': - assert 'x-watsup' in t['tags'] - assert t['tags']['x-watsup'] == 'nothin' + if t.get("tags", {}).get("node_id") == "test-id": + assert "x-watsup" in t["tags"] + assert t["tags"]["x-watsup"] == "nothin" class TracingTestLongClusterName(AmbassadorTest): @@ -145,7 +157,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -182,13 +195,16 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -196,34 +212,50 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) # Configure the TracingService. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TracingService name: tracing-longclustername service: zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411 driver: zipkin -""") +""" + ) def requirements(self): yield from super().requirements() - yield ("url", Query("http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/services")) + yield ( + "url", + Query( + "http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/services" + ), + ) def queries(self): # Speak through each Ambassador to the traced service... for i in range(100): - yield Query(self.url("target/"), phase=1) - + yield Query(self.url("target/"), phase=1) # ...then ask the Zipkin for services and spans. Including debug=True in these queries # is particularly helpful. - yield Query("http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/services", phase=check_phase) - yield Query("http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/spans?serviceName=tracingtestlongclustername-default", phase=check_phase) - yield Query("http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/traces?serviceName=tracingtestlongclustername-default", phase=check_phase) + yield Query( + "http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/services", + phase=check_phase, + ) + yield Query( + "http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/spans?serviceName=tracingtestlongclustername-default", + phase=check_phase, + ) + yield Query( + "http://zipkinservicenamewithoversixtycharacterstoforcenamecompression:9411/api/v2/traces?serviceName=tracingtestlongclustername-default", + phase=check_phase, + ) # The diagnostics page should load properly, even though our Tracing Service # has a long cluster name https://github.com/datawire/ambassador/issues/3021 @@ -236,17 +268,18 @@ def check(self): assert result.backend.name == self.target.path.k8s print(f"self.results[100] = {self.results[100]}") - assert self.results[100].backend is not None and self.results[100].backend.name == "raw", \ - f"unexpected self.results[100] = {self.results[100]}" + assert ( + self.results[100].backend is not None and self.results[100].backend.name == "raw" + ), f"unexpected self.results[100] = {self.results[100]}" assert len(self.results[100].backend.response) == 1 - assert self.results[100].backend.response[0] == 'tracingtestlongclustername-default' + assert self.results[100].backend.response[0] == "tracingtestlongclustername-default" assert self.results[101].backend assert self.results[101].backend.name == "raw" - tracelist = { x: True for x in self.results[101].backend.response } + tracelist = {x: True for x in self.results[101].backend.response} - assert 'router cluster_tracingtestlongclustername_http_default egress' in tracelist + assert "router cluster_tracingtestlongclustername_http_default egress" in tracelist # Look for the host that we actually queried, since that's what appears in the spans. assert self.results[0].backend @@ -255,15 +288,17 @@ def check(self): # Ensure we generate 128-bit traceids by default trace = self.results[102].json[0][0] - traceId = trace['traceId'] + traceId = trace["traceId"] assert len(traceId) == 32 + class TracingTestShortTraceId(AmbassadorTest): def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -300,13 +335,16 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -314,7 +352,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target-64/ service: {self.target.path.fqdn} -""") +""" + ) # Configure the TracingService. yield self, """ @@ -346,13 +385,13 @@ def queries(self): def check(self): # Ensure we generated 64-bit traceids trace = self.results[1].json[0][0] - traceId = trace['traceId'] + traceId = trace["traceId"] assert len(traceId) == 16 + # This test asserts that the external authorization server receives the proper tracing # headers when Ambassador is configured with an HTTP AuthService. class TracingExternalAuthTest(AmbassadorTest): - def init(self): if EDGE_STACK: self.xfail = "XFailing for now, custom AuthServices not supported in Edge Stack" @@ -360,7 +399,8 @@ def init(self): self.auth = AHTTP(name="auth") def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -397,10 +437,13 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -408,18 +451,22 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TracingService name: tracing-auth service: zipkin-auth:9411 driver: zipkin -""") +""" + ) - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: AuthService @@ -429,14 +476,17 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: allowed_request_headers: - Kat-Req-Extauth-Requested-Status - Kat-Req-Extauth-Requested-Header -""") +""" + ) def requirements(self): yield from super().requirements() yield ("url", Query("http://zipkin-auth:9411/api/v2/services")) def queries(self): - yield Query(self.url("target/"), headers={"Kat-Req-Extuath-Requested-Status": "200"}, expected=200) + yield Query( + self.url("target/"), headers={"Kat-Req-Extuath-Requested-Status": "200"}, expected=200 + ) def check(self): extauth_res = json.loads(self.results[0].headers["Extauth"][0]) @@ -446,7 +496,10 @@ def check(self): assert self.results[0].status == 200 assert self.results[0].headers["Server"] == ["envoy"] - assert extauth_res["request"]["headers"]["x-b3-parentspanid"] == request_headers["x-b3-parentspanid"] + assert ( + extauth_res["request"]["headers"]["x-b3-parentspanid"] + == request_headers["x-b3-parentspanid"] + ) assert extauth_res["request"]["headers"]["x-b3-sampled"] == request_headers["x-b3-sampled"] assert extauth_res["request"]["headers"]["x-b3-spanid"] == request_headers["x-b3-spanid"] assert extauth_res["request"]["headers"]["x-b3-traceid"] == request_headers["x-b3-traceid"] @@ -462,7 +515,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -499,13 +553,16 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -513,7 +570,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target-65/ service: {self.target.path.fqdn} -""") +""" + ) # Configure the TracingService. yield self, """ @@ -548,14 +606,15 @@ def check(self): print("%d traces obtained" % len(traces)) - #import json - #print(json.dumps(traces, indent=4, sort_keys=True)) + # import json + # print(json.dumps(traces, indent=4, sort_keys=True)) # We constantly find that Envoy's RNG isn't exactly predictable with small sample # sizes, so even though 10% of 100 is 10, we'll make this pass as long as we don't # go over 50 or under 1. assert 1 <= len(traces) <= 50 + class TracingTestZipkinV2(AmbassadorTest): """ Test for the "collector_endpoint_version" Zipkin config in TracingServices @@ -565,7 +624,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -602,12 +662,15 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -615,10 +678,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) # Configure the TracingService. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TracingService @@ -629,7 +694,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: collector_endpoint: /api/v2/spans collector_endpoint_version: HTTP_JSON collector_hostname: zipkin-v2 -""") +""" + ) def requirements(self): yield from super().requirements() @@ -639,14 +705,19 @@ def queries(self): # Speak through each Ambassador to the traced service... for i in range(100): - yield Query(self.url("target/"), phase=1) - + yield Query(self.url("target/"), phase=1) # ...then ask the Zipkin for services and spans. Including debug=True in these queries # is particularly helpful. yield Query("http://zipkin-v2:9411/api/v2/services", phase=check_phase) - yield Query("http://zipkin-v2:9411/api/v2/spans?serviceName=tracingtestzipkinv2-default", phase=check_phase) - yield Query("http://zipkin-v2:9411/api/v2/traces?serviceName=tracingtestzipkinv2-default", phase=check_phase) + yield Query( + "http://zipkin-v2:9411/api/v2/spans?serviceName=tracingtestzipkinv2-default", + phase=check_phase, + ) + yield Query( + "http://zipkin-v2:9411/api/v2/traces?serviceName=tracingtestzipkinv2-default", + phase=check_phase, + ) # The diagnostics page should load properly yield Query(self.url("ambassador/v0/diag/"), phase=check_phase) @@ -658,17 +729,18 @@ def check(self): assert result.backend.name == self.target.path.k8s print(f"self.results[100] = {self.results[100]}") - assert self.results[100].backend is not None and self.results[100].backend.name == "raw", \ - f"unexpected self.results[100] = {self.results[100]}" + assert ( + self.results[100].backend is not None and self.results[100].backend.name == "raw" + ), f"unexpected self.results[100] = {self.results[100]}" assert len(self.results[100].backend.response) == 1 - assert self.results[100].backend.response[0] == 'tracingtestzipkinv2-default' + assert self.results[100].backend.response[0] == "tracingtestzipkinv2-default" assert self.results[101].backend assert self.results[101].backend.name == "raw" - tracelist = { x: True for x in self.results[101].backend.response } + tracelist = {x: True for x in self.results[101].backend.response} - assert 'router cluster_tracingtestzipkinv2_http_default egress' in tracelist + assert "router cluster_tracingtestzipkinv2_http_default egress" in tracelist # Look for the host that we actually queried, since that's what appears in the spans. assert self.results[0].backend @@ -677,9 +749,10 @@ def check(self): # Ensure we generate 128-bit traceids by default trace = self.results[102].json[0][0] - traceId = trace['traceId'] + traceId = trace["traceId"] assert len(traceId) == 32 + class TracingTestZipkinV1(AmbassadorTest): """ Test for the "collector_endpoint_version" Zipkin config in TracingServices @@ -689,7 +762,8 @@ def init(self): self.target = HTTP() def manifests(self) -> str: - return """ + return ( + """ --- apiVersion: v1 kind: Service @@ -726,13 +800,16 @@ def manifests(self) -> str: ports: - name: http containerPort: 9411 -""" + super().manifests() +""" + + super().manifests() + ) def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: # Use self.target here, because we want this mapping to be annotated # on the service, not the Ambassador. - yield self.target, self.format(""" + yield self.target, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -740,10 +817,12 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: hostname: "*" prefix: /target/ service: {self.target.path.fqdn} -""") +""" + ) # Configure the TracingService. - yield self, self.format(""" + yield self, self.format( + """ --- apiVersion: getambassador.io/v3alpha1 kind: TracingService @@ -754,7 +833,8 @@ def config(self) -> Generator[Union[str, Tuple[Node, str]], None, None]: collector_endpoint: /api/v1/spans collector_endpoint_version: HTTP_JSON_V1 collector_hostname: zipkin-v1 -""") +""" + ) def requirements(self): yield from super().requirements() @@ -764,14 +844,20 @@ def queries(self): # Speak through each Ambassador to the traced service... for i in range(100): - yield Query(self.url("target/"), phase=1) + yield Query(self.url("target/"), phase=1) # result 100 - yield Query("http://zipkin-v1:9411/api/v2/services", phase=check_phase) + yield Query("http://zipkin-v1:9411/api/v2/services", phase=check_phase) # result 101 - yield Query("http://zipkin-v1:9411/api/v2/spans?serviceName=tracingtestzipkinv1-default", phase=check_phase) + yield Query( + "http://zipkin-v1:9411/api/v2/spans?serviceName=tracingtestzipkinv1-default", + phase=check_phase, + ) # result 102 - yield Query("http://zipkin-v1:9411/api/v2/traces?serviceName=tracingtestzipkinv1-default", phase=check_phase) + yield Query( + "http://zipkin-v1:9411/api/v2/traces?serviceName=tracingtestzipkinv1-default", + phase=check_phase, + ) # The diagnostics page should load properly yield Query(self.url("ambassador/v0/diag/"), phase=check_phase) @@ -793,4 +879,3 @@ def check(self): # verify no traces were captured traces = self.results[102].json assert len(traces) == 0 - diff --git a/python/tests/kat/test_ambassador.py b/python/tests/kat/test_ambassador.py index 2215cbd4c8..d2d1be3007 100644 --- a/python/tests/kat/test_ambassador.py +++ b/python/tests/kat/test_ambassador.py @@ -33,16 +33,17 @@ import t_lua_scripts import t_max_req_header_kb import t_no_ui -import t_mappingtests_default # mapping tests executed in the default namespace -import t_plain # t_plain include t_mappingtests_plain and t_optiontests as imports; these tests require each other and need to be executed as a set +import t_mappingtests_default # mapping tests executed in the default namespace +import t_plain # t_plain include t_mappingtests_plain and t_optiontests as imports; these tests require each other and need to be executed as a set import t_queryparameter_routing import t_ratelimit import t_redirect import t_regexrewrite_forwarding import t_request_header import t_retrypolicy -#import t_shadow -#import t_stats # t_stats has tests for statsd and dogstatsd. It's too flaky to run all the time. + +# import t_shadow +# import t_stats # t_stats has tests for statsd and dogstatsd. It's too flaky to run all the time. import t_tcpmapping import t_tls import t_tracing diff --git a/python/tests/kubeutils.py b/python/tests/kubeutils.py index ce3bfaef60..a2ccb1ab07 100644 --- a/python/tests/kubeutils.py +++ b/python/tests/kubeutils.py @@ -2,25 +2,26 @@ from tests.runutils import run_with_retry + def meta_action_kube_artifacts(namespace, artifacts, action, retries=0): temp_file = tempfile.NamedTemporaryFile() temp_file.write(artifacts.encode()) temp_file.flush() - command = ['tools/bin/kubectl', action, '-f', temp_file.name] + command = ["tools/bin/kubectl", action, "-f", temp_file.name] if namespace is None: - namespace = 'default' + namespace = "default" if namespace is not None: - command.extend(['-n', namespace]) + command.extend(["-n", namespace]) run_with_retry(command, retries=retries) temp_file.close() def apply_kube_artifacts(namespace, artifacts): - meta_action_kube_artifacts(namespace=namespace, artifacts=artifacts, action='apply', retries=1) + meta_action_kube_artifacts(namespace=namespace, artifacts=artifacts, action="apply", retries=1) def delete_kube_artifacts(namespace, artifacts): - meta_action_kube_artifacts(namespace=namespace, artifacts=artifacts, action='delete') + meta_action_kube_artifacts(namespace=namespace, artifacts=artifacts, action="delete") diff --git a/python/tests/manifests.py b/python/tests/manifests.py index ed93d01604..788c1a4cdd 100644 --- a/python/tests/manifests.py +++ b/python/tests/manifests.py @@ -1,4 +1,4 @@ -httpbin_manifests=""" +httpbin_manifests = """ --- apiVersion: v1 kind: Service @@ -75,7 +75,7 @@ """ -websocket_echo_server_manifests=""" +websocket_echo_server_manifests = """ --- apiVersion: v1 kind: Service diff --git a/python/tests/runutils.py b/python/tests/runutils.py index 923d3d5769..ecf67ce1ef 100644 --- a/python/tests/runutils.py +++ b/python/tests/runutils.py @@ -1,17 +1,19 @@ import subprocess import time + def run_and_assert(command, communicate=True): print(f"Running command {command}") output = subprocess.Popen(command, stdout=subprocess.PIPE) if communicate: stdout, stderr = output.communicate() - print('STDOUT', stdout.decode("utf-8") if stdout is not None else None) - print('STDERR', stderr.decode("utf-8") if stderr is not None else None) + print("STDOUT", stdout.decode("utf-8") if stdout is not None else None) + print("STDERR", stderr.decode("utf-8") if stderr is not None else None) assert output.returncode == 0, "non-zero exit status: %d" % output.returncode return stdout.decode("utf-8") if stdout is not None else None return None + def run_with_retry(command, retries=0): print(f"Running command {command}") returncode = -1 @@ -21,13 +23,13 @@ def run_with_retry(command, retries=0): while returncode != 0 and tries < max_tries: output = subprocess.Popen(command, stdout=subprocess.PIPE) if tries > 0: - print('SLEEPING 5 seconds, TRIES=%d' % tries) + print("SLEEPING 5 seconds, TRIES=%d" % tries) time.sleep(5) stdout, stderr = output.communicate() - print('STDOUT', stdout.decode("utf-8") if stdout is not None else None) - print('STDERR', stderr.decode("utf-8") if stderr is not None else None) + print("STDOUT", stdout.decode("utf-8") if stdout is not None else None) + print("STDERR", stderr.decode("utf-8") if stderr is not None else None) returncode = output.returncode decoded = stdout.decode("utf-8") if stdout is not None else None tries = tries + 1 assert returncode == 0, "non-zero exit status: %d" % output.returncode - return decoded \ No newline at end of file + return decoded diff --git a/python/tests/unit/test_acme_privatekey_secrets.py b/python/tests/unit/test_acme_privatekey_secrets.py index 48d2e46d4a..0b3261b9b0 100644 --- a/python/tests/unit/test_acme_privatekey_secrets.py +++ b/python/tests/unit/test_acme_privatekey_secrets.py @@ -15,10 +15,16 @@ # MemorySecretHandler is a degenerate SecretHandler that doesn't actually # cache anything to disk. It will never load a secret that isn't already # in the aconf. -class MemorySecretHandler (SecretHandler): - def cache_internal(self, name: str, namespace: str, - tls_crt: Optional[str], tls_key: Optional[str], - user_key: Optional[str], root_crt: Optional[str]) -> SavedSecret: +class MemorySecretHandler(SecretHandler): + def cache_internal( + self, + name: str, + namespace: str, + tls_crt: Optional[str], + tls_key: Optional[str], + user_key: Optional[str], + root_crt: Optional[str], + ) -> SavedSecret: # This is mostly ripped from ambassador.utils.SecretHandler.cache_internal, # just without actually saving anything. tls_crt_path = None @@ -29,11 +35,11 @@ def cache_internal(self, name: str, namespace: str, # Don't save if it has neither a tls_crt or a user_key or the root_crt if tls_crt or user_key or root_crt: - h = hashlib.new('sha1') + h = hashlib.new("sha1") for el in [tls_crt, tls_key, user_key]: if el: - h.update(el.encode('utf-8')) + h.update(el.encode("utf-8")) fp = h.hexdigest().upper() @@ -50,15 +56,19 @@ def cache_internal(self, name: str, namespace: str, root_crt_path = f"//test-secret-{fp}.root.crt" cert_data = { - 'tls_crt': tls_crt, - 'tls_key': tls_key, - 'user_key': user_key, - 'root_crt': root_crt, + "tls_crt": tls_crt, + "tls_key": tls_key, + "user_key": user_key, + "root_crt": root_crt, } - self.logger.debug(f"saved secret {name}.{namespace}: {tls_crt_path}, {tls_key_path}, {root_crt_path}") + self.logger.debug( + f"saved secret {name}.{namespace}: {tls_crt_path}, {tls_key_path}, {root_crt_path}" + ) - return SavedSecret(name, namespace, tls_crt_path, tls_key_path, user_key_path, root_crt_path, cert_data) + return SavedSecret( + name, namespace, tls_crt_path, tls_key_path, user_key_path, root_crt_path, cert_data + ) def _get_config_and_ir(logger: logging.Logger, watt: str) -> Tuple[Config, IR]: @@ -67,20 +77,23 @@ def _get_config_and_ir(logger: logging.Logger, watt: str) -> Tuple[Config, IR]: fetcher.parse_watt(watt) aconf.load_all(fetcher.sorted()) - secret_handler = MemorySecretHandler(logger, "/tmp/unit-test-source-root", "/tmp/unit-test-cache-dir", "0") + secret_handler = MemorySecretHandler( + logger, "/tmp/unit-test-source-root", "/tmp/unit-test-cache-dir", "0" + ) ir = IR(aconf, logger=logger, file_checker=lambda path: True, secret_handler=secret_handler) assert ir return aconf, ir + def _get_errors(caplog: pytest.LogCaptureFixture, logger_name: str, watt_data_filename: str): watt_data = open(watt_data_filename).read() - aconf, ir = _get_config_and_ir( - logging.getLogger(logger_name), - watt_data) + aconf, ir = _get_config_and_ir(logging.getLogger(logger_name), watt_data) - log_errors = [rec for rec in caplog.record_tuples if rec[0] == logger_name and rec[1] > logging.INFO] + log_errors = [ + rec for rec in caplog.record_tuples if rec[0] == logger_name and rec[1] > logging.INFO + ] aconf_errors = aconf.errors if "-global-" in aconf_errors: @@ -90,6 +103,7 @@ def _get_errors(caplog: pytest.LogCaptureFixture, logger_name: str, watt_data_fi return log_errors, aconf_errors + @pytest.mark.compilertest def test_acme_privatekey_secrets(caplog: pytest.LogCaptureFixture): caplog.set_level(logging.DEBUG) @@ -102,16 +116,35 @@ def test_acme_privatekey_secrets(caplog: pytest.LogCaptureFixture): # checking for errors in the correct place, we'll also run against a bad version of that file # and check that we *do* see errors. - badsnap_log_errors, badsnap_aconf_errors = _get_errors(caplog, "test_acme_privatekey_secrets-bad", os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "test_general_data", - "test-acme-private-key-snapshot-bad.json")) + badsnap_log_errors, badsnap_aconf_errors = _get_errors( + caplog, + "test_acme_privatekey_secrets-bad", + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "test_general_data", + "test-acme-private-key-snapshot-bad.json", + ), + ) assert badsnap_log_errors - assert not badsnap_aconf_errors, "Wanted no aconf errors but got:%s" % "".join([f"{nl} {err}" for err in badsnap_aconf_errors]) - - goodsnap_log_errors, goodsnap_aconf_errors = _get_errors(caplog, "test_acme_privatekey_secrets", os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "test_general_data", - "test-acme-private-key-snapshot.json")) - assert not goodsnap_log_errors, "Wanted no logged errors bug got:%s" % "".join([f"{nl} {logging.getLevelName(rec[1])}{tab}{rec[0]}:{rec[2]}" for rec in goodsnap_log_errors]) - assert not goodsnap_aconf_errors, "Wanted no aconf errors but got:%s" % "".join([f"{nl} {err}" for err in goodsnap_aconf_errors]) + assert not badsnap_aconf_errors, "Wanted no aconf errors but got:%s" % "".join( + [f"{nl} {err}" for err in badsnap_aconf_errors] + ) + + goodsnap_log_errors, goodsnap_aconf_errors = _get_errors( + caplog, + "test_acme_privatekey_secrets", + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "test_general_data", + "test-acme-private-key-snapshot.json", + ), + ) + assert not goodsnap_log_errors, "Wanted no logged errors bug got:%s" % "".join( + [ + f"{nl} {logging.getLevelName(rec[1])}{tab}{rec[0]}:{rec[2]}" + for rec in goodsnap_log_errors + ] + ) + assert not goodsnap_aconf_errors, "Wanted no aconf errors but got:%s" % "".join( + [f"{nl} {err}" for err in goodsnap_aconf_errors] + ) diff --git a/python/tests/unit/test_ambassador_module_validation.py b/python/tests/unit/test_ambassador_module_validation.py index 9589e7e1bb..dd2a4822d1 100644 --- a/python/tests/unit/test_ambassador_module_validation.py +++ b/python/tests/unit/test_ambassador_module_validation.py @@ -7,7 +7,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -15,9 +15,11 @@ from ambassador import Cache, IR from ambassador.compile import Compile + def require_no_errors(ir: IR): assert ir.aconf.errors == {} + def require_errors(ir: IR, errors: List[Tuple[str, str]]): flattened_ir_errors: List[str] = [] @@ -25,12 +27,11 @@ def require_errors(ir: IR, errors: List[Tuple[str, str]]): for error in ir.aconf.errors[key]: flattened_ir_errors.append(f"{key}: {error['error']}") - flattened_wanted_errors: List[str] = [ - f"{key}: {error}" for key, error in errors - ] + flattened_wanted_errors: List[str] = [f"{key}: {error}" for key, error in errors] assert sorted(flattened_ir_errors) == sorted(flattened_wanted_errors) + @pytest.mark.compilertest def test_valid_forward_client_cert_details(): yaml = """ @@ -52,6 +53,7 @@ def test_valid_forward_client_cert_details(): require_no_errors(r1["ir"]) require_no_errors(r2["ir"]) + @pytest.mark.compilertest def test_invalid_forward_client_cert_details(): yaml = """ @@ -70,12 +72,25 @@ def test_invalid_forward_client_cert_details(): r1 = Compile(logger, yaml, k8s=True) r2 = Compile(logger, yaml, k8s=True, cache=cache) - require_errors(r1["ir"], [ - ( "ambassador.default.1", "'forward_client_cert_details' may not be set to 'SANITIZE_INVALID'; it may only be set to one of: SANITIZE, FORWARD_ONLY, APPEND_FORWARD, SANITIZE_SET, ALWAYS_FORWARD_ONLY") - ]) - require_errors(r2["ir"], [ - ( "ambassador.default.1", "'forward_client_cert_details' may not be set to 'SANITIZE_INVALID'; it may only be set to one of: SANITIZE, FORWARD_ONLY, APPEND_FORWARD, SANITIZE_SET, ALWAYS_FORWARD_ONLY") - ]) + require_errors( + r1["ir"], + [ + ( + "ambassador.default.1", + "'forward_client_cert_details' may not be set to 'SANITIZE_INVALID'; it may only be set to one of: SANITIZE, FORWARD_ONLY, APPEND_FORWARD, SANITIZE_SET, ALWAYS_FORWARD_ONLY", + ) + ], + ) + require_errors( + r2["ir"], + [ + ( + "ambassador.default.1", + "'forward_client_cert_details' may not be set to 'SANITIZE_INVALID'; it may only be set to one of: SANITIZE, FORWARD_ONLY, APPEND_FORWARD, SANITIZE_SET, ALWAYS_FORWARD_ONLY", + ) + ], + ) + @pytest.mark.compilertest def test_valid_set_current_client_cert_details(): @@ -100,6 +115,7 @@ def test_valid_set_current_client_cert_details(): require_no_errors(r1["ir"]) require_no_errors(r2["ir"]) + @pytest.mark.compilertest def test_invalid_set_current_client_cert_details_key(): yaml = """ @@ -121,12 +137,25 @@ def test_invalid_set_current_client_cert_details_key(): logger.info("R1 IR: %s", r1["ir"].as_json()) - require_errors(r1["ir"], [ - ( "ambassador.default.1", "'set_current_client_cert_details' may not contain key 'invalid'; it may only contain keys: subject, cert, chain, dns, uri") - ]) - require_errors(r2["ir"], [ - ( "ambassador.default.1", "'set_current_client_cert_details' may not contain key 'invalid'; it may only contain keys: subject, cert, chain, dns, uri") - ]) + require_errors( + r1["ir"], + [ + ( + "ambassador.default.1", + "'set_current_client_cert_details' may not contain key 'invalid'; it may only contain keys: subject, cert, chain, dns, uri", + ) + ], + ) + require_errors( + r2["ir"], + [ + ( + "ambassador.default.1", + "'set_current_client_cert_details' may not contain key 'invalid'; it may only contain keys: subject, cert, chain, dns, uri", + ) + ], + ) + @pytest.mark.compilertest def test_invalid_set_current_client_cert_details_value(): @@ -147,12 +176,25 @@ def test_invalid_set_current_client_cert_details_value(): r1 = Compile(logger, yaml, k8s=True) r2 = Compile(logger, yaml, k8s=True, cache=cache) - require_errors(r1["ir"], [ - ( "ambassador.default.1", "'set_current_client_cert_details' value for key 'subject' may only be 'true' or 'false', not 'invalid'") - ]) - require_errors(r2["ir"], [ - ( "ambassador.default.1", "'set_current_client_cert_details' value for key 'subject' may only be 'true' or 'false', not 'invalid'") - ]) + require_errors( + r1["ir"], + [ + ( + "ambassador.default.1", + "'set_current_client_cert_details' value for key 'subject' may only be 'true' or 'false', not 'invalid'", + ) + ], + ) + require_errors( + r2["ir"], + [ + ( + "ambassador.default.1", + "'set_current_client_cert_details' value for key 'subject' may only be 'true' or 'false', not 'invalid'", + ) + ], + ) + @pytest.mark.compilertest def test_valid_grpc_stats_all_methods(): @@ -180,8 +222,8 @@ def test_valid_grpc_stats_all_methods(): stats_filters = [f for f in ir["filters"] if f["name"] == "grpc_stats"] assert len(stats_filters) == 1 assert stats_filters[0]["config"] == { - 'enable_upstream_stats': False, - 'stats_for_all_methods': True + "enable_upstream_stats": False, + "stats_for_all_methods": True, } @@ -213,18 +255,13 @@ def test_valid_grpc_stats_services(): stats_filters = [f for f in ir["filters"] if f["name"] == "grpc_stats"] assert len(stats_filters) == 1 assert stats_filters[0]["config"] == { - 'enable_upstream_stats': False, - 'individual_method_stats_allowlist': { - 'services': [ - { - 'name': 'echo.EchoService', - 'method_names': ['Echo'] - - } - ] - } + "enable_upstream_stats": False, + "individual_method_stats_allowlist": { + "services": [{"name": "echo.EchoService", "method_names": ["Echo"]}] + }, } + @pytest.mark.compilertest def test_valid_grpc_stats_upstream(): yaml = """ @@ -251,10 +288,11 @@ def test_valid_grpc_stats_upstream(): stats_filters = [f for f in ir["filters"] if f["name"] == "grpc_stats"] assert len(stats_filters) == 1 assert stats_filters[0]["config"] == { - 'enable_upstream_stats': True, - 'stats_for_all_methods': False + "enable_upstream_stats": True, + "stats_for_all_methods": False, } + @pytest.mark.compilertest def test_invalid_grpc_stats(): yaml = """ @@ -306,6 +344,6 @@ def test_valid_grpc_stats_empty(): stats_filters = [f for f in ir["filters"] if f["name"] == "grpc_stats"] assert len(stats_filters) == 1 assert stats_filters[0]["config"] == { - 'enable_upstream_stats': False, - 'stats_for_all_methods': False + "enable_upstream_stats": False, + "stats_for_all_methods": False, } diff --git a/python/tests/unit/test_bootstrap.py b/python/tests/unit/test_bootstrap.py index b0d0bebec4..96e3ec84e2 100644 --- a/python/tests/unit/test_bootstrap.py +++ b/python/tests/unit/test_bootstrap.py @@ -3,15 +3,16 @@ import pytest + def _test_bootstrap(yaml, expectations={}): # Compile an envoy config econf = econf_compile(yaml) # Get just the bootstrap config... - bootstrap = econf['bootstrap'] + bootstrap = econf["bootstrap"] # ...and make sure that Envoy thinks it is valid (it doesn't like the @type field) - bootstrap.pop('@type', None) + bootstrap.pop("@type", None) assert_valid_envoy_config(bootstrap) for key, expected in expectations.items(): @@ -19,6 +20,7 @@ def _test_bootstrap(yaml, expectations={}): assert key not in bootstrap else: import json + assert key in bootstrap assert bootstrap[key] == expected @@ -34,7 +36,7 @@ def _test_dd_entity_id(val, expected): # Run the bootstrap test. We don't need any special yaml # since all of this behavior is driven by env vars. yaml = module_and_mapping_manifests(None, []) - _test_bootstrap(yaml, expectations={'stats_config': expected}) + _test_bootstrap(yaml, expectations={"stats_config": expected}) # Teardown by removing dd / statsd vars del os.environ["STATSD_ENABLED"] @@ -56,23 +58,26 @@ def test_dd_entity_id_empty(): def test_dd_entity_id_set(): # If we set the env var, then it should show up the config. - _test_dd_entity_id("my.cool.1234.entity-id", { - 'stats_tags': [ - { - 'tag_name':'dd.internal.entity_id', - 'fixed_value':'my.cool.1234.entity-id' - } - ] - }) + _test_dd_entity_id( + "my.cool.1234.entity-id", + { + "stats_tags": [ + {"tag_name": "dd.internal.entity_id", "fixed_value": "my.cool.1234.entity-id"} + ] + }, + ) def test_dd_entity_id_set_typical(): # If we set the env var to a typical pod UID, then it should show up int he config. - _test_dd_entity_id("1fb8f8d8-00b3-44ef-bc8b-3659e4a3c2bd", { - 'stats_tags': [ - { - 'tag_name':'dd.internal.entity_id', - 'fixed_value':'1fb8f8d8-00b3-44ef-bc8b-3659e4a3c2bd' - } - ] - }) + _test_dd_entity_id( + "1fb8f8d8-00b3-44ef-bc8b-3659e4a3c2bd", + { + "stats_tags": [ + { + "tag_name": "dd.internal.entity_id", + "fixed_value": "1fb8f8d8-00b3-44ef-bc8b-3659e4a3c2bd", + } + ] + }, + ) diff --git a/python/tests/unit/test_buffer_limit_bytes.py b/python/tests/unit/test_buffer_limit_bytes.py index c40994d1a9..3b7c3d4c5b 100644 --- a/python/tests/unit/test_buffer_limit_bytes.py +++ b/python/tests/unit/test_buffer_limit_bytes.py @@ -5,7 +5,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -16,6 +16,7 @@ from tests.utils import default_listener_manifests + def _get_envoy_config(yaml): aconf = Config() fetcher = ResourceFetcher(logger, aconf) @@ -61,15 +62,17 @@ def test_setting_buffer_limit(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - per_connection_buffer_limit_bytes = listener.get('per_connection_buffer_limit_bytes', None) - assert per_connection_buffer_limit_bytes is not None, \ - f"per_connection_buffer_limit_bytes not found on listener: {listener.name}" + for listener in conf["static_resources"]["listeners"]: + per_connection_buffer_limit_bytes = listener.get("per_connection_buffer_limit_bytes", None) + assert ( + per_connection_buffer_limit_bytes is not None + ), f"per_connection_buffer_limit_bytes not found on listener: {listener.name}" print(f"Found per_connection_buffer_limit_bytes = {per_connection_buffer_limit_bytes}") key_found = True - assert expected == int(per_connection_buffer_limit_bytes), \ - "per_connection_buffer_limit_bytes must equal the value set on the ambassador Module" - assert key_found, 'per_connection_buffer_limit_bytes must be found in the envoy config' + assert expected == int( + per_connection_buffer_limit_bytes + ), "per_connection_buffer_limit_bytes must equal the value set on the ambassador Module" + assert key_found, "per_connection_buffer_limit_bytes must be found in the envoy config" @pytest.mark.compilertest @@ -103,15 +106,18 @@ def test_setting_buffer_limit_V3(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - per_connection_buffer_limit_bytes = listener.get('per_connection_buffer_limit_bytes', None) - assert per_connection_buffer_limit_bytes is not None, \ - f"per_connection_buffer_limit_bytes not found on listener: {listener.name}" + for listener in conf["static_resources"]["listeners"]: + per_connection_buffer_limit_bytes = listener.get("per_connection_buffer_limit_bytes", None) + assert ( + per_connection_buffer_limit_bytes is not None + ), f"per_connection_buffer_limit_bytes not found on listener: {listener.name}" print(f"Found per_connection_buffer_limit_bytes = {per_connection_buffer_limit_bytes}") key_found = True - assert expected == int(per_connection_buffer_limit_bytes), \ - "per_connection_buffer_limit_bytes must equal the value set on the ambassador Module" - assert key_found, 'per_connection_buffer_limit_bytes must be found in the envoy config' + assert expected == int( + per_connection_buffer_limit_bytes + ), "per_connection_buffer_limit_bytes must equal the value set on the ambassador Module" + assert key_found, "per_connection_buffer_limit_bytes must be found in the envoy config" + # Tests that the default value of per_connection_buffer_limit_bytes is disabled when there is not Module config for it. @pytest.mark.compilertest @@ -132,10 +138,11 @@ def test_default_buffer_limit(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - per_connection_buffer_limit_bytes = listener.get('per_connection_buffer_limit_bytes', None) - assert per_connection_buffer_limit_bytes is None, \ - f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" + for listener in conf["static_resources"]["listeners"]: + per_connection_buffer_limit_bytes = listener.get("per_connection_buffer_limit_bytes", None) + assert ( + per_connection_buffer_limit_bytes is None + ), f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" @pytest.mark.compilertest @@ -156,10 +163,12 @@ def test_default_buffer_limit_V3(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - per_connection_buffer_limit_bytes = listener.get('per_connection_buffer_limit_bytes', None) - assert per_connection_buffer_limit_bytes is None, \ - f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" + for listener in conf["static_resources"]["listeners"]: + per_connection_buffer_limit_bytes = listener.get("per_connection_buffer_limit_bytes", None) + assert ( + per_connection_buffer_limit_bytes is None + ), f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" + # Tests that the default value of per_connection_buffer_limit_bytes is disabled when there is not Module config for it (and that there are no issues when we dont make a listener). @pytest.mark.compilertest @@ -180,10 +189,11 @@ def test_buffer_limit_no_listener(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - per_connection_buffer_limit_bytes = listener.get('per_connection_buffer_limit_bytes', None) - assert per_connection_buffer_limit_bytes is None, \ - f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" + for listener in conf["static_resources"]["listeners"]: + per_connection_buffer_limit_bytes = listener.get("per_connection_buffer_limit_bytes", None) + assert ( + per_connection_buffer_limit_bytes is None + ), f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" @pytest.mark.compilertest @@ -204,7 +214,8 @@ def test_buffer_limit_no_listener_V3(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - per_connection_buffer_limit_bytes = listener.get('per_connection_buffer_limit_bytes', None) - assert per_connection_buffer_limit_bytes is None, \ - f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" + for listener in conf["static_resources"]["listeners"]: + per_connection_buffer_limit_bytes = listener.get("per_connection_buffer_limit_bytes", None) + assert ( + per_connection_buffer_limit_bytes is None + ), f"per_connection_buffer_limit_bytes found on listener (should not exist unless configured in the module): {listener.name}" diff --git a/python/tests/unit/test_cache.py b/python/tests/unit/test_cache.py index 16de5bb1ee..eff13a2b60 100644 --- a/python/tests/unit/test_cache.py +++ b/python/tests/unit/test_cache.py @@ -15,7 +15,7 @@ logging.basicConfig( level=logging.DEBUG, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -28,14 +28,12 @@ class Builder: - def __init__(self, logger: logging.Logger, tmpdir: Path, yaml_file: str, - enable_cache=True) -> None: + def __init__( + self, logger: logging.Logger, tmpdir: Path, yaml_file: str, enable_cache=True + ) -> None: self.logger = logger - self.test_dir = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "test_cache_data" - ) + self.test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_cache_data") self.cache: Optional[Cache] = None @@ -51,7 +49,9 @@ def __init__(self, logger: logging.Logger, tmpdir: Path, yaml_file: str, # Load the initial YAML. self.apply_yaml(yaml_file, allow_updates=False) - self.secret_handler = NullSecretHandler(logger, str(tmpdir/"secrets"/"src"), str(tmpdir/"secrets"/"cache"), "0") + self.secret_handler = NullSecretHandler( + logger, str(tmpdir / "secrets" / "src"), str(tmpdir / "secrets" / "cache"), "0" + ) # Save builds to make this simpler to call. self.builds: List[Tuple[IR, EnvoyConfig]] = [] @@ -67,10 +67,10 @@ def apply_yaml(self, yaml_file: str, allow_updates=True) -> None: def apply_yaml_string(self, yaml_data: str, allow_updates=True) -> None: for rsrc in yaml.safe_load_all(yaml_data): # We require kind, metadata.name, and metadata.namespace here. - kind = rsrc['kind'] - metadata = rsrc['metadata'] - name = metadata['name'] - namespace = metadata['namespace'] + kind = rsrc["kind"] + metadata = rsrc["metadata"] + name = metadata["name"] + namespace = metadata["namespace"] key = f"{kind}-v2-{name}-{namespace}" @@ -93,9 +93,9 @@ def apply_yaml_string(self, yaml_data: str, allow_updates=True) -> None: "metadata": { "name": name, "namespace": namespace, - "creationTimestamp": metadata.get("creationTimestamp", "2021-11-19T15:11:45Z") + "creationTimestamp": metadata.get("creationTimestamp", "2021-11-19T15:11:45Z"), }, - "deltaType": dtype + "deltaType": dtype, } def delete_yaml(self, yaml_file: str) -> None: @@ -106,15 +106,15 @@ def delete_yaml(self, yaml_file: str) -> None: def delete_yaml_string(self, yaml_data: str) -> None: for rsrc in yaml.safe_load_all(yaml_data): # We require kind, metadata.name, and metadata.namespace here. - kind = rsrc['kind'] - metadata = rsrc['metadata'] - name = metadata['name'] - namespace = metadata['namespace'] + kind = rsrc["kind"] + metadata = rsrc["metadata"] + name = metadata["name"] + namespace = metadata["namespace"] key = f"{kind}-v2-{name}-{namespace}" if key in self.resources: - del(self.resources[key]) + del self.resources[key] # if self.cache is not None: # self.cache.invalidate(key) @@ -125,18 +125,17 @@ def delete_yaml_string(self, yaml_data: str) -> None: "metadata": { "name": name, "namespace": namespace, - "creationTimestamp": metadata.get("creationTimestamp", "2021-11-19T15:11:45Z") + "creationTimestamp": metadata.get( + "creationTimestamp", "2021-11-19T15:11:45Z" + ), }, - "deltaType": "delete" + "deltaType": "delete", } def build(self) -> Tuple[IR, EnvoyConfig]: # Do a build, return IR & econf, but also stash them in self.builds. - watt: Dict[str, Any] = { - "Kubernetes": {}, - "Deltas": list(self.deltas.values()) - } + watt: Dict[str, Any] = {"Kubernetes": {}, "Deltas": list(self.deltas.values())} # Clear deltas for the next build. self.deltas = {} @@ -146,12 +145,12 @@ def build(self) -> Tuple[IR, EnvoyConfig]: # may not be. for rsrc in self.resources.values(): - kind = rsrc['kind'] + kind = rsrc["kind"] - if kind not in watt['Kubernetes']: - watt['Kubernetes'][kind] = [] + if kind not in watt["Kubernetes"]: + watt["Kubernetes"][kind] = [] - watt['Kubernetes'][kind].append(rsrc) + watt["Kubernetes"][kind].append(rsrc) watt_json = json.dumps(watt, sort_keys=True, indent=4) @@ -171,23 +170,37 @@ def build(self) -> Tuple[IR, EnvoyConfig]: aconf.load_all(fetcher.sorted()) # Next up: What kind of reconfiguration are we doing? - config_type, reset_cache, invalidate_groups_for = IR.check_deltas(self.logger, fetcher, self.cache) + config_type, reset_cache, invalidate_groups_for = IR.check_deltas( + self.logger, fetcher, self.cache + ) # For the tests in this file, we should see cache resets and full reconfigurations # IFF we have no cache. if self.cache is None: - assert config_type == "complete", "check_deltas wants an incremental reconfiguration with no cache, which it shouldn't" - assert reset_cache, "check_deltas with no cache does not want to reset the cache, but it should" + assert ( + config_type == "complete" + ), "check_deltas wants an incremental reconfiguration with no cache, which it shouldn't" + assert ( + reset_cache + ), "check_deltas with no cache does not want to reset the cache, but it should" else: - assert config_type == "incremental", "check_deltas with a cache wants a complete reconfiguration, which it shouldn't" - assert not reset_cache, "check_deltas with a cache wants to reset the cache, which it shouldn't" + assert ( + config_type == "incremental" + ), "check_deltas with a cache wants a complete reconfiguration, which it shouldn't" + assert ( + not reset_cache + ), "check_deltas with a cache wants to reset the cache, which it shouldn't" # Once that's done, compile the IR. - ir = IR(aconf, logger=self.logger, - cache=self.cache, invalidate_groups_for=invalidate_groups_for, - file_checker=lambda path: True, - secret_handler=self.secret_handler) + ir = IR( + aconf, + logger=self.logger, + cache=self.cache, + invalidate_groups_for=invalidate_groups_for, + file_checker=lambda path: True, + secret_handler=self.secret_handler, + ) assert ir, "could not create an IR" @@ -195,7 +208,7 @@ def build(self) -> Tuple[IR, EnvoyConfig]: assert econf, "could not create an econf" - self.builds.append(( ir, econf )) + self.builds.append((ir, econf)) return ir, econf @@ -205,9 +218,14 @@ def invalidate(self, key) -> None: self.cache.invalidate(key) - def check(self, what: str, b1: Tuple[IR, EnvoyConfig], b2: Tuple[IR, EnvoyConfig], - strip_cache_keys=False) -> bool: - for kind, idx in [ ( "IR", 0 ), ( "econf", 1 ) ]: + def check( + self, + what: str, + b1: Tuple[IR, EnvoyConfig], + b2: Tuple[IR, EnvoyConfig], + strip_cache_keys=False, + ) -> bool: + for kind, idx in [("IR", 0), ("econf", 1)]: if strip_cache_keys and (idx == 0): x1 = self.strip_cache_keys(b1[idx].as_dict()) j1 = json.dumps(x1, sort_keys=True, indent=4) @@ -218,7 +236,7 @@ def check(self, what: str, b1: Tuple[IR, EnvoyConfig], b2: Tuple[IR, EnvoyConfig j1 = b1[idx].as_json() j2 = b2[idx].as_json() - match = (j1 == j2) + match = j1 == j2 output = "" @@ -252,14 +270,14 @@ def strip_cache_keys(self, node: Any) -> Any: if isinstance(node, dict): output = {} for k, v in node.items(): - if k == '_cache_key': + if k == "_cache_key": continue output[k] = self.strip_cache_keys(v) return output elif isinstance(node, list): - return [ self.strip_cache_keys(x) for x in node ] + return [self.strip_cache_keys(x) for x in node] return node @@ -277,13 +295,13 @@ def test_circular_link(tmp_path): assert m # ...then walk the link chain until we get to a V2-Cluster. - worklist = [ m.cache_key ] + worklist = [m.cache_key] cluster_key: Optional[str] = None while worklist: key = worklist.pop(0) - if key.startswith('V3-Cluster'): + if key.startswith("V3-Cluster"): cluster_key = key break @@ -408,7 +426,7 @@ def test_delta_3(tmp_path): builder1.check("baseline", b1, b2, strip_cache_keys=True) # Load up five delta files and apply them in a random order. - deltas = [ f"cache_random_{i}.yaml" for i in [ 1, 2, 3, 4, 5 ] ] + deltas = [f"cache_random_{i}.yaml" for i in [1, 2, 3, 4, 5]] random.shuffle(deltas) for delta in deltas: @@ -489,10 +507,9 @@ def test_mappings_same_name_delta(tmp_path): # to ensure their clusters were generated properly. cluster1_ok = False cluster2_ok = False - for cluster in econf['static_resources']['clusters']: - cname = cluster.get('name', None) - assert cname is not None, \ - f"Error, cluster missing cluster name in econf" + for cluster in econf["static_resources"]["clusters"]: + cname = cluster.get("name", None) + assert cname is not None, f"Error, cluster missing cluster name in econf" # The 6666 in the cluster name comes from the Mapping.spec.service's port if cname == "cluster_bar_0_example_com_6666_bar0": cluster1_ok = True @@ -500,7 +517,7 @@ def test_mappings_same_name_delta(tmp_path): cluster2_ok = True if cluster1_ok and cluster2_ok: break - assert cluster1_ok and cluster2_ok, 'clusters could not be found with the correct envoy config' + assert cluster1_ok and cluster2_ok, "clusters could not be found with the correct envoy config" # Update the yaml for these Mappings to simulate a reconfiguration # We should properly remove the cache entries for these clusters when that happens. @@ -510,10 +527,9 @@ def test_mappings_same_name_delta(tmp_path): cluster1_ok = False cluster2_ok = False - for cluster in econf['static_resources']['clusters']: - cname = cluster.get('name', None) - assert cname is not None, \ - f"Error, cluster missing cluster name in econf" + for cluster in econf["static_resources"]["clusters"]: + cname = cluster.get("name", None) + assert cname is not None, f"Error, cluster missing cluster name in econf" # We can check the cluster name to identify if the clusters were updated properly # because in the deltas for the yaml we applied, we changed the port to 7777 # If there was an issue removing the initial ones from the cache then we should see @@ -524,7 +540,9 @@ def test_mappings_same_name_delta(tmp_path): cluster2_ok = True if cluster1_ok and cluster2_ok: break - assert cluster1_ok and cluster2_ok, 'clusters could not be found with the correct econf after updating their config' + assert ( + cluster1_ok and cluster2_ok + ), "clusters could not be found with the correct econf after updating their config" MadnessVerifier = Callable[[Tuple[IR, EnvoyConfig]], bool] @@ -541,7 +559,7 @@ def __init__(self, name, pfx, svc) -> None: self.service = svc # This is only OK for service names without any weirdnesses. - self.cluster = "cluster_" + re.sub(r'[^0-9A-Za-z_]', '_', self.service) + "_default" + self.cluster = "cluster_" + re.sub(r"[^0-9A-Za-z_]", "_", self.service) + "_default" def __str__(self) -> str: return f"MadnessMapping {self.name}: {self.pfx} => {self.service}" @@ -566,7 +584,14 @@ class MadnessOp: verifiers: List[MadnessVerifier] tmpdir: Path - def __init__(self, name: str, op: str, mapping: MadnessMapping, verifiers: List[MadnessVerifier], tmpdir: Path) -> None: + def __init__( + self, + name: str, + op: str, + mapping: MadnessMapping, + verifiers: List[MadnessVerifier], + tmpdir: Path, + ) -> None: self.name = name self.op = op self.mapping = mapping @@ -576,7 +601,7 @@ def __init__(self, name: str, op: str, mapping: MadnessMapping, verifiers: List[ def __str__(self) -> str: return self.name - def exec(self, builder1: Builder, builder2: Builder, dumpfile: Optional[str]=None) -> bool: + def exec(self, builder1: Builder, builder2: Builder, dumpfile: Optional[str] = None) -> bool: verifiers: List[MadnessVerifier] = [] if self.op == "apply": @@ -607,8 +632,18 @@ def exec(self, builder1: Builder, builder2: Builder, dumpfile: Optional[str]=Non logger.info("IR: %s" % json.dumps(b2[0].as_dict(), indent=2, sort_keys=True)) if dumpfile: - json.dump(b1[0].as_dict(), open(str(self.tmpdir/f"{dumpfile}-1.json"), "w"), indent=2, sort_keys=True) - json.dump(b2[0].as_dict(), open(str(self.tmpdir/f"{dumpfile}-2.json"), "w"), indent=2, sort_keys=True) + json.dump( + b1[0].as_dict(), + open(str(self.tmpdir / f"{dumpfile}-1.json"), "w"), + indent=2, + sort_keys=True, + ) + json.dump( + b2[0].as_dict(), + open(str(self.tmpdir / f"{dumpfile}-2.json"), "w"), + indent=2, + sort_keys=True, + ) if not builder1.check(self.name, b1, b2, strip_cache_keys=True): return False @@ -617,7 +652,7 @@ def exec(self, builder1: Builder, builder2: Builder, dumpfile: Optional[str]=Non for v in verifiers: # for b in [ b1 ]: - for b in [ b1, b2 ]: + for b in [b1, b2]: # The verifiers are meant to do assertions. The return value is # about short-circuiting the loop, not logging the errors. if not v(b): @@ -629,7 +664,9 @@ def _cluster_present(self, b: Tuple[IR, EnvoyConfig]) -> bool: ir, econf = b ir_has_cluster = ir.has_cluster(self.mapping.cluster) - assert ir_has_cluster, f"{self.name}: needed IR cluster {self.mapping.cluster}, have only {', '.join(ir.clusters.keys())}" + assert ( + ir_has_cluster + ), f"{self.name}: needed IR cluster {self.mapping.cluster}, have only {', '.join(ir.clusters.keys())}" return ir_has_cluster @@ -637,11 +674,15 @@ def _cluster_absent(self, b: Tuple[IR, EnvoyConfig]) -> bool: ir, econf = b ir_has_cluster = ir.has_cluster(self.mapping.cluster) - assert not ir_has_cluster, f"{self.name}: needed no IR cluster {self.mapping.cluster}, but found it" + assert ( + not ir_has_cluster + ), f"{self.name}: needed no IR cluster {self.mapping.cluster}, but found it" return not ir_has_cluster - def check_group(self, b: Tuple[IR, EnvoyConfig], current_mappings: Dict[MadnessMapping, bool]) -> bool: + def check_group( + self, b: Tuple[IR, EnvoyConfig], current_mappings: Dict[MadnessMapping, bool] + ) -> bool: ir, econf = b match = False @@ -650,34 +691,45 @@ def check_group(self, b: Tuple[IR, EnvoyConfig], current_mappings: Dict[MadnessM if current_mappings: # There are some active mappings. Make sure that the group exists, that it has the # correct mappings, and that the mappings have sane weights. - assert group, f"{self.name}: needed group 3644d75eb336f323bec43e48d4cfd8a950157607, but none found" + assert ( + group + ), f"{self.name}: needed group 3644d75eb336f323bec43e48d4cfd8a950157607, but none found" # We expect the mappings to be sorted in the group, because every change to the # mappings that are part of the group should result in the whole group being torn # down and recreated. - wanted_services = sorted([ m.service for m in current_mappings.keys() ]) - found_services = [ m.service for m in group.mappings ] + wanted_services = sorted([m.service for m in current_mappings.keys()]) + found_services = [m.service for m in group.mappings] - match1 = (wanted_services == found_services) - assert match1, f"{self.name}: wanted services {wanted_services}, but found {found_services}" + match1 = wanted_services == found_services + assert ( + match1 + ), f"{self.name}: wanted services {wanted_services}, but found {found_services}" weight_delta = 100 // len(current_mappings) - wanted_weights: List[int] = [ (i + 1) * weight_delta for i in range(len(current_mappings)) ] + wanted_weights: List[int] = [ + (i + 1) * weight_delta for i in range(len(current_mappings)) + ] wanted_weights[-1] = 100 - found_weights: List[int] = [ m._weight for m in group.mappings ] + found_weights: List[int] = [m._weight for m in group.mappings] - match2 = (wanted_weights == found_weights) - assert match2, f"{self.name}: wanted weights {wanted_weights}, but found {found_weights}" + match2 = wanted_weights == found_weights + assert ( + match2 + ), f"{self.name}: wanted weights {wanted_weights}, but found {found_weights}" return match1 and match2 else: # There are no active mappings, so make sure that the group doesn't exist. - assert not group, f"{self.name}: needed no group 3644d75eb336f323bec43e48d4cfd8a950157607, but found one" + assert ( + not group + ), f"{self.name}: needed no group 3644d75eb336f323bec43e48d4cfd8a950157607, but found one" match = True return match + @pytest.mark.compilertest def test_cache_madness(tmp_path): builder1 = Builder(logger, tmp_path, "/dev/null") @@ -722,17 +774,27 @@ def test_cache_madness(tmp_path): op: MadnessOp if mapping in current_mappings: - del(current_mappings[mapping]) - op = MadnessOp(name=f"delete {mapping.pfx} -> {mapping.service}", op="delete", mapping=mapping, - verifiers=[ lambda b: op.check_group(b, current_mappings) ], - tmpdir=tmp_path) + del current_mappings[mapping] + op = MadnessOp( + name=f"delete {mapping.pfx} -> {mapping.service}", + op="delete", + mapping=mapping, + verifiers=[lambda b: op.check_group(b, current_mappings)], + tmpdir=tmp_path, + ) else: current_mappings[mapping] = True - op = MadnessOp(name=f"apply {mapping.pfx} -> {mapping.service}", op="apply", mapping=mapping, - verifiers=[ lambda b: op.check_group(b, current_mappings) ], - tmpdir=tmp_path) - - print("==== EXEC %d: %s => %s" % (i, op, sorted([ m.service for m in current_mappings.keys() ]))) + op = MadnessOp( + name=f"apply {mapping.pfx} -> {mapping.service}", + op="apply", + mapping=mapping, + verifiers=[lambda b: op.check_group(b, current_mappings)], + tmpdir=tmp_path, + ) + + print( + "==== EXEC %d: %s => %s" % (i, op, sorted([m.service for m in current_mappings.keys()])) + ) logger.info("======== EXEC %d: %s", i, op) # if not op.exec(builder1, None, dumpfile=f"ir{i}"): @@ -740,5 +802,5 @@ def test_cache_madness(tmp_path): break -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_cidrrange.py b/python/tests/unit/test_cidrrange.py index 5cdce6cab3..bc1fd24b8f 100644 --- a/python/tests/unit/test_cidrrange.py +++ b/python/tests/unit/test_cidrrange.py @@ -6,27 +6,34 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") from ambassador.envoy.v3.v3cidrrange import CIDRRange + @pytest.mark.compilertest def test_cidrrange(): for spec, wanted_result, wanted_address, wanted_prefix_len, wanted_error in [ - ( "127.0.0.1", True, "127.0.0.1", 32, None ), # IPv4 exact - ( "::1", True, "::1", 128, None ), # IPv6 exact - ( "192.168.0.0/16", True, "192.168.0.0", 16, None ), # IPv4 range - ( "2001:2000::/64", True, "2001:2000::", 64, None ), # IPv6 range - ( "2001:2000:0:0:0::/64", True, "2001:2000::", 64, None ), # IPv6 range - ( "10", False, None, None, "Invalid IP address 10" ), - ( "10/8", False, None, None, "Invalid IP address 10" ), - ( "10.0.0.0/a", False, None, None, "CIDR range 10.0.0.0/a has an invalid length, ignoring" ), - ( "10.0.0.0/99", False, None, None, "Invalid prefix length for IPv4 address 10.0.0.0/99" ), - ( "2001:2000::/99", True, "2001:2000::", 99, None ), - ( "2001:2000::/199", False, None, None, "Invalid prefix length for IPv6 address 2001:2000::/199" ) + ("127.0.0.1", True, "127.0.0.1", 32, None), # IPv4 exact + ("::1", True, "::1", 128, None), # IPv6 exact + ("192.168.0.0/16", True, "192.168.0.0", 16, None), # IPv4 range + ("2001:2000::/64", True, "2001:2000::", 64, None), # IPv6 range + ("2001:2000:0:0:0::/64", True, "2001:2000::", 64, None), # IPv6 range + ("10", False, None, None, "Invalid IP address 10"), + ("10/8", False, None, None, "Invalid IP address 10"), + ("10.0.0.0/a", False, None, None, "CIDR range 10.0.0.0/a has an invalid length, ignoring"), + ("10.0.0.0/99", False, None, None, "Invalid prefix length for IPv4 address 10.0.0.0/99"), + ("2001:2000::/99", True, "2001:2000::", 99, None), + ( + "2001:2000::/199", + False, + None, + None, + "Invalid prefix length for IPv6 address 2001:2000::/199", + ), ]: c = CIDRRange(spec) @@ -47,17 +54,23 @@ def test_cidrrange(): @pytest.mark.compilertest def test_cidrrange_v3(): for spec, wanted_result, wanted_address, wanted_prefix_len, wanted_error in [ - ( "127.0.0.1", True, "127.0.0.1", 32, None ), # IPv4 exact - ( "::1", True, "::1", 128, None ), # IPv6 exact - ( "192.168.0.0/16", True, "192.168.0.0", 16, None ), # IPv4 range - ( "2001:2000::/64", True, "2001:2000::", 64, None ), # IPv6 range - ( "2001:2000:0:0:0::/64", True, "2001:2000::", 64, None ), # IPv6 range - ( "10", False, None, None, "Invalid IP address 10" ), - ( "10/8", False, None, None, "Invalid IP address 10" ), - ( "10.0.0.0/a", False, None, None, "CIDR range 10.0.0.0/a has an invalid length, ignoring" ), - ( "10.0.0.0/99", False, None, None, "Invalid prefix length for IPv4 address 10.0.0.0/99" ), - ( "2001:2000::/99", True, "2001:2000::", 99, None ), - ( "2001:2000::/199", False, None, None, "Invalid prefix length for IPv6 address 2001:2000::/199" ) + ("127.0.0.1", True, "127.0.0.1", 32, None), # IPv4 exact + ("::1", True, "::1", 128, None), # IPv6 exact + ("192.168.0.0/16", True, "192.168.0.0", 16, None), # IPv4 range + ("2001:2000::/64", True, "2001:2000::", 64, None), # IPv6 range + ("2001:2000:0:0:0::/64", True, "2001:2000::", 64, None), # IPv6 range + ("10", False, None, None, "Invalid IP address 10"), + ("10/8", False, None, None, "Invalid IP address 10"), + ("10.0.0.0/a", False, None, None, "CIDR range 10.0.0.0/a has an invalid length, ignoring"), + ("10.0.0.0/99", False, None, None, "Invalid prefix length for IPv4 address 10.0.0.0/99"), + ("2001:2000::/99", True, "2001:2000::", 99, None), + ( + "2001:2000::/199", + False, + None, + None, + "Invalid prefix length for IPv6 address 2001:2000::/199", + ), ]: c = CIDRRange(spec) @@ -74,5 +87,6 @@ def test_cidrrange_v3(): assert c.prefix_len == None assert c.error == wanted_error -if __name__ == '__main__': + +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_cluster_options.py b/python/tests/unit/test_cluster_options.py index 53b49d37bf..62317eaf2e 100644 --- a/python/tests/unit/test_cluster_options.py +++ b/python/tests/unit/test_cluster_options.py @@ -17,6 +17,7 @@ def check(cluster): econf_foreach_cluster(econf, check) + # Tests a setting in a cluster that has it's own fields. Example: common_http_protocol_options has multiple subfields def _test_cluster_subfields(yaml, setting, expectations={}, exists=True): econf = econf_compile(yaml) @@ -33,21 +34,22 @@ def check(cluster): econf_foreach_cluster(econf, check) + # Test dns_type setting in Mapping @pytest.mark.compilertest def test_logical_dns_type(): yaml = module_and_mapping_manifests(None, ["dns_type: logical_dns"]) # The dns type is listed as just "type" - _test_cluster_setting(yaml, setting="type", - expected="LOGICAL_DNS", exists=True) + _test_cluster_setting(yaml, setting="type", expected="LOGICAL_DNS", exists=True) + @pytest.mark.compilertest def test_strict_dns_type(): # Make sure we can configure strict dns as well even though it's the default yaml = module_and_mapping_manifests(None, ["dns_type: strict_dns"]) # The dns type is listed as just "type" - _test_cluster_setting(yaml, setting="type", - expected="STRICT_DNS", exists=True) + _test_cluster_setting(yaml, setting="type", expected="STRICT_DNS", exists=True) + @pytest.mark.compilertest def test_dns_type_wrong(): @@ -55,29 +57,28 @@ def test_dns_type_wrong(): # This is preferable to invalid config and an error is logged yaml = module_and_mapping_manifests(None, ["dns_type: something_new"]) # The dns type is listed as just "type" - _test_cluster_setting(yaml, setting="type", - expected="STRICT_DNS", exists=True) + _test_cluster_setting(yaml, setting="type", expected="STRICT_DNS", exists=True) + @pytest.mark.compilertest def test_logical_dns_type_endpoints(): # Ensure we use endpoint discovery instead of this value when using the endpoint resolver. yaml = module_and_mapping_manifests(None, ["dns_type: logical_dns", "resolver: endpoint"]) # The dns type is listed as just "type" - _test_cluster_setting(yaml, setting="type", - expected="EDS", exists=True) + _test_cluster_setting(yaml, setting="type", expected="EDS", exists=True) + @pytest.mark.compilertest def test_dns_ttl_module(): # Test configuring the respect_dns_ttl generates an Envoy config yaml = module_and_mapping_manifests(None, ["respect_dns_ttl: true"]) # The dns type is listed as just "type" - _test_cluster_setting(yaml, setting="respect_dns_ttl", - expected=True, exists=True) + _test_cluster_setting(yaml, setting="respect_dns_ttl", expected=True, exists=True) + @pytest.mark.compilertest def test_dns_ttl_mapping(): # Test dns_ttl is not configured when not applied in the Mapping yaml = module_and_mapping_manifests(None, None) # The dns type is listed as just "type" - _test_cluster_setting(yaml, setting="respect_dns_ttl", - expected=False, exists=False) + _test_cluster_setting(yaml, setting="respect_dns_ttl", expected=False, exists=False) diff --git a/python/tests/unit/test_common_http_protocol_options.py b/python/tests/unit/test_common_http_protocol_options.py index d26e6c954c..adbce13e65 100644 --- a/python/tests/unit/test_common_http_protocol_options.py +++ b/python/tests/unit/test_common_http_protocol_options.py @@ -2,43 +2,52 @@ import pytest + def _test_common_http_protocol_options(yaml, expectations={}): econf = econf_compile(yaml) def check(cluster): if expectations: - assert 'common_http_protocol_options' in cluster + assert "common_http_protocol_options" in cluster else: - assert 'common_http_protocol_options' not in cluster + assert "common_http_protocol_options" not in cluster for key, expected in expectations.items(): print("checking key %s" % key) - assert key in cluster['common_http_protocol_options'] - assert cluster['common_http_protocol_options'][key] == expected + assert key in cluster["common_http_protocol_options"] + assert cluster["common_http_protocol_options"][key] == expected + econf_foreach_cluster(econf, check) + @pytest.mark.compilertest def test_cluster_max_connection_lifetime_ms_missing(): # If we do not set the config, it should not appear in the Envoy conf. yaml = module_and_mapping_manifests(None, []) _test_common_http_protocol_options(yaml, expectations={}) + @pytest.mark.compilertest def test_cluster_max_connection_lifetime_ms_module_only(): # If we only set the config on the Module, it should show up. yaml = module_and_mapping_manifests(["cluster_max_connection_lifetime_ms: 2005"], []) - _test_common_http_protocol_options(yaml, expectations={'max_connection_duration':'2.005s'}) + _test_common_http_protocol_options(yaml, expectations={"max_connection_duration": "2.005s"}) + @pytest.mark.compilertest def test_cluster_max_connection_lifetime_ms_mapping_only(): # If we only set the config on the Mapping, it should show up. yaml = module_and_mapping_manifests(None, ["cluster_max_connection_lifetime_ms: 2005"]) - _test_common_http_protocol_options(yaml, expectations={'max_connection_duration':'2.005s'}) + _test_common_http_protocol_options(yaml, expectations={"max_connection_duration": "2.005s"}) + @pytest.mark.compilertest def test_cluster_max_connection_lifetime_ms_mapping_override(): # If we set the config on the Module and Mapping, the Mapping value wins. - yaml = module_and_mapping_manifests(["cluster_max_connection_lifetime_ms: 2005"], ["cluster_max_connection_lifetime_ms: 17005"]) - _test_common_http_protocol_options(yaml, expectations={'max_connection_duration':'17.005s'}) + yaml = module_and_mapping_manifests( + ["cluster_max_connection_lifetime_ms: 2005"], ["cluster_max_connection_lifetime_ms: 17005"] + ) + _test_common_http_protocol_options(yaml, expectations={"max_connection_duration": "17.005s"}) + @pytest.mark.compilertest def test_cluster_idle_timeout_ms_missing(): @@ -46,47 +55,58 @@ def test_cluster_idle_timeout_ms_missing(): yaml = module_and_mapping_manifests(None, []) _test_common_http_protocol_options(yaml, expectations={}) + @pytest.mark.compilertest def test_cluster_idle_timeout_ms_module_only(): # If we only set the config on the Module, it should show up. yaml = module_and_mapping_manifests(["cluster_idle_timeout_ms: 4005"], []) - _test_common_http_protocol_options(yaml, expectations={'idle_timeout':'4.005s'}) + _test_common_http_protocol_options(yaml, expectations={"idle_timeout": "4.005s"}) + @pytest.mark.compilertest def test_cluster_idle_timeout_ms_mapping_only(): # If we only set the config on the Mapping, it should show up. yaml = module_and_mapping_manifests(None, ["cluster_idle_timeout_ms: 4005"]) - _test_common_http_protocol_options(yaml, expectations={'idle_timeout':'4.005s'}) + _test_common_http_protocol_options(yaml, expectations={"idle_timeout": "4.005s"}) + @pytest.mark.compilertest def test_cluster_idle_timeout_ms_mapping_override(): # If we set the config on the Module and Mapping, the Mapping value wins. - yaml = module_and_mapping_manifests(["cluster_idle_timeout_ms: 4005"], ["cluster_idle_timeout_ms: 19105"]) - _test_common_http_protocol_options(yaml, expectations={'idle_timeout':'19.105s'}) + yaml = module_and_mapping_manifests( + ["cluster_idle_timeout_ms: 4005"], ["cluster_idle_timeout_ms: 19105"] + ) + _test_common_http_protocol_options(yaml, expectations={"idle_timeout": "19.105s"}) + @pytest.mark.compilertest def test_both_module(): # If we set both configs on the Module, both should show up. - yaml = module_and_mapping_manifests(["cluster_idle_timeout_ms: 4005", "cluster_max_connection_lifetime_ms: 2005"], None) - _test_common_http_protocol_options(yaml, expectations={ - 'max_connection_duration': '2.005s', - 'idle_timeout': '4.005s' - }) + yaml = module_and_mapping_manifests( + ["cluster_idle_timeout_ms: 4005", "cluster_max_connection_lifetime_ms: 2005"], None + ) + _test_common_http_protocol_options( + yaml, expectations={"max_connection_duration": "2.005s", "idle_timeout": "4.005s"} + ) + @pytest.mark.compilertest def test_both_mapping(): # If we set both configs on the Mapping, both should show up. - yaml = module_and_mapping_manifests(None, ["cluster_idle_timeout_ms: 4005", "cluster_max_connection_lifetime_ms: 2005"]) - _test_common_http_protocol_options(yaml, expectations={ - 'max_connection_duration': '2.005s', - 'idle_timeout': '4.005s' - }) + yaml = module_and_mapping_manifests( + None, ["cluster_idle_timeout_ms: 4005", "cluster_max_connection_lifetime_ms: 2005"] + ) + _test_common_http_protocol_options( + yaml, expectations={"max_connection_duration": "2.005s", "idle_timeout": "4.005s"} + ) + @pytest.mark.compilertest def test_both_one_module_one_mapping(): # If we set both configs, one on a Module, one on a Mapping, both should show up. - yaml = module_and_mapping_manifests(["cluster_idle_timeout_ms: 4005"], ["cluster_max_connection_lifetime_ms: 2005"]) - _test_common_http_protocol_options(yaml, expectations={ - 'max_connection_duration': '2.005s', - 'idle_timeout': '4.005s' - }) + yaml = module_and_mapping_manifests( + ["cluster_idle_timeout_ms: 4005"], ["cluster_max_connection_lifetime_ms: 2005"] + ) + _test_common_http_protocol_options( + yaml, expectations={"max_connection_duration": "2.005s", "idle_timeout": "4.005s"} + ) diff --git a/python/tests/unit/test_envoy_stats.py b/python/tests/unit/test_envoy_stats.py index 93716e453d..52a176d7ff 100644 --- a/python/tests/unit/test_envoy_stats.py +++ b/python/tests/unit/test_envoy_stats.py @@ -12,7 +12,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -22,13 +22,12 @@ class EnvoyStatsMocker: def __init__(self) -> None: - current_test = os.environ.get('PYTEST_CURRENT_TEST') + current_test = os.environ.get("PYTEST_CURRENT_TEST") assert current_test is not None, "PYTEST_CURRENT_TEST is not set??" self.test_dir = os.path.join( - os.path.dirname(current_test.split("::")[0]), - "test_envoy_stats_data" + os.path.dirname(current_test.split("::")[0]), "test_envoy_stats_data" ) self.log_idx = 0 @@ -60,36 +59,74 @@ def slow_fetch_stats(self) -> Optional[str]: def test_levels(): mocker = EnvoyStatsMocker() - esm = EnvoyStatsMgr(logger, - fetch_log_levels=mocker.fetch_log_levels, - fetch_envoy_stats=mocker.fetch_envoy_stats) + esm = EnvoyStatsMgr( + logger, fetch_log_levels=mocker.fetch_log_levels, fetch_envoy_stats=mocker.fetch_envoy_stats + ) esm.update() - assert esm.loginfo == { 'all': 'error' } + assert esm.loginfo == {"all": "error"} # This one may be a bit more fragile than we'd like. esm.update() assert esm.loginfo == { - 'error': [ 'admin', 'aws', 'assert', 'backtrace', 'cache_filter', - 'client', 'config', 'connection', 'conn_handler', - 'decompression', 'envoy_bug', 'ext_authz', 'rocketmq', - 'file', 'filter', 'forward_proxy', 'grpc', 'hc', 'health_checker', - 'http', 'http2', 'hystrix', 'init', 'io', 'jwt', 'kafka', - 'main', 'misc', 'mongo', 'quic', 'quic_stream', 'pool', - 'rbac', 'redis', 'router', 'runtime', 'stats', 'secret', - 'tap', 'testing', 'thrift', 'tracing', 'upstream', - 'udp', 'wasm' ], - 'info': [ 'dubbo' ], - 'warning': [ 'lua' ] + "error": [ + "admin", + "aws", + "assert", + "backtrace", + "cache_filter", + "client", + "config", + "connection", + "conn_handler", + "decompression", + "envoy_bug", + "ext_authz", + "rocketmq", + "file", + "filter", + "forward_proxy", + "grpc", + "hc", + "health_checker", + "http", + "http2", + "hystrix", + "init", + "io", + "jwt", + "kafka", + "main", + "misc", + "mongo", + "quic", + "quic_stream", + "pool", + "rbac", + "redis", + "router", + "runtime", + "stats", + "secret", + "tap", + "testing", + "thrift", + "tracing", + "upstream", + "udp", + "wasm", + ], + "info": ["dubbo"], + "warning": ["lua"], } def test_stats(): mocker = EnvoyStatsMocker() - esm = EnvoyStatsMgr(logger, - fetch_log_levels=mocker.fetch_log_levels, - fetch_envoy_stats=mocker.fetch_envoy_stats) + esm = EnvoyStatsMgr( + logger, fetch_log_levels=mocker.fetch_log_levels, fetch_envoy_stats=mocker.fetch_envoy_stats + ) esm.update() stats = esm.get_stats() @@ -102,31 +139,31 @@ def test_stats(): assert stats.last_update > stats.last_attempt assert stats.update_errors == 0 - assert stats.requests == { 'total': 19, '4xx': 19, '5xx': 0, 'bad': 19, 'ok': 0 } - assert stats.clusters['cluster_127_0_0_1_8500_ambassador'] == { - 'healthy_members': 1, - 'total_members': 1, - 'healthy_percent': 100, - 'update_attempts': 4220, - 'update_successes': 4220, - 'update_percent': 100, - 'upstream_ok': 14, - 'upstream_4xx': 14, - 'upstream_5xx': 0, - 'upstream_bad': 0 + assert stats.requests == {"total": 19, "4xx": 19, "5xx": 0, "bad": 19, "ok": 0} + assert stats.clusters["cluster_127_0_0_1_8500_ambassador"] == { + "healthy_members": 1, + "total_members": 1, + "healthy_percent": 100, + "update_attempts": 4220, + "update_successes": 4220, + "update_percent": 100, + "upstream_ok": 14, + "upstream_4xx": 14, + "upstream_5xx": 0, + "upstream_bad": 0, } - assert stats.clusters['cluster_identity_api_jennifer_testing_sv-0'] == { - 'healthy_members': 1, - 'total_members': 1, - 'healthy_percent': None, - 'update_attempts': 4216, - 'update_successes': 4216, - 'update_percent': 100, - 'upstream_ok': 0, - 'upstream_4xx': 0, - 'upstream_5xx': 0, - 'upstream_bad': 0 + assert stats.clusters["cluster_identity_api_jennifer_testing_sv-0"] == { + "healthy_members": 1, + "total_members": 1, + "healthy_percent": None, + "update_attempts": 4216, + "update_successes": 4216, + "update_percent": 100, + "upstream_ok": 0, + "upstream_4xx": 0, + "upstream_5xx": 0, + "upstream_bad": 0, } assert stats.envoy["cluster_manager"] == { @@ -138,7 +175,7 @@ def test_stats(): "update_rejected": 0, "update_success": 14, "update_time": 1602023101467, - "version": 11975404232982186540 + "version": 11975404232982186540, }, "cluster_added": 336, "cluster_modified": 0, @@ -147,13 +184,13 @@ def test_stats(): "cluster_updated_via_merge": 0, "update_merge_cancelled": 0, "update_out_of_merge_window": 0, - "warming_clusters": 0 + "warming_clusters": 0, } assert stats.envoy["control_plane"] == { "connected_state": 1, "pending_requests": 0, - "rate_limit_enforced": 0 + "rate_limit_enforced": 0, } assert stats.envoy["listener_manager"] == { @@ -164,7 +201,7 @@ def test_stats(): "update_rejected": 17, "update_success": 14, "update_time": 1602023102107, - "version": 11975404232982186540 + "version": 11975404232982186540, }, "listener_added": 2, "listener_create_failure": 0, @@ -177,7 +214,7 @@ def test_stats(): "total_listeners_active": 2, "total_listeners_draining": 0, "total_listeners_warming": 0, - "workers_started": 1 + "workers_started": 1, } esm.update() @@ -189,10 +226,13 @@ def test_stats(): def test_locks(): mocker = EnvoyStatsMocker() - esm = EnvoyStatsMgr(logger, - max_live_age=3, max_ready_age=3, - fetch_log_levels=mocker.fetch_log_levels, - fetch_envoy_stats=mocker.slow_fetch_stats) + esm = EnvoyStatsMgr( + logger, + max_live_age=3, + max_ready_age=3, + fetch_log_levels=mocker.fetch_log_levels, + fetch_envoy_stats=mocker.slow_fetch_stats, + ) def slow_background(): esm.update() @@ -210,7 +250,8 @@ def check_get_stats(): # At this point, we should be able to get stats very quickly, and see # alive but not ready. - sys.stdout.write("1"); sys.stdout.flush() + sys.stdout.write("1") + sys.stdout.flush() stats1 = check_get_stats() assert stats1.is_alive() assert not stats1.is_ready() @@ -218,7 +259,8 @@ def check_get_stats(): # Wait 2 seconds. We should get the _same_ stats object, and again, # alive but not ready. time.sleep(2) - sys.stdout.write("2"); sys.stdout.flush() + sys.stdout.write("2") + sys.stdout.flush() stats2 = check_get_stats() assert id(stats2) == id(stats1) assert stats2.is_alive() @@ -230,7 +272,8 @@ def check_get_stats(): # Wait 2 more seconds. We should get the same stats object, but it should # now say neither alive nor ready. time.sleep(2) - sys.stdout.write("3"); sys.stdout.flush() + sys.stdout.write("3") + sys.stdout.flush() stats3 = check_get_stats() assert id(stats3) == id(stats1) assert not stats3.is_alive() @@ -239,7 +282,8 @@ def check_get_stats(): # Wait 2 more seconds. At this point, we should get a new stats object, # and we should see alive and ready. time.sleep(2) - sys.stdout.write("4"); sys.stdout.flush() + sys.stdout.write("4") + sys.stdout.flush() stats4 = check_get_stats() assert id(stats4) != id(stats1) assert stats4.is_alive() @@ -252,12 +296,13 @@ def check_get_stats(): # Finally, if we wait four more seconds, we should still have the same # stats object as last time, but we should see neither alive nor ready. time.sleep(4) - sys.stdout.write("5"); sys.stdout.flush() + sys.stdout.write("5") + sys.stdout.flush() stats5 = check_get_stats() assert id(stats5) == id(stats4) assert not stats5.is_alive() assert not stats5.is_ready() -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_envvar_expansion.py b/python/tests/unit/test_envvar_expansion.py index 9e9b0621ed..b0016afe73 100644 --- a/python/tests/unit/test_envvar_expansion.py +++ b/python/tests/unit/test_envvar_expansion.py @@ -9,7 +9,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -20,7 +20,7 @@ from ambassador.ir import IRResource from ambassador.ir.irbuffer import IRBuffer -yaml = ''' +yaml = """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -28,7 +28,7 @@ hostname: "*" prefix: /test/ service: ${TEST_SERVICE}:9999 -''' +""" def test_envvar_expansion(): @@ -47,5 +47,5 @@ def test_envvar_expansion(): assert test_mapping.service == "foo:9999" -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_error_response.py b/python/tests/unit/test_error_response.py index 0ecacfd8c0..4823a10518 100644 --- a/python/tests/unit/test_error_response.py +++ b/python/tests/unit/test_error_response.py @@ -7,7 +7,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -20,77 +20,81 @@ def _status_code_filter_eq_obj(status_code): return { - 'status_code_filter': { - 'comparison': { - 'op': 'EQ', - 'value': { - 'default_value': f'{status_code}', - 'runtime_key': '_donotsetthiskey' - } + "status_code_filter": { + "comparison": { + "op": "EQ", + "value": {"default_value": f"{status_code}", "runtime_key": "_donotsetthiskey"}, } } } def _json_format_obj(json_format, content_type=None): - return { - 'json_format': json_format - } + return {"json_format": json_format} def _text_format_obj(text, content_type=None): - obj = { - 'text_format': f'{text}' - } + obj = {"text_format": f"{text}"} if content_type is not None: - obj['content_type'] = content_type + obj["content_type"] = content_type return obj def _text_format_source_obj(filename, content_type=None): - obj = { - 'text_format_source': { - 'filename': filename - } - } + obj = {"text_format_source": {"filename": filename}} if content_type is not None: - obj['content_type'] = content_type + obj["content_type"] = content_type return obj def _ambassador_module_config(): - return ''' + return """ --- apiVersion: getambassador.io/v3alpha1 kind: Module name: ambassador config: -''' +""" def _ambassador_module_onemapper(status_code, body_kind, body_value, content_type=None): - mod = _ambassador_module_config() + f''' + mod = ( + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: "{status_code}" body: -''' - if body_kind == 'text_format_source': - mod = mod + f''' +""" + ) + if body_kind == "text_format_source": + mod = ( + mod + + f""" {body_kind}: filename: "{body_value}" -''' - elif body_kind == 'json_format': - mod = mod + f''' +""" + ) + elif body_kind == "json_format": + mod = ( + mod + + f""" {body_kind}: {body_value} -''' +""" + ) else: - mod = mod + f''' + mod = ( + mod + + f""" {body_kind}: "{body_value}" -''' +""" + ) if content_type is not None: - mod = mod + f''' + mod = ( + mod + + f""" content_type: "{content_type}" -''' +""" + ) return mod @@ -106,9 +110,9 @@ def _test_errorresponse(yaml, expectations, expect_fail=False): ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) - error_response = IRErrorResponse(ir, aconf, - ir.ambassador_module.get('error_response_overrides', None), - ir.ambassador_module) + error_response = IRErrorResponse( + ir, aconf, ir.ambassador_module.get("error_response_overrides", None), ir.ambassador_module + ) error_response.setup(ir, aconf) if aconf.errors: @@ -121,58 +125,65 @@ def _test_errorresponse(yaml, expectations, expect_fail=False): assert ir_conf # There should be no default body format override - body_format = ir_conf.get('body_format', None) + body_format = ir_conf.get("body_format", None) assert body_format is None - mappers = ir_conf.get('mappers', None) + mappers = ir_conf.get("mappers", None) assert mappers - assert len(mappers) == len(expectations), \ - f"unexpected len(mappers) {len(expectations)} != len(expectations) {len(expectations)}" + assert len(mappers) == len( + expectations + ), f"unexpected len(mappers) {len(expectations)} != len(expectations) {len(expectations)}" for i in range(len(expectations)): expected_filter, expected_body_format_override = expectations[i] m = mappers[i] - print("checking with expected_body_format_override %s and expected_filter %s" % - (expected_body_format_override, expected_filter)) + print( + "checking with expected_body_format_override %s and expected_filter %s" + % (expected_body_format_override, expected_filter) + ) print("checking m: ", m) - actual_filter = m['filter'] - assert m['filter'] == expected_filter + actual_filter = m["filter"] + assert m["filter"] == expected_filter if expected_body_format_override: - actual_body_format_override = m['body_format_override'] + actual_body_format_override = m["body_format_override"] assert actual_body_format_override == expected_body_format_override def _test_errorresponse_onemapper(yaml, expected_filter, expected_body_format_override, fail=False): - return _test_errorresponse(yaml, [ (expected_filter, expected_body_format_override) ], expect_fail=fail) + return _test_errorresponse( + yaml, [(expected_filter, expected_body_format_override)], expect_fail=fail + ) def _test_errorresponse_twomappers(yaml, expectation1, expectation2, fail=False): - return _test_errorresponse(yaml, [ expectation1, expectation2 ], expect_fail=fail) + return _test_errorresponse(yaml, [expectation1, expectation2], expect_fail=fail) def _test_errorresponse_onemapper_onstatuscode_textformat(status_code, text_format, fail=False): _test_errorresponse_onemapper( - _ambassador_module_onemapper(status_code, 'text_format', text_format), + _ambassador_module_onemapper(status_code, "text_format", text_format), _status_code_filter_eq_obj(status_code), _text_format_obj(text_format), - fail=fail + fail=fail, ) def _test_errorresponse_onemapper_onstatuscode_textformat_contenttype( - status_code, text_format, content_type): + status_code, text_format, content_type +): _test_errorresponse_onemapper( _ambassador_module_onemapper( - status_code, 'text_format', text_format, content_type=content_type), + status_code, "text_format", text_format, content_type=content_type + ), _status_code_filter_eq_obj(status_code), - _text_format_obj(text_format, content_type=content_type) + _text_format_obj(text_format, content_type=content_type), ) def _test_errorresponse_onemapper_onstatuscode_textformat_datasource( - status_code, text_format, source, content_type): - + status_code, text_format, source, content_type +): # in order for tests to pass the files (all located in /tmp) need to exist try: @@ -182,8 +193,9 @@ def _test_errorresponse_onemapper_onstatuscode_textformat_datasource( pass _test_errorresponse_onemapper( - _ambassador_module_onemapper(status_code, 'text_format_source', source, - content_type=content_type), + _ambassador_module_onemapper( + status_code, "text_format_source", source, content_type=content_type + ), _status_code_filter_eq_obj(status_code), _text_format_source_obj(source, content_type=content_type), ) @@ -201,7 +213,7 @@ def _sanitize_json(json_format): def _test_errorresponse_onemapper_onstatuscode_jsonformat(status_code, json_format): _test_errorresponse_onemapper( - _ambassador_module_onemapper(status_code, 'json_format', json_format), + _ambassador_module_onemapper(status_code, "json_format", json_format), _status_code_filter_eq_obj(status_code), # We expect the output json to be sanitized and contain the string representation # of every value. We provide a basic implementation of string sanitizatino in this @@ -212,7 +224,7 @@ def _test_errorresponse_onemapper_onstatuscode_jsonformat(status_code, json_form def _test_errorresponse_twomappers_onstatuscode_textformat(code1, text1, code2, text2, fail=False): _test_errorresponse_twomappers( -f''' + f""" --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -225,10 +237,10 @@ def _test_errorresponse_twomappers_onstatuscode_textformat(code1, text1, code2, - on_status_code: {code2} body: text_format: {text2} -''', +""", (_status_code_filter_eq_obj(code1), _text_format_obj(text1)), (_status_code_filter_eq_obj(code2), _text_format_obj(text2)), - fail=fail + fail=fail, ) @@ -239,196 +251,232 @@ def _test_errorresponse_invalid_configs(yaml): @pytest.mark.compilertest def test_errorresponse_twomappers_onstatuscode_textformat(): _test_errorresponse_twomappers_onstatuscode_textformat( - '400', 'bad request my friend', '504', 'waited too long for an upstream resonse' - ) - _test_errorresponse_twomappers_onstatuscode_textformat( - '503', 'boom', '403', 'go away' + "400", "bad request my friend", "504", "waited too long for an upstream resonse" ) + _test_errorresponse_twomappers_onstatuscode_textformat("503", "boom", "403", "go away") @pytest.mark.compilertest def test_errorresponse_onemapper_onstatuscode_textformat(): - _test_errorresponse_onemapper_onstatuscode_textformat(429, '429 the int') - _test_errorresponse_onemapper_onstatuscode_textformat('501', 'five oh one') - _test_errorresponse_onemapper_onstatuscode_textformat('400', 'bad req') + _test_errorresponse_onemapper_onstatuscode_textformat(429, "429 the int") + _test_errorresponse_onemapper_onstatuscode_textformat("501", "five oh one") + _test_errorresponse_onemapper_onstatuscode_textformat("400", "bad req") _test_errorresponse_onemapper_onstatuscode_textformat( - '429', 'too fast, too furious on host %REQ(:authority)%' + "429", "too fast, too furious on host %REQ(:authority)%" ) @pytest.mark.compilertest def test_errorresponse_invalid_envoy_operator(): - _test_errorresponse_onemapper_onstatuscode_textformat(404, '%FAILME%', fail=True) + _test_errorresponse_onemapper_onstatuscode_textformat(404, "%FAILME%", fail=True) @pytest.mark.compilertest def test_errorresponse_onemapper_onstatuscode_textformat_contenttype(): - _test_errorresponse_onemapper_onstatuscode_textformat_contenttype('503', 'oops', 'text/what') + _test_errorresponse_onemapper_onstatuscode_textformat_contenttype("503", "oops", "text/what") _test_errorresponse_onemapper_onstatuscode_textformat_contenttype( - '429', 'too fast, too furious on host %REQ(:authority)%', 'text/html' + "429", "too fast, too furious on host %REQ(:authority)%", "text/html" ) _test_errorresponse_onemapper_onstatuscode_textformat_contenttype( - '404', "{\'error\':\'notfound\'}", 'application/json' + "404", "{'error':'notfound'}", "application/json" ) @pytest.mark.compilertest def test_errorresponse_onemapper_onstatuscode_jsonformat(): - _test_errorresponse_onemapper_onstatuscode_jsonformat('501', + _test_errorresponse_onemapper_onstatuscode_jsonformat( + "501", { - 'response_code': '%RESPONSE_CODE%', - 'upstream_cluster': '%UPSTREAM_CLUSTER%', - 'badness': 'yup' - } + "response_code": "%RESPONSE_CODE%", + "upstream_cluster": "%UPSTREAM_CLUSTER%", + "badness": "yup", + }, ) # Test both a JSON object whose Python type has non-string primitives... - _test_errorresponse_onemapper_onstatuscode_jsonformat('401', + _test_errorresponse_onemapper_onstatuscode_jsonformat( + "401", { - 'unauthorized': 'yeah', - 'your_address': '%DOWNSTREAM_REMOTE_ADDRESS%', - 'security_level': 9000, - 'awesome': True, - 'floaty': 0.75 - } + "unauthorized": "yeah", + "your_address": "%DOWNSTREAM_REMOTE_ADDRESS%", + "security_level": 9000, + "awesome": True, + "floaty": 0.75, + }, ) # ...and a JSON object where the Python type already has strings - _test_errorresponse_onemapper_onstatuscode_jsonformat('403', + _test_errorresponse_onemapper_onstatuscode_jsonformat( + "403", { - 'whoareyou': 'dunno', - 'your_address': '%DOWNSTREAM_REMOTE_ADDRESS%', - 'security_level': '11000', - 'awesome': 'false', - 'floaty': '0.95' - } + "whoareyou": "dunno", + "your_address": "%DOWNSTREAM_REMOTE_ADDRESS%", + "security_level": "11000", + "awesome": "false", + "floaty": "0.95", + }, ) @pytest.mark.compilertest def test_errorresponse_onemapper_onstatuscode_textformatsource(tmp_path: Path): _test_errorresponse_onemapper_onstatuscode_textformat_datasource( - '400', 'badness', str(tmp_path/'badness'), 'text/plain') + "400", "badness", str(tmp_path / "badness"), "text/plain" + ) _test_errorresponse_onemapper_onstatuscode_textformat_datasource( - '404', 'badness', str(tmp_path/'notfound.dat'), 'application/specialsauce') + "404", "badness", str(tmp_path / "notfound.dat"), "application/specialsauce" + ) _test_errorresponse_onemapper_onstatuscode_textformat_datasource( - '429', '2fast', str(tmp_path/'2fast.html'), 'text/html' ) + "429", "2fast", str(tmp_path / "2fast.html"), "text/html" + ) _test_errorresponse_onemapper_onstatuscode_textformat_datasource( - '503', 'something went wrong', str(tmp_path/'503.html'), 'text/html; charset=UTF-8' ) + "503", "something went wrong", str(tmp_path / "503.html"), "text/html; charset=UTF-8" + ) @pytest.mark.compilertest def test_errorresponse_invalid_configs(): # status code must be an int _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: bad body: text_format: 'good' -''') +""" + ) # cannot match on code < 400 nor >= 600 _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 200 body: text_format: 'good' -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 399 body: text_format: 'good' -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 600 body: text_format: 'good' -''') +""" + ) # body must be a dict _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: 'bad' -''') +""" + ) # body requires a valid format field _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: bad: 'good' -''') +""" + ) # body field must be present _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 501 bad: text_format: 'good' -''') +""" + ) # body field cannot be an empty dict _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 501 body: {{}} -''') +""" + ) # response override must be a non-empty array _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: [] -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: 'great sadness' -''') +""" + ) # (not an array, bad) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: on_status_code: 401 body: text_format: 'good' -''') +""" + ) # text_format_source must have a single string 'filename' _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: text_format_source: "this obviously cannot be a string" -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: text_format_source: filename: [] -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: text_format_source: notfilename: "/tmp/good" -''') +""" + ) # json_format field must be an object field _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: json_format: "this also cannot be a string" -''') +""" + ) # json_format cannot have values that do not cast to string trivially _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: @@ -436,36 +484,44 @@ def test_errorresponse_invalid_configs(): "x": "yo": 1 "field": "good" -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: json_format: "a": [] "x": true -''') +""" + ) # content type, if it exists, must be a string _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: text_format: "good" content_type: [] -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: text_format: "good" content_type: 4.2 -''') +""" + ) # only one of text_format, json_format, or text_format_source may be set _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: @@ -473,16 +529,20 @@ def test_errorresponse_invalid_configs(): json_format: "bad": 1 "invalid": "bad" -''') +""" + ) _test_errorresponse_invalid_configs( - _ambassador_module_config() + f''' + _ambassador_module_config() + + f""" error_response_overrides: - on_status_code: 401 body: text_format: "goodgood" text_format_source: filename: "/etc/issue" -''') +""" + ) + -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_fetch.py b/python/tests/unit/test_fetch.py index b9579382c3..7a93303511 100644 --- a/python/tests/unit/test_fetch.py +++ b/python/tests/unit/test_fetch.py @@ -6,14 +6,19 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") from ambassador import Config from ambassador.fetch import ResourceFetcher -from ambassador.fetch.dependency import DependencyManager, ServiceDependency, SecretDependency, IngressClassesDependency +from ambassador.fetch.dependency import ( + DependencyManager, + ServiceDependency, + SecretDependency, + IngressClassesDependency, +) from ambassador.fetch.location import LocationManager from ambassador.fetch.resource import NormalizedResource, ResourceManager from ambassador.fetch.k8sobject import ( @@ -37,7 +42,8 @@ def k8s_object_from_yaml(yaml: str) -> KubernetesObject: return KubernetesObject(parse_yaml(yaml)[0]) -valid_knative_ingress = k8s_object_from_yaml(''' +valid_knative_ingress = k8s_object_from_yaml( + """ --- apiVersion: networking.internal.knative.dev/v1alpha1 kind: Ingress @@ -77,18 +83,22 @@ def k8s_object_from_yaml(yaml: str) -> KubernetesObject: ingress: - domainInternal: ambassador.ambassador-webhook.svc.cluster.local observedGeneration: 2 -''') +""" +) -valid_ingress_class = k8s_object_from_yaml(''' +valid_ingress_class = k8s_object_from_yaml( + """ apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: name: external-lb spec: controller: getambassador.io/ingress-controller -''') +""" +) -valid_mapping = k8s_object_from_yaml(''' +valid_mapping = k8s_object_from_yaml( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -99,9 +109,11 @@ def k8s_object_from_yaml(yaml: str) -> KubernetesObject: hostname: "*" prefix: /test/ service: test.default -''') +""" +) -valid_mapping_v1 = k8s_object_from_yaml(''' +valid_mapping_v1 = k8s_object_from_yaml( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -112,78 +124,82 @@ def k8s_object_from_yaml(yaml: str) -> KubernetesObject: hostname: "*" prefix: /test/ service: test.default -''') +""" +) class TestKubernetesGVK: - def test_legacy(self): - gvk = KubernetesGVK('v1', 'Service') + gvk = KubernetesGVK("v1", "Service") - assert gvk.api_version == 'v1' - assert gvk.kind == 'Service' + assert gvk.api_version == "v1" + assert gvk.kind == "Service" assert gvk.api_group is None - assert gvk.version == 'v1' - assert gvk.domain == 'service' + assert gvk.version == "v1" + assert gvk.domain == "service" def test_group(self): - gvk = KubernetesGVK.for_ambassador('Mapping', version='v3alpha1') + gvk = KubernetesGVK.for_ambassador("Mapping", version="v3alpha1") - assert gvk.api_version == 'getambassador.io/v3alpha1' - assert gvk.kind == 'Mapping' - assert gvk.api_group == 'getambassador.io' - assert gvk.version == 'v3alpha1' - assert gvk.domain == 'mapping.getambassador.io' + assert gvk.api_version == "getambassador.io/v3alpha1" + assert gvk.kind == "Mapping" + assert gvk.api_group == "getambassador.io" + assert gvk.version == "v3alpha1" + assert gvk.domain == "mapping.getambassador.io" class TestKubernetesObject: - def test_valid(self): - assert valid_knative_ingress.gvk == KubernetesGVK.for_knative_networking('Ingress') - assert valid_knative_ingress.namespace == 'test' - assert valid_knative_ingress.name == 'helloworld-go' + assert valid_knative_ingress.gvk == KubernetesGVK.for_knative_networking("Ingress") + assert valid_knative_ingress.namespace == "test" + assert valid_knative_ingress.name == "helloworld-go" assert valid_knative_ingress.scope == KubernetesObjectScope.NAMESPACE - assert valid_knative_ingress.key == KubernetesObjectKey(valid_knative_ingress.gvk, 'test', 'helloworld-go') + assert valid_knative_ingress.key == KubernetesObjectKey( + valid_knative_ingress.gvk, "test", "helloworld-go" + ) assert valid_knative_ingress.generation == 2 assert len(valid_knative_ingress.annotations) == 2 - assert valid_knative_ingress.ambassador_id == 'webhook' + assert valid_knative_ingress.ambassador_id == "webhook" assert len(valid_knative_ingress.labels) == 3 - assert valid_knative_ingress.spec['rules'][0]['hosts'][0] == 'helloworld-go.test.svc.cluster.local' - assert valid_knative_ingress.status['observedGeneration'] == 2 + assert ( + valid_knative_ingress.spec["rules"][0]["hosts"][0] + == "helloworld-go.test.svc.cluster.local" + ) + assert valid_knative_ingress.status["observedGeneration"] == 2 def test_valid_cluster_scoped(self): - assert valid_ingress_class.name == 'external-lb' + assert valid_ingress_class.name == "external-lb" assert valid_ingress_class.scope == KubernetesObjectScope.CLUSTER - assert valid_ingress_class.key == KubernetesObjectKey(valid_ingress_class.gvk, None, 'external-lb') + assert valid_ingress_class.key == KubernetesObjectKey( + valid_ingress_class.gvk, None, "external-lb" + ) assert valid_ingress_class.key.namespace is None with pytest.raises(AttributeError): valid_ingress_class.namespace def test_invalid(self): - with pytest.raises(ValueError, match='not a valid Kubernetes object'): - k8s_object_from_yaml('apiVersion: v1') + with pytest.raises(ValueError, match="not a valid Kubernetes object"): + k8s_object_from_yaml("apiVersion: v1") class TestNormalizedResource: - def test_kubernetes_object_conversion(self): resource = NormalizedResource.from_kubernetes_object(valid_mapping) - assert resource.rkey == f'{valid_mapping.name}.{valid_mapping.namespace}' - assert resource.object['apiVersion'] == valid_mapping.gvk.api_version - assert resource.object['kind'] == valid_mapping.kind - assert resource.object['name'] == valid_mapping.name - assert resource.object['namespace'] == valid_mapping.namespace - assert resource.object['generation'] == valid_mapping.generation - assert len(resource.object['metadata_labels']) == 1 - assert resource.object['metadata_labels']['ambassador_crd'] == resource.rkey - assert resource.object['prefix'] == valid_mapping.spec['prefix'] - assert resource.object['service'] == valid_mapping.spec['service'] + assert resource.rkey == f"{valid_mapping.name}.{valid_mapping.namespace}" + assert resource.object["apiVersion"] == valid_mapping.gvk.api_version + assert resource.object["kind"] == valid_mapping.kind + assert resource.object["name"] == valid_mapping.name + assert resource.object["namespace"] == valid_mapping.namespace + assert resource.object["generation"] == valid_mapping.generation + assert len(resource.object["metadata_labels"]) == 1 + assert resource.object["metadata_labels"]["ambassador_crd"] == resource.rkey + assert resource.object["prefix"] == valid_mapping.spec["prefix"] + assert resource.object["service"] == valid_mapping.spec["service"] class TestLocationManager: - def test_context_manager(self): lm = LocationManager() @@ -192,18 +208,18 @@ def test_context_manager(self): assert lm.current.filename is None assert lm.current.ocount == 1 - with lm.push(filename='test', ocount=2) as loc: + with lm.push(filename="test", ocount=2) as loc: assert len(lm.previous) == 1 assert lm.current == loc - assert loc.filename == 'test' + assert loc.filename == "test" assert loc.ocount == 2 with lm.push_reset() as rloc: assert len(lm.previous) == 2 assert lm.current == rloc - assert rloc.filename == 'test' + assert rloc.filename == "test" assert rloc.ocount == 1 assert len(lm.previous) == 0 @@ -212,7 +228,7 @@ def test_context_manager(self): assert lm.current.ocount == 1 -class FinalizingKubernetesProcessor (KubernetesProcessor): +class FinalizingKubernetesProcessor(KubernetesProcessor): finalized: bool = False @@ -221,7 +237,6 @@ def finalize(self): class TestAmbassadorProcessor: - def test_mapping(self): aconf = Config() mgr = ResourceManager(logger, aconf, DependencyManager([])) @@ -232,7 +247,7 @@ def test_mapping(self): aconf.load_all(mgr.elements) assert len(aconf.errors) == 0 - mappings = aconf.get_config('mappings') + mappings = aconf.get_config("mappings") assert mappings assert len(mappings) == 1 @@ -240,8 +255,8 @@ def test_mapping(self): assert mapping.apiVersion == valid_mapping.gvk.api_version assert mapping.name == valid_mapping.name assert mapping.namespace == valid_mapping.namespace - assert mapping.prefix == valid_mapping.spec['prefix'] - assert mapping.service == valid_mapping.spec['service'] + assert mapping.prefix == valid_mapping.spec["prefix"] + assert mapping.service == valid_mapping.spec["service"] def test_mapping_v1(self): aconf = Config() @@ -254,7 +269,7 @@ def test_mapping_v1(self): aconf.load_all(mgr.elements) assert len(aconf.errors) == 0 - mappings = aconf.get_config('mappings') + mappings = aconf.get_config("mappings") assert mappings assert len(mappings) == 1 @@ -262,70 +277,73 @@ def test_mapping_v1(self): assert mapping.apiVersion == valid_mapping_v1.gvk.api_version assert mapping.name == valid_mapping_v1.name assert mapping.namespace == valid_mapping_v1.namespace - assert mapping.prefix == valid_mapping_v1.spec['prefix'] - assert mapping.service == valid_mapping_v1.spec['service'] + assert mapping.prefix == valid_mapping_v1.spec["prefix"] + assert mapping.service == valid_mapping_v1.spec["service"] class TestAggregateKubernetesProcessor: - def test_aggregation(self): aconf = Config() fp = FinalizingKubernetesProcessor() - p = AggregateKubernetesProcessor([ - CountingKubernetesProcessor(aconf, valid_knative_ingress.gvk, 'test_1'), - CountingKubernetesProcessor(aconf, valid_mapping.gvk, 'test_2'), - fp, - ]) + p = AggregateKubernetesProcessor( + [ + CountingKubernetesProcessor(aconf, valid_knative_ingress.gvk, "test_1"), + CountingKubernetesProcessor(aconf, valid_mapping.gvk, "test_2"), + fp, + ] + ) assert len(p.kinds()) == 2 assert p.try_process(valid_knative_ingress) assert p.try_process(valid_mapping) - assert aconf.get_count('test_1') == 1 - assert aconf.get_count('test_2') == 1 + assert aconf.get_count("test_1") == 1 + assert aconf.get_count("test_2") == 1 p.finalize() - assert fp.finalized, 'Aggregate processor did not call finalizers' + assert fp.finalized, "Aggregate processor did not call finalizers" class TestDeduplicatingKubernetesProcessor: - def test_deduplication(self): aconf = Config() - p = DeduplicatingKubernetesProcessor(CountingKubernetesProcessor(aconf, valid_mapping.gvk, 'test')) + p = DeduplicatingKubernetesProcessor( + CountingKubernetesProcessor(aconf, valid_mapping.gvk, "test") + ) assert p.try_process(valid_mapping) assert p.try_process(valid_mapping) assert p.try_process(valid_mapping) - assert aconf.get_count('test') == 1 + assert aconf.get_count("test") == 1 class TestCountingKubernetesProcessor: - def test_count(self): aconf = Config() - p = CountingKubernetesProcessor(aconf, valid_mapping.gvk, 'test') + p = CountingKubernetesProcessor(aconf, valid_mapping.gvk, "test") - assert p.try_process(valid_mapping), 'Processor rejected matching resource' - assert p.try_process(valid_mapping), 'Processor rejected matching resource (again)' - assert not p.try_process(valid_knative_ingress), 'Processor accepted non-matching resource' + assert p.try_process(valid_mapping), "Processor rejected matching resource" + assert p.try_process(valid_mapping), "Processor rejected matching resource (again)" + assert not p.try_process(valid_knative_ingress), "Processor accepted non-matching resource" - assert aconf.get_count('test') == 2, 'Processor did not increment counter' + assert aconf.get_count("test") == 2, "Processor did not increment counter" -class TestDependencyManager: +class TestDependencyManager: def setup(self): - self.deps = DependencyManager([ - SecretDependency(), - ServiceDependency(), - IngressClassesDependency(), - ]) + self.deps = DependencyManager( + [ + SecretDependency(), + ServiceDependency(), + IngressClassesDependency(), + ] + ) def test_cyclic(self): a = self.deps.for_instance(object()) @@ -353,8 +371,8 @@ def test_sort(self): b.provide(SecretDependency) c.provide(ServiceDependency) - assert self.deps.sorted_watt_keys() == ['secret', 'service', 'ingressclasses'] + assert self.deps.sorted_watt_keys() == ["secret", "service", "ingressclasses"] -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_hcm.py b/python/tests/unit/test_hcm.py index d3d4c192ff..af1076cadb 100644 --- a/python/tests/unit/test_hcm.py +++ b/python/tests/unit/test_hcm.py @@ -2,6 +2,7 @@ import pytest + def _test_hcm(yaml, expectations={}): # Compile an envoy config econf = econf_compile(yaml) @@ -15,76 +16,89 @@ def check(typed_config): assert key in typed_config assert typed_config[key] == expected return True + econf_foreach_hcm(econf, check) + @pytest.mark.compilertest def test_strip_matching_host_port_missing(): # If we do not set the config, it should be missing (noted in this test as None). yaml = module_and_mapping_manifests(None, []) - _test_hcm(yaml, expectations={'strip_matching_host_port': None}) + _test_hcm(yaml, expectations={"strip_matching_host_port": None}) + @pytest.mark.compilertest def test_strip_matching_host_port_module_false(): # If we set the config to false, it should be missing (noted in this test as None). - yaml = module_and_mapping_manifests(['strip_matching_host_port: false'], []) - _test_hcm(yaml, expectations={'strip_matching_host_port': None}) + yaml = module_and_mapping_manifests(["strip_matching_host_port: false"], []) + _test_hcm(yaml, expectations={"strip_matching_host_port": None}) + @pytest.mark.compilertest def test_strip_matching_host_port_module_true(): # If we set the config to true, it should show up as true. - yaml = module_and_mapping_manifests(['strip_matching_host_port: true'], []) - _test_hcm(yaml, expectations={'strip_matching_host_port': True}) + yaml = module_and_mapping_manifests(["strip_matching_host_port: true"], []) + _test_hcm(yaml, expectations={"strip_matching_host_port": True}) + @pytest.mark.compilertest def test_merge_slashes_missing(): # If we do not set the config, it should be missing (noted in this test as None). yaml = module_and_mapping_manifests(None, []) - _test_hcm(yaml, expectations={'merge_slashes': None}) + _test_hcm(yaml, expectations={"merge_slashes": None}) + @pytest.mark.compilertest def test_merge_slashes_module_false(): # If we set the config to false, it should be missing (noted in this test as None). - yaml = module_and_mapping_manifests(['merge_slashes: false'], []) - _test_hcm(yaml, expectations={'merge_slashes': None}) + yaml = module_and_mapping_manifests(["merge_slashes: false"], []) + _test_hcm(yaml, expectations={"merge_slashes": None}) + @pytest.mark.compilertest def test_merge_slashes_module_true(): # If we set the config to true, it should show up as true. - yaml = module_and_mapping_manifests(['merge_slashes: true'], []) - _test_hcm(yaml, expectations={'merge_slashes': True}) + yaml = module_and_mapping_manifests(["merge_slashes: true"], []) + _test_hcm(yaml, expectations={"merge_slashes": True}) + @pytest.mark.compilertest def test_reject_requests_with_escaped_slashes_missing(): # If we set the config to false, the action should be missing. yaml = module_and_mapping_manifests(None, []) - _test_hcm(yaml, expectations={'path_with_escaped_slashes_action': None}) + _test_hcm(yaml, expectations={"path_with_escaped_slashes_action": None}) + @pytest.mark.compilertest def test_reject_requests_with_escaped_slashes_false(): # If we set the config to false, the action should be missing. - yaml = module_and_mapping_manifests(['reject_requests_with_escaped_slashes: false'], []) - _test_hcm(yaml, expectations={'path_with_escaped_slashes_action': None}) + yaml = module_and_mapping_manifests(["reject_requests_with_escaped_slashes: false"], []) + _test_hcm(yaml, expectations={"path_with_escaped_slashes_action": None}) + @pytest.mark.compilertest def test_reject_requests_with_escaped_slashes_true(): # If we set the config to true, the action should show up as "REJECT_REQUEST". - yaml = module_and_mapping_manifests(['reject_requests_with_escaped_slashes: true'], []) - _test_hcm(yaml, expectations={'path_with_escaped_slashes_action': 'REJECT_REQUEST'}) + yaml = module_and_mapping_manifests(["reject_requests_with_escaped_slashes: true"], []) + _test_hcm(yaml, expectations={"path_with_escaped_slashes_action": "REJECT_REQUEST"}) + @pytest.mark.compilertest def test_preserve_external_request_id_missing(): # If we do not set the config, it should be false yaml = module_and_mapping_manifests(None, []) - _test_hcm(yaml, expectations={'preserve_external_request_id': False}) + _test_hcm(yaml, expectations={"preserve_external_request_id": False}) + @pytest.mark.compilertest def test_preserve_external_request_id_module_false(): # If we set the config to false, it should be false - yaml = module_and_mapping_manifests(['preserve_external_request_id: false'], []) - _test_hcm(yaml, expectations={'preserve_external_request_id': False}) + yaml = module_and_mapping_manifests(["preserve_external_request_id: false"], []) + _test_hcm(yaml, expectations={"preserve_external_request_id": False}) + @pytest.mark.compilertest def test_preserve_external_request_id_module_true(): # If we set the config to true, it should show up as true. - yaml = module_and_mapping_manifests(['preserve_external_request_id: true'], []) - _test_hcm(yaml, expectations={'preserve_external_request_id': True}) + yaml = module_and_mapping_manifests(["preserve_external_request_id: true"], []) + _test_hcm(yaml, expectations={"preserve_external_request_id": True}) diff --git a/python/tests/unit/test_host_redirect_errors.py b/python/tests/unit/test_host_redirect_errors.py index c6ec79a543..e5758b636e 100644 --- a/python/tests/unit/test_host_redirect_errors.py +++ b/python/tests/unit/test_host_redirect_errors.py @@ -13,7 +13,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -22,9 +22,11 @@ from ambassador.utils import NullSecretHandler from ambassador.compile import Compile + def require_no_errors(ir: IR): assert ir.aconf.errors == {} + def require_errors(ir: IR, errors: List[Tuple[str, str]]): flattened_ir_errors: List[str] = [] @@ -32,12 +34,11 @@ def require_errors(ir: IR, errors: List[Tuple[str, str]]): for error in ir.aconf.errors[key]: flattened_ir_errors.append(f"{key}: {error['error']}") - flattened_wanted_errors: List[str] = [ - f"{key}: {error}" for key, error in errors - ] + flattened_wanted_errors: List[str] = [f"{key}: {error}" for key, error in errors] assert sorted(flattened_ir_errors) == sorted(flattened_wanted_errors) + @pytest.mark.compilertest def test_hr_good_1(): yaml = """ @@ -72,6 +73,7 @@ def test_hr_good_1(): require_no_errors(r1["ir"]) require_no_errors(r2["ir"]) + @pytest.mark.compilertest def test_hr_error_1(): yaml = """ @@ -104,13 +106,26 @@ def test_hr_error_1(): r2 = Compile(logger, yaml, k8s=True, cache=cache) # XXX Why are these showing up tagged with "mapping-1.default.1" rather than "mapping-2.default.1"? - require_errors(r1["ir"], [ - ( "mapping-1.default.1", "cannot accept mapping-2 as second host_redirect after mapping-1") - ]) + require_errors( + r1["ir"], + [ + ( + "mapping-1.default.1", + "cannot accept mapping-2 as second host_redirect after mapping-1", + ) + ], + ) + + require_errors( + r2["ir"], + [ + ( + "mapping-1.default.1", + "cannot accept mapping-2 as second host_redirect after mapping-1", + ) + ], + ) - require_errors(r2["ir"], [ - ( "mapping-1.default.1", "cannot accept mapping-2 as second host_redirect after mapping-1") - ]) @pytest.mark.compilertest def test_hr_error_2(): @@ -143,13 +158,26 @@ def test_hr_error_2(): r2 = Compile(logger, yaml, k8s=True, cache=cache) # FIXME(lukeshu): These should not show up as "-global-". - require_errors(r1["ir"], [ - ( "-global-", "cannot accept mapping-2 without host_redirect after mapping-1 with host_redirect") - ]) + require_errors( + r1["ir"], + [ + ( + "-global-", + "cannot accept mapping-2 without host_redirect after mapping-1 with host_redirect", + ) + ], + ) + + require_errors( + r2["ir"], + [ + ( + "-global-", + "cannot accept mapping-2 without host_redirect after mapping-1 with host_redirect", + ) + ], + ) - require_errors(r2["ir"], [ - ( "-global-", "cannot accept mapping-2 without host_redirect after mapping-1 with host_redirect") - ]) @pytest.mark.compilertest def test_hr_error_3(): @@ -182,13 +210,26 @@ def test_hr_error_3(): r2 = Compile(logger, yaml, k8s=True, cache=cache) # XXX Why are these showing up tagged with "mapping-1.default.1" rather than "mapping-2.default.1"? - require_errors(r1["ir"], [ - ( "mapping-1.default.1", "cannot accept mapping-2 with host_redirect after mappings without host_redirect (eg mapping-1)") - ]) + require_errors( + r1["ir"], + [ + ( + "mapping-1.default.1", + "cannot accept mapping-2 with host_redirect after mappings without host_redirect (eg mapping-1)", + ) + ], + ) + + require_errors( + r2["ir"], + [ + ( + "mapping-1.default.1", + "cannot accept mapping-2 with host_redirect after mappings without host_redirect (eg mapping-1)", + ) + ], + ) - require_errors(r2["ir"], [ - ( "mapping-1.default.1", "cannot accept mapping-2 with host_redirect after mappings without host_redirect (eg mapping-1)") - ]) @pytest.mark.compilertest def test_hr_error_4(): @@ -242,9 +283,21 @@ def test_hr_error_4(): r1 = Compile(logger, yaml, k8s=True) r2 = Compile(logger, yaml, k8s=True, cache=cache) - for r in [ r1, r2 ]: - require_errors(r["ir"], [ - ( "mapping-1.default.1", "Cannot specify both path_redirect and prefix_redirect. Using path_redirect and ignoring prefix_redirect."), - ( "mapping-2.default.1", "Cannot specify both path_redirect and regex_redirect. Using path_redirect and ignoring regex_redirect."), - ( "mapping-3.default.1", "Cannot specify both prefix_redirect and regex_redirect. Using prefix_redirect and ignoring regex_redirect.") - ]) + for r in [r1, r2]: + require_errors( + r["ir"], + [ + ( + "mapping-1.default.1", + "Cannot specify both path_redirect and prefix_redirect. Using path_redirect and ignoring prefix_redirect.", + ), + ( + "mapping-2.default.1", + "Cannot specify both path_redirect and regex_redirect. Using path_redirect and ignoring regex_redirect.", + ), + ( + "mapping-3.default.1", + "Cannot specify both prefix_redirect and regex_redirect. Using prefix_redirect and ignoring regex_redirect.", + ), + ], + ) diff --git a/python/tests/unit/test_hostglob_matches.py b/python/tests/unit/test_hostglob_matches.py index b01dbb43bb..8f332c73f6 100644 --- a/python/tests/unit/test_hostglob_matches.py +++ b/python/tests/unit/test_hostglob_matches.py @@ -6,55 +6,57 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") from ambassador.ir.irutils import hostglob_matches + @pytest.mark.compilertest def test_hostglob_matches(): for v1, v2, wanted_result in [ - ( "a.example.com", "a.example.com", True ), - ( "a.example.com", "b.example.com", False ), - ( "*", "foo.example.com", True ), - ( "*.example.com", "a.example.com", True ), - ( "*example.com", "b.example.com", True ), + ("a.example.com", "a.example.com", True), + ("a.example.com", "b.example.com", False), + ("*", "foo.example.com", True), + ("*.example.com", "a.example.com", True), + ("*example.com", "b.example.com", True), # This is never OK: the "*" can't match a bare ".". - ( "*example.com", ".example.com", False ), + ("*example.com", ".example.com", False), # This is OK, because DNS allows names to end with a "." - ( "foo.example*", "foo.example.com.", True ), + ("foo.example*", "foo.example.com.", True), # This is never OK: the "*" cannot match an empty string. - ( "*example.com", "example.com", False ), - ( "*ple.com", "b.example.com", True ), - ( "*.example.com", "a.example.org", False ), - ( "*example.com", "a.example.org", False ), - ( "*ple.com", "a.example.org", False ), - ( "a.example.*", "a.example.com", True ), - ( "a.example*", "a.example.com", True ), - ( "a.exa*", "a.example.com", True ), - ( "a.example.*", "a.example.org", True ), - ( "a.example.*", "b.example.com", False ), - ( "a.example*", "b.example.com", False ), - ( "a.exa*", "b.example.com", False ), + ("*example.com", "example.com", False), + ("*ple.com", "b.example.com", True), + ("*.example.com", "a.example.org", False), + ("*example.com", "a.example.org", False), + ("*ple.com", "a.example.org", False), + ("a.example.*", "a.example.com", True), + ("a.example*", "a.example.com", True), + ("a.exa*", "a.example.com", True), + ("a.example.*", "a.example.org", True), + ("a.example.*", "b.example.com", False), + ("a.example*", "b.example.com", False), + ("a.exa*", "b.example.com", False), # '*' has to appear at the beginning or the end, not in the middle. - ( "a.*.com", "a.example.com", False ), + ("a.*.com", "a.example.com", False), # Various DNS glob situations disagree about whether "*" can cross subdomain # boundaries. We follow what Envoy does, which is to allow crossing. - ( "*.com", "a.example.com", True ), - ( "*.com", "a.example.org", False ), - ( "*.example.com", "*.example.com", True ), + ("*.com", "a.example.com", True), + ("*.com", "a.example.org", False), + ("*.example.com", "*.example.com", True), # This looks wrong but it's OK: both match e.g. foo.example.com. - ( "*example.com", "*.example.com", True ), + ("*example.com", "*.example.com", True), # These are ugly corner cases, but they should still work! - ( "*.example.com", "a.example.*", True ), - ( "*.example.com", "a.b.example.*", True ), - ( "*.example.baz.com", "a.b.example.*", True ), - ( "*.foo.bar", "baz.zing.*", True ), + ("*.example.com", "a.example.*", True), + ("*.example.com", "a.b.example.*", True), + ("*.example.baz.com", "a.b.example.*", True), + ("*.foo.bar", "baz.zing.*", True), ]: assert hostglob_matches(v1, v2) == wanted_result, f"1. {v1} ~ {v2} != {wanted_result}" assert hostglob_matches(v2, v1) == wanted_result, f"2. {v2} ~ {v1} != {wanted_result}" -if __name__ == '__main__': + +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_ir.py b/python/tests/unit/test_ir.py index 9a2eec7198..b7ae0c8e80 100644 --- a/python/tests/unit/test_ir.py +++ b/python/tests/unit/test_ir.py @@ -1,45 +1,61 @@ from dataclasses import dataclass -from tests.utils import (Compile, logger, default_listener_manifests, - default_http3_listener_manifest, default_udp_listener_manifest, default_tcp_listener_manifest) +from tests.utils import ( + Compile, + logger, + default_listener_manifests, + default_http3_listener_manifest, + default_udp_listener_manifest, + default_tcp_listener_manifest, +) from typing import List, Optional import pytest import logging + def http3_quick_start_manifests(): - return default_listener_manifests() + default_http3_listener_manifest() + return default_listener_manifests() + default_http3_listener_manifest() + class TestIR: - def test_http3_enabled(self, caplog): - caplog.set_level(logging.WARNING, logger="ambassador") - - @dataclass - class TestCase: - name:str - inputYaml: str - expected: dict[str,bool] - expectedLog: Optional[str] = None - - testcases = [ - TestCase("quick-start", default_listener_manifests(), { "tcp-0.0.0.0-8080": False, "tcp-0.0.0.0-8443": False }), - TestCase("quick-start-with_http3", http3_quick_start_manifests(), { "tcp-0.0.0.0-8080": False, "tcp-0.0.0.0-8443": True, "udp-0.0.0.0-8443": True }), - TestCase("http3-only", default_http3_listener_manifest(),{ "udp-0.0.0.0-8443": True }), - TestCase("raw-udp", default_udp_listener_manifest(),{}), - TestCase("raw-tcp", default_tcp_listener_manifest(),{ "tcp-0.0.0.0-8443": False }), - ] - - for case in testcases: - compiled_ir = Compile(logger, case.inputYaml, k8s=True) - result_ir = compiled_ir['ir'] - - listeners = result_ir.listeners - - assert len(case.expected.items()) == len(listeners) - - for listener_id, http3_enabled in case.expected.items(): - listener = listeners.get(listener_id, None) - assert listener is not None - assert listener.http3_enabled == http3_enabled - - if case.expectedLog != None: - assert case.expectedLog in caplog.text + def test_http3_enabled(self, caplog): + caplog.set_level(logging.WARNING, logger="ambassador") + + @dataclass + class TestCase: + name: str + inputYaml: str + expected: dict[str, bool] + expectedLog: Optional[str] = None + + testcases = [ + TestCase( + "quick-start", + default_listener_manifests(), + {"tcp-0.0.0.0-8080": False, "tcp-0.0.0.0-8443": False}, + ), + TestCase( + "quick-start-with_http3", + http3_quick_start_manifests(), + {"tcp-0.0.0.0-8080": False, "tcp-0.0.0.0-8443": True, "udp-0.0.0.0-8443": True}, + ), + TestCase("http3-only", default_http3_listener_manifest(), {"udp-0.0.0.0-8443": True}), + TestCase("raw-udp", default_udp_listener_manifest(), {}), + TestCase("raw-tcp", default_tcp_listener_manifest(), {"tcp-0.0.0.0-8443": False}), + ] + + for case in testcases: + compiled_ir = Compile(logger, case.inputYaml, k8s=True) + result_ir = compiled_ir["ir"] + + listeners = result_ir.listeners + + assert len(case.expected.items()) == len(listeners) + + for listener_id, http3_enabled in case.expected.items(): + listener = listeners.get(listener_id, None) + assert listener is not None + assert listener.http3_enabled == http3_enabled + + if case.expectedLog != None: + assert case.expectedLog in caplog.text diff --git a/python/tests/unit/test_irauth.py b/python/tests/unit/test_irauth.py index d743224aa2..907a1eaea3 100644 --- a/python/tests/unit/test_irauth.py +++ b/python/tests/unit/test_irauth.py @@ -8,7 +8,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -19,12 +19,13 @@ from tests.utils import default_listener_manifests + def _get_ext_auth_config(yaml): - for listener in yaml['static_resources']['listeners']: - for filter_chain in listener['filter_chains']: - for f in filter_chain['filters']: - for http_filter in f['typed_config']['http_filters']: - if http_filter['name'] == 'envoy.filters.http.ext_authz': + for listener in yaml["static_resources"]["listeners"]: + for filter_chain in listener["filter_chains"]: + for f in filter_chain["filters"]: + for http_filter in f["typed_config"]["http_filters"]: + if http_filter["name"] == "envoy.filters.http.ext_authz": return http_filter return False @@ -69,9 +70,11 @@ def test_irauth_grpcservice_version_v2(): assert ext_auth_config == False - errors = econf.ir.aconf.errors['mycoolauthservice.default.1'] - assert errors[0]['error'] == 'AuthService: protocol_version v2 is unsupported, protocol_version must be "v3"' - + errors = econf.ir.aconf.errors["mycoolauthservice.default.1"] + assert ( + errors[0]["error"] + == 'AuthService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) def test_irauth_grpcservice_version_v3(): @@ -94,10 +97,13 @@ def test_irauth_grpcservice_version_v3(): ext_auth_config = _get_ext_auth_config(conf) assert ext_auth_config - assert ext_auth_config['typed_config']['grpc_service']['envoy_grpc']['cluster_name'] == 'cluster_extauth_someservice_default' - assert ext_auth_config['typed_config']['transport_api_version'] == 'V3' + assert ( + ext_auth_config["typed_config"]["grpc_service"]["envoy_grpc"]["cluster_name"] + == "cluster_extauth_someservice_default" + ) + assert ext_auth_config["typed_config"]["transport_api_version"] == "V3" - assert 'mycoolauthservice.default.1' not in econf.ir.aconf.errors + assert "mycoolauthservice.default.1" not in econf.ir.aconf.errors @pytest.mark.compilertest @@ -124,5 +130,8 @@ def test_irauth_grpcservice_version_default(): assert ext_auth_config == False - errors = econf.ir.aconf.errors['mycoolauthservice.default.1'] - assert errors[0]['error'] == 'AuthService: protocol_version v2 is unsupported, protocol_version must be "v3"' + errors = econf.ir.aconf.errors["mycoolauthservice.default.1"] + assert ( + errors[0]["error"] + == 'AuthService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) diff --git a/python/tests/unit/test_irlogservice.py b/python/tests/unit/test_irlogservice.py index 39c24fb706..bb7dbbbe2b 100644 --- a/python/tests/unit/test_irlogservice.py +++ b/python/tests/unit/test_irlogservice.py @@ -8,7 +8,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("emissary-ingress") @@ -19,15 +19,15 @@ from tests.utils import default_listener_manifests -SERVICE_NAME = 'cool-log-svcname' +SERVICE_NAME = "cool-log-svcname" def _get_log_config(yaml, driver: Literal["http", "tcp"]): - for listener in yaml['static_resources']['listeners']: - for filter_chain in listener['filter_chains']: - for f in filter_chain['filters']: - for log_filter in f['typed_config']['access_log']: - if log_filter['name'] == f'envoy.access_loggers.{driver}_grpc': + for listener in yaml["static_resources"]["listeners"]: + for filter_chain in listener["filter_chains"]: + for f in filter_chain["filters"]: + for log_filter in f["typed_config"]["access_log"]: + if log_filter["name"] == f"envoy.access_loggers.{driver}_grpc": return log_filter return False @@ -49,46 +49,46 @@ def _get_envoy_config(yaml): def _get_logfilter_http_default_conf(): return { - '@type': f'type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig', - 'common_config': { - 'transport_api_version': 'V3', - 'log_name': 'logservice', - 'grpc_service': { - 'envoy_grpc': { - 'cluster_name': 'cluster_logging_cool_log_svcname_default' - } + "@type": f"type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig", + "common_config": { + "transport_api_version": "V3", + "log_name": "logservice", + "grpc_service": { + "envoy_grpc": {"cluster_name": "cluster_logging_cool_log_svcname_default"} }, - 'buffer_flush_interval': '1s', - 'buffer_size_bytes': 16384 + "buffer_flush_interval": "1s", + "buffer_size_bytes": 16384, }, - 'additional_request_headers_to_log': [], - 'additional_response_headers_to_log': [], - 'additional_response_trailers_to_log': [] + "additional_request_headers_to_log": [], + "additional_response_headers_to_log": [], + "additional_response_trailers_to_log": [], } + def _get_logfilter_tcp_default_conf(): return { - '@type': f'type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.TcpGrpcAccessLogConfig', - 'common_config': { - 'transport_api_version': 'V3', - 'log_name': 'logservice', - 'grpc_service': { - 'envoy_grpc': { - 'cluster_name': 'cluster_logging_cool_log_svcname_default' - } + "@type": f"type.googleapis.com/envoy.extensions.access_loggers.grpc.v3.TcpGrpcAccessLogConfig", + "common_config": { + "transport_api_version": "V3", + "log_name": "logservice", + "grpc_service": { + "envoy_grpc": {"cluster_name": "cluster_logging_cool_log_svcname_default"} }, - 'buffer_flush_interval': '1s', - 'buffer_size_bytes': 16384 - } + "buffer_flush_interval": "1s", + "buffer_size_bytes": 16384, + }, } + ###################### unit test covering http driver ########################### + @pytest.mark.compilertest def test_irlogservice_http_defaults(): """tests defaults for log service when http driver is used and ensures that transport protocol is v3""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -96,28 +96,36 @@ def test_irlogservice_http_defaults(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: http driver_config: {} grpc: true """ + ) - driver: Literal['http', 'tcp'] = 'http' + driver: Literal["http", "tcp"] = "http" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf == False - errors = econf.ir.aconf.errors - assert 'ir.logservice' in errors - assert errors['ir.logservice'][0]['error'] == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + errors = econf.ir.aconf.errors + assert "ir.logservice" in errors + assert ( + errors["ir.logservice"][0]["error"] + == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) + @pytest.mark.compilertest def test_irlogservice_http_default_overrides(): """tests default overrides for log service and ensures that transport protocol is v3""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -125,7 +133,9 @@ def test_irlogservice_http_default_overrides(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: http grpc: true protocol_version: "v3" @@ -147,31 +157,32 @@ def test_irlogservice_http_default_overrides(): during_response: false during_trailer: true """ + ) - driver: Literal['http', 'tcp'] = 'http' + driver: Literal["http", "tcp"] = "http" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf config = _get_logfilter_http_default_conf() - config['common_config']['buffer_flush_interval'] = '33s' - config['common_config']['buffer_size_bytes'] = 9999 - config['additional_request_headers_to_log'] = ['x-dino-power', 'x-dino-request-power'] - config['additional_response_headers_to_log'] = ['x-dino-power', 'x-dino-response-power'] - config['additional_response_trailers_to_log'] = ['x-dino-power', 'x-dino-trailer-power'] - + config["common_config"]["buffer_flush_interval"] = "33s" + config["common_config"]["buffer_size_bytes"] = 9999 + config["additional_request_headers_to_log"] = ["x-dino-power", "x-dino-request-power"] + config["additional_response_headers_to_log"] = ["x-dino-power", "x-dino-response-power"] + config["additional_response_trailers_to_log"] = ["x-dino-power", "x-dino-trailer-power"] - assert conf.get('typed_config') == config + assert conf.get("typed_config") == config - assert 'ir.logservice' not in econf.ir.aconf.errors + assert "ir.logservice" not in econf.ir.aconf.errors @pytest.mark.compilertest def test_irlogservice_http_v2(): """ensures that no longer supported v2 transport protocol is defaulted to v3""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -179,30 +190,37 @@ def test_irlogservice_http_v2(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: http driver_config: {} grpc: true protocol_version: "v2" """ + ) - driver: Literal['http', 'tcp'] = 'http' + driver: Literal["http", "tcp"] = "http" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf == False - errors = econf.ir.aconf.errors - assert 'ir.logservice' in errors - assert errors['ir.logservice'][0]['error'] == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + errors = econf.ir.aconf.errors + assert "ir.logservice" in errors + assert ( + errors["ir.logservice"][0]["error"] + == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) @pytest.mark.compilertest def test_irlogservice_http_v3(): """ensures that when transport protocol v3 is provided, nothing is logged""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -210,23 +228,25 @@ def test_irlogservice_http_v3(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: http driver_config: {} grpc: true protocol_version: "v3" """ + ) - driver: Literal['http', 'tcp'] = 'http' + driver: Literal["http", "tcp"] = "http" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf - assert conf.get('typed_config') == _get_logfilter_http_default_conf() - - assert 'ir.logservice' not in econf.ir.aconf.errors + assert conf.get("typed_config") == _get_logfilter_http_default_conf() + assert "ir.logservice" not in econf.ir.aconf.errors ############### unit test covering tcp driver ####################### @@ -234,7 +254,8 @@ def test_irlogservice_http_v3(): def test_irlogservice_tcp_defaults(): """tests defaults for log service using tcp driver and ensures that transport protocol is v3""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -242,29 +263,36 @@ def test_irlogservice_tcp_defaults(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: tcp driver_config: {} grpc: true """ + ) - driver: Literal['http', 'tcp'] = 'tcp' + driver: Literal["http", "tcp"] = "tcp" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf == False - errors = econf.ir.aconf.errors - assert 'ir.logservice' in errors - assert errors['ir.logservice'][0]['error'] == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + errors = econf.ir.aconf.errors + assert "ir.logservice" in errors + assert ( + errors["ir.logservice"][0]["error"] + == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) @pytest.mark.compilertest def test_irlogservice_tcp_default_overrides(): """tests default overrides for log service with tcp driver and ensures that transport protocol is v3""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -272,7 +300,9 @@ def test_irlogservice_tcp_default_overrides(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: tcp driver_config: {} grpc: true @@ -280,27 +310,29 @@ def test_irlogservice_tcp_default_overrides(): flush_interval_time: 33 flush_interval_byte_size: 9999 """ + ) - driver: Literal['http', 'tcp'] = 'tcp' + driver: Literal["http", "tcp"] = "tcp" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf config = _get_logfilter_tcp_default_conf() - config['common_config']['buffer_flush_interval'] = '33s' - config['common_config']['buffer_size_bytes'] = 9999 + config["common_config"]["buffer_flush_interval"] = "33s" + config["common_config"]["buffer_size_bytes"] = 9999 - assert conf.get('typed_config') == config + assert conf.get("typed_config") == config - assert 'ir.logservice' not in econf.ir.aconf.errors + assert "ir.logservice" not in econf.ir.aconf.errors @pytest.mark.compilertest def test_irlogservice_tcp_v2(): """ensures that no longer supported v2 transport protocol is defaulted to v3""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -308,30 +340,37 @@ def test_irlogservice_tcp_v2(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: tcp driver_config: {} grpc: true protocol_version: "v2" """ + ) - driver: Literal['http', 'tcp'] = 'tcp' + driver: Literal["http", "tcp"] = "tcp" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf == False - errors = econf.ir.aconf.errors - assert 'ir.logservice' in errors - assert errors['ir.logservice'][0]['error'] == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + errors = econf.ir.aconf.errors + assert "ir.logservice" in errors + assert ( + errors["ir.logservice"][0]["error"] + == 'LogService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) @pytest.mark.compilertest def test_irlogservice_tcp_v3(): """ensures that when transport protocol v3 is provided, nothing is logged""" - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: LogService @@ -339,19 +378,22 @@ def test_irlogservice_tcp_v3(): name: myls namespace: default spec: - service: """ + SERVICE_NAME + """ + service: """ + + SERVICE_NAME + + """ driver: tcp driver_config: {} grpc: true protocol_version: "v3" """ + ) - driver: Literal['http', 'tcp'] = 'tcp' + driver: Literal["http", "tcp"] = "tcp" econf = _get_envoy_config(yaml) conf = _get_log_config(econf.as_dict(), driver) assert conf - assert conf.get('typed_config') == _get_logfilter_tcp_default_conf() + assert conf.get("typed_config") == _get_logfilter_tcp_default_conf() - assert 'ir.logservice' not in econf.ir.aconf.errors + assert "ir.logservice" not in econf.ir.aconf.errors diff --git a/python/tests/unit/test_irmapping.py b/python/tests/unit/test_irmapping.py index b26804248d..4f3a008164 100644 --- a/python/tests/unit/test_irmapping.py +++ b/python/tests/unit/test_irmapping.py @@ -8,7 +8,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -18,6 +18,7 @@ from ambassador.ir.irbasemappinggroup import IRBaseMappingGroup from ambassador.utils import NullSecretHandler + def _get_ir_config(yaml): aconf = Config() fetcher = ResourceFetcher(logger, aconf) @@ -52,7 +53,7 @@ def test_ir_mapping(): for i in conf.groups.values(): all_mappings = all_mappings + i.mappings - slowsvc_mappings = [x for x in all_mappings if x['name'] == 'slowsvc-slow'] - assert(len(slowsvc_mappings) == 1) + slowsvc_mappings = [x for x in all_mappings if x["name"] == "slowsvc-slow"] + assert len(slowsvc_mappings) == 1 print(slowsvc_mappings[0].as_dict()) - assert(slowsvc_mappings[0].docs['timeout_ms'] == 8000) + assert slowsvc_mappings[0].docs["timeout_ms"] == 8000 diff --git a/python/tests/unit/test_irratelimit.py b/python/tests/unit/test_irratelimit.py index 325b4cf0b7..9bb4a964bf 100644 --- a/python/tests/unit/test_irratelimit.py +++ b/python/tests/unit/test_irratelimit.py @@ -7,7 +7,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -18,15 +18,15 @@ from tests.utils import default_listener_manifests -SERVICE_NAME = 'coolsvcname' +SERVICE_NAME = "coolsvcname" def _get_rl_config(yaml): - for listener in yaml['static_resources']['listeners']: - for filter_chain in listener['filter_chains']: - for f in filter_chain['filters']: - for http_filter in f['typed_config']['http_filters']: - if http_filter['name'] == 'envoy.filters.http.ratelimit': + for listener in yaml["static_resources"]["listeners"]: + for filter_chain in listener["filter_chains"]: + for f in filter_chain["filters"]: + for http_filter in f["typed_config"]["http_filters"]: + if http_filter["name"] == "envoy.filters.http.ratelimit": return http_filter return False @@ -48,18 +48,16 @@ def _get_envoy_config(yaml): def _get_ratelimit_default_conf(): return { - '@type': 'type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit', - 'domain': 'ambassador', - 'request_type': 'both', - 'timeout': '0.020s', - 'rate_limit_service': { - 'transport_api_version': 'V3', - 'grpc_service': { - 'envoy_grpc': { - 'cluster_name': 'cluster_{}_default'.format(SERVICE_NAME) - } - } - } + "@type": "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit", + "domain": "ambassador", + "request_type": "both", + "timeout": "0.020s", + "rate_limit_service": { + "transport_api_version": "V3", + "grpc_service": { + "envoy_grpc": {"cluster_name": "cluster_{}_default".format(SERVICE_NAME)} + }, + }, } @@ -75,17 +73,21 @@ def test_irratelimit_defaults(): namespace: default spec: service: {} -""".format(SERVICE_NAME) +""".format( + SERVICE_NAME + ) econf = _get_envoy_config(yaml) conf = _get_rl_config(econf.as_dict()) assert conf == False - errors = econf.ir.aconf.errors - assert 'ir.ratelimit' in errors - assert errors['ir.ratelimit'][0]['error'] == 'RateLimitService: protocol_version v2 is unsupported, protocol_version must be "v3"' - + errors = econf.ir.aconf.errors + assert "ir.ratelimit" in errors + assert ( + errors["ir.ratelimit"][0]["error"] + == 'RateLimitService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) @pytest.mark.compilertest @@ -101,15 +103,17 @@ def test_irratelimit_grpcsvc_version_v3(): spec: service: {} protocol_version: "v3" -""".format(SERVICE_NAME) +""".format( + SERVICE_NAME + ) econf = _get_envoy_config(yaml) conf = _get_rl_config(econf.as_dict()) assert conf - assert conf.get('typed_config') == _get_ratelimit_default_conf() + assert conf.get("typed_config") == _get_ratelimit_default_conf() - assert 'ir.ratelimit' not in econf.ir.aconf.errors + assert "ir.ratelimit" not in econf.ir.aconf.errors @pytest.mark.compilertest @@ -125,21 +129,26 @@ def test_irratelimit_grpcsvc_version_v2(): spec: service: {} protocol_version: "v2" -""".format(SERVICE_NAME) +""".format( + SERVICE_NAME + ) econf = _get_envoy_config(yaml) conf = _get_rl_config(econf.as_dict()) assert conf == False - errors = econf.ir.aconf.errors - assert 'ir.ratelimit' in errors - assert errors['ir.ratelimit'][0]['error'] == 'RateLimitService: protocol_version v2 is unsupported, protocol_version must be "v3"' + errors = econf.ir.aconf.errors + assert "ir.ratelimit" in errors + assert ( + errors["ir.ratelimit"][0]["error"] + == 'RateLimitService: protocol_version v2 is unsupported, protocol_version must be "v3"' + ) @pytest.mark.compilertest def test_irratelimit_error(): - """ Test error no valid spec with service name """ + """Test error no valid spec with service name""" yaml = """ --- @@ -156,15 +165,14 @@ def test_irratelimit_error(): assert not conf - errors = econf.ir.aconf.errors - assert 'ir.ratelimit' in errors - assert errors['ir.ratelimit'][0]['error'] == 'service is required in RateLimitService' - + errors = econf.ir.aconf.errors + assert "ir.ratelimit" in errors + assert errors["ir.ratelimit"][0]["error"] == "service is required in RateLimitService" @pytest.mark.compilertest def test_irratelimit_overrides(): - """ Test that default are properly overriden """ + """Test that default are properly overriden""" yaml = """ --- @@ -179,19 +187,22 @@ def test_irratelimit_overrides(): timeout_ms: 500 tls: rl-tls-context protocol_version: v3 -""".format(SERVICE_NAME) - +""".format( + SERVICE_NAME + ) config = _get_ratelimit_default_conf() - config['rate_limit_service']['grpc_service']['envoy_grpc']['cluster_name'] = 'cluster_{}_someotherns'.format(SERVICE_NAME) - config['timeout'] = '0.500s' - config['domain'] = 'otherdomain' + config["rate_limit_service"]["grpc_service"]["envoy_grpc"][ + "cluster_name" + ] = "cluster_{}_someotherns".format(SERVICE_NAME) + config["timeout"] = "0.500s" + config["domain"] = "otherdomain" econf = _get_envoy_config(yaml) conf = _get_rl_config(econf.as_dict()) assert conf - assert conf.get('typed_config') == config + assert conf.get("typed_config") == config - errors = econf.ir.aconf.errors - assert 'ir.ratelimit' not in errors + errors = econf.ir.aconf.errors + assert "ir.ratelimit" not in errors diff --git a/python/tests/unit/test_listener.py b/python/tests/unit/test_listener.py index 14e2fe4244..871656be47 100644 --- a/python/tests/unit/test_listener.py +++ b/python/tests/unit/test_listener.py @@ -11,32 +11,34 @@ import pytest + def _ensure_alt_svc_header_injected(listener, expectedAltSvc): """helper function to ensure that the alt-svc header is getting injected properly""" - filter_chains = listener['filter_chains'] + filter_chains = listener["filter_chains"] for filter_chain in filter_chains: - hcm_typed_config = filter_chain['filters'][0]['typed_config'] - virtual_hosts = hcm_typed_config['route_config']['virtual_hosts'] + hcm_typed_config = filter_chain["filters"][0]["typed_config"] + virtual_hosts = hcm_typed_config["route_config"]["virtual_hosts"] for host in virtual_hosts: - response_headers_to_add = host['response_headers_to_add'] + response_headers_to_add = host["response_headers_to_add"] assert len(response_headers_to_add) == 1 - header = response_headers_to_add[0]['header'] - assert header['key'] == 'alt-svc' - assert header['value'] == expectedAltSvc + header = response_headers_to_add[0]["header"] + assert header["key"] == "alt-svc" + assert header["value"] == expectedAltSvc + def _verify_no_added_response_headers(listener): - """helper function to ensure response_headers_to_add do not exist """ - filter_chains = listener['filter_chains'] + """helper function to ensure response_headers_to_add do not exist""" + filter_chains = listener["filter_chains"] for filter_chain in filter_chains: - hcm_typed_config = filter_chain['filters'][0]['typed_config'] - virtual_hosts = hcm_typed_config['route_config']['virtual_hosts'] + hcm_typed_config = filter_chain["filters"][0]["typed_config"] + virtual_hosts = hcm_typed_config["route_config"]["virtual_hosts"] for host in virtual_hosts: - assert 'response_headers_to_add' not in host + assert "response_headers_to_add" not in host -def _generateListener(name: str, protocol: Optional[str], protocol_stack:Optional[List[str]]): +def _generateListener(name: str, protocol: Optional[str], protocol_stack: Optional[List[str]]): yaml = f""" apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -53,8 +55,8 @@ def _generateListener(name: str, protocol: Optional[str], protocol_stack:Optiona """ return yaml -class TestListener: +class TestListener: @pytest.mark.compilertest def test_socket_protocol(self): """ensure that we can identify the listener socket protocol based on the provided protocol and protocolStack""" @@ -67,25 +69,71 @@ class TestCase: expectedSocketProtocol: Optional[str] testcases = [ - # test with emissary defined protcolStacks via pre-definied protocol enum - TestCase(name="http_protocol", protocol="HTTP", protocolStack=None, expectedSocketProtocol="TCP"), - TestCase(name="https_protocol", protocol="HTTPS", protocolStack=None, expectedSocketProtocol="TCP"), - TestCase(name="httpproxy_protocol", protocol="HTTPPROXY", protocolStack=None, expectedSocketProtocol="TCP"), - TestCase(name="httpsproxy_protocol", protocol="HTTPSPROXY", protocolStack=None, expectedSocketProtocol="TCP"), - TestCase(name="tcp_protocol", protocol="TCP", protocolStack=None, expectedSocketProtocol="TCP"), - TestCase(name="tls_protocol", protocol="TLS", protocolStack=None, expectedSocketProtocol="TCP"), - - # test with custom stacks - TestCase(name="tcp_stack", protocol=None, protocolStack=["TLS", "HTTP", "TCP"], expectedSocketProtocol="TCP"), - TestCase(name="udp_stack", protocol=None, protocolStack=["TLS", "HTTP", "UDP"], expectedSocketProtocol="UDP"), - TestCase(name="invalid_stack", protocol=None, protocolStack=["TLS", "HTTP"], expectedSocketProtocol=None), - TestCase(name="empty_stack", protocol=None, protocolStack=[], expectedSocketProtocol=None), - ] + # test with emissary defined protcolStacks via pre-definied protocol enum + TestCase( + name="http_protocol", + protocol="HTTP", + protocolStack=None, + expectedSocketProtocol="TCP", + ), + TestCase( + name="https_protocol", + protocol="HTTPS", + protocolStack=None, + expectedSocketProtocol="TCP", + ), + TestCase( + name="httpproxy_protocol", + protocol="HTTPPROXY", + protocolStack=None, + expectedSocketProtocol="TCP", + ), + TestCase( + name="httpsproxy_protocol", + protocol="HTTPSPROXY", + protocolStack=None, + expectedSocketProtocol="TCP", + ), + TestCase( + name="tcp_protocol", + protocol="TCP", + protocolStack=None, + expectedSocketProtocol="TCP", + ), + TestCase( + name="tls_protocol", + protocol="TLS", + protocolStack=None, + expectedSocketProtocol="TCP", + ), + # test with custom stacks + TestCase( + name="tcp_stack", + protocol=None, + protocolStack=["TLS", "HTTP", "TCP"], + expectedSocketProtocol="TCP", + ), + TestCase( + name="udp_stack", + protocol=None, + protocolStack=["TLS", "HTTP", "UDP"], + expectedSocketProtocol="UDP", + ), + TestCase( + name="invalid_stack", + protocol=None, + protocolStack=["TLS", "HTTP"], + expectedSocketProtocol=None, + ), + TestCase( + name="empty_stack", protocol=None, protocolStack=[], expectedSocketProtocol=None + ), + ] for case in testcases: yaml = _generateListener(case.name, case.protocol, case.protocolStack) compiled_ir = Compile(logger, yaml, k8s=True) - result_ir = compiled_ir['ir'] + result_ir = compiled_ir["ir"] listeners = list(result_ir.listeners.values()) errors = result_ir.aconf.errors @@ -103,37 +151,38 @@ def test_http3_valid_quic_listener(self): yaml = default_http3_listener_manifest() econf = econf_compile(yaml) - listeners = econf['static_resources']['listeners'] + listeners = econf["static_resources"]["listeners"] assert len(listeners) == 1 # verify listener options listener = listeners[0] - assert 'udp_listener_config' in listener - assert 'quic_options' in listener['udp_listener_config'] - assert listener['udp_listener_config']["downstream_socket_config"]['prefer_gro'] == True + assert "udp_listener_config" in listener + assert "quic_options" in listener["udp_listener_config"] + assert listener["udp_listener_config"]["downstream_socket_config"]["prefer_gro"] == True # verify filter chains - filter_chains = listener['filter_chains'] + filter_chains = listener["filter_chains"] assert len(filter_chains) == 1 filter_chain = filter_chains[0] - assert filter_chain['filter_chain_match']['transport_protocol'] == 'quic' - assert filter_chain['transport_socket']['name'] == 'envoy.transport_sockets.quic' - + assert filter_chain["filter_chain_match"]["transport_protocol"] == "quic" + assert filter_chain["transport_socket"]["name"] == "envoy.transport_sockets.quic" + # verify HCM typed_config - typed_config = filter_chain['filters'][0]['typed_config'] - assert typed_config['codec_type'] == "HTTP3" - assert 'http3_protocol_options' in typed_config + typed_config = filter_chain["filters"][0]["typed_config"] + assert typed_config["codec_type"] == "HTTP3" + assert "http3_protocol_options" in typed_config @pytest.mark.compilertest def test_http3_missing_tls_context(self): - """UDP listener supporting the Quic protocol requires that a the "transport_socket" be set + """UDP listener supporting the Quic protocol requires that a the "transport_socket" be set in the filter_chains due to the fact that QUIC requires TLS. Envoy will reject the configuration if it is not found. This test ensures that the HTTP/3 Listener is dropped when a valid TLSContext is not available. """ - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -147,32 +196,34 @@ def test_http3_missing_tls_context(self): hostBinding: namespace: from: ALL -""" + default_http3_listener_manifest() +""" + + default_http3_listener_manifest() + ) ## we don't use the Compile utils here because we want to make sure that a fake secret is not injected aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(yaml, k8s=True) aconf.load_all(fetcher.sorted()) - secret_handler = EmptySecretHandler(logger, source_root=None, cache_dir=None, version='V3') + secret_handler = EmptySecretHandler(logger, source_root=None, cache_dir=None, version="V3") ir = IR(aconf, secret_handler=secret_handler) econf = EnvoyConfig.generate(ir, cache=None).as_dict() - - # the tcp/tls is more forgiving and doesn't crash envoy which is the current behavior + + # the tcp/tls is more forgiving and doesn't crash envoy which is the current behavior # we observe pre v3. So we just verify that the only listener is the TCP listener. - listeners = econf['static_resources']['listeners'] + listeners = econf["static_resources"]["listeners"] assert len(listeners) == 1 tcp_listener = listeners[0] - assert tcp_listener['address']['socket_address']['protocol'] == "TCP" - + assert tcp_listener["address"]["socket_address"]["protocol"] == "TCP" @pytest.mark.compilertest def test_http3_companion_listeners(self): """ensure that when we have companion http3 (udp)/tcp listeners bound to same port that we properly set - port reuse, and ensure the TCP listener broadcast http/3 support + port reuse, and ensure the TCP listener broadcast http/3 support """ - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -186,30 +237,31 @@ def test_http3_companion_listeners(self): hostBinding: namespace: from: ALL -""" + default_http3_listener_manifest() +""" + + default_http3_listener_manifest() + ) econf = econf_compile(yaml) - listeners = econf['static_resources']['listeners'] + listeners = econf["static_resources"]["listeners"] assert len(listeners) == 2 ## check TCP Listener tcp_listener = listeners[0] - assert tcp_listener['address']['socket_address']['protocol'] == "TCP" + assert tcp_listener["address"]["socket_address"]["protocol"] == "TCP" - tcp_filter_chains = tcp_listener['filter_chains'] + tcp_filter_chains = tcp_listener["filter_chains"] assert len(tcp_filter_chains) == 2 - - default_alt_svc = "h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400" + default_alt_svc = 'h3=":443"; ma=86400, h3-29=":443"; ma=86400' _ensure_alt_svc_header_injected(tcp_listener, default_alt_svc) - + ## check UDP Listener udp_listener = listeners[1] - assert udp_listener['address']['socket_address']['protocol'] == "UDP" + assert udp_listener["address"]["socket_address"]["protocol"] == "UDP" - udp_filter_chains = udp_listener['filter_chains'] + udp_filter_chains = udp_listener["filter_chains"] assert len(udp_filter_chains) == 1 _verify_no_added_response_headers(udp_listener) @@ -217,12 +269,13 @@ def test_http3_companion_listeners(self): @pytest.mark.compilertest def test_http3_non_matching_ports(self): """support having the http (tcp) listener to be bound to different address:port, by default - the alt-svc will not be injected. Note, this test ensures that envoy can be configured - this way and will not crash. However, due to developer not setting the `alt-svc` most clients - will not be able to upgrade to HTTP/3. + the alt-svc will not be injected. Note, this test ensures that envoy can be configured + this way and will not crash. However, due to developer not setting the `alt-svc` most clients + will not be able to upgrade to HTTP/3. """ - yaml = """ + yaml = ( + """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -236,29 +289,30 @@ def test_http3_non_matching_ports(self): hostBinding: namespace: from: ALL -""" + default_http3_listener_manifest() +""" + + default_http3_listener_manifest() + ) econf = econf_compile(yaml) - listeners = econf['static_resources']['listeners'] + listeners = econf["static_resources"]["listeners"] assert len(listeners) == 2 ## check TCP Listener tcp_listener = listeners[0] - assert tcp_listener['address']['socket_address']['protocol'] == "TCP" + assert tcp_listener["address"]["socket_address"]["protocol"] == "TCP" - tcp_filter_chains = tcp_listener['filter_chains'] + tcp_filter_chains = tcp_listener["filter_chains"] assert len(tcp_filter_chains) == 2 _verify_no_added_response_headers(tcp_listener) - + ## check UDP Listener udp_listener = listeners[1] - assert udp_listener['address']['socket_address']['protocol'] == "UDP" + assert udp_listener["address"]["socket_address"]["protocol"] == "UDP" - udp_filter_chains = udp_listener['filter_chains'] + udp_filter_chains = udp_listener["filter_chains"] assert len(udp_filter_chains) == 1 _verify_no_added_response_headers(udp_listener) - diff --git a/python/tests/unit/test_listener_common_http_protocol_options.py b/python/tests/unit/test_listener_common_http_protocol_options.py index 039c4b61dc..036f929a8e 100644 --- a/python/tests/unit/test_listener_common_http_protocol_options.py +++ b/python/tests/unit/test_listener_common_http_protocol_options.py @@ -2,6 +2,7 @@ import pytest + def _test_listener_common_http_protocol_options(yaml, expectations={}): # Compile an envoy config econf = econf_compile(yaml) @@ -10,32 +11,41 @@ def _test_listener_common_http_protocol_options(yaml, expectations={}): def check(typed_config): for key, expected in expectations.items(): if expected is None: - assert key not in typed_config['common_http_protocol_options'] + assert key not in typed_config["common_http_protocol_options"] else: - assert key in typed_config['common_http_protocol_options'] - assert typed_config['common_http_protocol_options'][key] == expected + assert key in typed_config["common_http_protocol_options"] + assert typed_config["common_http_protocol_options"][key] == expected return True + econf_foreach_hcm(econf, check) + @pytest.mark.compilertest def test_headers_with_underscores_action_unset(): yaml = module_and_mapping_manifests(None, []) _test_listener_common_http_protocol_options(yaml, expectations={}) + @pytest.mark.compilertest def test_headers_with_underscores_action_reject(): yaml = module_and_mapping_manifests(["headers_with_underscores_action: REJECT_REQUEST"], []) - _test_listener_common_http_protocol_options(yaml, expectations={'headers_with_underscores_action': 'REJECT_REQUEST'}) + _test_listener_common_http_protocol_options( + yaml, expectations={"headers_with_underscores_action": "REJECT_REQUEST"} + ) + @pytest.mark.compilertest def test_listener_idle_timeout_ms(): yaml = module_and_mapping_manifests(["listener_idle_timeout_ms: 150000"], []) - _test_listener_common_http_protocol_options(yaml, expectations={'idle_timeout': '150.000s'}) + _test_listener_common_http_protocol_options(yaml, expectations={"idle_timeout": "150.000s"}) + @pytest.mark.compilertest def test_all_listener_common_http_protocol_options(): - yaml = module_and_mapping_manifests(["headers_with_underscores_action: DROP_HEADER", "listener_idle_timeout_ms: 4005"], []) - _test_listener_common_http_protocol_options(yaml, expectations={ - 'headers_with_underscores_action': 'DROP_HEADER', - 'idle_timeout': '4.005s' - }) + yaml = module_and_mapping_manifests( + ["headers_with_underscores_action: DROP_HEADER", "listener_idle_timeout_ms: 4005"], [] + ) + _test_listener_common_http_protocol_options( + yaml, + expectations={"headers_with_underscores_action": "DROP_HEADER", "idle_timeout": "4.005s"}, + ) diff --git a/python/tests/unit/test_listener_http_protocol_options.py b/python/tests/unit/test_listener_http_protocol_options.py index 05cb3e5465..1c8fe3078a 100644 --- a/python/tests/unit/test_listener_http_protocol_options.py +++ b/python/tests/unit/test_listener_http_protocol_options.py @@ -2,6 +2,7 @@ import pytest + def _test_listener_http_protocol_options(yaml, expectations={}): econf = econf_compile(yaml) @@ -9,39 +10,50 @@ def _test_listener_http_protocol_options(yaml, expectations={}): def check(typed_config): for key, expected in expectations.items(): if expected is None: - assert key not in typed_config['http_protocol_options'] + assert key not in typed_config["http_protocol_options"] else: - assert key in typed_config['http_protocol_options'] - assert typed_config['http_protocol_options'][key] == expected + assert key in typed_config["http_protocol_options"] + assert typed_config["http_protocol_options"][key] == expected return True + econf_foreach_hcm(econf, check) + @pytest.mark.compilertest def test_emptiness(): yaml = module_and_mapping_manifests([], []) _test_listener_http_protocol_options(yaml, expectations={}) + @pytest.mark.compilertest def test_proper_case_false(): yaml = module_and_mapping_manifests(["proper_case: false"], []) _test_listener_http_protocol_options(yaml, expectations={}) + @pytest.mark.compilertest def test_proper_case_true(): yaml = module_and_mapping_manifests(["proper_case: true"], []) - _test_listener_http_protocol_options(yaml, expectations={'header_key_format': {'proper_case_words': {}}}) + _test_listener_http_protocol_options( + yaml, expectations={"header_key_format": {"proper_case_words": {}}} + ) + @pytest.mark.compilertest def test_proper_case_and_enable_http_10(): yaml = module_and_mapping_manifests(["proper_case: true", "enable_http10: true"], []) - _test_listener_http_protocol_options(yaml, expectations={'accept_http_10': True, 'header_key_format': {'proper_case_words': {}}}) + _test_listener_http_protocol_options( + yaml, expectations={"accept_http_10": True, "header_key_format": {"proper_case_words": {}}} + ) + @pytest.mark.compilertest def test_allow_chunked_length_false(): yaml = module_and_mapping_manifests(["allow_chunked_length: false"], []) - _test_listener_http_protocol_options(yaml, expectations={'allow_chunked_length': False}) + _test_listener_http_protocol_options(yaml, expectations={"allow_chunked_length": False}) + @pytest.mark.compilertest def test_allow_chunked_length_true(): yaml = module_and_mapping_manifests(["allow_chunked_length: true"], []) - _test_listener_http_protocol_options(yaml, expectations={'allow_chunked_length': True}) + _test_listener_http_protocol_options(yaml, expectations={"allow_chunked_length": True}) diff --git a/python/tests/unit/test_listener_statsprefix.py b/python/tests/unit/test_listener_statsprefix.py index d2ffaf5e00..0afbe53e45 100644 --- a/python/tests/unit/test_listener_statsprefix.py +++ b/python/tests/unit/test_listener_statsprefix.py @@ -1,10 +1,16 @@ -from tests.utils import econf_compile, econf_foreach_listener, econf_foreach_listener_chain, EnvoyHCMInfo, EnvoyTCPInfo +from tests.utils import ( + econf_compile, + econf_foreach_listener, + econf_foreach_listener_chain, + EnvoyHCMInfo, + EnvoyTCPInfo, +) import pytest import json # This manifest set is from a test setup Flynn used when testing this by hand. -manifests = ''' +manifests = """ --- apiVersion: v1 kind: Secret @@ -118,7 +124,7 @@ authority: none tlsSecret: name: tls-cert -''' +""" def check_filter(abbrev, typed_config, expected_stat_prefix): @@ -126,7 +132,10 @@ def check_filter(abbrev, typed_config, expected_stat_prefix): print(f"------ {abbrev}, stat_prefix {stat_prefix}") - assert stat_prefix == expected_stat_prefix, f"wanted stat_prefix {expected_stat_prefix}, got {stat_prefix}" + assert ( + stat_prefix == expected_stat_prefix + ), f"wanted stat_prefix {expected_stat_prefix}, got {stat_prefix}" + def check_listener(listener): port = listener["address"]["socket_address"]["port_value"] @@ -141,11 +150,11 @@ def check_listener(listener): # Ports < 9000 use HTTP, not TCP. check_info = { - 8080: ( "HCM", 1, EnvoyHCMInfo, "ingress_http" ), - 8443: ( "HCM", 2, EnvoyHCMInfo, "ingress_https" ), - 8888: ( "HCM", 2, EnvoyHCMInfo, "alternate" ), - 9998: ( "TCP", 1, EnvoyTCPInfo, "ingress_tcp_9998" ), - 9999: ( "TCP", 1, EnvoyTCPInfo, "ingress_tls_9999" ) + 8080: ("HCM", 1, EnvoyHCMInfo, "ingress_http"), + 8443: ("HCM", 2, EnvoyHCMInfo, "ingress_https"), + 8888: ("HCM", 2, EnvoyHCMInfo, "alternate"), + 9998: ("TCP", 1, EnvoyTCPInfo, "ingress_tcp_9998"), + 9999: ("TCP", 1, EnvoyTCPInfo, "ingress_tls_9999"), } abbrev, chain_count, filter_info, expected_stat_prefix = check_info[port] @@ -155,9 +164,13 @@ def checker(typed_config): return check_filter(abbrev, typed_config, expected_stat_prefix) econf_foreach_listener_chain( - listener, checker, chain_count=chain_count, + listener, + checker, + chain_count=chain_count, need_name=filter_info.name, - need_type=filter_info.type) + need_type=filter_info.type, + ) + @pytest.mark.compilertest def listener_stats_prefix(): diff --git a/python/tests/unit/test_lookup.py b/python/tests/unit/test_lookup.py index fbdfc2a390..ea93031f03 100644 --- a/python/tests/unit/test_lookup.py +++ b/python/tests/unit/test_lookup.py @@ -8,7 +8,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -19,7 +19,7 @@ from ambassador.ir import IRResource from ambassador.ir.irbuffer import IRBuffer -yaml = ''' +yaml = """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -39,19 +39,23 @@ hostname: "*" prefix: /test/ service: test:9999 -''' +""" class IRTestResource(IRBuffer): - def __init__(self, ir: 'IR', aconf: Config, - rkey: str="ir.testresource", - name: str="ir.testresource", - kind: str="IRTestResource", - **kwargs) -> None: + def __init__( + self, + ir: "IR", + aconf: Config, + rkey: str = "ir.testresource", + name: str = "ir.testresource", + kind: str = "IRTestResource", + **kwargs + ) -> None: super().__init__(ir=ir, aconf=aconf, rkey=rkey, kind=kind, name=name, **kwargs) - self.default_class = 'test_resource' + self.default_class = "test_resource" def test_lookup(): @@ -66,61 +70,71 @@ def test_lookup(): ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) - t1 = IRBuffer(ir, aconf, rkey='-foo-', name='buffer', max_request_bytes=4096) - - t2 = IRTestResource(ir, aconf, rkey='-foo-', name='buffer', max_request_bytes=8192) - - assert t1.lookup('max_request_bytes') == 4096 - assert t1.lookup('max_request_bytes', 57) == 4096 - assert t1.lookup('max_request_bytes2', 57) == 57 - - assert t1.lookup('max_request_words') == 1 - assert t1.lookup('max_request_words', 77) == 1 - assert t1.lookup('max_request_words', default_key='altered') == 2 - assert t1.lookup('max_request_words', 77, default_key='altered') == 2 - assert t1.lookup('max_request_words', default_key='altered2') == None - assert t1.lookup('max_request_words', 77, default_key='altered2') == 77 - - assert t1.lookup('max_request_words', default_class='test_resource') == 3 - assert t1.lookup('max_request_words', 77, default_class='test_resource') == 3 - assert t1.lookup('max_request_words', 77, default_class='test_resource2') == 1 - assert t1.lookup('max_request_words', default_key='altered', default_class='test_resource') == 4 - assert t1.lookup('max_request_words', 77, default_key='altered', default_class='test_resource') == 4 - assert t1.lookup('max_request_words', default_key='altered2', default_class='test_resource') == None - assert t1.lookup('max_request_words', 77, default_key='altered2', default_class='test_resource') == 77 - - assert t1.lookup('funk') == None - assert t1.lookup('funk', 77) == 77 - - assert t1.lookup('funk', default_class='test_resource') == 8 - assert t1.lookup('funk', 77, default_class='test_resource') == 8 - assert t1.lookup('funk', 77, default_class='test_resource2') == 77 - - assert t2.lookup('max_request_bytes') == 8192 - assert t2.lookup('max_request_bytes', 57) == 8192 - assert t2.lookup('max_request_bytes2', 57) == 57 - - assert t2.lookup('max_request_words') == 3 - assert t2.lookup('max_request_words', 77) == 3 - assert t2.lookup('max_request_words', default_key='altered') == 4 - assert t2.lookup('max_request_words', 77, default_key='altered') == 4 - assert t2.lookup('max_request_words', default_key='altered2') == None - assert t2.lookup('max_request_words', 77, default_key='altered2') == 77 - - assert t2.lookup('max_request_words', default_class='/') == 1 - assert t2.lookup('max_request_words', 77, default_class='/') == 1 - assert t2.lookup('max_request_words', 77, default_class='/2') == 1 - assert t2.lookup('max_request_words', default_key='altered', default_class='/') == 2 - assert t2.lookup('max_request_words', 77, default_key='altered', default_class='/') == 2 - assert t2.lookup('max_request_words', default_key='altered2', default_class='/') == None - assert t2.lookup('max_request_words', 77, default_key='altered2', default_class='/') == 77 - - assert t2.lookup('funk') == 8 - assert t2.lookup('funk', 77) == 8 - - assert t2.lookup('funk', default_class='test_resource') == 8 - assert t2.lookup('funk', 77, default_class='test_resource') == 8 - assert t2.lookup('funk', 77, default_class='test_resource2') == 77 - -if __name__ == '__main__': + t1 = IRBuffer(ir, aconf, rkey="-foo-", name="buffer", max_request_bytes=4096) + + t2 = IRTestResource(ir, aconf, rkey="-foo-", name="buffer", max_request_bytes=8192) + + assert t1.lookup("max_request_bytes") == 4096 + assert t1.lookup("max_request_bytes", 57) == 4096 + assert t1.lookup("max_request_bytes2", 57) == 57 + + assert t1.lookup("max_request_words") == 1 + assert t1.lookup("max_request_words", 77) == 1 + assert t1.lookup("max_request_words", default_key="altered") == 2 + assert t1.lookup("max_request_words", 77, default_key="altered") == 2 + assert t1.lookup("max_request_words", default_key="altered2") == None + assert t1.lookup("max_request_words", 77, default_key="altered2") == 77 + + assert t1.lookup("max_request_words", default_class="test_resource") == 3 + assert t1.lookup("max_request_words", 77, default_class="test_resource") == 3 + assert t1.lookup("max_request_words", 77, default_class="test_resource2") == 1 + assert t1.lookup("max_request_words", default_key="altered", default_class="test_resource") == 4 + assert ( + t1.lookup("max_request_words", 77, default_key="altered", default_class="test_resource") + == 4 + ) + assert ( + t1.lookup("max_request_words", default_key="altered2", default_class="test_resource") + == None + ) + assert ( + t1.lookup("max_request_words", 77, default_key="altered2", default_class="test_resource") + == 77 + ) + + assert t1.lookup("funk") == None + assert t1.lookup("funk", 77) == 77 + + assert t1.lookup("funk", default_class="test_resource") == 8 + assert t1.lookup("funk", 77, default_class="test_resource") == 8 + assert t1.lookup("funk", 77, default_class="test_resource2") == 77 + + assert t2.lookup("max_request_bytes") == 8192 + assert t2.lookup("max_request_bytes", 57) == 8192 + assert t2.lookup("max_request_bytes2", 57) == 57 + + assert t2.lookup("max_request_words") == 3 + assert t2.lookup("max_request_words", 77) == 3 + assert t2.lookup("max_request_words", default_key="altered") == 4 + assert t2.lookup("max_request_words", 77, default_key="altered") == 4 + assert t2.lookup("max_request_words", default_key="altered2") == None + assert t2.lookup("max_request_words", 77, default_key="altered2") == 77 + + assert t2.lookup("max_request_words", default_class="/") == 1 + assert t2.lookup("max_request_words", 77, default_class="/") == 1 + assert t2.lookup("max_request_words", 77, default_class="/2") == 1 + assert t2.lookup("max_request_words", default_key="altered", default_class="/") == 2 + assert t2.lookup("max_request_words", 77, default_key="altered", default_class="/") == 2 + assert t2.lookup("max_request_words", default_key="altered2", default_class="/") == None + assert t2.lookup("max_request_words", 77, default_key="altered2", default_class="/") == 77 + + assert t2.lookup("funk") == 8 + assert t2.lookup("funk", 77) == 8 + + assert t2.lookup("funk", default_class="test_resource") == 8 + assert t2.lookup("funk", 77, default_class="test_resource") == 8 + assert t2.lookup("funk", 77, default_class="test_resource2") == 77 + + +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_mapping.py b/python/tests/unit/test_mapping.py index 1dc709bc08..0a2e6a1369 100644 --- a/python/tests/unit/test_mapping.py +++ b/python/tests/unit/test_mapping.py @@ -3,6 +3,7 @@ from tests.utils import compile_with_cachecheck + @pytest.mark.compilertest def test_mapping_host_star_error(): test_yaml = """ @@ -35,6 +36,7 @@ def test_mapping_host_star_error(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_host_authority_star_error(): test_yaml = """ @@ -61,13 +63,16 @@ def test_mapping_host_authority_star_error(): assert len(errors) == 1, f"Expected 1 error but got {len(errors)}" assert errors[0]["ok"] == False - assert errors[0]["error"] == ":authority exact-match '*' contains *, which cannot match anything." + assert ( + errors[0]["error"] == ":authority exact-match '*' contains *, which cannot match anything." + ) for g in ir.groups.values(): assert g.prefix != "/star/" # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_host_ok(): test_yaml = """ @@ -88,7 +93,9 @@ def test_mapping_host_ok(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 @@ -101,6 +108,7 @@ def test_mapping_host_ok(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_host_authority_ok(): test_yaml = """ @@ -122,7 +130,9 @@ def test_mapping_host_authority_ok(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 @@ -135,6 +145,7 @@ def test_mapping_host_authority_ok(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_host_authority_and_host(): test_yaml = """ @@ -157,7 +168,9 @@ def test_mapping_host_authority_and_host(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 @@ -170,6 +183,7 @@ def test_mapping_host_authority_and_host(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_hostname_ok(): test_yaml = """ @@ -190,7 +204,9 @@ def test_mapping_hostname_ok(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 @@ -203,6 +219,7 @@ def test_mapping_hostname_ok(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_hostname_and_host(): test_yaml = """ @@ -224,7 +241,9 @@ def test_mapping_hostname_and_host(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 @@ -237,6 +256,7 @@ def test_mapping_hostname_and_host(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_hostname_and_authority(): test_yaml = """ @@ -259,7 +279,9 @@ def test_mapping_hostname_and_authority(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 @@ -272,6 +294,7 @@ def test_mapping_hostname_and_authority(): # print(json.dumps(ir.as_dict(), sort_keys=True, indent=4)) + @pytest.mark.compilertest def test_mapping_hostname_and_host_and_authority(): test_yaml = """ @@ -295,7 +318,9 @@ def test_mapping_hostname_and_host_and_authority(): ir = r["ir"] errors = ir.aconf.errors - assert len(errors) == 0, "Expected no errors but got %s" % (json.dumps(errors, sort_keys=True, indent=4)) + assert len(errors) == 0, "Expected no errors but got %s" % ( + json.dumps(errors, sort_keys=True, indent=4) + ) found = 0 diff --git a/python/tests/unit/test_max_request_header.py b/python/tests/unit/test_max_request_header.py index 2c3c766888..c4e39f41b9 100644 --- a/python/tests/unit/test_max_request_header.py +++ b/python/tests/unit/test_max_request_header.py @@ -5,7 +5,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -62,18 +62,20 @@ def test_set_max_request_header(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - for filter_chain in listener['filter_chains']: - for f in filter_chain['filters']: - max_req_headers = f['typed_config'].get('max_request_headers_kb', None) - assert max_req_headers is not None, \ - f"max_request_headers_kb not found on typed_config: {f['typed_config']}" + for listener in conf["static_resources"]["listeners"]: + for filter_chain in listener["filter_chains"]: + for f in filter_chain["filters"]: + max_req_headers = f["typed_config"].get("max_request_headers_kb", None) + assert ( + max_req_headers is not None + ), f"max_request_headers_kb not found on typed_config: {f['typed_config']}" print(f"Found max_req_headers = {max_req_headers}") key_found = True - assert expected == int(max_req_headers), \ - "max_request_headers_kb must equal the value set on the ambassador Module" - assert key_found, 'max_request_headers_kb must be found in the envoy config' + assert expected == int( + max_req_headers + ), "max_request_headers_kb must equal the value set on the ambassador Module" + assert key_found, "max_request_headers_kb must be found in the envoy config" @pytest.mark.compilertest @@ -105,15 +107,17 @@ def test_set_max_request_header_v3(): conf = econf.as_dict() - for listener in conf['static_resources']['listeners']: - for filter_chain in listener['filter_chains']: - for f in filter_chain['filters']: - max_req_headers = f['typed_config'].get('max_request_headers_kb', None) - assert max_req_headers is not None, \ - f"max_request_headers_kb not found on typed_config: {f['typed_config']}" + for listener in conf["static_resources"]["listeners"]: + for filter_chain in listener["filter_chains"]: + for f in filter_chain["filters"]: + max_req_headers = f["typed_config"].get("max_request_headers_kb", None) + assert ( + max_req_headers is not None + ), f"max_request_headers_kb not found on typed_config: {f['typed_config']}" print(f"Found max_req_headers = {max_req_headers}") key_found = True - assert expected == int(max_req_headers), \ - "max_request_headers_kb must equal the value set on the ambassador Module" - assert key_found, 'max_request_headers_kb must be found in the envoy config' + assert expected == int( + max_req_headers + ), "max_request_headers_kb must equal the value set on the ambassador Module" + assert key_found, "max_request_headers_kb must be found in the envoy config" diff --git a/python/tests/unit/test_qualify_service.py b/python/tests/unit/test_qualify_service.py index fe5967f044..817c8dd45f 100644 --- a/python/tests/unit/test_qualify_service.py +++ b/python/tests/unit/test_qualify_service.py @@ -8,7 +8,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -19,16 +19,20 @@ from ambassador.ir import IRResource from ambassador.ir.irbasemapping import normalize_service_name -yaml = ''' +yaml = """ --- apiVersion: getambassador.io/v3alpha1 kind: Module name: ambassador config: {} -''' +""" + + +def qualify_service_name( + ir: "IR", service: str, namespace: Optional[str], rkey: Optional[str] = None +) -> str: + return normalize_service_name(ir, service, namespace, "KubernetesTestResolver", rkey=rkey) -def qualify_service_name(ir: 'IR', service: str, namespace: Optional[str], rkey: Optional[str]=None) -> str: - return normalize_service_name(ir, service, namespace, 'KubernetesTestResolver', rkey=rkey) def test_qualify_service(): """ @@ -55,12 +59,21 @@ def test_qualify_service(): assert qualify_service_name(ir, "backoffice.otherns", "default") == "backoffice.otherns" assert qualify_service_name(ir, "backoffice.otherns", "otherns") == "backoffice.otherns" - assert normalize_service_name(ir, "backoffice", None, 'ConsulResolver') == "backoffice" - assert normalize_service_name(ir, "backoffice", "default", 'ConsulResolver') == "backoffice" - assert normalize_service_name(ir, "backoffice", "otherns", 'ConsulResolver') == "backoffice" - assert normalize_service_name(ir, "backoffice.otherns", None, 'ConsulResolver') == "backoffice.otherns" - assert normalize_service_name(ir, "backoffice.otherns", "default", 'ConsulResolver') == "backoffice.otherns" - assert normalize_service_name(ir, "backoffice.otherns", "otherns", 'ConsulResolver') == "backoffice.otherns" + assert normalize_service_name(ir, "backoffice", None, "ConsulResolver") == "backoffice" + assert normalize_service_name(ir, "backoffice", "default", "ConsulResolver") == "backoffice" + assert normalize_service_name(ir, "backoffice", "otherns", "ConsulResolver") == "backoffice" + assert ( + normalize_service_name(ir, "backoffice.otherns", None, "ConsulResolver") + == "backoffice.otherns" + ) + assert ( + normalize_service_name(ir, "backoffice.otherns", "default", "ConsulResolver") + == "backoffice.otherns" + ) + assert ( + normalize_service_name(ir, "backoffice.otherns", "otherns", "ConsulResolver") + == "backoffice.otherns" + ) assert qualify_service_name(ir, "backoffice:80", None) == "backoffice:80" assert qualify_service_name(ir, "backoffice:80", "default") == "backoffice:80" @@ -69,79 +82,234 @@ def test_qualify_service(): assert qualify_service_name(ir, "backoffice.otherns:80", "default") == "backoffice.otherns:80" assert qualify_service_name(ir, "backoffice.otherns:80", "otherns") == "backoffice.otherns:80" - assert qualify_service_name(ir, "[fe80::e022:9cff:fecc:c7c4]", None) == "[fe80::e022:9cff:fecc:c7c4]" - assert qualify_service_name(ir, "[fe80::e022:9cff:fecc:c7c4]", "default") == "[fe80::e022:9cff:fecc:c7c4]" - assert qualify_service_name(ir, "[fe80::e022:9cff:fecc:c7c4]", "other") == "[fe80::e022:9cff:fecc:c7c4]" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]", None) == "https://[fe80::e022:9cff:fecc:c7c4]" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]", "default") == "https://[fe80::e022:9cff:fecc:c7c4]" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]", "other") == "https://[fe80::e022:9cff:fecc:c7c4]" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]:443", None) == "https://[fe80::e022:9cff:fecc:c7c4]:443" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]:443", "default") == "https://[fe80::e022:9cff:fecc:c7c4]:443" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]:443", "other") == "https://[fe80::e022:9cff:fecc:c7c4]:443" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4%25zone]:443", "other") == "https://[fe80::e022:9cff:fecc:c7c4%25zone]:443" - - assert normalize_service_name(ir, "backoffice:80", None, 'ConsulResolver') == "backoffice:80" - assert normalize_service_name(ir, "backoffice:80", "default", 'ConsulResolver') == "backoffice:80" - assert normalize_service_name(ir, "backoffice:80", "otherns", 'ConsulResolver') == "backoffice:80" - assert normalize_service_name(ir, "backoffice.otherns:80", None, 'ConsulResolver') == "backoffice.otherns:80" - assert normalize_service_name(ir, "backoffice.otherns:80", "default", 'ConsulResolver') == "backoffice.otherns:80" - assert normalize_service_name(ir, "backoffice.otherns:80", "otherns", 'ConsulResolver') == "backoffice.otherns:80" + assert ( + qualify_service_name(ir, "[fe80::e022:9cff:fecc:c7c4]", None) + == "[fe80::e022:9cff:fecc:c7c4]" + ) + assert ( + qualify_service_name(ir, "[fe80::e022:9cff:fecc:c7c4]", "default") + == "[fe80::e022:9cff:fecc:c7c4]" + ) + assert ( + qualify_service_name(ir, "[fe80::e022:9cff:fecc:c7c4]", "other") + == "[fe80::e022:9cff:fecc:c7c4]" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]", None) + == "https://[fe80::e022:9cff:fecc:c7c4]" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]", "default") + == "https://[fe80::e022:9cff:fecc:c7c4]" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]", "other") + == "https://[fe80::e022:9cff:fecc:c7c4]" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]:443", None) + == "https://[fe80::e022:9cff:fecc:c7c4]:443" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]:443", "default") + == "https://[fe80::e022:9cff:fecc:c7c4]:443" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4]:443", "other") + == "https://[fe80::e022:9cff:fecc:c7c4]:443" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4%25zone]:443", "other") + == "https://[fe80::e022:9cff:fecc:c7c4%25zone]:443" + ) + + assert normalize_service_name(ir, "backoffice:80", None, "ConsulResolver") == "backoffice:80" + assert ( + normalize_service_name(ir, "backoffice:80", "default", "ConsulResolver") == "backoffice:80" + ) + assert ( + normalize_service_name(ir, "backoffice:80", "otherns", "ConsulResolver") == "backoffice:80" + ) + assert ( + normalize_service_name(ir, "backoffice.otherns:80", None, "ConsulResolver") + == "backoffice.otherns:80" + ) + assert ( + normalize_service_name(ir, "backoffice.otherns:80", "default", "ConsulResolver") + == "backoffice.otherns:80" + ) + assert ( + normalize_service_name(ir, "backoffice.otherns:80", "otherns", "ConsulResolver") + == "backoffice.otherns:80" + ) assert qualify_service_name(ir, "http://backoffice", None) == "http://backoffice" assert qualify_service_name(ir, "http://backoffice", "default") == "http://backoffice" assert qualify_service_name(ir, "http://backoffice", "otherns") == "http://backoffice.otherns" - assert qualify_service_name(ir, "http://backoffice.otherns", None) == "http://backoffice.otherns" - assert qualify_service_name(ir, "http://backoffice.otherns", "default") == "http://backoffice.otherns" - assert qualify_service_name(ir, "http://backoffice.otherns", "otherns") == "http://backoffice.otherns" - - assert normalize_service_name(ir, "http://backoffice", None, 'ConsulResolver') == "http://backoffice" - assert normalize_service_name(ir, "http://backoffice", "default", 'ConsulResolver') == "http://backoffice" - assert normalize_service_name(ir, "http://backoffice", "otherns", 'ConsulResolver') == "http://backoffice" - assert normalize_service_name(ir, "http://backoffice.otherns", None, 'ConsulResolver') == "http://backoffice.otherns" - assert normalize_service_name(ir, "http://backoffice.otherns", "default", 'ConsulResolver') == "http://backoffice.otherns" - assert normalize_service_name(ir, "http://backoffice.otherns", "otherns", 'ConsulResolver') == "http://backoffice.otherns" + assert ( + qualify_service_name(ir, "http://backoffice.otherns", None) == "http://backoffice.otherns" + ) + assert ( + qualify_service_name(ir, "http://backoffice.otherns", "default") + == "http://backoffice.otherns" + ) + assert ( + qualify_service_name(ir, "http://backoffice.otherns", "otherns") + == "http://backoffice.otherns" + ) + + assert ( + normalize_service_name(ir, "http://backoffice", None, "ConsulResolver") + == "http://backoffice" + ) + assert ( + normalize_service_name(ir, "http://backoffice", "default", "ConsulResolver") + == "http://backoffice" + ) + assert ( + normalize_service_name(ir, "http://backoffice", "otherns", "ConsulResolver") + == "http://backoffice" + ) + assert ( + normalize_service_name(ir, "http://backoffice.otherns", None, "ConsulResolver") + == "http://backoffice.otherns" + ) + assert ( + normalize_service_name(ir, "http://backoffice.otherns", "default", "ConsulResolver") + == "http://backoffice.otherns" + ) + assert ( + normalize_service_name(ir, "http://backoffice.otherns", "otherns", "ConsulResolver") + == "http://backoffice.otherns" + ) assert qualify_service_name(ir, "http://backoffice:80", None) == "http://backoffice:80" assert qualify_service_name(ir, "http://backoffice:80", "default") == "http://backoffice:80" - assert qualify_service_name(ir, "http://backoffice:80", "otherns") == "http://backoffice.otherns:80" - assert qualify_service_name(ir, "http://backoffice.otherns:80", None) == "http://backoffice.otherns:80" - assert qualify_service_name(ir, "http://backoffice.otherns:80", "default") == "http://backoffice.otherns:80" - assert qualify_service_name(ir, "http://backoffice.otherns:80", "otherns") == "http://backoffice.otherns:80" - - assert normalize_service_name(ir, "http://backoffice:80", None, 'ConsulResolver') == "http://backoffice:80" - assert normalize_service_name(ir, "http://backoffice:80", "default", 'ConsulResolver') == "http://backoffice:80" - assert normalize_service_name(ir, "http://backoffice:80", "otherns", 'ConsulResolver') == "http://backoffice:80" - assert normalize_service_name(ir, "http://backoffice.otherns:80", None, 'ConsulResolver') == "http://backoffice.otherns:80" - assert normalize_service_name(ir, "http://backoffice.otherns:80", "default", 'ConsulResolver') == "http://backoffice.otherns:80" - assert normalize_service_name(ir, "http://backoffice.otherns:80", "otherns", 'ConsulResolver') == "http://backoffice.otherns:80" + assert ( + qualify_service_name(ir, "http://backoffice:80", "otherns") + == "http://backoffice.otherns:80" + ) + assert ( + qualify_service_name(ir, "http://backoffice.otherns:80", None) + == "http://backoffice.otherns:80" + ) + assert ( + qualify_service_name(ir, "http://backoffice.otherns:80", "default") + == "http://backoffice.otherns:80" + ) + assert ( + qualify_service_name(ir, "http://backoffice.otherns:80", "otherns") + == "http://backoffice.otherns:80" + ) + + assert ( + normalize_service_name(ir, "http://backoffice:80", None, "ConsulResolver") + == "http://backoffice:80" + ) + assert ( + normalize_service_name(ir, "http://backoffice:80", "default", "ConsulResolver") + == "http://backoffice:80" + ) + assert ( + normalize_service_name(ir, "http://backoffice:80", "otherns", "ConsulResolver") + == "http://backoffice:80" + ) + assert ( + normalize_service_name(ir, "http://backoffice.otherns:80", None, "ConsulResolver") + == "http://backoffice.otherns:80" + ) + assert ( + normalize_service_name(ir, "http://backoffice.otherns:80", "default", "ConsulResolver") + == "http://backoffice.otherns:80" + ) + assert ( + normalize_service_name(ir, "http://backoffice.otherns:80", "otherns", "ConsulResolver") + == "http://backoffice.otherns:80" + ) assert qualify_service_name(ir, "https://backoffice", None) == "https://backoffice" assert qualify_service_name(ir, "https://backoffice", "default") == "https://backoffice" assert qualify_service_name(ir, "https://backoffice", "otherns") == "https://backoffice.otherns" - assert qualify_service_name(ir, "https://backoffice.otherns", None) == "https://backoffice.otherns" - assert qualify_service_name(ir, "https://backoffice.otherns", "default") == "https://backoffice.otherns" - assert qualify_service_name(ir, "https://backoffice.otherns", "otherns") == "https://backoffice.otherns" - - assert normalize_service_name(ir, "https://backoffice", None, 'ConsulResolver') == "https://backoffice" - assert normalize_service_name(ir, "https://backoffice", "default", 'ConsulResolver') == "https://backoffice" - assert normalize_service_name(ir, "https://backoffice", "otherns", 'ConsulResolver') == "https://backoffice" - assert normalize_service_name(ir, "https://backoffice.otherns", None, 'ConsulResolver') == "https://backoffice.otherns" - assert normalize_service_name(ir, "https://backoffice.otherns", "default", 'ConsulResolver') == "https://backoffice.otherns" - assert normalize_service_name(ir, "https://backoffice.otherns", "otherns", 'ConsulResolver') == "https://backoffice.otherns" + assert ( + qualify_service_name(ir, "https://backoffice.otherns", None) == "https://backoffice.otherns" + ) + assert ( + qualify_service_name(ir, "https://backoffice.otherns", "default") + == "https://backoffice.otherns" + ) + assert ( + qualify_service_name(ir, "https://backoffice.otherns", "otherns") + == "https://backoffice.otherns" + ) + + assert ( + normalize_service_name(ir, "https://backoffice", None, "ConsulResolver") + == "https://backoffice" + ) + assert ( + normalize_service_name(ir, "https://backoffice", "default", "ConsulResolver") + == "https://backoffice" + ) + assert ( + normalize_service_name(ir, "https://backoffice", "otherns", "ConsulResolver") + == "https://backoffice" + ) + assert ( + normalize_service_name(ir, "https://backoffice.otherns", None, "ConsulResolver") + == "https://backoffice.otherns" + ) + assert ( + normalize_service_name(ir, "https://backoffice.otherns", "default", "ConsulResolver") + == "https://backoffice.otherns" + ) + assert ( + normalize_service_name(ir, "https://backoffice.otherns", "otherns", "ConsulResolver") + == "https://backoffice.otherns" + ) assert qualify_service_name(ir, "https://backoffice:443", None) == "https://backoffice:443" assert qualify_service_name(ir, "https://backoffice:443", "default") == "https://backoffice:443" - assert qualify_service_name(ir, "https://backoffice:443", "otherns") == "https://backoffice.otherns:443" - assert qualify_service_name(ir, "https://backoffice.otherns:443", None) == "https://backoffice.otherns:443" - assert qualify_service_name(ir, "https://backoffice.otherns:443", "default") == "https://backoffice.otherns:443" - assert qualify_service_name(ir, "https://backoffice.otherns:443", "otherns") == "https://backoffice.otherns:443" - - assert normalize_service_name(ir, "https://backoffice:443", None, 'ConsulResolver') == "https://backoffice:443" - assert normalize_service_name(ir, "https://backoffice:443", "default", 'ConsulResolver') == "https://backoffice:443" - assert normalize_service_name(ir, "https://backoffice:443", "otherns", 'ConsulResolver') == "https://backoffice:443" - assert normalize_service_name(ir, "https://backoffice.otherns:443", None, 'ConsulResolver') == "https://backoffice.otherns:443" - assert normalize_service_name(ir, "https://backoffice.otherns:443", "default", 'ConsulResolver') == "https://backoffice.otherns:443" - assert normalize_service_name(ir, "https://backoffice.otherns:443", "otherns", 'ConsulResolver') == "https://backoffice.otherns:443" + assert ( + qualify_service_name(ir, "https://backoffice:443", "otherns") + == "https://backoffice.otherns:443" + ) + assert ( + qualify_service_name(ir, "https://backoffice.otherns:443", None) + == "https://backoffice.otherns:443" + ) + assert ( + qualify_service_name(ir, "https://backoffice.otherns:443", "default") + == "https://backoffice.otherns:443" + ) + assert ( + qualify_service_name(ir, "https://backoffice.otherns:443", "otherns") + == "https://backoffice.otherns:443" + ) + + assert ( + normalize_service_name(ir, "https://backoffice:443", None, "ConsulResolver") + == "https://backoffice:443" + ) + assert ( + normalize_service_name(ir, "https://backoffice:443", "default", "ConsulResolver") + == "https://backoffice:443" + ) + assert ( + normalize_service_name(ir, "https://backoffice:443", "otherns", "ConsulResolver") + == "https://backoffice:443" + ) + assert ( + normalize_service_name(ir, "https://backoffice.otherns:443", None, "ConsulResolver") + == "https://backoffice.otherns:443" + ) + assert ( + normalize_service_name(ir, "https://backoffice.otherns:443", "default", "ConsulResolver") + == "https://backoffice.otherns:443" + ) + assert ( + normalize_service_name(ir, "https://backoffice.otherns:443", "otherns", "ConsulResolver") + == "https://backoffice.otherns:443" + ) assert qualify_service_name(ir, "localhost", None) == "localhost" assert qualify_service_name(ir, "localhost", "default") == "localhost" @@ -151,13 +319,22 @@ def test_qualify_service(): assert qualify_service_name(ir, "localhost.otherns", "default") == "localhost.otherns" assert qualify_service_name(ir, "localhost.otherns", "otherns") == "localhost.otherns" - assert normalize_service_name(ir, "localhost", None, 'ConsulResolver') == "localhost" - assert normalize_service_name(ir, "localhost", "default", 'ConsulResolver') == "localhost" - assert normalize_service_name(ir, "localhost", "otherns", 'ConsulResolver') == "localhost" + assert normalize_service_name(ir, "localhost", None, "ConsulResolver") == "localhost" + assert normalize_service_name(ir, "localhost", "default", "ConsulResolver") == "localhost" + assert normalize_service_name(ir, "localhost", "otherns", "ConsulResolver") == "localhost" # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert normalize_service_name(ir, "localhost.otherns", None, 'ConsulResolver') == "localhost.otherns" - assert normalize_service_name(ir, "localhost.otherns", "default", 'ConsulResolver') == "localhost.otherns" - assert normalize_service_name(ir, "localhost.otherns", "otherns", 'ConsulResolver') == "localhost.otherns" + assert ( + normalize_service_name(ir, "localhost.otherns", None, "ConsulResolver") + == "localhost.otherns" + ) + assert ( + normalize_service_name(ir, "localhost.otherns", "default", "ConsulResolver") + == "localhost.otherns" + ) + assert ( + normalize_service_name(ir, "localhost.otherns", "otherns", "ConsulResolver") + == "localhost.otherns" + ) assert qualify_service_name(ir, "localhost:80", None) == "localhost:80" assert qualify_service_name(ir, "localhost:80", "default") == "localhost:80" @@ -167,106 +344,278 @@ def test_qualify_service(): assert qualify_service_name(ir, "localhost.otherns:80", "default") == "localhost.otherns:80" assert qualify_service_name(ir, "localhost.otherns:80", "otherns") == "localhost.otherns:80" - assert normalize_service_name(ir, "localhost:80", None, 'ConsulResolver') == "localhost:80" - assert normalize_service_name(ir, "localhost:80", "default", 'ConsulResolver') == "localhost:80" - assert normalize_service_name(ir, "localhost:80", "otherns", 'ConsulResolver') == "localhost:80" + assert normalize_service_name(ir, "localhost:80", None, "ConsulResolver") == "localhost:80" + assert normalize_service_name(ir, "localhost:80", "default", "ConsulResolver") == "localhost:80" + assert normalize_service_name(ir, "localhost:80", "otherns", "ConsulResolver") == "localhost:80" # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert normalize_service_name(ir, "localhost.otherns:80", None, 'ConsulResolver') == "localhost.otherns:80" - assert normalize_service_name(ir, "localhost.otherns:80", "default", 'ConsulResolver') == "localhost.otherns:80" - assert normalize_service_name(ir, "localhost.otherns:80", "otherns", 'ConsulResolver') == "localhost.otherns:80" + assert ( + normalize_service_name(ir, "localhost.otherns:80", None, "ConsulResolver") + == "localhost.otherns:80" + ) + assert ( + normalize_service_name(ir, "localhost.otherns:80", "default", "ConsulResolver") + == "localhost.otherns:80" + ) + assert ( + normalize_service_name(ir, "localhost.otherns:80", "otherns", "ConsulResolver") + == "localhost.otherns:80" + ) assert qualify_service_name(ir, "http://localhost", None) == "http://localhost" assert qualify_service_name(ir, "http://localhost", "default") == "http://localhost" assert qualify_service_name(ir, "http://localhost", "otherns") == "http://localhost" # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. assert qualify_service_name(ir, "http://localhost.otherns", None) == "http://localhost.otherns" - assert qualify_service_name(ir, "http://localhost.otherns", "default") == "http://localhost.otherns" - assert qualify_service_name(ir, "http://localhost.otherns", "otherns") == "http://localhost.otherns" - - assert normalize_service_name(ir, "http://localhost", None, 'ConsulResolver') == "http://localhost" - assert normalize_service_name(ir, "http://localhost", "default", 'ConsulResolver') == "http://localhost" - assert normalize_service_name(ir, "http://localhost", "otherns", 'ConsulResolver') == "http://localhost" + assert ( + qualify_service_name(ir, "http://localhost.otherns", "default") + == "http://localhost.otherns" + ) + assert ( + qualify_service_name(ir, "http://localhost.otherns", "otherns") + == "http://localhost.otherns" + ) + + assert ( + normalize_service_name(ir, "http://localhost", None, "ConsulResolver") == "http://localhost" + ) + assert ( + normalize_service_name(ir, "http://localhost", "default", "ConsulResolver") + == "http://localhost" + ) + assert ( + normalize_service_name(ir, "http://localhost", "otherns", "ConsulResolver") + == "http://localhost" + ) # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert normalize_service_name(ir, "http://localhost.otherns", None, 'ConsulResolver') == "http://localhost.otherns" - assert normalize_service_name(ir, "http://localhost.otherns", "default", 'ConsulResolver') == "http://localhost.otherns" - assert normalize_service_name(ir, "http://localhost.otherns", "otherns", 'ConsulResolver') == "http://localhost.otherns" + assert ( + normalize_service_name(ir, "http://localhost.otherns", None, "ConsulResolver") + == "http://localhost.otherns" + ) + assert ( + normalize_service_name(ir, "http://localhost.otherns", "default", "ConsulResolver") + == "http://localhost.otherns" + ) + assert ( + normalize_service_name(ir, "http://localhost.otherns", "otherns", "ConsulResolver") + == "http://localhost.otherns" + ) assert qualify_service_name(ir, "http://localhost:80", None) == "http://localhost:80" assert qualify_service_name(ir, "http://localhost:80", "default") == "http://localhost:80" assert qualify_service_name(ir, "http://localhost:80", "otherns") == "http://localhost:80" # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert qualify_service_name(ir, "http://localhost.otherns:80", None) == "http://localhost.otherns:80" - assert qualify_service_name(ir, "http://localhost.otherns:80", "default") == "http://localhost.otherns:80" - assert qualify_service_name(ir, "http://localhost.otherns:80", "otherns") == "http://localhost.otherns:80" - - assert normalize_service_name(ir, "http://localhost:80", None, 'ConsulResolver') == "http://localhost:80" - assert normalize_service_name(ir, "http://localhost:80", "default", 'ConsulResolver') == "http://localhost:80" - assert normalize_service_name(ir, "http://localhost:80", "otherns", 'ConsulResolver') == "http://localhost:80" + assert ( + qualify_service_name(ir, "http://localhost.otherns:80", None) + == "http://localhost.otherns:80" + ) + assert ( + qualify_service_name(ir, "http://localhost.otherns:80", "default") + == "http://localhost.otherns:80" + ) + assert ( + qualify_service_name(ir, "http://localhost.otherns:80", "otherns") + == "http://localhost.otherns:80" + ) + + assert ( + normalize_service_name(ir, "http://localhost:80", None, "ConsulResolver") + == "http://localhost:80" + ) + assert ( + normalize_service_name(ir, "http://localhost:80", "default", "ConsulResolver") + == "http://localhost:80" + ) + assert ( + normalize_service_name(ir, "http://localhost:80", "otherns", "ConsulResolver") + == "http://localhost:80" + ) # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert normalize_service_name(ir, "http://localhost.otherns:80", None, 'ConsulResolver') == "http://localhost.otherns:80" - assert normalize_service_name(ir, "http://localhost.otherns:80", "default", 'ConsulResolver') == "http://localhost.otherns:80" - assert normalize_service_name(ir, "http://localhost.otherns:80", "otherns", 'ConsulResolver') == "http://localhost.otherns:80" + assert ( + normalize_service_name(ir, "http://localhost.otherns:80", None, "ConsulResolver") + == "http://localhost.otherns:80" + ) + assert ( + normalize_service_name(ir, "http://localhost.otherns:80", "default", "ConsulResolver") + == "http://localhost.otherns:80" + ) + assert ( + normalize_service_name(ir, "http://localhost.otherns:80", "otherns", "ConsulResolver") + == "http://localhost.otherns:80" + ) assert qualify_service_name(ir, "https://localhost", None) == "https://localhost" assert qualify_service_name(ir, "https://localhost", "default") == "https://localhost" assert qualify_service_name(ir, "https://localhost", "otherns") == "https://localhost" # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert qualify_service_name(ir, "https://localhost.otherns", None) == "https://localhost.otherns" - assert qualify_service_name(ir, "https://localhost.otherns", "default") == "https://localhost.otherns" - assert qualify_service_name(ir, "https://localhost.otherns", "otherns") == "https://localhost.otherns" - - assert normalize_service_name(ir, "https://localhost", None, 'ConsulResolver') == "https://localhost" - assert normalize_service_name(ir, "https://localhost", "default", 'ConsulResolver') == "https://localhost" - assert normalize_service_name(ir, "https://localhost", "otherns", 'ConsulResolver') == "https://localhost" + assert ( + qualify_service_name(ir, "https://localhost.otherns", None) == "https://localhost.otherns" + ) + assert ( + qualify_service_name(ir, "https://localhost.otherns", "default") + == "https://localhost.otherns" + ) + assert ( + qualify_service_name(ir, "https://localhost.otherns", "otherns") + == "https://localhost.otherns" + ) + + assert ( + normalize_service_name(ir, "https://localhost", None, "ConsulResolver") + == "https://localhost" + ) + assert ( + normalize_service_name(ir, "https://localhost", "default", "ConsulResolver") + == "https://localhost" + ) + assert ( + normalize_service_name(ir, "https://localhost", "otherns", "ConsulResolver") + == "https://localhost" + ) # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert normalize_service_name(ir, "https://localhost.otherns", None, 'ConsulResolver') == "https://localhost.otherns" - assert normalize_service_name(ir, "https://localhost.otherns", "default", 'ConsulResolver') == "https://localhost.otherns" - assert normalize_service_name(ir, "https://localhost.otherns", "otherns", 'ConsulResolver') == "https://localhost.otherns" + assert ( + normalize_service_name(ir, "https://localhost.otherns", None, "ConsulResolver") + == "https://localhost.otherns" + ) + assert ( + normalize_service_name(ir, "https://localhost.otherns", "default", "ConsulResolver") + == "https://localhost.otherns" + ) + assert ( + normalize_service_name(ir, "https://localhost.otherns", "otherns", "ConsulResolver") + == "https://localhost.otherns" + ) assert qualify_service_name(ir, "https://localhost:443", None) == "https://localhost:443" assert qualify_service_name(ir, "https://localhost:443", "default") == "https://localhost:443" assert qualify_service_name(ir, "https://localhost:443", "otherns") == "https://localhost:443" # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert qualify_service_name(ir, "https://localhost.otherns:443", None) == "https://localhost.otherns:443" - assert qualify_service_name(ir, "https://localhost.otherns:443", "default") == "https://localhost.otherns:443" - assert qualify_service_name(ir, "https://localhost.otherns:443", "otherns") == "https://localhost.otherns:443" - - assert normalize_service_name(ir, "https://localhost:443", None, 'ConsulResolver') == "https://localhost:443" - assert normalize_service_name(ir, "https://localhost:443", "default", 'ConsulResolver') == "https://localhost:443" - assert normalize_service_name(ir, "https://localhost:443", "otherns", 'ConsulResolver') == "https://localhost:443" + assert ( + qualify_service_name(ir, "https://localhost.otherns:443", None) + == "https://localhost.otherns:443" + ) + assert ( + qualify_service_name(ir, "https://localhost.otherns:443", "default") + == "https://localhost.otherns:443" + ) + assert ( + qualify_service_name(ir, "https://localhost.otherns:443", "otherns") + == "https://localhost.otherns:443" + ) + + assert ( + normalize_service_name(ir, "https://localhost:443", None, "ConsulResolver") + == "https://localhost:443" + ) + assert ( + normalize_service_name(ir, "https://localhost:443", "default", "ConsulResolver") + == "https://localhost:443" + ) + assert ( + normalize_service_name(ir, "https://localhost:443", "otherns", "ConsulResolver") + == "https://localhost:443" + ) # It's not meaningful to actually say "localhost.otherns", but it should passed through unchanged. - assert normalize_service_name(ir, "https://localhost.otherns:443", None, 'ConsulResolver') == "https://localhost.otherns:443" - assert normalize_service_name(ir, "https://localhost.otherns:443", "default", 'ConsulResolver') == "https://localhost.otherns:443" - assert normalize_service_name(ir, "https://localhost.otherns:443", "otherns", 'ConsulResolver') == "https://localhost.otherns:443" - - assert qualify_service_name(ir, "ambassador://foo.ns", "otherns") == "ambassador://foo.ns" # let's not introduce silly semantics - assert qualify_service_name(ir, "//foo.ns:1234", "otherns") == "foo.ns:1234" # we tell people "URL-ish", actually support URL-ish + assert ( + normalize_service_name(ir, "https://localhost.otherns:443", None, "ConsulResolver") + == "https://localhost.otherns:443" + ) + assert ( + normalize_service_name(ir, "https://localhost.otherns:443", "default", "ConsulResolver") + == "https://localhost.otherns:443" + ) + assert ( + normalize_service_name(ir, "https://localhost.otherns:443", "otherns", "ConsulResolver") + == "https://localhost.otherns:443" + ) + + assert ( + qualify_service_name(ir, "ambassador://foo.ns", "otherns") == "ambassador://foo.ns" + ) # let's not introduce silly semantics + assert ( + qualify_service_name(ir, "//foo.ns:1234", "otherns") == "foo.ns:1234" + ) # we tell people "URL-ish", actually support URL-ish assert qualify_service_name(ir, "foo.ns:1234", "otherns") == "foo.ns:1234" - assert normalize_service_name(ir, "ambassador://foo.ns", "otherns", 'ConsulResolver') == "ambassador://foo.ns" # let's not introduce silly semantics - assert normalize_service_name(ir, "//foo.ns:1234", "otherns", 'ConsulResolver') == "foo.ns:1234" # we tell people "URL-ish", actually support URL-ish - assert normalize_service_name(ir, "foo.ns:1234", "otherns", 'ConsulResolver') == "foo.ns:1234" + assert ( + normalize_service_name(ir, "ambassador://foo.ns", "otherns", "ConsulResolver") + == "ambassador://foo.ns" + ) # let's not introduce silly semantics + assert ( + normalize_service_name(ir, "//foo.ns:1234", "otherns", "ConsulResolver") == "foo.ns:1234" + ) # we tell people "URL-ish", actually support URL-ish + assert normalize_service_name(ir, "foo.ns:1234", "otherns", "ConsulResolver") == "foo.ns:1234" assert not ir.aconf.errors - assert qualify_service_name(ir, "https://bad-service:443:443", "otherns") == "https://bad-service:443:443" - assert qualify_service_name(ir, "https://bad-service:443:443", "otherns", rkey="test-rkey") == "https://bad-service:443:443" + assert ( + qualify_service_name(ir, "https://bad-service:443:443", "otherns") + == "https://bad-service:443:443" + ) + assert ( + qualify_service_name(ir, "https://bad-service:443:443", "otherns", rkey="test-rkey") + == "https://bad-service:443:443" + ) assert qualify_service_name(ir, "bad-service:443:443", "otherns") == "bad-service:443:443" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4:443", "otherns") == "https://[fe80::e022:9cff:fecc:c7c4:443" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4", "otherns") == "https://[fe80::e022:9cff:fecc:c7c4" - assert qualify_service_name(ir, "https://fe80::e022:9cff:fecc:c7c4", "otherns") == "https://fe80::e022:9cff:fecc:c7c4" + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4:443", "otherns") + == "https://[fe80::e022:9cff:fecc:c7c4:443" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4", "otherns") + == "https://[fe80::e022:9cff:fecc:c7c4" + ) + assert ( + qualify_service_name(ir, "https://fe80::e022:9cff:fecc:c7c4", "otherns") + == "https://fe80::e022:9cff:fecc:c7c4" + ) assert qualify_service_name(ir, "https://bad-service:-1", "otherns") == "https://bad-service:-1" - assert qualify_service_name(ir, "https://bad-service:70000", "otherns") == "https://bad-service:70000" - - assert normalize_service_name(ir, "https://bad-service:443:443", "otherns", 'ConsulResolver') == "https://bad-service:443:443" - assert normalize_service_name(ir, "https://bad-service:443:443", "otherns", 'ConsulResolver', rkey="test-rkey") == "https://bad-service:443:443" - assert normalize_service_name(ir, "bad-service:443:443", "otherns", 'ConsulResolver') == "bad-service:443:443" - assert normalize_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4:443", "otherns", 'ConsulResolver') == "https://[fe80::e022:9cff:fecc:c7c4:443" - assert normalize_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4", "otherns", 'ConsulResolver') == "https://[fe80::e022:9cff:fecc:c7c4" - assert normalize_service_name(ir, "https://fe80::e022:9cff:fecc:c7c4", "otherns", 'ConsulResolver') == "https://fe80::e022:9cff:fecc:c7c4" - assert normalize_service_name(ir, "https://bad-service:-1", "otherns", 'ConsulResolver') == "https://bad-service:-1" - assert normalize_service_name(ir, "https://bad-service:70000", "otherns", 'ConsulResolver') == "https://bad-service:70000" - assert qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4%zone]:443", "other") == "https://[fe80::e022:9cff:fecc:c7c4%zone]:443" + assert ( + qualify_service_name(ir, "https://bad-service:70000", "otherns") + == "https://bad-service:70000" + ) + + assert ( + normalize_service_name(ir, "https://bad-service:443:443", "otherns", "ConsulResolver") + == "https://bad-service:443:443" + ) + assert ( + normalize_service_name( + ir, "https://bad-service:443:443", "otherns", "ConsulResolver", rkey="test-rkey" + ) + == "https://bad-service:443:443" + ) + assert ( + normalize_service_name(ir, "bad-service:443:443", "otherns", "ConsulResolver") + == "bad-service:443:443" + ) + assert ( + normalize_service_name( + ir, "https://[fe80::e022:9cff:fecc:c7c4:443", "otherns", "ConsulResolver" + ) + == "https://[fe80::e022:9cff:fecc:c7c4:443" + ) + assert ( + normalize_service_name( + ir, "https://[fe80::e022:9cff:fecc:c7c4", "otherns", "ConsulResolver" + ) + == "https://[fe80::e022:9cff:fecc:c7c4" + ) + assert ( + normalize_service_name(ir, "https://fe80::e022:9cff:fecc:c7c4", "otherns", "ConsulResolver") + == "https://fe80::e022:9cff:fecc:c7c4" + ) + assert ( + normalize_service_name(ir, "https://bad-service:-1", "otherns", "ConsulResolver") + == "https://bad-service:-1" + ) + assert ( + normalize_service_name(ir, "https://bad-service:70000", "otherns", "ConsulResolver") + == "https://bad-service:70000" + ) + assert ( + qualify_service_name(ir, "https://[fe80::e022:9cff:fecc:c7c4%zone]:443", "other") + == "https://[fe80::e022:9cff:fecc:c7c4%zone]:443" + ) aconf_errors = ir.aconf.errors assert "-global-" in aconf_errors @@ -278,63 +627,123 @@ def test_qualify_service(): # integer value as" to keep pytest working on peoples up-to-date laptops with Python 3.8, and let's recognize # "invalid literal for int() with base 10:" for the Python 3.7 in the builder container. assert not errors[0]["ok"] - assert (errors[0]["error"] == "Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" or - errors[0]["error"] == "Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'") + assert ( + errors[0]["error"] + == "Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" + or errors[0]["error"] + == "Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'" + ) assert not errors[1]["ok"] - assert (errors[1]["error"] == "test-rkey: Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" or - errors[1]["error"] == "test-rkey: Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'") + assert ( + errors[1]["error"] + == "test-rkey: Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" + or errors[1]["error"] + == "test-rkey: Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'" + ) assert not errors[2]["ok"] - assert (errors[2]["error"] == "Malformed service 'bad-service:443:443': Port could not be cast to integer value as '443:443'" or - errors[2]["error"] == "Malformed service 'bad-service:443:443': invalid literal for int() with base 10: '443:443'") + assert ( + errors[2]["error"] + == "Malformed service 'bad-service:443:443': Port could not be cast to integer value as '443:443'" + or errors[2]["error"] + == "Malformed service 'bad-service:443:443': invalid literal for int() with base 10: '443:443'" + ) assert not errors[3]["ok"] - assert errors[3]["error"] == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4:443': Invalid IPv6 URL" + assert ( + errors[3]["error"] + == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4:443': Invalid IPv6 URL" + ) assert not errors[4]["ok"] - assert errors[4]["error"] == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4': Invalid IPv6 URL" + assert ( + errors[4]["error"] + == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4': Invalid IPv6 URL" + ) assert not errors[5]["ok"] - assert (errors[5]["error"] == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': Port could not be cast to integer value as ':e022:9cff:fecc:c7c4'" or - errors[5]["error"] == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': invalid literal for int() with base 10: ':e022:9cff:fecc:c7c4'") + assert ( + errors[5]["error"] + == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': Port could not be cast to integer value as ':e022:9cff:fecc:c7c4'" + or errors[5]["error"] + == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': invalid literal for int() with base 10: ':e022:9cff:fecc:c7c4'" + ) assert not errors[6]["ok"] - assert errors[6]["error"] == "Malformed service 'https://bad-service:-1': Port out of range 0-65535" + assert ( + errors[6]["error"] + == "Malformed service 'https://bad-service:-1': Port out of range 0-65535" + ) assert not errors[7]["ok"] - assert errors[7]["error"] == "Malformed service 'https://bad-service:70000': Port out of range 0-65535" + assert ( + errors[7]["error"] + == "Malformed service 'https://bad-service:70000': Port out of range 0-65535" + ) assert not errors[8]["ok"] - assert (errors[8]["error"] == "Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" or - errors[8]["error"] == "Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'") + assert ( + errors[8]["error"] + == "Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" + or errors[8]["error"] + == "Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'" + ) assert not errors[9]["ok"] - assert (errors[9]["error"] == "test-rkey: Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" or - errors[9]["error"] == "test-rkey: Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'") + assert ( + errors[9]["error"] + == "test-rkey: Malformed service 'https://bad-service:443:443': Port could not be cast to integer value as '443:443'" + or errors[9]["error"] + == "test-rkey: Malformed service 'https://bad-service:443:443': invalid literal for int() with base 10: '443:443'" + ) assert not errors[10]["ok"] - assert (errors[10]["error"] == "Malformed service 'bad-service:443:443': Port could not be cast to integer value as '443:443'" or - errors[10]["error"] == "Malformed service 'bad-service:443:443': invalid literal for int() with base 10: '443:443'") + assert ( + errors[10]["error"] + == "Malformed service 'bad-service:443:443': Port could not be cast to integer value as '443:443'" + or errors[10]["error"] + == "Malformed service 'bad-service:443:443': invalid literal for int() with base 10: '443:443'" + ) assert not errors[11]["ok"] - assert errors[11]["error"] == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4:443': Invalid IPv6 URL" + assert ( + errors[11]["error"] + == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4:443': Invalid IPv6 URL" + ) assert not errors[12]["ok"] - assert errors[12]["error"] == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4': Invalid IPv6 URL" + assert ( + errors[12]["error"] + == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4': Invalid IPv6 URL" + ) assert not errors[13]["ok"] - assert (errors[13]["error"] == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': Port could not be cast to integer value as ':e022:9cff:fecc:c7c4'" or - errors[13]["error"] == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': invalid literal for int() with base 10: ':e022:9cff:fecc:c7c4'") + assert ( + errors[13]["error"] + == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': Port could not be cast to integer value as ':e022:9cff:fecc:c7c4'" + or errors[13]["error"] + == "Malformed service 'https://fe80::e022:9cff:fecc:c7c4': invalid literal for int() with base 10: ':e022:9cff:fecc:c7c4'" + ) assert not errors[14]["ok"] - assert errors[14]["error"] == "Malformed service 'https://bad-service:-1': Port out of range 0-65535" + assert ( + errors[14]["error"] + == "Malformed service 'https://bad-service:-1': Port out of range 0-65535" + ) assert not errors[15]["ok"] - assert errors[15]["error"] == "Malformed service 'https://bad-service:70000': Port out of range 0-65535" + assert ( + errors[15]["error"] + == "Malformed service 'https://bad-service:70000': Port out of range 0-65535" + ) assert not errors[16]["ok"] - assert errors[16]["error"] == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4%zone]:443': Invalid percent-escape in hostname: %zo" + assert ( + errors[16]["error"] + == "Malformed service 'https://[fe80::e022:9cff:fecc:c7c4%zone]:443': Invalid percent-escape in hostname: %zo" + ) + -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_reconfig_stats.py b/python/tests/unit/test_reconfig_stats.py index 78cd45ff09..bea7fae219 100644 --- a/python/tests/unit/test_reconfig_stats.py +++ b/python/tests/unit/test_reconfig_stats.py @@ -5,6 +5,7 @@ from ambassador_diag.diagd import ReconfigStats + def assert_checks(r: ReconfigStats, when: float, want_check: bool, want_timers: bool) -> None: got_check = r.needs_check(when) assert got_check == want_check, f"{when}: wanted check {want_check}, got {got_check}" @@ -17,7 +18,7 @@ def test_reconfig_stats(): logging.basicConfig( level=logging.DEBUG, format="%(asctime)s ffs %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ffs") @@ -28,7 +29,7 @@ def test_reconfig_stats(): max_incr_between_checks=5, max_time_between_checks=20, max_config_between_timers=2, - max_time_between_timers=10 + max_time_between_timers=10, ) r.dump() @@ -37,7 +38,7 @@ def test_reconfig_stats(): assert_checks(r, 11, False, False) assert_checks(r, 12, False, False) r.mark("incremental", 10) - assert_checks(r, 14, False, True) # Need timers from outstanding + assert_checks(r, 14, False, True) # Need timers from outstanding assert_checks(r, 18, False, True) r.mark_timers_logged(20) assert_checks(r, 20, False, False) @@ -95,6 +96,7 @@ def test_reconfig_stats(): assert r.errors == 1 -if __name__ == '__main__': +if __name__ == "__main__": import sys + pytest.main(sys.argv) diff --git a/python/tests/unit/test_route.py b/python/tests/unit/test_route.py index 1ce9426b5f..c69953aba3 100644 --- a/python/tests/unit/test_route.py +++ b/python/tests/unit/test_route.py @@ -2,28 +2,29 @@ import pytest + def _test_route(yaml, expectations={}): econf = econf_compile(yaml) def check(typed_config): # Find the one and virtual host in the route config - vhosts = typed_config['route_config']['virtual_hosts'] + vhosts = typed_config["route_config"]["virtual_hosts"] assert len(vhosts) == 1 # Find the httpbin route. Run our expectations over that. - routes = vhosts[0]['routes'] + routes = vhosts[0]["routes"] for r in routes: # Keep going until we find a real route - if 'route' not in r: + if "route" not in r: continue # Keep going until we find a prefix match for /httpbin/ - match = r['match'] - if 'prefix' not in match or match['prefix'] != '/httpbin/': + match = r["match"] + if "prefix" not in match or match["prefix"] != "/httpbin/": continue - assert 'route' in r - route = r['route'] + assert "route" in r + route = r["route"] for key, expected in expectations.items(): print("checking key %s" % key) assert key in route @@ -32,26 +33,30 @@ def check(typed_config): econf_foreach_hcm(econf, check) + @pytest.mark.compilertest def test_timeout_ms(): # If we do not set the config, we should get the default 3000ms. yaml = module_and_mapping_manifests(None, []) - _test_route(yaml, expectations={'timeout':'3.000s'}) + _test_route(yaml, expectations={"timeout": "3.000s"}) + @pytest.mark.compilertest def test_timeout_ms_module(): # If we set a default on the Module, it should override the usual default of 3000ms. yaml = module_and_mapping_manifests(["cluster_request_timeout_ms: 4000"], []) - _test_route(yaml, expectations={'timeout':'4.000s'}) + _test_route(yaml, expectations={"timeout": "4.000s"}) + @pytest.mark.compilertest def test_timeout_ms_mapping(): # If we set a default on the Module, it should override the usual default of 3000ms. yaml = module_and_mapping_manifests(None, ["timeout_ms: 1234"]) - _test_route(yaml, expectations={'timeout':'1.234s'}) + _test_route(yaml, expectations={"timeout": "1.234s"}) + @pytest.mark.compilertest def test_timeout_ms_both(): # If we set a default on the Module, it should override the usual default of 3000ms. yaml = module_and_mapping_manifests(["cluster_request_timeout_ms: 9000"], ["timeout_ms: 5001"]) - _test_route(yaml, expectations={'timeout':'5.001s'}) + _test_route(yaml, expectations={"timeout": "5.001s"}) diff --git a/python/tests/unit/test_router.py b/python/tests/unit/test_router.py index dcb402a345..1e4acffaaf 100644 --- a/python/tests/unit/test_router.py +++ b/python/tests/unit/test_router.py @@ -1,34 +1,45 @@ -from tests.utils import econf_compile, econf_foreach_hcm, module_and_mapping_manifests, zipkin_tracing_service_manifest +from tests.utils import ( + econf_compile, + econf_foreach_hcm, + module_and_mapping_manifests, + zipkin_tracing_service_manifest, +) import pytest + def _test_router(yaml, expectations={}): econf = econf_compile(yaml) def check(typed_config): - http_filters = typed_config['http_filters'] + http_filters = typed_config["http_filters"] assert len(http_filters) == 2 # Find the typed router config, and run our uexpecations over that. for http_filter in http_filters: - if http_filter['name'] != 'envoy.filters.http.router': + if http_filter["name"] != "envoy.filters.http.router": continue # If we expect nothing, then the typed config should be missing entirely. if len(expectations) == 0: - assert 'typed_config' not in http_filter + assert "typed_config" not in http_filter break - assert 'typed_config' in http_filter - typed_config = http_filter['typed_config'] - assert typed_config['@type'] == 'type.googleapis.com/envoy.extensions.filters.http.router.v3.Router' + assert "typed_config" in http_filter + typed_config = http_filter["typed_config"] + assert ( + typed_config["@type"] + == "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + ) for key, expected in expectations.items(): print("checking key %s" % key) assert key in typed_config assert typed_config[key] == expected break + econf_foreach_hcm(econf, check) + @pytest.mark.compilertest def test_suppress_envoy_headers(): # If we do not set the config, it should not appear. @@ -36,23 +47,29 @@ def test_suppress_envoy_headers(): _test_router(yaml, expectations={}) # If we set the config to false, it should not appear. - yaml = module_and_mapping_manifests(['suppress_envoy_headers: false'], []) + yaml = module_and_mapping_manifests(["suppress_envoy_headers: false"], []) _test_router(yaml, expectations={}) # If we set the config to true, it should appear. - yaml = module_and_mapping_manifests(['suppress_envoy_headers: true'], []) - _test_router(yaml, expectations={'suppress_envoy_headers': True}) + yaml = module_and_mapping_manifests(["suppress_envoy_headers: true"], []) + _test_router(yaml, expectations={"suppress_envoy_headers": True}) + @pytest.mark.compilertest def test_tracing_service(): # If we have a tracing service, we should see start_child_span yaml = module_and_mapping_manifests(None, []) + "\n" + zipkin_tracing_service_manifest() - _test_router(yaml, expectations={'start_child_span': True}) + _test_router(yaml, expectations={"start_child_span": True}) + @pytest.mark.compilertest def test_tracing_service_and_suppress_envoy_headers(): # If we set both suppress_envoy_headers and include a TracingService, # we should see both suppress_envoy_headers and the default start_child_span # value (True). - yaml = module_and_mapping_manifests(['suppress_envoy_headers: true'], []) + "\n" + zipkin_tracing_service_manifest() - _test_router(yaml, expectations={'start_child_span': True, 'suppress_envoy_headers': True}) + yaml = ( + module_and_mapping_manifests(["suppress_envoy_headers: true"], []) + + "\n" + + zipkin_tracing_service_manifest() + ) + _test_router(yaml, expectations={"start_child_span": True, "suppress_envoy_headers": True}) diff --git a/python/tests/unit/test_shadow.py b/python/tests/unit/test_shadow.py index fde6d40408..a82bcc04ac 100644 --- a/python/tests/unit/test_shadow.py +++ b/python/tests/unit/test_shadow.py @@ -11,7 +11,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -22,21 +22,30 @@ from ambassador.utils import SecretHandler, SecretInfo if TYPE_CHECKING: - from ambassador.ir.irresource import IRResource # pragma: no cover + from ambassador.ir.irresource import IRResource # pragma: no cover + class MockSecretHandler(SecretHandler): - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: - return SecretInfo('fallback-self-signed-cert', 'ambassador', "mocked-fallback-secret", - TLSCerts["acook"].pubcert, TLSCerts["acook"].privkey, decode_b64=False) + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: + return SecretInfo( + "fallback-self-signed-cert", + "ambassador", + "mocked-fallback-secret", + TLSCerts["acook"].pubcert, + TLSCerts["acook"].privkey, + decode_b64=False, + ) def get_mirrored_config(ads_config): - for l in ads_config.get('static_resources', {}).get('listeners'): - for fc in l.get('filter_chains'): - for f in fc.get('filters'): - for vh in f['typed_config']['route_config']['virtual_hosts']: - for r in vh.get('routes'): - if r['match']['prefix'] == '/httpbin/': + for l in ads_config.get("static_resources", {}).get("listeners"): + for fc in l.get("filter_chains"): + for f in fc.get("filters"): + for vh in f["typed_config"]["route_config"]["virtual_hosts"]: + for r in vh.get("routes"): + if r["match"]["prefix"] == "/httpbin/": return r return None @@ -45,7 +54,7 @@ def get_mirrored_config(ads_config): def test_shadow(tmp_path: Path): aconf = Config() - yaml = ''' + yaml = """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -81,14 +90,15 @@ def test_shadow(tmp_path: Path): prefix: /httpbin/ shadow: true weight: 10 -''' +""" fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(yaml, k8s=True) aconf.load_all(fetcher.sorted()) - - secret_handler = MockSecretHandler(logger, "mockery", str(tmp_path/"ambassador"/"snapshots"), "v1") + secret_handler = MockSecretHandler( + logger, "mockery", str(tmp_path / "ambassador" / "snapshots"), "v1" + ) ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) assert ir @@ -96,14 +106,16 @@ def test_shadow(tmp_path: Path): econf = EnvoyConfig.generate(ir) bootstrap_config, ads_config, _ = econf.split_config() - ads_config.pop('@type', None) + ads_config.pop("@type", None) mirrored_config = get_mirrored_config(ads_config) - assert 'request_mirror_policies' in mirrored_config['route'] - assert len(mirrored_config['route']['request_mirror_policies']) == 1 - mirror_policy = mirrored_config['route']['request_mirror_policies'][0] - assert mirror_policy['cluster'] == 'cluster_shadow_httpbin_shadow_default' - assert mirror_policy['runtime_fraction']['default_value']['numerator'] == 10 - assert mirror_policy['runtime_fraction']['default_value']['denominator'] == 'HUNDRED' - assert_valid_envoy_config(ads_config, extra_dirs=[str(tmp_path/"ambassador"/"snapshots")]) - assert_valid_envoy_config(bootstrap_config, extra_dirs=[str(tmp_path/"ambassador"/"snapshots")]) + assert "request_mirror_policies" in mirrored_config["route"] + assert len(mirrored_config["route"]["request_mirror_policies"]) == 1 + mirror_policy = mirrored_config["route"]["request_mirror_policies"][0] + assert mirror_policy["cluster"] == "cluster_shadow_httpbin_shadow_default" + assert mirror_policy["runtime_fraction"]["default_value"]["numerator"] == 10 + assert mirror_policy["runtime_fraction"]["default_value"]["denominator"] == "HUNDRED" + assert_valid_envoy_config(ads_config, extra_dirs=[str(tmp_path / "ambassador" / "snapshots")]) + assert_valid_envoy_config( + bootstrap_config, extra_dirs=[str(tmp_path / "ambassador" / "snapshots")] + ) diff --git a/python/tests/unit/test_statsd.py b/python/tests/unit/test_statsd.py index e02c0a754f..d2950bbddf 100644 --- a/python/tests/unit/test_statsd.py +++ b/python/tests/unit/test_statsd.py @@ -10,7 +10,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -41,19 +41,15 @@ def del_env(env_key): def teardown_function(function): - del_env('STATSD_ENABLED') - del_env('DOGSTATSD') - del_env('STATSD_HOST') + del_env("STATSD_ENABLED") + del_env("DOGSTATSD") + del_env("STATSD_HOST") @pytest.mark.compilertest @httpretty.activate def test_statsd_default(): - httpretty.register_uri( - httpretty.GET, - "statsd-sink", - body='{"origin": "127.0.0.1"}' - ) + httpretty.register_uri(httpretty.GET, "statsd-sink", body='{"origin": "127.0.0.1"}') yaml = """ apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -66,41 +62,33 @@ def test_statsd_default(): """ expected_stats_sinks = { - "name":"envoy.stats_sinks.statsd", - "typed_config":{ - "@type":"type.googleapis.com/envoy.config.metrics.v3.StatsdSink", - "address":{ - "socket_address":{ - "protocol":"UDP", - "address":"127.0.0.1", - "port_value":8125 - } - } - } - } - - os.environ['STATSD_ENABLED'] = 'true' + "name": "envoy.stats_sinks.statsd", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", + "address": { + "socket_address": {"protocol": "UDP", "address": "127.0.0.1", "port_value": 8125} + }, + }, + } + + os.environ["STATSD_ENABLED"] = "true" econf = _get_envoy_config(yaml) assert econf econf_dict = econf.as_dict() - assert 'stats_sinks' in econf_dict['bootstrap'] - assert len(econf_dict['bootstrap']['stats_sinks']) > 0 - assert econf_dict['bootstrap']['stats_sinks'][0] == expected_stats_sinks - assert 'stats_flush_interval' in econf_dict['bootstrap'] - assert econf_dict['bootstrap']['stats_flush_interval']['seconds'] == '1' + assert "stats_sinks" in econf_dict["bootstrap"] + assert len(econf_dict["bootstrap"]["stats_sinks"]) > 0 + assert econf_dict["bootstrap"]["stats_sinks"][0] == expected_stats_sinks + assert "stats_flush_interval" in econf_dict["bootstrap"] + assert econf_dict["bootstrap"]["stats_flush_interval"]["seconds"] == "1" @pytest.mark.compilertest @httpretty.activate def test_statsd_other(): - httpretty.register_uri( - httpretty.GET, - "other-statsd-sink", - body='{"origin": "127.0.0.1"}' - ) + httpretty.register_uri(httpretty.GET, "other-statsd-sink", body='{"origin": "127.0.0.1"}') yaml = """ apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -113,42 +101,34 @@ def test_statsd_other(): """ expected_stats_sinks = { - "name":"envoy.stats_sinks.statsd", - "typed_config":{ - "@type":"type.googleapis.com/envoy.config.metrics.v3.StatsdSink", - "address":{ - "socket_address":{ - "protocol":"UDP", - "address":"127.0.0.1", - "port_value":8125 - } - } - } - } - - os.environ['STATSD_ENABLED'] = 'true' - os.environ['STATSD_HOST'] = 'other-statsd-sink' + "name": "envoy.stats_sinks.statsd", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.StatsdSink", + "address": { + "socket_address": {"protocol": "UDP", "address": "127.0.0.1", "port_value": 8125} + }, + }, + } + + os.environ["STATSD_ENABLED"] = "true" + os.environ["STATSD_HOST"] = "other-statsd-sink" econf = _get_envoy_config(yaml) assert econf econf_dict = econf.as_dict() - assert 'stats_sinks' in econf_dict['bootstrap'] - assert len(econf_dict['bootstrap']['stats_sinks']) > 0 - assert econf_dict['bootstrap']['stats_sinks'][0] == expected_stats_sinks - assert 'stats_flush_interval' in econf_dict['bootstrap'] - assert econf_dict['bootstrap']['stats_flush_interval']['seconds'] == '1' + assert "stats_sinks" in econf_dict["bootstrap"] + assert len(econf_dict["bootstrap"]["stats_sinks"]) > 0 + assert econf_dict["bootstrap"]["stats_sinks"][0] == expected_stats_sinks + assert "stats_flush_interval" in econf_dict["bootstrap"] + assert econf_dict["bootstrap"]["stats_flush_interval"]["seconds"] == "1" @pytest.mark.compilertest @httpretty.activate def test_dogstatsd(): - httpretty.register_uri( - httpretty.GET, - "statsd-sink", - body='{"origin": "127.0.0.1"}' - ) + httpretty.register_uri(httpretty.GET, "statsd-sink", body='{"origin": "127.0.0.1"}') yaml = """ apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -161,30 +141,25 @@ def test_dogstatsd(): """ expected_stats_sinks = { - "name":"envoy.stat_sinks.dog_statsd", - "typed_config":{ - "@type":"type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", - "address":{ - "socket_address":{ - "protocol":"UDP", - "address":"127.0.0.1", - "port_value":8125 - } - } - } - } - - - os.environ['STATSD_ENABLED'] = 'true' - os.environ['DOGSTATSD'] = 'true' + "name": "envoy.stat_sinks.dog_statsd", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.metrics.v3.DogStatsdSink", + "address": { + "socket_address": {"protocol": "UDP", "address": "127.0.0.1", "port_value": 8125} + }, + }, + } + + os.environ["STATSD_ENABLED"] = "true" + os.environ["DOGSTATSD"] = "true" econf = _get_envoy_config(yaml) assert econf econf_dict = econf.as_dict() - assert 'stats_sinks' in econf_dict['bootstrap'] - assert len(econf_dict['bootstrap']['stats_sinks']) > 0 - assert econf_dict['bootstrap']['stats_sinks'][0] == expected_stats_sinks - assert 'stats_flush_interval' in econf_dict['bootstrap'] - assert econf_dict['bootstrap']['stats_flush_interval']['seconds'] == '1' + assert "stats_sinks" in econf_dict["bootstrap"] + assert len(econf_dict["bootstrap"]["stats_sinks"]) > 0 + assert econf_dict["bootstrap"]["stats_sinks"][0] == expected_stats_sinks + assert "stats_flush_interval" in econf_dict["bootstrap"] + assert econf_dict["bootstrap"]["stats_flush_interval"]["seconds"] == "1" diff --git a/python/tests/unit/test_timer.py b/python/tests/unit/test_timer.py index e9e3c1b942..7eb6edd4bc 100644 --- a/python/tests/unit/test_timer.py +++ b/python/tests/unit/test_timer.py @@ -8,6 +8,7 @@ epsilon = 0.0001 + def feq(x: float, y: float) -> bool: return abs(x - y) <= epsilon @@ -120,5 +121,5 @@ def test_Timer(): assert feq(t1.average, 150), f"t1.average must be 150, got {t1.average}" -if __name__ == '__main__': +if __name__ == "__main__": pytest.main(sys.argv) diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 0a2a472240..efe94b8cd2 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -11,7 +11,7 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s test %(levelname)s: %(message)s", - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) logger = logging.getLogger("ambassador") @@ -23,16 +23,25 @@ from tests.utils import default_listener_manifests if TYPE_CHECKING: - from ambassador.ir.irresource import IRResource # pragma: no cover + from ambassador.ir.irresource import IRResource # pragma: no cover + class MockSecretHandler(SecretHandler): - def load_secret(self, resource: 'IRResource', secret_name: str, namespace: str) -> Optional[SecretInfo]: - return SecretInfo('fallback-self-signed-cert', 'ambassador', "mocked-fallback-secret", - TLSCerts["acook"].pubcert, TLSCerts["acook"].privkey, decode_b64=False) + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: + return SecretInfo( + "fallback-self-signed-cert", + "ambassador", + "mocked-fallback-secret", + TLSCerts["acook"].pubcert, + TLSCerts["acook"].privkey, + decode_b64=False, + ) def _get_envoy_config(yaml): - + aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(default_listener_manifests() + yaml, k8s=True) @@ -63,6 +72,7 @@ def lightstep_tracing_service_manifest(): propagation_modes: ["ENVOY", "TRACE_CONTEXT"] """ + @pytest.mark.compilertest def test_tracing_config_v3(tmp_path: Path): aconf = Config() @@ -73,7 +83,9 @@ def test_tracing_config_v3(tmp_path: Path): aconf.load_all(fetcher.sorted()) - secret_handler = MockSecretHandler(logger, "mockery", str(tmp_path/"ambassador"/"snapshots"), "v1") + secret_handler = MockSecretHandler( + logger, "mockery", str(tmp_path / "ambassador" / "snapshots"), "v1" + ) ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) assert ir @@ -89,14 +101,16 @@ def test_tracing_config_v3(tmp_path: Path): "@type": "type.googleapis.com/envoy.config.trace.v3.LightstepConfig", "access_token_file": "/lightstep-credentials/access-token", "collector_cluster": "cluster_tracing_lightstep_80_ambassador", - "propagation_modes": ["ENVOY", "TRACE_CONTEXT"] - } + "propagation_modes": ["ENVOY", "TRACE_CONTEXT"], + }, } } - ads_config.pop('@type', None) - assert_valid_envoy_config(ads_config, extra_dirs=[str(tmp_path/"ambassador"/"snapshots")]) - assert_valid_envoy_config(bootstrap_config, extra_dirs=[str(tmp_path/"ambassador"/"snapshots")]) + ads_config.pop("@type", None) + assert_valid_envoy_config(ads_config, extra_dirs=[str(tmp_path / "ambassador" / "snapshots")]) + assert_valid_envoy_config( + bootstrap_config, extra_dirs=[str(tmp_path / "ambassador" / "snapshots")] + ) @pytest.mark.compilertest @@ -120,19 +134,19 @@ def test_tracing_zipkin_defaults(): assert "tracing" in bootstrap_config assert bootstrap_config["tracing"] == { - 'http': { - 'name': 'envoy.zipkin', - 'typed_config': { - '@type': 'type.googleapis.com/envoy.config.trace.v3.ZipkinConfig', - 'collector_endpoint': '/api/v2/spans', - 'collector_endpoint_version': 'HTTP_JSON', - 'trace_id_128bit': True, - 'collector_cluster': 'cluster_tracing_zipkin_test_9411_default' - - } + "http": { + "name": "envoy.zipkin", + "typed_config": { + "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig", + "collector_endpoint": "/api/v2/spans", + "collector_endpoint_version": "HTTP_JSON", + "trace_id_128bit": True, + "collector_cluster": "cluster_tracing_zipkin_test_9411_default", + }, } } + @pytest.mark.compilertest def test_tracing_zipkin_invalid_collector_version(): """test to ensure that providing an improper value will result in an error and the tracer not included""" diff --git a/python/tests/utils.py b/python/tests/utils.py index e69cdd495f..d57f6878a9 100644 --- a/python/tests/utils.py +++ b/python/tests/utils.py @@ -25,6 +25,7 @@ logger = logging.getLogger("ambassador") + def zipkin_tracing_service_manifest(): return """ --- @@ -39,6 +40,7 @@ def zipkin_tracing_service_manifest(): config: {} """ + def default_listener_manifests(): return """ --- @@ -69,8 +71,9 @@ def default_listener_manifests(): from: ALL """ + def default_http3_listener_manifest(): - return """ + return """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -89,8 +92,9 @@ def default_http3_listener_manifest(): from: ALL """ + def default_udp_listener_manifest(): - return """ + return """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -107,8 +111,10 @@ def default_udp_listener_manifest(): namespace: from: ALL """ + + def default_tcp_listener_manifest(): - return """ + return """ --- apiVersion: getambassador.io/v3alpha1 kind: Listener @@ -126,8 +132,11 @@ def default_tcp_listener_manifest(): from: ALL """ + def module_and_mapping_manifests(module_confs, mapping_confs): - yaml = default_listener_manifests() + """ + yaml = ( + default_listener_manifests() + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Module @@ -136,15 +145,23 @@ def module_and_mapping_manifests(module_confs, mapping_confs): namespace: default spec: config:""" + ) if module_confs: for module_conf in module_confs: - yaml = yaml + """ + yaml = ( + yaml + + """ {} -""".format(module_conf) +""".format( + module_conf + ) + ) else: yaml = yaml + " {}\n" - yaml = yaml + """ + yaml = ( + yaml + + """ --- apiVersion: getambassador.io/v3alpha1 kind: Mapping @@ -155,20 +172,29 @@ def module_and_mapping_manifests(module_confs, mapping_confs): hostname: "*" prefix: /httpbin/ service: httpbin""" + ) if mapping_confs: for mapping_conf in mapping_confs: - yaml = yaml + """ - {}""".format(mapping_conf) + yaml = ( + yaml + + """ + {}""".format( + mapping_conf + ) + ) return yaml + def _require_no_errors(ir: IR): assert ir.aconf.errors == {} + def _secret_handler(): source_root = tempfile.TemporaryDirectory(prefix="null-secret-", suffix="-source") cache_dir = tempfile.TemporaryDirectory(prefix="null-secret-", suffix="-cache") return NullSecretHandler(logger, source_root.name, cache_dir.name, "fake") + def compile_with_cachecheck(yaml, errors_ok=False): # Compile with and without a cache. Neither should produce errors. cache = Cache(logger) @@ -181,53 +207,63 @@ def compile_with_cachecheck(yaml, errors_ok=False): _require_no_errors(r2["ir"]) # Both should produce equal Envoy config as sorted json. - r1j = json.dumps(r1['xds'].as_dict(), sort_keys=True, indent=2) - r2j = json.dumps(r2['xds'].as_dict(), sort_keys=True, indent=2) + r1j = json.dumps(r1["xds"].as_dict(), sort_keys=True, indent=2) + r2j = json.dumps(r2["xds"].as_dict(), sort_keys=True, indent=2) assert r1j == r2j # All good. return r1 -EnvoyFilterInfo = namedtuple('EnvoyFilterInfo', [ 'name', 'type' ]) + +EnvoyFilterInfo = namedtuple("EnvoyFilterInfo", ["name", "type"]) EnvoyHCMInfo = EnvoyFilterInfo( name="envoy.filters.network.http_connection_manager", - type="type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + type="type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", ) EnvoyTCPInfo = EnvoyFilterInfo( name="envoy.filters.network.tcp_proxy", - type="type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy" + type="type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", ) + def econf_compile(yaml): compiled = compile_with_cachecheck(yaml) - return compiled['xds'].as_dict() + return compiled["xds"].as_dict() + def econf_foreach_listener(econf, fn, listener_count=1): - listeners = econf['static_resources']['listeners'] + listeners = econf["static_resources"]["listeners"] wanted_plural = "" if (listener_count == 1) else "s" - assert len(listeners) == listener_count, f"Expected {listener_count} listener{wanted_plural}, got {len(listeners)}" + assert ( + len(listeners) == listener_count + ), f"Expected {listener_count} listener{wanted_plural}, got {len(listeners)}" for listener in listeners: fn(listener) -def econf_foreach_listener_chain(listener, fn, chain_count=2, need_name=None, need_type=None, dump_info=None): + +def econf_foreach_listener_chain( + listener, fn, chain_count=2, need_name=None, need_type=None, dump_info=None +): # We need a specific number of filter chains. Normally it's 2, # since the compiler tests don't generally supply Listeners or Hosts, # so we get secure and insecure chains. - filter_chains = listener['filter_chains'] + filter_chains = listener["filter_chains"] if dump_info: dump_info(filter_chains) wanted_plural = "" if (chain_count == 1) else "s" - assert len(filter_chains) == chain_count, f"Expected {chain_count} filter chain{wanted_plural}, got {len(filter_chains)}" + assert ( + len(filter_chains) == chain_count + ), f"Expected {chain_count} filter chain{wanted_plural}, got {len(filter_chains)}" for chain in filter_chains: # We expect one filter on this chain. - filters = chain['filters'] + filters = chain["filters"] got_count = len(filters) got_plural = "" if (got_count == 1) else "s" assert got_count == 1, f"Expected just one filter, got {got_count} filter{got_plural}" @@ -236,26 +272,30 @@ def econf_foreach_listener_chain(listener, fn, chain_count=2, need_name=None, ne filter = filters[0] if need_name: - assert filter['name'] == need_name + assert filter["name"] == need_name - typed_config = filter['typed_config'] + typed_config = filter["typed_config"] if need_type: - assert typed_config['@type'] == need_type, f"bad type: got {repr(typed_config['@type'])} but expected {repr(need_type)}" + assert ( + typed_config["@type"] == need_type + ), f"bad type: got {repr(typed_config['@type'])} but expected {repr(need_type)}" fn(typed_config) + def econf_foreach_hcm(econf, fn, chain_count=2): - for listener in econf['static_resources']['listeners']: + for listener in econf["static_resources"]["listeners"]: hcm_info = EnvoyHCMInfo econf_foreach_listener_chain( - listener, fn, chain_count=chain_count, - need_name=hcm_info.name, need_type=hcm_info.type) + listener, fn, chain_count=chain_count, need_name=hcm_info.name, need_type=hcm_info.type + ) -def econf_foreach_cluster(econf, fn, name='cluster_httpbin_default'): - for cluster in econf['static_resources']['clusters']: - if cluster['name'] != name: + +def econf_foreach_cluster(econf, fn, name="cluster_httpbin_default"): + for cluster in econf["static_resources"]["clusters"]: + if cluster["name"] != name: continue found_cluster = True @@ -264,22 +304,26 @@ def econf_foreach_cluster(econf, fn, name='cluster_httpbin_default'): break assert found_cluster + def assert_valid_envoy_config(config_dict, extra_dirs=[]): with tempfile.TemporaryDirectory() as tmpdir: - econf = open(os.path.join(tmpdir, 'econf.json'), 'xt') + econf = open(os.path.join(tmpdir, "econf.json"), "xt") econf.write(json.dumps(config_dict)) econf.close() - img = os.environ.get('ENVOY_DOCKER_TAG') + img = os.environ.get("ENVOY_DOCKER_TAG") assert img cmd = [ - 'docker', 'run', - '--rm', + "docker", + "run", + "--rm", f"--volume={tmpdir}:/ambassador:ro", *[f"--volume={extra_dir}:{extra_dir}:ro" for extra_dir in extra_dirs], img, - '/usr/local/bin/envoy-static-stripped', - '--config-path', '/ambassador/econf.json', - '--mode', 'validate', + "/usr/local/bin/envoy-static-stripped", + "--config-path", + "/ambassador/econf.json", + "--mode", + "validate", ] p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if p.returncode != 0: @@ -295,7 +339,7 @@ def create_crl_pem_b64(issuerCert, issuerKey, revokedCerts): for revokedCert in revokedCerts: clientCert = crypto.load_certificate(crypto.FILETYPE_PEM, bytes(revokedCert, "utf-8")) r = crypto.Revoked() - r.set_serial(bytes('{:x}'.format(clientCert.get_serial_number()), "ascii")) + r.set_serial(bytes("{:x}".format(clientCert.get_serial_number()), "ascii")) r.set_rev_date(when) r.set_reason(None) crl.add_revoked(r) @@ -303,4 +347,6 @@ def create_crl_pem_b64(issuerCert, issuerKey, revokedCerts): cert = crypto.load_certificate(crypto.FILETYPE_PEM, bytes(issuerCert, "utf-8")) key = crypto.load_privatekey(crypto.FILETYPE_PEM, bytes(issuerKey, "utf-8")) crl.sign(cert, key, b"sha256") - return b64encode((crypto.dump_crl(crypto.FILETYPE_PEM, crl).decode("utf-8")+"\n").encode('utf-8')).decode('utf-8') + return b64encode( + (crypto.dump_crl(crypto.FILETYPE_PEM, crl).decode("utf-8") + "\n").encode("utf-8") + ).decode("utf-8") diff --git a/python/watch_hook.py b/python/watch_hook.py index 39366344f5..a3b733b210 100644 --- a/python/watch_hook.py +++ b/python/watch_hook.py @@ -15,7 +15,7 @@ from ambassador.utils import SecretInfo, SavedSecret, SecretHandler, dump_json if TYPE_CHECKING: - from ambassador.ir.irresource import IRResource # pragma: no cover + from ambassador.ir.irresource import IRResource # pragma: no cover # default AES's Secret name # (by default, we assume it will be in the same namespace as Ambassador) @@ -34,16 +34,20 @@ # Fake SecretHandler for our fake IR, below. + class SecretRecorder(SecretHandler): def __init__(self, logger: logging.Logger) -> None: super().__init__(logger, "-source_root-", "-cache_dir-", "0") self.needed: Dict[Tuple[str, str], SecretInfo] = {} # Record what was requested, and always return success. - def load_secret(self, resource: 'IRResource', - secret_name: str, namespace: str) -> Optional[SecretInfo]: - self.logger.debug("SecretRecorder (%s %s): load secret %s in namespace %s" % - (resource.kind, resource.name, secret_name, namespace)) + def load_secret( + self, resource: "IRResource", secret_name: str, namespace: str + ) -> Optional[SecretInfo]: + self.logger.debug( + "SecretRecorder (%s %s): load secret %s in namespace %s" + % (resource.kind, resource.name, secret_name, namespace) + ) return self.record_secret(secret_name, namespace) @@ -51,24 +55,36 @@ def record_secret(self, secret_name: str, namespace: str) -> Optional[SecretInfo secret_key = (secret_name, namespace) if secret_key not in self.needed: - self.needed[secret_key] = SecretInfo(secret_name, namespace, 'needed-secret', '-crt-', '-key-', - decode_b64=False) + self.needed[secret_key] = SecretInfo( + secret_name, namespace, "needed-secret", "-crt-", "-key-", decode_b64=False + ) return self.needed[secret_key] # Secrets that're still needed also get recorded. - def still_needed(self, resource: 'IRResource', secret_name: str, namespace: str) -> None: - self.logger.debug("SecretRecorder (%s %s): secret %s in namespace %s is still needed" % - (resource.kind, resource.name, secret_name, namespace)) + def still_needed(self, resource: "IRResource", secret_name: str, namespace: str) -> None: + self.logger.debug( + "SecretRecorder (%s %s): secret %s in namespace %s is still needed" + % (resource.kind, resource.name, secret_name, namespace) + ) self.record_secret(secret_name, namespace) # Never cache anything. - def cache_secret(self, resource: 'IRResource', secret_info: SecretInfo): - self.logger.debug("SecretRecorder (%s %s): skipping cache step for secret %s in namespace %s" % - (resource.kind, resource.name, secret_info.name, secret_info.namespace)) - - return SavedSecret(secret_info.name, secret_info.namespace, '-crt-path-', '-key-path-', '-user-path-', - '-root-crt-path', { 'tls.crt': '-crt-', 'tls.key': '-key-', 'user.key': '-user-' }) + def cache_secret(self, resource: "IRResource", secret_info: SecretInfo): + self.logger.debug( + "SecretRecorder (%s %s): skipping cache step for secret %s in namespace %s" + % (resource.kind, resource.name, secret_info.name, secret_info.namespace) + ) + + return SavedSecret( + secret_info.name, + secret_info.namespace, + "-crt-path-", + "-key-path-", + "-user-path-", + "-root-crt-path", + {"tls.crt": "-crt-", "tls.key": "-key-", "user.key": "-user-"}, + ) # XXX Sooooo there's some ugly stuff here. @@ -80,6 +96,7 @@ def cache_secret(self, resource: 'IRResource', secret_info: SecretInfo): # The solution here is to subclass the IR and take advantage of the watch_only # initialization keyword, which skips the hard parts of building an IR. + class FakeIR(IR): def __init__(self, aconf: Config, logger=None) -> None: # If we're asked about a secret, record interest in that secret. @@ -88,12 +105,17 @@ def __init__(self, aconf: Config, logger=None) -> None: # If we're asked about a file, it's good. file_checker = lambda path: True - super().__init__(aconf, logger=logger, watch_only=True, - secret_handler=self.secret_recorder, file_checker=file_checker) + super().__init__( + aconf, + logger=logger, + watch_only=True, + secret_handler=self.secret_recorder, + file_checker=file_checker, + ) # Don't bother actually saving resources that come up when working with # the faked modules. - def save_resource(self, resource: 'IRResource') -> 'IRResource': + def save_resource(self, resource: "IRResource") -> "IRResource": return resource @@ -108,9 +130,15 @@ def __init__(self, logger, yaml_stream) -> None: self.load_yaml(yaml_stream) - def add_kube_watch(self, what: str, kind: str, namespace: Optional[str], - field_selector: Optional[str]=None, label_selector: Optional[str]=None) -> None: - watch = { "kind": kind } + def add_kube_watch( + self, + what: str, + kind: str, + namespace: Optional[str], + field_selector: Optional[str] = None, + label_selector: Optional[str] = None, + ) -> None: + watch = {"kind": kind} if namespace: watch["namespace"] = namespace @@ -133,7 +161,7 @@ def load_yaml(self, yaml_stream): self.aconf.load_all(fetcher.sorted()) # We can lift mappings straight from the aconf... - mappings = self.aconf.get_config('mappings') or {} + mappings = self.aconf.get_config("mappings") or {} # ...but we need the fake IR to deal with resolvers and TLS contexts. self.fake = FakeIR(self.aconf, logger=self.logger) @@ -143,35 +171,55 @@ def load_yaml(self, yaml_stream): resolvers = self.fake.resolvers contexts = self.fake.tls_contexts - self.logger.debug(f'mappings: {len(mappings)}') - self.logger.debug(f'resolvers: {len(resolvers)}') - self.logger.debug(f'contexts: {len(contexts)}') - - global_resolver = self.fake.ambassador_module.get('resolver', None) - - global_label_selector = os.environ.get('AMBASSADOR_LABEL_SELECTOR', '') - self.logger.debug('label-selector: %s' % global_label_selector) - - cloud_connect_token_resource_name = os.getenv(ENV_CLOUD_CONNECT_TOKEN_RESOURCE_NAME, DEFAULT_CLOUD_CONNECT_TOKEN_RESOURCE_NAME) - cloud_connect_token_resource_namespace = os.getenv(ENV_CLOUD_CONNECT_TOKEN_RESOURCE_NAMESPACE, Config.ambassador_namespace) - self.logger.debug(f'cloud-connect-token: need configmap/secret {cloud_connect_token_resource_name}.{cloud_connect_token_resource_namespace}') - self.add_kube_watch(f'ConfigMap {cloud_connect_token_resource_name}', 'configmap', namespace=cloud_connect_token_resource_namespace, - field_selector=f"metadata.name={cloud_connect_token_resource_name}") - self.add_kube_watch(f'Secret {cloud_connect_token_resource_name}', 'secret', namespace=cloud_connect_token_resource_namespace, - field_selector=f"metadata.name={cloud_connect_token_resource_name}") + self.logger.debug(f"mappings: {len(mappings)}") + self.logger.debug(f"resolvers: {len(resolvers)}") + self.logger.debug(f"contexts: {len(contexts)}") + + global_resolver = self.fake.ambassador_module.get("resolver", None) + + global_label_selector = os.environ.get("AMBASSADOR_LABEL_SELECTOR", "") + self.logger.debug("label-selector: %s" % global_label_selector) + + cloud_connect_token_resource_name = os.getenv( + ENV_CLOUD_CONNECT_TOKEN_RESOURCE_NAME, DEFAULT_CLOUD_CONNECT_TOKEN_RESOURCE_NAME + ) + cloud_connect_token_resource_namespace = os.getenv( + ENV_CLOUD_CONNECT_TOKEN_RESOURCE_NAMESPACE, Config.ambassador_namespace + ) + self.logger.debug( + f"cloud-connect-token: need configmap/secret {cloud_connect_token_resource_name}.{cloud_connect_token_resource_namespace}" + ) + self.add_kube_watch( + f"ConfigMap {cloud_connect_token_resource_name}", + "configmap", + namespace=cloud_connect_token_resource_namespace, + field_selector=f"metadata.name={cloud_connect_token_resource_name}", + ) + self.add_kube_watch( + f"Secret {cloud_connect_token_resource_name}", + "secret", + namespace=cloud_connect_token_resource_namespace, + field_selector=f"metadata.name={cloud_connect_token_resource_name}", + ) # watch the AES Secret if the edge stack is running if self.fake.edge_stack_allowed: aes_secret_name = os.getenv(ENV_AES_SECRET_NAME, DEFAULT_AES_SECRET_NAME) aes_secret_namespace = os.getenv(ENV_AES_SECRET_NAMESPACE, Config.ambassador_namespace) - self.logger.debug(f'edge stack detected: need secret {aes_secret_name}.{aes_secret_namespace}') - self.add_kube_watch(f'Secret {aes_secret_name}', 'secret', namespace=aes_secret_namespace, - field_selector=f"metadata.name={aes_secret_name}") + self.logger.debug( + f"edge stack detected: need secret {aes_secret_name}.{aes_secret_namespace}" + ) + self.add_kube_watch( + f"Secret {aes_secret_name}", + "secret", + namespace=aes_secret_namespace, + field_selector=f"metadata.name={aes_secret_name}", + ) # Walk hosts. for host in self.fake.get_hosts(): - sel = host.get('selector') or {} - match_labels = sel.get('matchLabels') or {} + sel = host.get("selector") or {} + match_labels = sel.get("matchLabels") or {} label_selectors: List[str] = [] @@ -179,36 +227,38 @@ def load_yaml(self, yaml_stream): label_selectors.append(global_label_selector) if match_labels: - label_selectors += [ f"{l}={v}" for l, v in match_labels.items() ] + label_selectors += [f"{l}={v}" for l, v in match_labels.items()] - label_selector = ','.join(label_selectors) if label_selectors else None + label_selector = ",".join(label_selectors) if label_selectors else None - for wanted_kind in ['service', 'secret']: - self.add_kube_watch(f"Host {host.name}", wanted_kind, host.namespace, - label_selector=label_selector) + for wanted_kind in ["service", "secret"]: + self.add_kube_watch( + f"Host {host.name}", wanted_kind, host.namespace, label_selector=label_selector + ) for mname, mapping in mappings.items(): - res_name = mapping.get('resolver', None) - res_source = 'mapping' + res_name = mapping.get("resolver", None) + res_source = "mapping" if not res_name: res_name = global_resolver - res_source = 'defaults' + res_source = "defaults" - ctx_name = mapping.get('tls', None) + ctx_name = mapping.get("tls", None) self.logger.debug( - f'Mapping {mname}: resolver {res_name} from {res_source}, service {mapping.service}, tls {ctx_name}') + f"Mapping {mname}: resolver {res_name} from {res_source}, service {mapping.service}, tls {ctx_name}" + ) if res_name: resolver = resolvers.get(res_name, None) - self.logger.debug(f'-> resolver {resolver}') + self.logger.debug(f"-> resolver {resolver}") if resolver: svc = Service(logger, mapping.service, ctx_name) - if resolver.kind == 'ConsulResolver': - self.logger.debug(f'Mapping {mname} uses Consul resolver {res_name}') + if resolver.kind == "ConsulResolver": + self.logger.debug(f"Mapping {mname} uses Consul resolver {res_name}") # At the moment, we stuff the resolver's datacenter into the association # ID for this watch. The ResourceFetcher relies on that. @@ -222,73 +272,99 @@ def load_yaml(self, yaml_stream): "id": resolver.datacenter, "consul-address": resolver.address, "datacenter": resolver.datacenter, - "service-name": svc.hostname + "service-name": svc.hostname, } ) - elif resolver.kind == 'KubernetesEndpointResolver': + elif resolver.kind == "KubernetesEndpointResolver": hostname = svc.hostname namespace = Config.ambassador_namespace if not hostname: # This is really kind of impossible. - self.logger.error(f"KubernetesEndpointResolver {res_name} has no 'hostname'") + self.logger.error( + f"KubernetesEndpointResolver {res_name} has no 'hostname'" + ) continue if "." in hostname: (hostname, namespace) = hostname.split(".", 2)[0:2] - self.logger.debug(f'...kube endpoints: svc {svc.hostname} -> host {hostname} namespace {namespace}') + self.logger.debug( + f"...kube endpoints: svc {svc.hostname} -> host {hostname} namespace {namespace}" + ) - self.add_kube_watch(f"endpoint", "endpoints", namespace, - label_selector=global_label_selector, - field_selector=f"metadata.name={hostname}") + self.add_kube_watch( + f"endpoint", + "endpoints", + namespace, + label_selector=global_label_selector, + field_selector=f"metadata.name={hostname}", + ) for secret_key, secret_info in self.fake.secret_recorder.needed.items(): - self.logger.debug(f'need secret {secret_info.name}.{secret_info.namespace}') + self.logger.debug(f"need secret {secret_info.name}.{secret_info.namespace}") - self.add_kube_watch(f"needed secret", "secret", secret_info.namespace, - label_selector=global_label_selector, - field_selector=f"metadata.name={secret_info.name}") + self.add_kube_watch( + f"needed secret", + "secret", + secret_info.namespace, + label_selector=global_label_selector, + field_selector=f"metadata.name={secret_info.name}", + ) if self.fake.edge_stack_allowed: # If the edge stack is allowed, make sure we watch for our fallback context. - self.add_kube_watch("Fallback TLSContext", "TLSContext", namespace=Config.ambassador_namespace) + self.add_kube_watch( + "Fallback TLSContext", "TLSContext", namespace=Config.ambassador_namespace + ) - ambassador_basedir = os.environ.get('AMBASSADOR_CONFIG_BASE_DIR', '/ambassador') + ambassador_basedir = os.environ.get("AMBASSADOR_CONFIG_BASE_DIR", "/ambassador") - if os.path.exists(os.path.join(ambassador_basedir, '.ambassadorinstallations_ok')): - self.add_kube_watch("AmbassadorInstallations", "ambassadorinstallations.getambassador.io", Config.ambassador_namespace) + if os.path.exists(os.path.join(ambassador_basedir, ".ambassadorinstallations_ok")): + self.add_kube_watch( + "AmbassadorInstallations", + "ambassadorinstallations.getambassador.io", + Config.ambassador_namespace, + ) - ambassador_knative_requested = (os.environ.get("AMBASSADOR_KNATIVE_SUPPORT", "-unset-").lower() == 'true') + ambassador_knative_requested = ( + os.environ.get("AMBASSADOR_KNATIVE_SUPPORT", "-unset-").lower() == "true" + ) if ambassador_knative_requested: - self.logger.debug('Looking for Knative support...') + self.logger.debug("Looking for Knative support...") - if os.path.exists(os.path.join(ambassador_basedir, '.knative_clusteringress_ok')): + if os.path.exists(os.path.join(ambassador_basedir, ".knative_clusteringress_ok")): # Watch for clusteringresses.networking.internal.knative.dev in any namespace and with any labels. - self.logger.debug('watching for clusteringresses.networking.internal.knative.dev') - self.add_kube_watch("Knative clusteringresses", "clusteringresses.networking.internal.knative.dev", - None) + self.logger.debug("watching for clusteringresses.networking.internal.knative.dev") + self.add_kube_watch( + "Knative clusteringresses", + "clusteringresses.networking.internal.knative.dev", + None, + ) - if os.path.exists(os.path.join(ambassador_basedir, '.knative_ingress_ok')): + if os.path.exists(os.path.join(ambassador_basedir, ".knative_ingress_ok")): # Watch for ingresses.networking.internal.knative.dev in any namespace and # with any labels. - self.add_kube_watch("Knative ingresses", "ingresses.networking.internal.knative.dev", None) + self.add_kube_watch( + "Knative ingresses", "ingresses.networking.internal.knative.dev", None + ) self.watchset: Dict[str, List[Dict[str, str]]] = { "kubernetes-watches": self.kube_watches, - "consul-watches": self.consul_watches + "consul-watches": self.consul_watches, } - save_dir = os.environ.get('AMBASSADOR_WATCH_DIR', '/tmp') + save_dir = os.environ.get("AMBASSADOR_WATCH_DIR", "/tmp") if save_dir: watchset = dump_json(self.watchset) - with open(os.path.join(save_dir, 'watch.json'), "w") as output: + with open(os.path.join(save_dir, "watch.json"), "w") as output: output.write(watchset) + #### Mainline. if __name__ == "__main__": @@ -297,22 +373,22 @@ def load_yaml(self, yaml_stream): args = sys.argv[1:] if args: - if args[0] == '--debug': + if args[0] == "--debug": loglevel = logging.DEBUG args.pop(0) - elif args[0].startswith('--'): - raise Exception(f'Usage: {os.path.basename(sys.argv[0])} [--debug] [path]') + elif args[0].startswith("--"): + raise Exception(f"Usage: {os.path.basename(sys.argv[0])} [--debug] [path]") logging.basicConfig( level=loglevel, format="%(asctime)s watch-hook %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" + datefmt="%Y-%m-%d %H:%M:%S", ) - alogger = logging.getLogger('ambassador') + alogger = logging.getLogger("ambassador") alogger.setLevel(logging.INFO) - logger = logging.getLogger('watch_hook') + logger = logging.getLogger("watch_hook") logger.setLevel(loglevel) yaml_stream = sys.stdin