Add grpc error intercept test

tests/data/virtual-server-grpc/tls-secret.yaml

@@ -4,5 +4,5 @@ metadata:
   name: virtual-server-tls-grpc-secret
 type: kubernetes.io/tls
 data:
-  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURwRENDQW93Q0NRRDhoTVNKOXdGN0JUQU5CZ2txaGtpRzl3MEJBUXNGQURDQmt6RUxNQWtHQTFVRUJoTUMKU1VVeERUQUxCZ05WQkFnTUJFTnZjbXN4RFRBTEJnTlZCQWNNQkVOdmNtc3hEakFNQmdOVkJBb01CVTVIU1U1WQpNUXd3Q2dZRFZRUUxEQU5MU1VNeEl6QWhCZ05WQkFNTUduWnBjblIxWVd3dGMyVnlkbVZ5TG1WNFlXMXdiR1V1ClkyOXRNU013SVFZSktvWklodmNOQVFrQkZoUnJkV0psY201bGRHVnpRRzVuYVc1NExtTnZiVEFlRncweU1URXcKTWpZeE5EVXhNRGxhRncweU1URXhNalV4TkRVeE1EbGFNSUdUTVFzd0NRWURWUVFHRXdKSlJURU5NQXNHQTFVRQpDQXdFUTI5eWF6RU5NQXNHQTFVRUJ3d0VRMjl5YXpFT01Bd0dBMVVFQ2d3RlRrZEpUbGd4RERBS0JnTlZCQXNNCkEwdEpRekVqTUNFR0ExVUVBd3dhZG1seWRIVmhiQzF6WlhKMlpYSXVaWGhoYlhCc1pTNWpiMjB4SXpBaEJna3EKaGtpRzl3MEJDUUVXRkd0MVltVnlibVYwWlhOQWJtZHBibmd1WTI5dE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRgpBQU9DQVE4QU1JSUJDZ0tDQVFFQXZFTW9zQW40aHJ4bEpiVUtGUXBpUG1PbUlnbVZpd0lxUlh1a2FYUzFoWHVMClpXalZIb0phZ3k2Qlp0TmtZTXVxT1hkSGJHVjVrSnBOMmVxWVl0SnJQSXBNT0JEWEwveEYzTXhMMnB2eFZOVFAKZzZNUXdESDdaZGI2TUlGMlB2T2dRaU1TUlhxb3pEVmkvTW5XWTYwR2pUK01sUXNPeGVMRWRuWUN2RTQ2VEphcQpDSVM0eXRDZ0tCMzU5MnhLU1hoZHc0cGdsWHpoMmlqYnhYaEZkUGN0elZUTlRFbHJuVUI3MGt6S3RtcDFzM2l1CnpXMXlRN2I3OFZ6YVhqMnZ6T2Y3THhyUXNGSEtFL0VlNDRkYkc3dEJkUUpPcCtrS21IcVRvYXYrdHMwU0h4ZTYKRk0wbW5YQTBJbW9FdWVyWitncFFsTUFYSGtma0hRckRlNVMrMkpwZVpRSURBUUFCTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQmFQWVkwYlprT3ZhL1BBVHpSaHBRUkRKV0ljQlFra0F3UXgyWkh4ek9lcU9pVWxucDdxRFR3Cm9ybUdYaEt1ZFhWUGZJSjVGcXVSZS83UEtGVXFDQUJNTFNUb294c2U4dGVIclJMY0paZUNMekJVNUovaDI2eDYKQjlFb0NQcU54L1Vya1VRVDRYVW9ET0tFK01pVEw4bktmVVowbkN4KzJFRkxKbjk0SkdlODN4N3lHL0dTTFhzcgpXMnRBaDR0NzFLUzgwQjhEb29Pa1lmREQxcm9JdklrQmRDM2EzRzZwWUNEamJIT2FPcXQ4cWhZZUFlMjdvQ0J6CmQzd1k3RW9yL3d6UHM0YldPMWE0QXl4LzBsZnphVUp3ak9pT3dGNXlBbC9uOGxCQVJENElneFBCRnpsNXUyelAKdDA1ZEpSQUdsK1ljZkNoQ0QxT3ppT3FYKytJS3crMGUKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
-  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktrd2dnU2xBZ0VBQW9JQkFRQzhReWl3Q2ZpR3ZHVWwKdFFvVkNtSStZNllpQ1pXTEFpcEZlNlJwZExXRmU0dGxhTlVlZ2xxRExvRm0wMlJneTZvNWQwZHNaWG1RbWszWgo2cGhpMG1zOGlrdzRFTmN2L0VYY3pFdmFtL0ZVMU0rRG94REFNZnRsMXZvd2dYWSs4NkJDSXhKRmVxak1OV0w4CnlkWmpyUWFOUDR5VkN3N0Y0c1IyZGdLOFRqcE1scW9JaExqSzBLQW9IZm4zYkVwSmVGM0RpbUNWZk9IYUtOdkYKZUVWMDl5M05WTTFNU1d1ZFFIdlNUTXEyYW5XemVLN05iWEpEdHZ2eFhOcGVQYS9NNS9zdkd0Q3dVY29UOFI3agpoMXNidTBGMUFrNm42UXFZZXBPaHEvNjJ6UklmRjdvVXpTYWRjRFFpYWdTNTZ0bjZDbENVd0JjZVIrUWRDc043CmxMN1ltbDVsQWdNQkFBRUNnZ0VCQUs5RnUyREJ0NjhCUVE0dEVianJGUEpPNWdJMGVCU1dMSExLSFJUTlFwNkcKbWc0TTNyWHIySWJmU2p4SVBPRGRYSnJwMFFZN0owV1lVemk1NzZ1NWlYc3dxcXRjQ211emEvdVgvRnNINmxQMwpXVWVPRG05UldXUEVGT1FKOCtQQ0FBYnpEZlMvc04yR0txVjg2R3VlVUVZNnp4Q0gzZllnSEpiSllkeWMzQ3dtClZFbFBwSmFDUlBxMkJ1ZVEwc3U2UHVGcUtNdTlBYnY2SDNzSUNZdU9SekhGRXFHN1dBOWNlUkNBUVBpb0VOUmQKWXR6dS91dlZDK3dHSks2WDl5QjVmMGoyN0xIdEdoWVVWZlhmcHdOSFhaR3UrOWw2TUlqRlc3dVdmOW1QMWU4agp5c2FRV2dpM21jMUpsclkxRjFxSHFVajlxd1FpZW9rZnN0cno2OUlTTXpVQ2dZRUEzMzFKZklQYjU0SFF0TTA3Ckp4dkgwZ1Fsd0Y0YWZkL3p1M3NsTEUzNUhIWmFiK2tXcVp3YzhJcy9qcXFxVFNrVmlMMkJmNCtuSkgwM21abjAKd0Qxb29ReU5pcnZ4aUtiWHZYZGRNeGh1TzNRZEE2TGd2c3luaTZKM3U4Y3JPOGpWL1RZWVBWR2kybGxwWjIxOQowTUc3VmhFSkVZZG45ZUxnUWtZS0hvck1YRU1DZ1lFQTE2WUZhdk1LOFoxVTJGSnF5d1pVWnR4Tkc2M2IzY2xsClBZbXZqRUFMc0NBY0hoQVJ0WkhWVnJ3ck9TNDlNU3RYYUg3U25vbmV3dSs4MkRDMkhnQXpFQmlNWnVoOWp1Y3IKeEJJWThSM2xVS1lZTFJjQnJ3SDBFT1JZdUYxaFVXdjZvU3cyajArVWJnaTJXY3B1ZzhZQVJzOTF2S1BGSy81ZQpPckdUVHRwZ2hEY0NnWUVBbU1VcythZXRKN2l1Y1JrbGlWdE5JWHZpVG5oRlJheFVFT0ZpRE1JVU9tZVE5SlR4Ci8ra09RMFA4bzBwNGRaeFM4eEVQdDZIOEVFNWtObVJ1VVFicFFjV3g3NWIveVhpN1d2R0FUUkF6TWdMeVhtejEKa3BlVkpEZGYvNFZyUVVmTk4zVEY1d3lwOVBaUHZmcXptWU1FeXhXRURHNXlHendHTG1kd1BXYUZ3YjhDZ1lCdgpQcVhzMWhQL01EcGtuWitadkZvdDlkZVJQODc4U3JoL1ZsZkk4Z0VWaW5yMnh6TmJoeStXM3RzdFZEMGM4RTdYCkNSVjkwNEdtN2lWdElUUFJwaWl4VlBpWTNiWm4wYmt5SEdQaGZwcWRVQkdJc1NXZE1sVEZvZ1ppbFVsdXgxVXoKYjZ0MW9vZmZlMzhXTXVkdjh1U1JvdFFKekRJYXo4MnNXWTYwVHRNYllRS0JnUUN0UHpZYTRMWElkL1FEald0WAppTTRYL2hhTGlacUsxLzVYanZ0dlNEN2VwTG50L1dYaTFlTnliNjloN1E5TnJCc21iaGVzK1pmZlcxb1RFTmdzCklEZjNWdVFiZEVETFI5cEpXM085N3ludStuY3d6WGNzZklOc1R3dzlCR0RsZERUa2Rmd1dkbEZvRXhUYXV2YU4KYTV1QTZwOHZlYUlRZ1E5UWNwQzFQMkdXVnc9PQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tCg==
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURzakNDQXBvQ0NRREdqUVE5L3hnejZqQU5CZ2txaGtpRzl3MEJBUXNGQURDQm1qRUxNQWtHQTFVRUJoTUMKU1VVeERUQUxCZ05WQkFnTUJFTnZjbXN4RFRBTEJnTlZCQWNNQkVOdmNtc3hFekFSQmdOVkJBb01DazVIU1U1WQpMQ0JKYm1NeERqQU1CZ05WQkFzTUJVNUhTVTVZTVNNd0lRWURWUVFEREJwMmFYSjBkV0ZzTFhObGNuWmxjaTVsCmVHRnRjR3hsTG1OdmJURWpNQ0VHQ1NxR1NJYjNEUUVKQVJZVWEzVmlaWEp1WlhSbGMwQnVaMmx1ZUM1amIyMHcKSGhjTk1qRXhNVEkxTVRrek5qUXdXaGNOTWpNeE1ESTJNVGt6TmpRd1dqQ0JtakVMTUFrR0ExVUVCaE1DU1VVeApEVEFMQmdOVkJBZ01CRU52Y21zeERUQUxCZ05WQkFjTUJFTnZjbXN4RXpBUkJnTlZCQW9NQ2s1SFNVNVlMQ0JKCmJtTXhEakFNQmdOVkJBc01CVTVIU1U1WU1TTXdJUVlEVlFRRERCcDJhWEowZFdGc0xYTmxjblpsY2k1bGVHRnQKY0d4bExtTnZiVEVqTUNFR0NTcUdTSWIzRFFFSkFSWVVhM1ZpWlhKdVpYUmxjMEJ1WjJsdWVDNWpiMjB3Z2dFaQpNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUNoYnlha0tlbzZzR2Q0eFBva3dQaHpjMGcvCnNiNXZpZnNnbHJvc2IyUjhjaGJtQlJRV1lGZlNzbldidWtkejB2RFVaWHhlY1hOZUdKVFc0WWdOOUZUSERnQXcKNjhxNWFBS3RiY1d2ejlDVEd0RkthdW5GcE1GU21pcWU5VHFQM1JaVkJGa1gwWFkzYTFWbmlQNDFxV2wxdnVKQgpMM2JwKzlzNVVadnMraER4UFFuMG5xSFJCQ0t2VDNUZWpHbnJPelprQ2ljUGhSS0hhNnExNlhiNGhGd2E4RXB2CmhYMFZkWVNDNFMzL2xSNThkNFJxL0lJSmUyZlhpZTJwcFF5UUpsNTJ6SGZvUnVFK3VvM3NsMmREdDF6dC96Yk0Ka2M3bmZiYzJqZFVoT0FONGhOOEowS0NvWEI4U3FwcjcxNXlKdklJc3ZSRC9RRFNyd1BQeXh2N1VnMTZoQWdNQgpBQUV3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUY3SFg5OUY5Q2FsMEplZzJScktSS3lxTXBrWGNKWjhEd2RJCkdmMm9UL2I5OGdmRWxhbFgxMHBQc2lHQ2NVOFJjRW9wdzJ6bHdUNXNSTDVXWVVyN3FnS1RKOXR0cm9rRUhPL0IKYUdORWsveFpNZ1JkOFpVL1NTOGdsQ0tjUGdOekI4TmVhQmV2U3lhT0NJYndjMWZSalF0YlVjYkp3N3JWUlp6cApaU1JJZWVPVVV2ZVg1RDZaNGQyN2RGd1pKU1B3TkV3eVlnZERKQWxhelMwMnJOZDZRUktwTFViNUJSUlB4aVYvCmtvNGUxK1h0V3FvVFpWMi9IbDZRWnB6MzdyMjZXQ0dxbUpHYnJzM0JmRkwxUFkvTEFqY1NZOVRrNU9kdU80SXcKM09PN3p5UTV3eDhuc3V4SHlQOUJUMFh3aU5hMzU1RXpwd3V6N1hyTzI0ZElNYjVsZWRnPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRQ2hieWFrS2VvNnNHZDQKeFBva3dQaHpjMGcvc2I1dmlmc2dscm9zYjJSOGNoYm1CUlFXWUZmU3NuV2J1a2R6MHZEVVpYeGVjWE5lR0pUVwo0WWdOOUZUSERnQXc2OHE1YUFLdGJjV3Z6OUNUR3RGS2F1bkZwTUZTbWlxZTlUcVAzUlpWQkZrWDBYWTNhMVZuCmlQNDFxV2wxdnVKQkwzYnArOXM1VVp2cytoRHhQUW4wbnFIUkJDS3ZUM1Rlakduck96WmtDaWNQaFJLSGE2cTEKNlhiNGhGd2E4RXB2aFgwVmRZU0M0UzMvbFI1OGQ0UnEvSUlKZTJmWGllMnBwUXlRSmw1MnpIZm9SdUUrdW8zcwpsMmREdDF6dC96Yk1rYzduZmJjMmpkVWhPQU40aE44SjBLQ29YQjhTcXByNzE1eUp2SUlzdlJEL1FEU3J3UFB5Cnh2N1VnMTZoQWdNQkFBRUNnZ0VBWm5KWlBWajBNaVo4bzZHdGRPR1pTZnJnNExyMXRXY0ZIVnRKN3FVS1NnZEYKRE5nd05Uc1N3TDFMOFhXM25vTkJIaWtCVWhZQk5yZTJ6TjczTHBQZHNTenJaaUJjMkdodk9vd3RKak5sazlVeQorRno4MmRhQ2NOOHhLUXRMREwwclRPeWpkWUFSMjMyY0IwWml2TDgwRStyOVBveldsQXFteHF0Sm5vdmJjSnRmCmpBK3RkTTcvTTJ0ckJqRmJXKzlwTERoS3orVkw2TUc0b1BGYmtVcTJ3bVFYRE0vQ1hKZnE5WXVMTmhVODFsbEwKYzZwN0trckNwU3M1QWRHWitwU1IxNWlSRWUwblRMT1JkVWFFdWRkVm0xQllmZWVhcVVia0VPUk1WTm5oSG1ORwpTV25KdEhDanNpUitDWmdBYktOT3ZkQ3IrekN5ZHdZUyt4RWxpSXE1T1FLQmdRRFVtM0p2WHhEbDloY2g5TUg1CkY4Y2lVS2NuVnJHZ1lHRXU5ZTlaOG5JeFVEUWlkQVlJSkI3WEZSL1RlY3RubzFjQjd3cVlWcHpvenNZaC9RNkcKYTVMeXA3cnc1NkFwek1qV0IxOGtzbTJHUmloK1VleEtiMVVNYWlpeEFrRjZ4cWVjSTFxUVN1T0hQTFlrRjNUWgpOcCtvRkhYUHBFMlE0cVNtVVl0ZHJOdWpZd0tCZ1FEQ1lmUjFGNHpjbkpnTWZJVGpaOVEzd1JmaW9YalVFak9ICjZLT2d2bUppRDFsdG5rT3JSNTlKMXIvdTU3cU4zbGlNakdGT2pod25qaUIrWk5BcDZRenN0bkhVV09yUHduQXYKUjJXNUoyRW9oSGgxaTJhWUg4SkNTRVpocmZBSXZIbEIvQ29XaUFiZzkvR0lpNG9oNFB0NEljOTJLemNaK0RVeQpBbzltV1Uxdkt3S0JnUUNnKy9OeWtURmlieXlrOFlmTzdVcERtWDU0TXhUY3N4M2pTU1dybmdFSmhnbHo3UmFFCkk1V1dsdEE2ZVFhanV2S3U3Q25Cb0JPLzFKSUNPbk05SlVkbnBjblBrQk9la3dtZnhvVXNiRTZ5VlgxajZQUmEKaUdLRnUveUR5NGw1UmVLMFA3RGJnVmszbGFqMU95Mm5LODFJbi9WMC9Kd2ZFUDVMVVlPTnNzMjhzUUtCZ1FDRQp3NWJPS3VtZy9LdTFTNDhRS3loOWREczJKWWQ3Z1hzRXh0YUx3YjA4c0xNcDljRE9TYnI0R2Q4NTg3Z3RrY0gxCkxTU0JIUHNKNFQ4OFZPc0ExUlpvenl2c0YxYzUwOW4vME1vZnJrL2o5cWEzMGlDZW9vSng5eDlyTS93UVczcU8Kb1FhMklPNWgxYmQ0eGFYeEFkTi85OGZWTkNzTVo4VWRoVFlnZDdvMXhRS0JnQ1Fjdmw1Yy9ZY3NHRUd1YU02Mwp1NzNaNm9ha05mNkpEZ2lzdjZ2c3dHMHdLbFRQNGgwKzNTSmdmL2dnTGVMNVBxdm5pemxZbEJSQUd3KzQwYVFGCmZnZnZBOWVlaVFZdVdiRDV0RlpUTDZKcFBnd01PSlVQUFBIU0JianJSTGpoMzk5dlBBdlhqekNPT25ad01LcXcKTDBIZWF1M1J6a0ZkRDllNTk3c3U3MG90Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K

tests/suite/test_virtual_server_grpc.py

@@ -1,20 +1,19 @@
 import grpc
 import pytest
+import time
 from kubernetes.client.rest import ApiException
 
 from settings import TEST_DATA, DEPLOYMENTS
 from suite.custom_assertions import assert_event_starts_with_text_and_contains_errors, \
     assert_grpc_entries_exist, assert_proxy_entries_do_not_exist, assert_vs_conf_not_exists
-from suite.custom_resources_utils import read_custom_resource
 from suite.grpc.helloworld_pb2 import HelloRequest
 from suite.grpc.helloworld_pb2_grpc import GreeterStub
 from suite.resources_utils import create_example_app, wait_until_all_pods_are_ready, \
     delete_common_app, create_secret_from_yaml, replace_configmap_from_yaml, \
-    delete_items_from_yaml, get_first_pod_name, get_events
+    delete_items_from_yaml, get_first_pod_name, get_events, wait_before_test, scale_deployment
 from suite.ssl_utils import get_certificate
 from suite.vs_vsr_resources_utils import get_vs_nginx_template_conf, \
     patch_virtual_server_from_yaml
-from suite.resources_utils import wait_before_test
 
 
 @pytest.fixture(scope="function")
@@ -64,6 +63,7 @@ def fin():
 
 @pytest.mark.vs
 @pytest.mark.smoke
+@pytest.mark.ciara
 @pytest.mark.parametrize('crd_ingress_controller, virtual_server_setup',
                          [({"type": "complete", "extra_args": [f"-enable-custom-resources"]},
                            {"example": "virtual-server-grpc"})],
@@ -135,6 +135,42 @@ def test_connect_grpc_backend(self, kube_apis, ingress_controller_prerequisites,
                 print(e.details())
                 pytest.fail("RPC error was not expected during call, exiting...")
 
+    @pytest.mark.parametrize("backend_setup", [{"app_type": "grpc-vs"}], indirect=True)
+    def test_grpc_error_intercept(self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller, 
+                                  backend_setup, virtual_server_setup):
+        cert = get_certificate(virtual_server_setup.public_endpoint.public_ip,
+                               virtual_server_setup.vs_host,
+                               virtual_server_setup.public_endpoint.port_ssl)
+        target = f'{virtual_server_setup.public_endpoint.public_ip}:{virtual_server_setup.public_endpoint.port_ssl}'
+        credentials = grpc.ssl_channel_credentials(root_certificates=cert.encode())
+        options = (('grpc.ssl_target_name_override', virtual_server_setup.vs_host),)
+
+        with grpc.secure_channel(target, credentials, options) as channel:
+            stub = GreeterStub(channel)
+            response = ""
+            try:
+                response = stub.SayHello(HelloRequest(name=virtual_server_setup.public_endpoint.public_ip))
+                valid_message = "Hello {}".format(virtual_server_setup.public_endpoint.public_ip)
+                # no status has been returned in the response
+                assert valid_message in response.message
+            except grpc.RpcError as e:
+                print(e.details())
+                pytest.fail("RPC error was not expected during call, exiting...")
+
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "grpc1", virtual_server_setup.namespace, 0)
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "grpc2", virtual_server_setup.namespace, 0)
+        time.sleep(1)
+
+        with grpc.secure_channel(target, credentials, options) as channel:
+            stub = GreeterStub(channel)
+            try:
+                response = stub.SayHello(HelloRequest(name=virtual_server_setup.public_endpoint.public_ip))
+                # assert the grpc status has been returned in the header
+                assert response.status == 14
+                pytest.fail("RPC error was expected during call, exiting...")
+            except grpc.RpcError as e:
+                print(e)
+
     @pytest.mark.parametrize("backend_setup", [{"app_type": "grpc-vs"}], indirect=True)
     def test_config_after_enable_tls(self, kube_apis, ingress_controller_prerequisites,
                                      crd_ingress_controller, backend_setup, virtual_server_setup):
@@ -151,6 +187,7 @@ def test_config_after_enable_tls(self, kube_apis, ingress_controller_prerequisit
                                             ingress_controller_prerequisites.namespace)
         assert 'grpc_pass grpcs://' in config
 
+
 @pytest.mark.vs
 @pytest.mark.smoke
 @pytest.mark.skip_for_nginx_oss