From 67096f7f701b86408881637042152e3b2361542f Mon Sep 17 00:00:00 2001 From: Ninad Sinha <32181677+AHarmlessPyro@users.noreply.github.com> Date: Sat, 12 Nov 2022 10:55:08 +0530 Subject: [PATCH] Python testing fix (#103) --- .gitignore | 1 + ingestors/python/metlo/django.py | 128 ++++++++++++++++++------------- ingestors/python/metlo/flask.py | 119 +++++++++++++++------------- ingestors/python/setup.cfg | 2 +- 4 files changed, 141 insertions(+), 109 deletions(-) diff --git a/.gitignore b/.gitignore index c5eda650..c7727735 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ __pycache__ metlo-api-security-enterprise ingestors/kubernetes/example_deployment.yaml ingestors/python/metlo.egg-info +ingestors/python/.idea ingestors/java/spring/.gradle/ ingestors/java/spring/.idea/ diff --git a/ingestors/python/metlo/django.py b/ingestors/python/metlo/django.py index a670ac4d..7f36660a 100644 --- a/ingestors/python/metlo/django.py +++ b/ingestors/python/metlo/django.py @@ -2,15 +2,21 @@ from concurrent.futures import ThreadPoolExecutor from urllib.request import Request, urlopen from urllib.parse import urlparse +import logging from django.conf import settings endpoint = "api/v1/log-request/single" +logger = logging.getLogger("metlo") + class MetloDjango(object): def perform_request(self, data): - urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8")) + try: + urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8")) + except Exception as e: + logger.warn(e) def __init__(self, get_response): """ @@ -22,6 +28,8 @@ def __init__(self, get_response): max_workers=settings.METLO_CONFIG.get("workers", 4) ) + self.disabled = settings.METLO_CONFIG.get("DISABLED", False) + assert ( settings.METLO_CONFIG.get("METLO_HOST") is not None ), "METLO_CONFIG is missing METLO_HOST attribute" @@ -47,60 +55,70 @@ def __init__(self, get_response): def __call__(self, request): response = self.get_response(request) - params = request.GET if request.method == "GET" else request.POST - dest_ip = ( - request.META.get("SERVER_NAME") - if "1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME") - else "localhost" - ) - src_ip = ( - request.META.get("REMOTE_ADDR") - if "1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR") - else "localhost" - ) - source_port = request.environ["wsgi.input"].stream.raw._sock.getpeername()[1] - res_body = response.content.decode("utf-8") - data = { - "request": { - "url": { - "host": request._current_scheme_host - if request._current_scheme_host - else src_ip, - "path": request.path, - "parameters": list( - map(lambda x: {"name": x[0], "value": x[1]}, params.items()) - ), - }, - "headers": list( - map( - lambda x: {"name": x[0], "value": x[1]}, request.headers.items() - ) - ), - "body": request.body.decode("utf-8"), - "method": request.method, - }, - "response": { - "url": f"{dest_ip}:{request.META.get('SERVER_PORT')}", - "status": response.status_code, - "headers": list( - map( - lambda x: {"name": x[0], "value": x[1]}, - response.headers.items(), - ) - ), - "body": res_body, - }, - "meta": { - "environment": "production", - "incoming": True, - "source": src_ip, - "sourcePort": source_port, - "destination": dest_ip, - "destinationPort": request.META.get("SERVER_PORT"), - "metloSource": "python/django", - }, - } - self.pool.submit(self.perform_request, data=data) + if not self.disabled: + try: + params = request.GET if request.method == "GET" else request.POST + dest_ip = ( + request.META.get("SERVER_NAME") + if "1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME") + else "localhost" + ) + src_ip = ( + request.META.get("REMOTE_ADDR") + if "1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR") + else "localhost" + ) + source_port = request.environ[ + "wsgi.input" + ].stream.raw._sock.getpeername()[1] + res_body = response.content.decode("utf-8") + data = { + "request": { + "url": { + "host": request._current_scheme_host + if request._current_scheme_host + else src_ip, + "path": request.path, + "parameters": list( + map( + lambda x: {"name": x[0], "value": x[1]}, + params.items(), + ) + ), + }, + "headers": list( + map( + lambda x: {"name": x[0], "value": x[1]}, + request.headers.items(), + ) + ), + "body": request.body.decode("utf-8"), + "method": request.method, + }, + "response": { + "url": f"{dest_ip}:{request.META.get('SERVER_PORT')}", + "status": response.status_code, + "headers": list( + map( + lambda x: {"name": x[0], "value": x[1]}, + response.headers.items(), + ) + ), + "body": res_body, + }, + "meta": { + "environment": "production", + "incoming": True, + "source": src_ip, + "sourcePort": source_port, + "destination": dest_ip, + "destinationPort": request.META.get("SERVER_PORT"), + "metloSource": "python/django", + }, + } + self.pool.submit(self.perform_request, data=data) + except Exception as e: + logger.debug(e) return response def process_exception(self, request, exception): diff --git a/ingestors/python/metlo/flask.py b/ingestors/python/metlo/flask.py index ddc980e9..4df70921 100644 --- a/ingestors/python/metlo/flask.py +++ b/ingestors/python/metlo/flask.py @@ -2,15 +2,22 @@ from concurrent.futures import ThreadPoolExecutor from urllib.request import Request, urlopen from urllib.parse import urlparse +import logging from flask import request endpoint = "api/v1/log-request/single" +logger = logging.getLogger("metlo") + + class MetloFlask: def perform_request(self, data): - urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8")) + try: + urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8")) + except Exception as e: + logger.warn(e) def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs): """ @@ -21,6 +28,7 @@ def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs): """ self.app = app self.pool = ThreadPoolExecutor(max_workers=kwargs.get("workers", 4)) + self.disabled = kwargs.get("disabled", False) assert ( metlo_host is not None @@ -45,55 +53,60 @@ def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs): method="POST", ) - @app.after_request - def function(response, *args, **kwargs): - dst_host = ( - request.environ.get("HTTP_HOST") - or request.environ.get("HTTP_X_FORWARDED_FOR") - or request.environ.get("REMOTE_ADDR") - ) - data = { - "request": { - "url": { - "host": dst_host, - "path": request.path, - "parameters": list( - map( - lambda x: {"name": x[0], "value": x[1]}, - request.args.items(), - ) - ), - }, - "headers": list( - map( - lambda x: {"name": x[0], "value": x[1]}, - (request.headers).items(), - ) - ), - "body": request.data.decode("utf-8"), - "method": request.method, - }, - "response": { - "url": f"{request.environ.get('SERVER_NAME')}:{request.environ.get('SERVER_PORT')}", - "status": response.status_code, - "headers": list( - map( - lambda x: {"name": x[0], "value": x[1]}, - (response.headers).items(), - ) - ), - "body": response.data.decode("utf-8"), - }, - "meta": { - "environment": "production", - "incoming": True, - "source": request.environ.get("HTTP_X_FORWARDED_FOR") - or request.environ.get("REMOTE_ADDR"), - "sourcePort": request.environ.get("REMOTE_PORT"), - "destination": request.environ.get("SERVER_NAME"), - "destinationPort": request.environ.get("SERVER_PORT"), - "metloSource": "python/flask", - }, - } - self.pool.submit(self.perform_request, data=data) - return response + if not self.disabled: + + @app.after_request + def function(response, *args, **kwargs): + try: + dst_host = ( + request.environ.get("HTTP_HOST") + or request.environ.get("HTTP_X_FORWARDED_FOR") + or request.environ.get("REMOTE_ADDR") + ) + data = { + "request": { + "url": { + "host": dst_host, + "path": request.path, + "parameters": list( + map( + lambda x: {"name": x[0], "value": x[1]}, + request.args.items(), + ) + ), + }, + "headers": list( + map( + lambda x: {"name": x[0], "value": x[1]}, + (request.headers).items(), + ) + ), + "body": request.data.decode("utf-8"), + "method": request.method, + }, + "response": { + "url": f"{request.environ.get('SERVER_NAME')}:{request.environ.get('SERVER_PORT')}", + "status": response.status_code, + "headers": list( + map( + lambda x: {"name": x[0], "value": x[1]}, + (response.headers).items(), + ) + ), + "body": response.data.decode("utf-8"), + }, + "meta": { + "environment": "production", + "incoming": True, + "source": request.environ.get("HTTP_X_FORWARDED_FOR") + or request.environ.get("REMOTE_ADDR"), + "sourcePort": request.environ.get("REMOTE_PORT"), + "destination": request.environ.get("SERVER_NAME"), + "destinationPort": request.environ.get("SERVER_PORT"), + "metloSource": "python/flask", + }, + } + self.pool.submit(self.perform_request, data=data) + except Exception as e: + logger.debug(e) + return response diff --git a/ingestors/python/setup.cfg b/ingestors/python/setup.cfg index 10bc2af8..cdb4e05a 100644 --- a/ingestors/python/setup.cfg +++ b/ingestors/python/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = metlo -version = 0.0.15 +version = 0.0.16 description = The Python Agent for Metlo long_description = file: README.md long_description_content_type = text/markdown