Skip to content

Commit

Permalink
Python testing fix (#103)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AHarmlessPyro
AHarmlessPyro authored Nov 12, 2022
1 parent d880329 commit 67096f7
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 109 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ __pycache__
metlo-api-security-enterprise
ingestors/kubernetes/example_deployment.yaml
ingestors/python/metlo.egg-info
ingestors/python/.idea

ingestors/java/spring/.gradle/
ingestors/java/spring/.idea/
Expand Down
128 changes: 73 additions & 55 deletions ingestors/python/metlo/django.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,21 @@
from concurrent.futures import ThreadPoolExecutor
from urllib.request import Request, urlopen
from urllib.parse import urlparse
import logging

from django.conf import settings

endpoint = "api/v1/log-request/single"

logger = logging.getLogger("metlo")


class MetloDjango(object):
def perform_request(self, data):
urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8"))
try:
urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8"))
except Exception as e:
logger.warn(e)

def __init__(self, get_response):
"""
Expand All @@ -22,6 +28,8 @@ def __init__(self, get_response):
max_workers=settings.METLO_CONFIG.get("workers", 4)
)

self.disabled = settings.METLO_CONFIG.get("DISABLED", False)

assert (
settings.METLO_CONFIG.get("METLO_HOST") is not None
), "METLO_CONFIG is missing METLO_HOST attribute"
Expand All @@ -47,60 +55,70 @@ def __init__(self, get_response):

def __call__(self, request):
response = self.get_response(request)
params = request.GET if request.method == "GET" else request.POST
dest_ip = (
request.META.get("SERVER_NAME")
if "1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME")
else "localhost"
)
src_ip = (
request.META.get("REMOTE_ADDR")
if "1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR")
else "localhost"
)
source_port = request.environ["wsgi.input"].stream.raw._sock.getpeername()[1]
res_body = response.content.decode("utf-8")
data = {
"request": {
"url": {
"host": request._current_scheme_host
if request._current_scheme_host
else src_ip,
"path": request.path,
"parameters": list(
map(lambda x: {"name": x[0], "value": x[1]}, params.items())
),
},
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]}, request.headers.items()
)
),
"body": request.body.decode("utf-8"),
"method": request.method,
},
"response": {
"url": f"{dest_ip}:{request.META.get('SERVER_PORT')}",
"status": response.status_code,
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
response.headers.items(),
)
),
"body": res_body,
},
"meta": {
"environment": "production",
"incoming": True,
"source": src_ip,
"sourcePort": source_port,
"destination": dest_ip,
"destinationPort": request.META.get("SERVER_PORT"),
"metloSource": "python/django",
},
}
self.pool.submit(self.perform_request, data=data)
if not self.disabled:
try:
params = request.GET if request.method == "GET" else request.POST
dest_ip = (
request.META.get("SERVER_NAME")
if "1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME")
else "localhost"
)
src_ip = (
request.META.get("REMOTE_ADDR")
if "1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR")
else "localhost"
)
source_port = request.environ[
"wsgi.input"
].stream.raw._sock.getpeername()[1]
res_body = response.content.decode("utf-8")
data = {
"request": {
"url": {
"host": request._current_scheme_host
if request._current_scheme_host
else src_ip,
"path": request.path,
"parameters": list(
map(
lambda x: {"name": x[0], "value": x[1]},
params.items(),
)
),
},
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
request.headers.items(),
)
),
"body": request.body.decode("utf-8"),
"method": request.method,
},
"response": {
"url": f"{dest_ip}:{request.META.get('SERVER_PORT')}",
"status": response.status_code,
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
response.headers.items(),
)
),
"body": res_body,
},
"meta": {
"environment": "production",
"incoming": True,
"source": src_ip,
"sourcePort": source_port,
"destination": dest_ip,
"destinationPort": request.META.get("SERVER_PORT"),
"metloSource": "python/django",
},
}
self.pool.submit(self.perform_request, data=data)
except Exception as e:
logger.debug(e)
return response

def process_exception(self, request, exception):
Expand Down
119 changes: 66 additions & 53 deletions ingestors/python/metlo/flask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,22 @@
from concurrent.futures import ThreadPoolExecutor
from urllib.request import Request, urlopen
from urllib.parse import urlparse
import logging

from flask import request

endpoint = "api/v1/log-request/single"


logger = logging.getLogger("metlo")


class MetloFlask:
def perform_request(self, data):
urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8"))
try:
urlopen(url=self.saved_request, data=json.dumps(data).encode("utf-8"))
except Exception as e:
logger.warn(e)

def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs):
"""
Expand All @@ -21,6 +28,7 @@ def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs):
"""
self.app = app
self.pool = ThreadPoolExecutor(max_workers=kwargs.get("workers", 4))
self.disabled = kwargs.get("disabled", False)

assert (
metlo_host is not None
Expand All @@ -45,55 +53,60 @@ def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs):
method="POST",
)

@app.after_request
def function(response, *args, **kwargs):
dst_host = (
request.environ.get("HTTP_HOST")
or request.environ.get("HTTP_X_FORWARDED_FOR")
or request.environ.get("REMOTE_ADDR")
)
data = {
"request": {
"url": {
"host": dst_host,
"path": request.path,
"parameters": list(
map(
lambda x: {"name": x[0], "value": x[1]},
request.args.items(),
)
),
},
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
(request.headers).items(),
)
),
"body": request.data.decode("utf-8"),
"method": request.method,
},
"response": {
"url": f"{request.environ.get('SERVER_NAME')}:{request.environ.get('SERVER_PORT')}",
"status": response.status_code,
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
(response.headers).items(),
)
),
"body": response.data.decode("utf-8"),
},
"meta": {
"environment": "production",
"incoming": True,
"source": request.environ.get("HTTP_X_FORWARDED_FOR")
or request.environ.get("REMOTE_ADDR"),
"sourcePort": request.environ.get("REMOTE_PORT"),
"destination": request.environ.get("SERVER_NAME"),
"destinationPort": request.environ.get("SERVER_PORT"),
"metloSource": "python/flask",
},
}
self.pool.submit(self.perform_request, data=data)
return response
if not self.disabled:

@app.after_request
def function(response, *args, **kwargs):
try:
dst_host = (
request.environ.get("HTTP_HOST")
or request.environ.get("HTTP_X_FORWARDED_FOR")
or request.environ.get("REMOTE_ADDR")
)
data = {
"request": {
"url": {
"host": dst_host,
"path": request.path,
"parameters": list(
map(
lambda x: {"name": x[0], "value": x[1]},
request.args.items(),
)
),
},
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
(request.headers).items(),
)
),
"body": request.data.decode("utf-8"),
"method": request.method,
},
"response": {
"url": f"{request.environ.get('SERVER_NAME')}:{request.environ.get('SERVER_PORT')}",
"status": response.status_code,
"headers": list(
map(
lambda x: {"name": x[0], "value": x[1]},
(response.headers).items(),
)
),
"body": response.data.decode("utf-8"),
},
"meta": {
"environment": "production",
"incoming": True,
"source": request.environ.get("HTTP_X_FORWARDED_FOR")
or request.environ.get("REMOTE_ADDR"),
"sourcePort": request.environ.get("REMOTE_PORT"),
"destination": request.environ.get("SERVER_NAME"),
"destinationPort": request.environ.get("SERVER_PORT"),
"metloSource": "python/flask",
},
}
self.pool.submit(self.perform_request, data=data)
except Exception as e:
logger.debug(e)
return response
2 changes: 1 addition & 1 deletion ingestors/python/setup.cfg
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[metadata]
name = metlo
version = 0.0.15
version = 0.0.16
description = The Python Agent for Metlo
long_description = file: README.md
long_description_content_type = text/markdown
Expand Down

0 comments on commit 67096f7

Please sign in to comment.