Python ingestor

ingestors/python/metlo/README.md

@@ -0,0 +1,108 @@
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://storage.googleapis.com/metlo-security-public-images/metlo_logo_horiz_negative%404x.png" height="80">
+    <img alt="logo" src="https://storage.googleapis.com/metlo-security-public-images/metlo_logo_horiz%404x.png" height="80">
+  </picture>
+  <h1 align="center">Metlo API Security</h1>
+  <p align="center">Secure Your API.</p>
+</p>
+
+---
+<div align="center">
+
+[![Prs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=shields)](http://makeapullrequest.com)
+[![Join Discord Server](https://img.shields.io/badge/discord%20community-join-blue)](https://discord.gg/4xhumff9BX)
+![Github Commit Activity](https://img.shields.io/github/commit-activity/m/metlo-labs/metlo)
+![GitHub Workflow Status](https://img.shields.io/github/workflow/status/metlo-labs/metlo/build)
+[![License](https://img.shields.io/badge/license-MIT-brightgreen)](/LICENSE)
+
+</div>
+
+---
+
+## Metlo is an open-source API security platform
+
+* Create an Inventory of all your API Endpoints.
+* Proactively test your APIs before they go into production.
+* Detect API attacks in real time.
+
+## Installation
+
+Currently Metlo ingestor for Python supports 2 servers :
+
+- Django
+- Flask
+
+It can be installed from `pypi` by running :
+
+```shell
+pip install metlo-python
+```
+
+If for some reason, a more manual method is required, then the code is available
+on [github](https://github.com/metlo-labs/metlo/tree/develop/ingestors/python)
+
+## Configuration
+
+### Django
+
+Once installed, METLO middleware can be added simply by modifying middlewares list (in the projects `settings.py`) like
+so :
+
+```python
+MIDDLEWARE = [
+    "......",
+    ...,
+    "......",
+    "metlo.django.middleware.Middleware",
+] 
+```
+
+and configuring a `METLO_CONFIG` attribute in the projects `settings.py` like this :
+
+```python
+METLO_CONFIG = {
+    "API_KEY": "<API-KEY-GOES-HERE>",
+    "METLO_HOST": "<METLO-COLLECTOR-URL>"
+}
+```
+
+`METLO_CONFIG` can take an optional key-value pair representing the max number of workers for communicating with METLO.
+
+### Flask
+
+Once installed, METLO middleware can be added simply like :
+
+```python
+from flask import Flask
+
+...
+from metlo.flask.middleware import FlaskMiddleware
+
+app = Flask(__name__)
+FlaskMiddleware(app, "<METLO-COLLECTOR-URL>", "<API-KEY-GOES-HERE>")
+```
+
+The Flask Middleware takes the flask app, METLO collector url, and the METLO API Key as parameters. As an optional
+parameter, a named value can be passed for max number of workers for communicating with METLO.
+
+```python
+FlaskMiddleware(app, "<METLO-COLLECTOR-URL>", "<API-KEY-GOES-HERE>", workers="<WORKER-COUNT>")
+```
+
+## Building wheels
+
+To start with, we need to install setuptools build, which can be done by :
+
+```shell
+pip install -q build
+```
+
+Followed by
+
+```shell
+python -m build
+```
+
+This should be done in the root directory (so right here)
+The build will be generated in the `dist/` folder.

ingestors/python/metlo/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"

ingestors/python/metlo/setup.cfg

@@ -0,0 +1,41 @@
+[metadata]
+name = metlo_python
+version = 0.1.0
+description = metlo-python is a middleware for python servers to ingest request traces into Metlo API Security
+readme = "README.md"
+repository = "https://github.com/metlo-labs/metlo"
+url = https://www.metlo.com
+license = MIT
+project_urls =
+    Documentation =https://docs.metlo.com/docs/python
+    Source Code = https://github.com/metlo-labs/metlo/
+    Issue Tracker = https://github.com/metlo-labs/metlo/issues/
+    Twitter =https://mobile.twitter.com/metlohq
+    Chat = https://discord.gg/gasqPDKEAC
+classifiers =
+    Development Status :: 3 - Alpha,
+    Intended Audience :: Developers,
+    Operating System :: OS Independent,
+    Topic :: Security :: Libraries :: Python Modules,
+    Programming Language :: Python :: 3,
+    Programming Language :: Python :: 3.4,
+    Programming Language :: Python :: 3.5,
+    Programming Language :: Python :: 3.6,
+    Programming Language :: Python :: 3.7,
+    Programming Language :: Python :: 3.8,
+    Programming Language :: Python :: 3.9,
+
+
+# Keywords description https://python-poetry.org/docs/pyproject/#keywords
+keywords = []  #! Update me
+
+
+[options]
+python_requires = >= 3.6
+package_dir = = src
+include_package_data = True
+packages =
+    metlo_python
+
+[options.packages.find]
+where = src

ingestors/python/metlo/src/__init__.py

@@ -0,0 +1,2 @@
+from .metlo_python.flask import Middleware as FlaskMiddleware
+from .metlo_python.django import Middleware as DjangoMiddleware

ingestors/python/metlo/src/metlo_python/django.py

@@ -0,0 +1,77 @@
+import json
+from concurrent.futures import ThreadPoolExecutor
+from urllib.request import Request, urlopen
+
+from django.conf import settings
+
+
+class Middleware(object):
+
+    def perform_request(self, data):
+        urlopen(
+            url=self.saved_request,
+            data=json.dumps(data).encode('utf-8')
+        )
+
+    def __init__(self, get_response):
+        """
+        Middleware for Django to communicate with METLO
+        :param get_response: Automatically populated by django
+        """
+        self.get_response = get_response
+        self.pool = ThreadPoolExecutor(max_workers=settings.METLO_CONFIG.get("workers", 4))
+
+        assert settings.METLO_CONFIG.get("METLO_HOST") is not None, "METLO_CONFIG is missing METLO_HOST attribute"
+        assert settings.METLO_CONFIG.get("API_KEY") is not None, "METLO_CONFIG is missing API_KEY attribute"
+
+        self.host = settings.METLO_CONFIG["METLO_HOST"]
+        self.key = settings.METLO_CONFIG["API_KEY"]
+        self.saved_request = Request(
+            url=self.host,
+            headers={
+                "Content-Type": "application/json; charset=utf-8",
+                "Authorization": self.key,
+            },
+            method="POST"
+        )
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        params = request.GET if request.method == "GET" else request.POST
+        dest_ip = request.META.get("SERVER_NAME") if \
+            "1.0.0.127.in-addr.arpa" not in request.META.get("SERVER_NAME") else "localhost"
+        src_ip = request.META.get("REMOTE_ADDR") if \
+            "1.0.0.127.in-addr.arpa" not in request.META.get("REMOTE_ADDR") else "localhost"
+        source_port = request.environ["wsgi.input"].stream.raw._sock.getpeername()[1]
+        res_body = response.content.decode("utf-8")
+        data = {
+            "request": {
+                "url": {
+                    "host": request._current_scheme_host if request._current_scheme_host else src_ip,
+                    "path": request.path,
+                    "parameters": list(map(lambda x: {"name": x[0], "value": x[1]}, params.items())),
+                },
+                "headers": dict(request.headers),
+                "body": request.body.decode("utf-8"),
+                "method": request.method,
+            },
+            "response": {
+                "url": f"{dest_ip}:{request.META.get('SERVER_PORT')}",
+                "status": response.status_code,
+                "headers": dict(response.headers),
+                "body": res_body,
+            },
+            "meta": {
+                "environment": "production",
+                "incoming": True,
+                "source": src_ip,
+                "sourcePort": source_port,
+                "destination": dest_ip,
+                "destinationPort": request.META.get('SERVER_PORT'),
+            }
+        }
+        self.pool.submit(self.perform_request, data=data)
+        return response
+
+    def process_exception(self, request, exception):
+        return None

ingestors/python/metlo/src/metlo_python/flask.py

@@ -0,0 +1,70 @@
+import json
+from concurrent.futures import ThreadPoolExecutor
+from urllib.request import Request, urlopen
+
+from flask import request
+
+
+class Middleware:
+    def perform_request(self, data):
+        urlopen(
+            url=self.saved_request,
+            data=json.dumps(data).encode('utf-8')
+        )
+
+    def __init__(self, app, metlo_host: str, metlo_api_key: str, **kwargs):
+        """
+        :param app: Instance of Flask app
+        :param metlo_host: Publicly accessible address of Metlo Collector
+        :param metlo_api_key: Metlo API Key
+        :param kwargs: optional parameter containing worker count for communicating with metlo
+        """
+        self.app = app
+        self.pool = ThreadPoolExecutor(max_workers=kwargs.get("workers", 4))
+
+        assert metlo_host is not None, "METLO for FLASK __init__ is missing metlo_host parameter"
+        assert metlo_api_key is not None, "METLO for FLASK __init__ is missing metlo_api_key parameter"
+
+        self.host = metlo_host
+        self.key = metlo_api_key
+        self.saved_request = Request(
+            url=self.host,
+            headers={
+                "Content-Type": "application/json; charset=utf-8",
+                "Authorization": self.key,
+            },
+            method="POST"
+        )
+
+        @app.after_request
+        def function(response, *args, **kwargs):
+            dst_host = request.environ.get("HTTP_HOST") or request.environ.get(
+                "HTTP_X_FORWARDED_FOR") or request.environ.get("REMOTE_ADDR")
+            data = {
+                "request": {
+                    "url": {
+                        "host": dst_host,
+                        "path": request.path,
+                        "parameters": list(map(lambda x: {"name": x[0], "value": x[1]}, request.args.items())),
+                    },
+                    "headers": dict(request.headers),
+                    "body": request.data.decode("utf-8"),
+                    "method": request.method,
+                },
+                "response": {
+                    "url": f"{request.environ.get('SERVER_NAME')}:{request.environ.get('SERVER_PORT')}",
+                    "status": response.status_code,
+                    "headers": dict(response.headers),
+                    "body": response.data.decode("utf-8"),
+                },
+                "meta": {
+                    "environment": "production",
+                    "incoming": True,
+                    "source": request.environ.get("HTTP_X_FORWARDED_FOR") or request.environ.get("REMOTE_ADDR"),
+                    "sourcePort": request.environ.get("REMOTE_PORT"),
+                    "destination": request.environ.get("SERVER_NAME"),
+                    "destinationPort": request.environ.get('SERVER_PORT'),
+                }
+            }
+            self.pool.submit(self.perform_request, data=data)
+            return response