The final piece: netlib -> mitproxy.net

.appveyor.yml

@@ -23,7 +23,7 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
+  - ps: "tox -- --cov mitmproxy --cov pathod -v"
 
 deploy_script:
   ps: |

.travis.yml

@@ -49,7 +49,7 @@ install:
     fi
   - pip install tox
 
-script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
+script: tox -- --cov mitmproxy --cov pathod -v
 
 after_success:
   - |

MANIFEST.in

@@ -1,4 +1,3 @@
 graft mitmproxy
 graft pathod
-graft netlib
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,8 +3,7 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as
-their shared networking library, **netlib**.
+This repository contains the **mitmproxy** and **pathod** projects.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
@@ -64,7 +63,7 @@ virtualenv_ installed (you can find installation instructions for virtualenv
 
 The *dev* script will create a virtualenv environment in a directory called
 "venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
+primary mitmproxy components - mitmproxy and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
@@ -144,7 +143,7 @@ PR checks will fail and block merging. We are using this command to check for st
 
 .. code-block:: text
 
-    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
+    flake8 --jobs 8 --count mitmproxy pathod examples test
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg

docs/conf.py

@@ -231,10 +231,7 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("netlib") > -1:
-        off = spath.rfind("netlib")
-        mpath = spath[off:]
-    elif spath.rfind("mitmproxy") > -1:
+    if spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/dev/testing.rst

@@ -10,7 +10,7 @@ suitable extension to the test suite.
 Our tests are written for the `py.test`_ or nose_ test frameworks.
 At the point where you send your pull request, a command like this:
 
->>> py.test --cov mitmproxy --cov netlib
+>>> py.test --cov mitmproxy
 
 Should give output something like this:
 

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
+** !_build/** ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

examples/har_dump.py

@@ -16,7 +16,7 @@
 
 from mitmproxy import version
 from mitmproxy.utils import strutils
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
 
 HAR = {}
 

mitmproxy/addons/stickycookie.py

@@ -1,7 +1,7 @@
 import collections
 from http import cookiejar
 
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
 
 from mitmproxy import exceptions
 from mitmproxy import flowfilter

mitmproxy/addons/streambodies.py

@@ -1,4 +1,4 @@
-from netlib.http import http1
+from mitmproxy.net.http import http1
 from mitmproxy import exceptions
 from mitmproxy import ctx
 

mitmproxy/addons/wsgiapp.py

@@ -1,7 +1,7 @@
 from mitmproxy import ctx
 from mitmproxy import exceptions
 
-from netlib import wsgi
+from mitmproxy.net import wsgi
 from mitmproxy import version
 
 

mitmproxy/certs.py

@@ -193,7 +193,7 @@ def expire(self, entry):
     @staticmethod
     def load_dhparam(path):
 
-        # netlib<=0.10 doesn't generate a dhparam file.
+        # mitmproxy<=0.10 doesn't generate a dhparam file.
         # Create it now if neccessary.
         if not os.path.exists(path):
             with open(path, "wb") as f:

mitmproxy/connections.py

@@ -5,7 +5,7 @@
 
 from mitmproxy import stateobject
 from mitmproxy import certs
-from netlib import tcp
+from mitmproxy.net import tcp
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):

mitmproxy/contentviews.py

@@ -33,10 +33,10 @@
 from PIL import Image
 from mitmproxy import exceptions
 from mitmproxy.contrib.wbxml import ASCommandResponse
-from netlib import http
+from mitmproxy.net import http
 from mitmproxy.types import multidict
 from mitmproxy.utils import strutils
-from netlib.http import url
+from mitmproxy.net.http import url
 
 try:
     import pyamf

mitmproxy/exceptions.py

@@ -112,7 +112,7 @@ class AddonHalt(MitmproxyException):
 
 class NetlibException(MitmproxyException):
     """
-    Base class for all exceptions thrown by netlib.
+    Base class for all exceptions thrown by mitmproxy.net.
     """
     def __init__(self, message=None):
         super().__init__(message)

mitmproxy/export.py

@@ -3,7 +3,7 @@
 import textwrap
 import urllib
 
-import netlib.http
+import mitmproxy.net.http
 
 
 def _native(s):
@@ -89,9 +89,9 @@ def python_code(flow):
     return code
 
 
-def is_json(headers: netlib.http.Headers, content: bytes) -> bool:
+def is_json(headers: mitmproxy.net.http.Headers, content: bytes) -> bool:
     if headers:
-        ct = netlib.http.parse_content_type(headers.get("content-type", ""))
+        ct = mitmproxy.net.http.parse_content_type(headers.get("content-type", ""))
         if ct and "%s/%s" % (ct[0], ct[1]) == "application/json":
             try:
                 return json.loads(content.decode("utf8", "surrogateescape"))

mitmproxy/http.py

@@ -1,9 +1,9 @@
 import cgi
 
 from mitmproxy import flow
-from netlib import http
+from mitmproxy.net import http
 from mitmproxy import version
-from netlib import tcp
+from mitmproxy.net import tcp
 
 
 class HTTPRequest(http.Request):
@@ -12,7 +12,7 @@ class HTTPRequest(http.Request):
     A mitmproxy HTTP request.
     """
 
-    # This is a very thin wrapper on top of :py:class:`netlib.http.Request` and
+    # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Request` and
     # may be removed in the future.
 
     def __init__(
@@ -73,7 +73,7 @@ def set_state(self, state):
     @classmethod
     def wrap(self, request):
         """
-        Wraps an existing :py:class:`netlib.http.Request`.
+        Wraps an existing :py:class:`mitmproxy.net.http.Request`.
         """
         req = HTTPRequest(
             first_line_format=request.data.first_line_format,
@@ -99,7 +99,7 @@ class HTTPResponse(http.Response):
     """
     A mitmproxy HTTP response.
     """
-    # This is a very thin wrapper on top of :py:class:`netlib.http.Response` and
+    # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and
     # may be removed in the future.
 
     def __init__(
@@ -131,7 +131,7 @@ def __init__(
     @classmethod
     def wrap(self, response):
         """
-        Wraps an existing :py:class:`netlib.http.Response`.
+        Wraps an existing :py:class:`mitmproxy.net.http.Response`.
         """
         resp = HTTPResponse(
             http_version=response.data.http_version,

mitmproxy/master.py

@@ -15,7 +15,7 @@
 from mitmproxy import io
 from mitmproxy.proxy.protocol import http_replay
 from mitmproxy.types import basethread
-import netlib.http
+import mitmproxy.net.http
 
 from . import ctx as mitmproxy_ctx
 
@@ -122,7 +122,7 @@ def create_request(self, method, scheme, host, port, path):
         s = connections.ServerConnection.make_dummy((host, port))
 
         f = http.HTTPFlow(c, s)
-        headers = netlib.http.Headers()
+        headers = mitmproxy.net.http.Headers()
 
         req = http.HTTPRequest(
             "absolute",

mitmproxy/net/http/__init__.py

@@ -0,0 +1,15 @@
+from mitmproxy.net.http.request import Request
+from mitmproxy.net.http.response import Response
+from mitmproxy.net.http.message import Message
+from mitmproxy.net.http.headers import Headers, parse_content_type
+from mitmproxy.net.http.message import decoded
+from mitmproxy.net.http import http1, http2, status_codes, multipart
+
+__all__ = [
+    "Request",
+    "Response",
+    "Message",
+    "Headers", "parse_content_type",
+    "decoded",
+    "http1", "http2", "status_codes", "multipart",
+]

netlib/http/http1/assemble.py → mitmproxy/net/http/http1/assemble.py

@@ -1,4 +1,4 @@
-import netlib.http.url
+import mitmproxy.net.http.url
 from mitmproxy import exceptions
 
 
@@ -44,7 +44,7 @@ def assemble_body(headers, body_chunks):
 def _assemble_request_line(request_data):
     """
     Args:
-        request_data (netlib.http.request.RequestData)
+        request_data (mitmproxy.net.http.request.RequestData)
     """
     form = request_data.first_line_format
     if form == "relative":
@@ -76,11 +76,11 @@ def _assemble_request_line(request_data):
 def _assemble_request_headers(request_data):
     """
     Args:
-        request_data (netlib.http.request.RequestData)
+        request_data (mitmproxy.net.http.request.RequestData)
     """
     headers = request_data.headers.copy()
     if "host" not in headers and request_data.scheme and request_data.host and request_data.port:
-        headers["host"] = netlib.http.url.hostport(
+        headers["host"] = mitmproxy.net.http.url.hostport(
             request_data.scheme,
             request_data.host,
             request_data.port

netlib/http/http1/read.py → mitmproxy/net/http/http1/read.py

@@ -2,11 +2,11 @@
 import sys
 import re
 
-from netlib.http import request
-from netlib.http import response
-from netlib.http import headers
-from netlib.http import url
-from netlib import check
+from mitmproxy.net.http import request
+from mitmproxy.net.http import response
+from mitmproxy.net.http import headers
+from mitmproxy.net.http import url
+from mitmproxy.net import check
 from mitmproxy import exceptions
 
 

mitmproxy/net/http/http2/__init__.py

@@ -0,0 +1,8 @@
+from mitmproxy.net.http.http2.framereader import read_raw_frame, parse_frame
+from mitmproxy.net.http.http2.utils import parse_headers
+
+__all__ = [
+    "read_raw_frame",
+    "parse_frame",
+    "parse_headers",
+]

netlib/http/http2/utils.py → mitmproxy/net/http/http2/utils.py

@@ -1,4 +1,4 @@
-from netlib.http import url
+from mitmproxy.net.http import url
 
 
 def parse_headers(headers):

netlib/http/message.py → mitmproxy/net/http/message.py

@@ -3,9 +3,9 @@
 from typing import Optional
 
 from mitmproxy.utils import strutils
-from netlib.http import encoding
+from mitmproxy.net.http import encoding
 from mitmproxy.types import serializable
-from netlib.http import headers
+from mitmproxy.net.http import headers
 
 
 # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
@@ -69,7 +69,7 @@ def headers(self):
         Message headers object
 
         Returns:
-            netlib.http.Headers
+            mitmproxy.net.http.Headers
         """
         return self.data.headers
 

netlib/http/multipart.py → mitmproxy/net/http/multipart.py

@@ -1,6 +1,6 @@
 import re
 
-from netlib.http import headers
+from mitmproxy.net.http import headers
 
 
 def decode(hdrs, content):