more tests

mitmaddon/networkthread.py

@@ -71,11 +71,13 @@ class bRequest:
         self.timestamp_end = flow["request"]["timestamp_end"]
 
         self.headers = []
-        for k, v in flow["request"]["headers"]:
+        for k, v in flow["request"].get("headers", {}):
             self.headers.append(bHeader(str(k), str(v)))
 
     def json(self) -> dict:
-        return {}
+        state = vars(self).copy()
+        state["headers"] = {h.key: h.value for h in self.headers}
+        return state
 
 
 @dataclass
@@ -99,11 +101,13 @@ class bResponse:
         self.timestamp_end = flow["response"]["timestamp_end"]
 
         self.headers = []
-        for k, v in flow["response"]["headers"]:
+        for k, v in flow["response"].get("headers", {}):
             self.headers.append(bHeader(k, v))
 
     def json(self) -> dict:
-        return {}
+        state = vars(self).copy()
+        state["headers"] = [{h.key: h.value} for h in self.headers]
+        return state
 
 
 @dataclass
@@ -133,18 +137,13 @@ class bPacket:
     flowid: str
     data: str
 
-    def __init__(self, json: Dict):
-        self.ptype = json["type"]
-        self.flowid = str(json["id"])
-        self.data = json["data"]
-
 
 @dataclass
 class FlowItem:
     state: bFlowState
     flow: Flow
     time: float = 0
-    retries_left: int = 5
+    retries: int = 5
 
 
 """
@@ -187,7 +186,7 @@ class NetworkThread(threading.Thread):
                 # get new self.flows that may occured
                 i, flowitem = self.q.get(block=False)
                 if self.flows.get(i, None):
-                    print(f"flow {i} doubled? ignoring...")
+                    raise ValueError(f"flow {i} doubled? ignoring...")
                     continue
                 else:
                     self.flows[i] = flowitem
@@ -199,40 +198,63 @@ class NetworkThread(threading.Thread):
         msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
         self.send(msg)
 
-    def send_http_request(self, id: int, request: bRequest):
-        pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
-        self.send_packet(pkg)
-        self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
-        self.flows[id].time = time.monotonic()
-
-    def send_http_response(self, id: int, response: bResponse):
-        pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
-        self.send_packet(pkg)
-        self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
-        self.flows[id].time = time.monotonic()
-
     # update all current self.flows
     # handles the state machine for each flow
     def update_flows(self):
-        for id, flow in self.flows.items():
+        # force copy of item list, so we can remove dict items in the loop
+        for id, flow in list(self.flows.items()):
             if self.flows[id].retries <= 0:
+                if self.flows[id].flow:
                     self.flows[id].flow.kill()
                 print(f"http flow {id} timed out! flow killed.")
+                del self.flows[id]
 
             delta = time.monotonic() - self.flows[id].time
 
             if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
                     flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
-                self.send_http_request(id, bRequest(flow.flow))
+                pkg = bPacket(bPacketType.HTTP_REQUEST, id, bRequest(flow.flow.get_state()).json())
+                self.send_packet(pkg)
+
+                self.flows[id].time = time.monotonic()
+                self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
                 self.flows[id].retries -= 1
 
-            if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
+            elif flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
                     flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
-                self.send_http_response(id, bResponse(flow.flow))
+                pkg = bPacket(bPacketType.HTTP_RESPONSE, id, bResponse(flow.flow.get_state()).json())
+                self.send_packet(pkg)
+
+                self.flows[id].time = time.monotonic()
+                self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
                 self.flows[id].retries -= 1
 
-            elif flow.state == bFlowState.ERROR:
+            if flow.state == bFlowState.ERROR:
                 print(f"error in flow {id}!")
+                del self.flows[id]
+
+    def handle_packet(self, pkg):
+        flow = self.flows.get(pkg.flowid, None)
+
+        # flow ACKed
+        if pkg.ptype == bPacketType.ACK:
+            if flow and flow.flow:
+                flow.flow.resume()
+            else:
+                raise ValueError("unknown flow")
+
+        # flow killed
+        elif pkg.ptype == bPacketType.KILL:
+            if flow and flow.flow:
+                flow.flow.kill()
+            else:
+                raise ValueError("unknown flow")
+
+            if flow:
+                del self.flows[pkg.flowid]
+
+        else:
+            print(f"got unexpected message {pkg.ptype}")
 
     # handle incoming packets / update the statemachine
     def handle_packets(self):
@@ -242,14 +264,7 @@ class NetworkThread(threading.Thread):
                 if msg:
                     result = json.loads(str(msg))
                     pkg = bPacket(json=result)
-                    # flow ACKed
+                    self.handle_packet(pkg)
-                    if pkg.ptype == bPacketType.ACK:
-                        continue
-                    # flow killed
-                    elif pkg.ptype == bPacketType.KILL:
-                        continue
-                    else:
-                        print(f"got unexpected message {pkg.ptype}")
             except json.JSONDecodeError:
                 print(f"malformed message received {msg}")
 

mitmaddon/test_bigsnitch.py

@@ -2,8 +2,10 @@
 
 import pdb
 import queue
-
+import uuid
 import pytest
+from mitmproxy.http import HTTPFlow, HTTPRequest, HTTPResponse
+from mitmproxy.connections import ClientConnection, ServerConnection
 from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
 import os
 import tempfile
@@ -13,22 +15,64 @@ import zmq
 from deepdiff import DeepDiff
 
 
+@pytest.fixture
+def flowstate_clientconn():
+    return dict(
+        id=str(uuid.uuid4()),
+        address=("address", 22),
+        source_address=("address", 22),
+        ip_address=("192.168.0.1", 22),
+        timestamp_start=946681202,
+        timestamp_tcp_setup=946681203,
+        timestamp_tls_setup=946681204,
+        timestamp_end=946681205,
+        tls_established=True,
+        sni="address",
+        alpn=None,
+        tls_version="TLSv1.2",
+        via=None,
+        state=0,
+        error=None,
+        tls=False,
+        certificate_list=[],
+        alpn_offers=[],
+        cipher_name=None,
+        cipher_list=[],
+        via2=None,
+    )
+
+
+@pytest.fixture
+def flowstate_serverconn():
+    return dict(
+        id=str(uuid.uuid4()),
+        address=("address", 22),
+        source_address=("address", 22),
+        ip_address=("192.168.0.1", 22),
+        timestamp_start=946681202,
+        timestamp_tcp_setup=946681203,
+        timestamp_tls_setup=946681204,
+        timestamp_end=946681205,
+        tls_established=True,
+        sni="address",
+        alpn=None,
+        tls_version="TLSv1.2",
+        via=None,
+        state=0,
+        error=None,
+        tls=False,
+        certificate_list=[],
+        alpn_offers=[],
+        cipher_name=None,
+        cipher_list=[],
+        via2=None,
+    )
+
+
 # usual flow state of the request with some big parts removed
 @pytest.fixture
-def flowstate_request():
+def flowstate_request(flowstate_clientconn, flowstate_serverconn):
-    return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0),
+    return {'client_conn': flowstate_clientconn,
-                            'alpn_proto_negotiated': b'http/1.1',
-                            'cipher_name': 'TLS_AES_256_GCM_SHA384',
-                            'clientcert': None,
-                            'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea',
-                            'mitmcert': '',
-                            'sni': 'yolo.jetzt',
-                            'timestamp_end': None,
-                            'timestamp_start': 1619390481.8003347,
-                            'timestamp_tls_setup': 1619390482.6879823,
-                            'tls_established': True,
-                            'tls_extensions': [],
-                            'tls_version': 'TLSv1.3'},
             'error': None,
             'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
             'intercepted': False,
@@ -36,8 +80,7 @@ def flowstate_request():
             'marked': False,
             'metadata': {},
             'mode': 'transparent',
-            'request': {'authority': b'',
+            'request': {'content': b'',
-                        'content': b'',
                         'headers': ((b'Host', b'yolo.jetzt'),
                                     (b'User-Agent', b'curl/7.75.0'),
                                     (b'Accept', b'*/*')),
@@ -49,41 +92,27 @@ def flowstate_request():
                         'scheme': b'https',
                         'timestamp_end': 1619390482.69,
                         'timestamp_start': 1619390482.6886377,
-                        'trailers': None},
+                        'authority': '',
+                        'trailers': '',
+                        },
             'response': None,
-            'server_conn': {'address': ('yolo.jetzt', 443),
+            "server_conn": flowstate_serverconn,
-                            'alpn_proto_negotiated': b'http/1.1',
-                            'cert': '',
-                            'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b',
-                            'ip_address': ('95.156.226.69', 443),
-                            'sni': 'yolo.jetzt',
-                            'source_address': ('192.168.42.182', 51514),
-                            'timestamp_end': None,
-                            'timestamp_start': 1619390481.8154442,
-                            'timestamp_tcp_setup': 1619390481.994565,
-                            'timestamp_tls_setup': 1619390482.6819758,
-                            'tls_established': True,
-                            'tls_version': 'TLSv1.2',
-                            'via': None},
             'type': 'http',
             'version': 9}
 
 
+@pytest.fixture
+def flow_request(flowstate_request):
+    c = ClientConnection.make_dummy("192.168.0.2")
+    s = ServerConnection.make_dummy("192.168.0.1")
+    flow = HTTPFlow(c, s)
+    flow.request = HTTPRequest.from_state(flowstate_request["request"])
+    return flow
+
 @pytest.fixture()
-def flowstate_response():
+def flowstate_response(flowstate_clientconn, flowstate_serverconn):
-    return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
+    return {
-                            'alpn_proto_negotiated': b'http/1.1',
+        "client_conn": flowstate_clientconn,
-                            'cipher_name': 'TLS_AES_256_GCM_SHA384',
-                            'clientcert': None,
-                            'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
-                            'mitmcert': '',
-                            'sni': 'yolo.jetzt',
-                            'timestamp_end': None,
-                            'timestamp_start': 1619461916.6160116,
-                            'timestamp_tls_setup': 1619461916.7581937,
-                            'tls_established': True,
-                            'tls_extensions': [],
-                            'tls_version': 'TLSv1.3'},
         'error': None,
         'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
         'intercepted': True,
@@ -181,24 +210,20 @@ def flowstate_response():
                      'timestamp_end': 1619461916.7979567,
                      'timestamp_start': 1619461916.7935555,
                      'trailers': None},
-            'server_conn': {'address': ('yolo.jetzt', 443),
+        'server_conn': flowstate_serverconn,
-                            'alpn_proto_negotiated': b'http/1.1',
-                            'cert': '',
-                            'id': '64c838e2-ab74-49c4-9ebe-70e8d17e9081',
-                            'ip_address': ('95.156.226.69', 443),
-                            'sni': 'yolo.jetzt',
-                            'source_address': ('10.23.42.102', 43399),
-                            'timestamp_end': None,
-                            'timestamp_start': 1619461916.63117,
-                            'timestamp_tcp_setup': 1619461916.6659565,
-                            'timestamp_tls_setup': 1619461916.7522914,
-                            'tls_established': True,
-                            'tls_version': 'TLSv1.2',
-                            'via': None},
         'type': 'http',
         'version': 9}
 
 
+@pytest.fixture
+def flow_response(flowstate_request, flowstate_response):
+    c = ClientConnection.make_dummy("192.168.0.2")
+    s = ServerConnection.make_dummy("192.168.0.1")
+    flow = HTTPFlow(c, s)
+    flow.request = HTTPRequest.from_state(flowstate_request["request"])
+    flow.response = HTTPResponse.from_state(flowstate_response["response"])
+    return flow
+
 class MitmAddonTestServer:
     def __init__(self, queue, path: str):
         self.queue = queue
@@ -235,6 +260,15 @@ def client_server():
     server.disconnect()
 
 
+@pytest.fixture
+def client_no_start():
+    queue = Queue()
+    sock = os.path.join(tempfile.mkdtemp(), "bigsnitchtest.sock")
+    sock = f"ipc://{sock}"
+    client = NetworkThread(name="testthread",queue=queue,path=sock)
+    return queue, client
+
+
 class TestBigSnitchWrapper:
     def test_request_convert(self, flowstate_request):
         req = bRequest(flow=flowstate_request)
@@ -247,7 +281,7 @@ class TestBigSnitchWrapper:
              'method': 'GET',
              'port': 443,
              'scheme': 'https',
-             'server_ip_address': '95.156.226.69',
+             'server_ip_address': '192.168.0.1',
              'timestamp_end': 1619390482.69,
              'timestamp_start': 1619390482.6886377,
              'tls': True}
@@ -405,6 +439,179 @@ class TestBigSnitchWrapper:
 
         assert not DeepDiff(res.__dict__, d)
 
+    def test_convert_request_to_json(self, flowstate_request):
+        j = bRequest(flowstate_request).json()
+        d = {'content': '',
+             'headers': {'Accept': '*/*',
+                         'Host': 'yolo.jetzt',
+                         'User-Agent': 'curl/7.75.0'},
+             'host': 'yolo.jetzt',
+             'http_version': 'HTTP/1.1',
+             'method': 'GET',
+             'port': 443,
+             'scheme': 'https',
+             'server_ip_address': '192.168.0.1',
+             'timestamp_end': 1619390482.69,
+             'timestamp_start': 1619390482.6886377,
+             'tls': True}
+
+        assert not DeepDiff(j, d)
+
+    def test_convert_response_to_json(self, flowstate_response):
+        j = bResponse(flowstate_response).json()
+        d = {'content': '<!DOCTYPE html">\n'
+                        '<html>\n'
+                        '<head>  \n'
+                        '<meta http-equiv="content-type" '
+                        'content="text/html;charset=ISO-8859-1">\n'
+                        '<title>todays yolo - 3026</title>  \n'
+                        '<style type="text/css" media="screen">\n'
+                        '<!--\n'
+                        'body \n'
+                        '{\n'
+                        'color: white;\n'
+                        'background-color: #003;\n'
+                        'margin: 0px\n'
+                        '}\n'
+                        '\n'
+                        '.yolo\n'
+                        '{\n'
+                        'font-family: Verdana, Geneva, Arial, sans-serif;\n'
+                        'font-weight: bold;\n'
+                        'font-size: 3em;\n'
+                        '}\n'
+                        '\n'
+                        'a:visited {\n'
+                        '\tcolor: blue\n'
+                        '}\n'
+                        '\n'
+                        '#content        \n'
+                        '{\n'
+                        'font-family: Verdana, Geneva, Arial, sans-serif;\n'
+                        'color: white;\n'
+                        'background-color: transparent;\n'
+                        'text-align: center;\n'
+                        'font-size: 2em;\n'
+                        'position: absolute;\n'
+                        'top: 50%;\n'
+                        'left: 50%;\n'
+                        '-webkit-transform: translateX(-50%) translateY(-50%);\n'
+                        '-moz-transform: translateX(-50%) translateY(-50%);\n'
+                        '-ms-transform: translateX(-50%) translateY(-50%);\n'
+                        'transform: translateX(-50%) translateY(-50%);\n'
+                        'overflow: visible;\n'
+                        'visibility: visible;\n'
+                        'display: block\n'
+                        '}\n'
+                        '\n'
+                        '#sponsors        \n'
+                        '{\n'
+                        'font-family: Verdana, Geneva, Arial, sans-serif;\n'
+                        'color: white;\n'
+                        'background-color: transparent;\n'
+                        'text-align: center;\n'
+                        'font-size: 1em;\n'
+                        'position: absolute;\n'
+                        'top: 90%;\n'
+                        'left: 50%;\n'
+                        '-webkit-transform: translateX(-50%) translateY(-50%);\n'
+                        '-moz-transform: translateX(-50%) translateY(-50%);\n'
+                        '-ms-transform: translateX(-50%) translateY(-50%);\n'
+                        'transform: translateX(-50%) translateY(-50%);\n'
+                        'overflow: visible;\n'
+                        'visibility: visible;\n'
+                        'display: block\n'
+                        '}\n'
+                        '\n'
+                        '#progress\n'
+                        '{\n'
+                        '\tbackground-color: #f08;\n'
+                        '\tposition: absolute;\n'
+                        '\tleft: 0px;\n'
+                        '\ttop: 0px;\n'
+                        '\theight: 10px;\n'
+                        '}\n'
+                        '\n'
+                        '-->\n'
+                        '</style>\n'
+                        '</head>\n'
+                        '<body>\n'
+                        '<div id="progress" class="progressBarDiv"></div>\n'
+                        '<div id="content">\n'
+                        'the yolo for today is<br>\n'
+                        '<span class="yolo" id="yoloValue" '
+                        'title="YOLOVALUE">3026</span><br>\n'
+                        '<br>\n'
+                        '</div>\n'
+                        '<div id="sponsors">\n'
+                        '\t<a href="https://cat.yolo.jetzt">Cat</a>\n'
+                        '\t<br>\n'
+                        '\tRegulation (EU) 2016/679 compliant\n'
+                        '</div>\n'
+                        '<script type="text/javascript">\n'
+                        'var currentYoloTS = 1619388000000; //Multiply by 1000 the YOLO '
+                        'way\n'
+                        'var tDiff = Date.now() - currentYoloTS;\n'
+                        '\n'
+                        'var dayMS = 24 * 60 * 60 * 1000;\n'
+                        '\n'
+                        'console.debug(currentYoloTS);\n'
+                        'console.debug(Date.now() - currentYoloTS);\n'
+                        'function checkTimestamps()\n'
+                        '{\n'
+                        '\ttDiff = Date.now() - currentYoloTS;\n'
+                        '\tupdateUi();\n'
+                        '\tif(tDiff > dayMS)\n'
+                        '\t{\n'
+                        '\t\tclearInterval(updateInterval);\n'
+                        '\t\tlocation.reload();\t\n'
+                        '\t}\n'
+                        '}\n'
+                        '\n'
+                        'function updateUi()\n'
+                        '{\n'
+                        '\tvalPercent = tDiff / dayMS * 100;\n'
+                        '\tif(valPercent > 100) valPercent = 100;\n'
+                        '\tdocument.getElementById("progress").style = "width: " + '
+                        'valPercent + "%;";\n'
+                        '\tif(tDiff > dayMS)\n'
+                        '\t{\n'
+                        '\t\tdocument.getElementById("yoloValue").innerHTML = "&#xfffd;";\n'
+                        '\t}\n'
+                        '}\n'
+                        '\n'
+                        'updateUi();\n'
+                        'var updateInterval = setInterval(checkTimestamps, 5000);\n'
+                        '\n'
+                        '//YOLODATE\n'
+                        '</script>\n'
+                        '</body>\n'
+                        '</html>\n',
+             'headers': [{'Server': 'nginx'},
+                         {'Date': 'Mon, 26 Apr 2021 18:31:56 GMT'},
+                         {'Content-Type': 'text/html'},
+                         {'Content-Length': '2460'},
+                         {'Last-Modified': 'Sun, 25 Apr 2021 22:00:00 GMT'},
+                         {'Connection': 'keep-alive'},
+                         {'ETag': '"6085e660-99c"'},
+                         {'Strict-Transport-Security': 'max-age=31536000; '
+                                                       'includeSubDomains; preload'},
+                         {'X-Xss-Protection': '1; mode=block'},
+                         {'X-Content-Type-Options': 'nosniff'},
+                         {'Content-Security-Policy': "default-src 'self'; script-src "
+                                                     "'self' 'unsafe-inline'; connect-src "
+                                                     "'self'; img-src 'self'; style-src "
+                                                     "'self' 'unsafe-inline';"},
+                         {'X-Frame-Options': 'SAMEORIGIN'},
+                         {'Referrer-Policy': 'no-referrer'},
+                         {'Accept-Ranges': 'bytes'}],
+             'http_version': 'HTTP/1.1',
+             'reason': 'OK',
+             'status_code': 200,
+             'timestamp_end': 1619461916.7979567,
+             'timestamp_start': 1619461916.7935555}
+
+        assert not DeepDiff(j, d)
 
 class TestMitmAddon:
     def test_get_new_flows_empty(self, client_server):
@@ -419,8 +626,37 @@ class TestMitmAddon:
         assert queue.empty()
         assert not len(client.flows)
 
-    def test_get_new_flows_single(self, client_server):
+    def test_get_new_flows_single(self, client_no_start, flow_request):
-        queue, client, server = client_server
+        queue, client = client_no_start
+
+        flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request)
+        queue.put_nowait((flow_request.id, flowitem))
+
+        client.get_new_flows()
+
+        assert queue.empty()
+        assert len(client.flows) == 1
+
+        assert flow_request.id in client.flows.keys()
+        assert not DeepDiff(flowitem, client.flows[flow_request.id])
+
+    def test_get_new_flows_double(self, client_no_start, flow_request, flow_response):
+        queue, client = client_no_start
+
+        flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request)
+        queue.put_nowait((flow_request.id, flowitem))
+        flowitem2 = FlowItem(bFlowState.UNSENT_HTTP_RESPONSE, flow_response)
+        queue.put_nowait((flow_response.id, flowitem2))
+
+        client.get_new_flows()
+
+        assert queue.empty()
+        assert len(client.flows) == 2
+
+        assert flow_request.id in client.flows.keys()
+        assert not DeepDiff(flowitem, client.flows[flow_request.id])
+        assert flow_response.id in client.flows.keys()
+        assert not DeepDiff(flowitem2, client.flows[flow_response.id])
 
     def test_request(self, client_server):
         queue, client, server = client_server