more tests

mitmaddon/bigsnitch.py

@@ -3,18 +3,18 @@
 
 import pdb
 
-from queue import Queue, Empty
+from queue import Queue
 from networkthread import bFlowState, FlowItem, NetworkThread
 
+
 class BigSnitchBridge:
     def __init__(self):
         print("BigSnitchBridge started")
         self.q = Queue()
-        #self.thread = NetworkThread("network", self.q)
-        #self.thread.start()
+        self.thread = NetworkThread("network", self.q)
+        self.thread.start()
 
     def request(self, flow):
-        pdb.set_trace()
         flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
         self.q.put_nowait((flow.id, flowitem))
         # intercept until ACK received
@@ -27,7 +27,7 @@ class BigSnitchBridge:
         flow.intercept()
 
     def error(self, flow):
-        flowitem = FlowItem(bFlowState.ERROR, flow, time.monotonic())
+        flowitem = FlowItem(bFlowState.ERROR, flow)
         self.q.put_nowait((flow.id, flowitem))
 
 

mitmaddon/networkthread.py

@@ -1,6 +1,5 @@
 import pdb
 
-from mitmproxy import ctx
 from mitmproxy.flow import Flow
 import threading
 import time
@@ -9,12 +8,12 @@ import json
 import os
 from enum import Enum
 from dataclasses import dataclass
-from typing import List, Dict
+from typing import List, Dict, Any
 from queue import Queue, Empty
 
-"""
+
 # this method is used to convert flow states (generated with get_state()) to json
-def convert_to_strings(obj):
+def convert_to_strings(obj: Any) -> Any:
     if isinstance(obj, dict):
         return {convert_to_strings(key): convert_to_strings(value)
                 for key, value in obj.items()}
@@ -29,13 +28,14 @@ def convert_to_strings(obj):
         return data
 
     return obj
-"""
+
 
 @dataclass
 class bHeader:
     key: str
     value: str
 
+
 @dataclass
 class bRequest:
     server_ip_address: str
@@ -57,6 +57,8 @@ class bRequest:
 
     # init from flow dict
     def __init__(self, flow: dict):
+        flow = convert_to_strings(flow)
+
         self.server_ip_address = flow["server_conn"]["ip_address"][0]
         self.tls = flow["server_conn"]["tls_established"]
         self.content = flow["request"]["content"]
@@ -69,8 +71,12 @@ class bRequest:
         self.timestamp_end = flow["request"]["timestamp_end"]
 
         self.headers = []
-        for k,v in flow["request"]["headers"]:
-            self.headers.append(bHeader(k,v))
+        for k, v in flow["request"]["headers"]:
+            self.headers.append(bHeader(str(k), str(v)))
+
+    def json(self) -> dict:
+        return {}
+
 
 @dataclass
 class bResponse:
@@ -84,15 +90,21 @@ class bResponse:
     headers: List[bHeader]
 
     def __init__(self, flow: dict):
-        self.status_code = flow["status_code"]
-        self.http_version = flow["http_version"]
-        self.reason = flow["reason"]
-        self.content = flow["content"]
-        self.timestamp_start = flow["timestamp_start"]
-        self.timestamp_end = flow["timestamp_end"]
+        flow = convert_to_strings(flow)
+        self.status_code = flow["response"]["status_code"]
+        self.http_version = flow["response"]["http_version"]
+        self.reason = flow["response"]["reason"]
+        self.content = flow["response"]["content"]
+        self.timestamp_start = flow["response"]["timestamp_start"]
+        self.timestamp_end = flow["response"]["timestamp_end"]
+
+        self.headers = []
+        for k, v in flow["response"]["headers"]:
+            self.headers.append(bHeader(k, v))
+
+    def json(self) -> dict:
+        return {}
 
-        for k,v in flow["headers"]:
-            self.headers.append(bHeader(k,v))
 
 @dataclass
 class bFlowState(Enum):
@@ -102,6 +114,7 @@ class bFlowState(Enum):
     UNSENT_HTTP_RESPONSE = 3
     SENT_HTTP_RESPONSE = 4
 
+
 @dataclass
 class bPacketType:
     NACK = 0
@@ -113,6 +126,7 @@ class bPacketType:
     HTTP_REQUEST = 6
     HTTP_RESPONSE = 7
 
+
 @dataclass
 class bPacket:
     ptype: bPacketType
@@ -124,6 +138,7 @@ class bPacket:
         self.flowid = str(json["id"])
         self.data = json["data"]
 
+
 @dataclass
 class FlowItem:
     state: bFlowState
@@ -131,6 +146,7 @@ class FlowItem:
     time: float = 0
     retries_left: int = 5
 
+
 """
 
 The network thread communicates with the bigsnitch plugin using zeromq.
@@ -152,7 +168,7 @@ class NetworkThread(threading.Thread):
         self.q = queue
         # for zmq use
         self.context = zmq.Context()
-                # all current flows being handled by mitmproxy
+        # all current self.flows being handled by mitmproxy
         self.flows: Dict[FlowItem] = {}
         # timer for sending pings to check if the connection broke
         self.timer = time.monotonic()
@@ -161,20 +177,20 @@ class NetworkThread(threading.Thread):
 
     # send a single message, no checks involved
     def send(self, msg):
-            a = convert_to_strings(msg)
-            self.socket.send(str.encode(json.dumps(a)))
+        #a = convert_to_strings(msg)
+        self.socket.send(str.encode(json.dumps(msg)))
 
-        # add new flows from the queue
+    # add new self.flows from the queue
     def get_new_flows(self):
         while True:
             try:
-                    # get new flows that may occured
+                # get new self.flows that may occured
                 i, flowitem = self.q.get(block=False)
                 if self.flows.get(i, None):
                     print(f"flow {i} doubled? ignoring...")
                     continue
                 else:
-                        self.flows.append(flowitem)
+                    self.flows[i] = flowitem
 
             except Empty:
                 break
@@ -184,54 +200,38 @@ class NetworkThread(threading.Thread):
         self.send(msg)
 
     def send_http_request(self, id: int, request: bRequest):
-            pkg = bPacket(bPacketType.HTTP_REQUEST, id, request.json())
+        pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
         self.send_packet(pkg)
-            flows[id].state = bFlowState.SENT_HTTP_REQUEST
-            flows[id].time = time.monotonic()
+        self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
+        self.flows[id].time = time.monotonic()
 
     def send_http_response(self, id: int, response: bResponse):
-            pkg = bPacket(bPacketType.HTTP_RESPONSE, id, response.json())
+        pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
         self.send_packet(pkg)
-            flows[id].state = bFlowState.SENT_HTTP_RESPONSE
-            flows[id].time = time.monotonic()
+        self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
+        self.flows[id].time = time.monotonic()
 
-        # update all current flows
+    # update all current self.flows
     # handles the state machine for each flow
     def update_flows(self):
         for id, flow in self.flows.items():
-                # send the request if not sent
-                if state == bFlowState.UNSENT_HTTP_REQUEST:
-                    self.send_request(id, bRequest(flow.flow))
+            if self.flows[id].retries <= 0:
+                self.flows[id].flow.kill()
+                print(f"http flow {id} timed out! flow killed.")
 
-                elif state == bFlowState.SENT_HTTP_REQUEST:
-                    # check timer, try resend
-                    delta = time.monotonic() - flows[id].time
-                    if delta > 5:
-                        self.send_request(id, bRequest(flow.flow))
-                        flows[id].retries -= 1
-                        continue
+            delta = time.monotonic() - self.flows[id].time
 
-                    if flows[id].retries <= 0:
-                        flows[id].flow.kill()
-                        print(f"http request {id} timed out! flow killed.")
+            if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
+                    flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
+                self.send_http_request(id, bRequest(flow.flow))
+                self.flows[id].retries -= 1
 
-                # send the response if not sent
-                elif state == bFlowState.UNSENT_HTTP_RESPONSE:
-                    self.send_response(id, bResponse(flow.flow))
+            if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
+                    flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
+                self.send_http_response(id, bResponse(flow.flow))
+                self.flows[id].retries -= 1
 
-                elif state == bFlowState.SENT_HTTP_RESPONSE:
-                    # check timer, try resend
-                    delta = time.monotonic() - flows[id].time
-                    if delta > 5:
-                        self.send_response(id, bResponse(flow.flow))
-                        flows[id].retries -= 1
-                        continue
-
-                    if flows[id].retries <= 0:
-                        flows[id].flow.kill()
-                        print(f"http response {id} timedout! flow killed.")
-
-                elif state == bFlowState.ERROR:
+            elif flow.state == bFlowState.ERROR:
                 print(f"error in flow {id}!")
 
     # handle incoming packets / update the statemachine
@@ -240,7 +240,7 @@ class NetworkThread(threading.Thread):
             msg = self.socket.recv()
             try:
                 if msg:
-                        result = json.loads(msg)
+                    result = json.loads(str(msg))
                     pkg = bPacket(json=result)
                     # flow ACKed
                     if pkg.ptype == bPacketType.ACK:

mitmaddon/requirements.txt

@@ -3,3 +3,4 @@ mitmdump
 pyzmq
 deepdiff
 pytest
+tox

mitmaddon/test_bigsnitch.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
 import pdb
+import queue
 
 import pytest
-from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread
+from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
 import os
 import tempfile
 from queue import Queue
@@ -11,6 +12,7 @@ import zmq
 
 from deepdiff import DeepDiff
 
+
 # usual flow state of the request with some big parts removed
 @pytest.fixture
 def flowstate_request():
@@ -66,6 +68,137 @@ def flowstate_request():
             'type': 'http',
             'version': 9}
 
+
+@pytest.fixture()
+def flowstate_response():
+    return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
+                            'alpn_proto_negotiated': b'http/1.1',
+                            'cipher_name': 'TLS_AES_256_GCM_SHA384',
+                            'clientcert': None,
+                            'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
+                            'mitmcert': '',
+                            'sni': 'yolo.jetzt',
+                            'timestamp_end': None,
+                            'timestamp_start': 1619461916.6160116,
+                            'timestamp_tls_setup': 1619461916.7581937,
+                            'tls_established': True,
+                            'tls_extensions': [],
+                            'tls_version': 'TLSv1.3'},
+            'error': None,
+            'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
+            'intercepted': True,
+            'is_replay': None,
+            'marked': False,
+            'metadata': {},
+            'mode': 'transparent',
+            'request': {'authority': b'',
+                        'content': b'',
+                        'headers': ((b'Host', b'yolo.jetzt'),
+                                    (b'User-Agent', b'curl/7.75.0'),
+                                    (b'Accept', b'*/*')),
+                        'host': 'yolo.jetzt',
+                        'http_version': b'HTTP/1.1',
+                        'method': b'GET',
+                        'path': b'/',
+                        'port': 443,
+                        'scheme': b'https',
+                        'timestamp_end': 1619461916.7603076,
+                        'timestamp_start': 1619461916.7588415,
+                        'trailers': None},
+            'response': {'content': b'<!DOCTYPE html">\n<html>\n<head>  \n<meta http-equi'
+                                    b'v="content-type" content="text/html;charset=ISO-8859'
+                                    b'-1">\n<title>todays yolo - 3026</title>  \n<style '
+                                    b'type="text/css" media="screen">\n<!--\nbody \n{\ncol'
+                                    b'or: white;\nbackground-color: #003;\nmargin: 0px\n}'
+                                    b'\n\n.yolo\n{\nfont-family: Verdana, Geneva, Arial, s'
+                                    b'ans-serif;\nfont-weight: bold;\nfont-size: 3em;\n}\n'
+                                    b'\na:visited {\n\tcolor: blue\n}\n\n#content       '
+                                    b' \n{\nfont-family: Verdana, Geneva, Arial, sans-se'
+                                    b'rif;\ncolor: white;\nbackground-color: transparent'
+                                    b';\ntext-align: center;\nfont-size: 2em;\nposition: '
+                                    b'absolute;\ntop: 50%;\nleft: 50%;\n-webkit-transform'
+                                    b': translateX(-50%) translateY(-50%);\n-moz-transf'
+                                    b'orm: translateX(-50%) translateY(-50%);\n-ms-tran'
+                                    b'sform: translateX(-50%) translateY(-50%);\ntransf'
+                                    b'orm: translateX(-50%) translateY(-50%);\noverflow'
+                                    b': visible;\nvisibility: visible;\ndisplay: block\n}'
+                                    b'\n\n#sponsors        \n{\nfont-family: Verdana, Gene'
+                                    b'va, Arial, sans-serif;\ncolor: white;\nbackground-'
+                                    b'color: transparent;\ntext-align: center;\nfont-siz'
+                                    b'e: 1em;\nposition: absolute;\ntop: 90%;\nleft: 50%;'
+                                    b'\n-webkit-transform: translateX(-50%) translateY('
+                                    b'-50%);\n-moz-transform: translateX(-50%) translat'
+                                    b'eY(-50%);\n-ms-transform: translateX(-50%) transl'
+                                    b'ateY(-50%);\ntransform: translateX(-50%) translat'
+                                    b'eY(-50%);\noverflow: visible;\nvisibility: visible'
+                                    b';\ndisplay: block\n}\n\n#progress\n{\n\tbackground-'
+                                    b'color: #f08;\n\tposition: absolute;\n\tleft: 0px'
+                                    b';\n\ttop: 0px;\n\theight: 10px;\n}\n\n-->\n</sty'
+                                    b'le>\n</head>\n<body>\n<div id="progress" class="pro'
+                                    b'gressBarDiv"></div>\n<div id="content">\nthe yolo '
+                                    b'for today is<br>\n<span class="yolo" id="yoloValu'
+                                    b'e" title="YOLOVALUE">3026</span><br>\n<br>\n</div>'
+                                    b'\n<div id="sponsors">\n\t<a href="https://cat.yolo.'
+                                    b'jetzt">Cat</a>\n\t<br>\n\tRegulation (EU) 2016/679 c'
+                                    b'ompliant\n</div>\n<script type="text/javascript">\n'
+                                    b'var currentYoloTS = 1619388000000; //Multiply by 100'
+                                    b'0 the YOLO way\nvar tDiff = Date.now() - currentY'
+                                    b'oloTS;\n\nvar dayMS = 24 * 60 * 60 * 1000;\n\nconsol'
+                                    b'e.debug(currentYoloTS);\nconsole.debug(Date.now()'
+                                    b' - currentYoloTS);\nfunction checkTimestamps()\n{\n'
+                                    b'\ttDiff = Date.now() - currentYoloTS;\n\tupdateUi()'
+                                    b';\n\tif(tDiff > dayMS)\n\t{\n\t\tclearInterval(upda'
+                                    b'teInterval);\n\t\tlocation.reload();\t\n\t}\n}\n'
+                                    b'\nfunction updateUi()\n{\n\tvalPercent = tDiff / day'
+                                    b'MS * 100;\n\tif(valPercent > 100) valPercent = 100'
+                                    b';\n\tdocument.getElementById("progress").style = "'
+                                    b'width: " + valPercent + "%;";\n\tif(tDiff > dayMS)'
+                                    b'\n\t{\n\t\tdocument.getElementById("yoloValue").i'
+                                    b'nnerHTML = "&#xfffd;";\n\t}\n}\n\nupdateUi();\nvar'
+                                    b' updateInterval = setInterval(checkTimestamps, 5000)'
+                                    b';\n\n//YOLODATE\n</script>\n</body>\n</html>\n',
+                         'headers': ((b'Server', b'nginx'),
+                                     (b'Date', b'Mon, 26 Apr 2021 18:31:56 GMT'),
+                                     (b'Content-Type', b'text/html'),
+                                     (b'Content-Length', b'2460'),
+                                     (b'Last-Modified', b'Sun, 25 Apr 2021 22:00:00 GMT'),
+                                     (b'Connection', b'keep-alive'),
+                                     (b'ETag', b'"6085e660-99c"'),
+                                     (b'Strict-Transport-Security',
+                                      b'max-age=31536000; includeSubDomains; preload'),
+                                     (b'X-Xss-Protection', b'1; mode=block'),
+                                     (b'X-Content-Type-Options', b'nosniff'),
+                                     (b'Content-Security-Policy',
+                                      b"default-src 'self'; script-src 'self' 'unsafe-in"
+                                      b"line'; connect-src 'self'; img-src 'self'; style"
+                                      b"-src 'self' 'unsafe-inline';"),
+                                     (b'X-Frame-Options', b'SAMEORIGIN'),
+                                     (b'Referrer-Policy', b'no-referrer'),
+                                     (b'Accept-Ranges', b'bytes')),
+                         'http_version': b'HTTP/1.1',
+                         'reason': b'OK',
+                         'status_code': 200,
+                         'timestamp_end': 1619461916.7979567,
+                         'timestamp_start': 1619461916.7935555,
+                         'trailers': None},
+            'server_conn': {'address': ('yolo.jetzt', 443),
+                            'alpn_proto_negotiated': b'http/1.1',
+                            'cert': '',
+                            'id': '64c838e2-ab74-49c4-9ebe-70e8d17e9081',
+                            'ip_address': ('95.156.226.69', 443),
+                            'sni': 'yolo.jetzt',
+                            'source_address': ('10.23.42.102', 43399),
+                            'timestamp_end': None,
+                            'timestamp_start': 1619461916.63117,
+                            'timestamp_tcp_setup': 1619461916.6659565,
+                            'timestamp_tls_setup': 1619461916.7522914,
+                            'tls_established': True,
+                            'tls_version': 'TLSv1.2',
+                            'via': None},
+            'type': 'http',
+            'version': 9}
+
+
 class MitmAddonTestServer:
     def __init__(self, queue, path: str):
         self.queue = queue
@@ -86,6 +219,7 @@ class MitmAddonTestServer:
         msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
         self.send(msg)
 
+
 @pytest.fixture
 def client_server():
     queue = Queue()
@@ -96,32 +230,200 @@ def client_server():
     client.start()
     server = MitmAddonTestServer(queue, sock)
     server.connect()
-    yield client, server
+    yield queue, client, server
     client.join(1)
     server.disconnect()
 
+
 class TestBigSnitchWrapper:
     def test_request_convert(self, flowstate_request):
         req = bRequest(flow=flowstate_request)
-        d = {'content': b'',
-         'headers': [bHeader(key=b'Host', value=b'yolo.jetzt'),
-                     bHeader(key=b'User-Agent', value=b'curl/7.75.0'),
-                     bHeader(key=b'Accept', value=b'*/*')],
+        d = {'content': '',
+             'headers': [bHeader(key='Host', value='yolo.jetzt'),
+                         bHeader(key='User-Agent', value='curl/7.75.0'),
+                         bHeader(key='Accept', value='*/*')],
              'host': 'yolo.jetzt',
-         'http_version': b'HTTP/1.1',
-         'method': b'GET',
+             'http_version': 'HTTP/1.1',
+             'method': 'GET',
              'port': 443,
-         'scheme': b'https',
+             'scheme': 'https',
              'server_ip_address': '95.156.226.69',
              'timestamp_end': 1619390482.69,
              'timestamp_start': 1619390482.6886377,
              'tls': True}
         assert not DeepDiff(req.__dict__, d)
-"""
+
+    def test_response_convert(self, flowstate_response):
+        res = bResponse(flow=flowstate_response)
+        d = {'content': '<!DOCTYPE html">\n'
+                        '<html>\n'
+                        '<head>  \n'
+                        '<meta http-equiv="content-type" '
+                        'content="text/html;charset=ISO-8859-1">\n'
+                        '<title>todays yolo - 3026</title>  \n'
+                        '<style type="text/css" media="screen">\n'
+                        '<!--\n'
+                        'body \n'
+                        '{\n'
+                        'color: white;\n'
+                        'background-color: #003;\n'
+                        'margin: 0px\n'
+                        '}\n'
+                        '\n'
+                        '.yolo\n'
+                        '{\n'
+                        'font-family: Verdana, Geneva, Arial, sans-serif;\n'
+                        'font-weight: bold;\n'
+                        'font-size: 3em;\n'
+                        '}\n'
+                        '\n'
+                        'a:visited {\n'
+                        '\tcolor: blue\n'
+                        '}\n'
+                        '\n'
+                        '#content        \n'
+                        '{\n'
+                        'font-family: Verdana, Geneva, Arial, sans-serif;\n'
+                        'color: white;\n'
+                        'background-color: transparent;\n'
+                        'text-align: center;\n'
+                        'font-size: 2em;\n'
+                        'position: absolute;\n'
+                        'top: 50%;\n'
+                        'left: 50%;\n'
+                        '-webkit-transform: translateX(-50%) translateY(-50%);\n'
+                        '-moz-transform: translateX(-50%) translateY(-50%);\n'
+                        '-ms-transform: translateX(-50%) translateY(-50%);\n'
+                        'transform: translateX(-50%) translateY(-50%);\n'
+                        'overflow: visible;\n'
+                        'visibility: visible;\n'
+                        'display: block\n'
+                        '}\n'
+                        '\n'
+                        '#sponsors        \n'
+                        '{\n'
+                        'font-family: Verdana, Geneva, Arial, sans-serif;\n'
+                        'color: white;\n'
+                        'background-color: transparent;\n'
+                        'text-align: center;\n'
+                        'font-size: 1em;\n'
+                        'position: absolute;\n'
+                        'top: 90%;\n'
+                        'left: 50%;\n'
+                        '-webkit-transform: translateX(-50%) translateY(-50%);\n'
+                        '-moz-transform: translateX(-50%) translateY(-50%);\n'
+                        '-ms-transform: translateX(-50%) translateY(-50%);\n'
+                        'transform: translateX(-50%) translateY(-50%);\n'
+                        'overflow: visible;\n'
+                        'visibility: visible;\n'
+                        'display: block\n'
+                        '}\n'
+                        '\n'
+                        '#progress\n'
+                        '{\n'
+                        '\tbackground-color: #f08;\n'
+                        '\tposition: absolute;\n'
+                        '\tleft: 0px;\n'
+                        '\ttop: 0px;\n'
+                        '\theight: 10px;\n'
+                        '}\n'
+                        '\n'
+                        '-->\n'
+                        '</style>\n'
+                        '</head>\n'
+                        '<body>\n'
+                        '<div id="progress" class="progressBarDiv"></div>\n'
+                        '<div id="content">\n'
+                        'the yolo for today is<br>\n'
+                        '<span class="yolo" id="yoloValue" '
+                        'title="YOLOVALUE">3026</span><br>\n'
+                        '<br>\n'
+                        '</div>\n'
+                        '<div id="sponsors">\n'
+                        '\t<a href="https://cat.yolo.jetzt">Cat</a>\n'
+                        '\t<br>\n'
+                        '\tRegulation (EU) 2016/679 compliant\n'
+                        '</div>\n'
+                        '<script type="text/javascript">\n'
+                        'var currentYoloTS = 1619388000000; //Multiply by 1000 the YOLO '
+                        'way\n'
+                        'var tDiff = Date.now() - currentYoloTS;\n'
+                        '\n'
+                        'var dayMS = 24 * 60 * 60 * 1000;\n'
+                        '\n'
+                        'console.debug(currentYoloTS);\n'
+                        'console.debug(Date.now() - currentYoloTS);\n'
+                        'function checkTimestamps()\n'
+                        '{\n'
+                        '\ttDiff = Date.now() - currentYoloTS;\n'
+                        '\tupdateUi();\n'
+                        '\tif(tDiff > dayMS)\n'
+                        '\t{\n'
+                        '\t\tclearInterval(updateInterval);\n'
+                        '\t\tlocation.reload();\t\n'
+                        '\t}\n'
+                        '}\n'
+                        '\n'
+                        'function updateUi()\n'
+                        '{\n'
+                        '\tvalPercent = tDiff / dayMS * 100;\n'
+                        '\tif(valPercent > 100) valPercent = 100;\n'
+                        '\tdocument.getElementById("progress").style = "width: " + '
+                        'valPercent + "%;";\n'
+                        '\tif(tDiff > dayMS)\n'
+                        '\t{\n'
+                        '\t\tdocument.getElementById("yoloValue").innerHTML = "&#xfffd;";\n'
+                        '\t}\n'
+                        '}\n'
+                        '\n'
+                        'updateUi();\n'
+                        'var updateInterval = setInterval(checkTimestamps, 5000);\n'
+                        '\n'
+                        '//YOLODATE\n'
+                        '</script>\n'
+                        '</body>\n'
+                        '</html>\n',
+             'headers': [bHeader(key='Server', value='nginx'),
+                         bHeader(key='Date', value='Mon, 26 Apr 2021 18:31:56 GMT'),
+                         bHeader(key='Content-Type', value='text/html'),
+                         bHeader(key='Content-Length', value='2460'),
+                         bHeader(key='Last-Modified', value='Sun, 25 Apr 2021 22:00:00 GMT'),
+                         bHeader(key='Connection', value='keep-alive'),
+                         bHeader(key='ETag', value='"6085e660-99c"'),
+                         bHeader(key='Strict-Transport-Security', value='max-age=31536000; includeSubDomains; preload'),
+                         bHeader(key='X-Xss-Protection', value='1; mode=block'),
+                         bHeader(key='X-Content-Type-Options', value='nosniff'),
+                         bHeader(key='Content-Security-Policy', value="default-src 'self'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self'; style-src 'self' 'unsafe-inline';"),
+                         bHeader(key='X-Frame-Options', value='SAMEORIGIN'),
+                         bHeader(key='Referrer-Policy', value='no-referrer'),
+                         bHeader(key='Accept-Ranges', value='bytes')],
+             'http_version': 'HTTP/1.1',
+             'reason': 'OK',
+             'status_code': 200,
+             'timestamp_end': 1619461916.7979567,
+             'timestamp_start': 1619461916.7935555}
+
+        assert not DeepDiff(res.__dict__, d)
+
+
 class TestMitmAddon:
+    def test_get_new_flows_empty(self, client_server):
+        queue, client, server = client_server
+        # queue empty, flows empty
+        assert queue.empty()
+        assert not len(client.flows)
+
+        client.get_new_flows()
+
+        # afterwards too
+        assert queue.empty()
+        assert not len(client.flows)
+
+    def test_get_new_flows_single(self, client_server):
+        queue, client, server = client_server
+
     def test_request(self, client_server):
-        self.client, self.server = client_server
+        queue, client, server = client_server
         # create request
-        flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
-        self.q.put_nowait((flow.id, flowitem))
-"""
+        #flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flowstate_request)
+        #self.q.put_nowait(('51215b69-c76f-4ac2-afcb-da3b823d9f88', flowitem))