diff --git a/mitmaddon/networkthread.py b/mitmaddon/networkthread.py
index b8a79c7..154259d 100644
--- a/mitmaddon/networkthread.py
+++ b/mitmaddon/networkthread.py
@@ -71,11 +71,13 @@ class bRequest:
self.timestamp_end = flow["request"]["timestamp_end"]
self.headers = []
- for k, v in flow["request"]["headers"]:
+ for k, v in flow["request"].get("headers", {}):
self.headers.append(bHeader(str(k), str(v)))
def json(self) -> dict:
- return {}
+ state = vars(self).copy()
+ state["headers"] = {h.key: h.value for h in self.headers}
+ return state
@dataclass
@@ -99,11 +101,13 @@ class bResponse:
self.timestamp_end = flow["response"]["timestamp_end"]
self.headers = []
- for k, v in flow["response"]["headers"]:
+ for k, v in flow["response"].get("headers", {}):
self.headers.append(bHeader(k, v))
def json(self) -> dict:
- return {}
+ state = vars(self).copy()
+ state["headers"] = [{h.key: h.value} for h in self.headers]
+ return state
@dataclass
@@ -133,18 +137,13 @@ class bPacket:
flowid: str
data: str
- def __init__(self, json: Dict):
- self.ptype = json["type"]
- self.flowid = str(json["id"])
- self.data = json["data"]
-
@dataclass
class FlowItem:
state: bFlowState
flow: Flow
time: float = 0
- retries_left: int = 5
+ retries: int = 5
"""
@@ -187,7 +186,7 @@ class NetworkThread(threading.Thread):
# get new self.flows that may occured
i, flowitem = self.q.get(block=False)
if self.flows.get(i, None):
- print(f"flow {i} doubled? ignoring...")
+ raise ValueError(f"flow {i} doubled? ignoring...")
continue
else:
self.flows[i] = flowitem
@@ -199,40 +198,63 @@ class NetworkThread(threading.Thread):
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
self.send(msg)
- def send_http_request(self, id: int, request: bRequest):
- pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
- self.send_packet(pkg)
- self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
- self.flows[id].time = time.monotonic()
-
- def send_http_response(self, id: int, response: bResponse):
- pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
- self.send_packet(pkg)
- self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
- self.flows[id].time = time.monotonic()
-
# update all current self.flows
# handles the state machine for each flow
def update_flows(self):
- for id, flow in self.flows.items():
+ # force copy of item list, so we can remove dict items in the loop
+ for id, flow in list(self.flows.items()):
if self.flows[id].retries <= 0:
- self.flows[id].flow.kill()
+ if self.flows[id].flow:
+ self.flows[id].flow.kill()
print(f"http flow {id} timed out! flow killed.")
+ del self.flows[id]
delta = time.monotonic() - self.flows[id].time
if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
- self.send_http_request(id, bRequest(flow.flow))
+ pkg = bPacket(bPacketType.HTTP_REQUEST, id, bRequest(flow.flow.get_state()).json())
+ self.send_packet(pkg)
+
+ self.flows[id].time = time.monotonic()
+ self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
self.flows[id].retries -= 1
- if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
+ elif flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
- self.send_http_response(id, bResponse(flow.flow))
+ pkg = bPacket(bPacketType.HTTP_RESPONSE, id, bResponse(flow.flow.get_state()).json())
+ self.send_packet(pkg)
+
+ self.flows[id].time = time.monotonic()
+ self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
self.flows[id].retries -= 1
- elif flow.state == bFlowState.ERROR:
+ if flow.state == bFlowState.ERROR:
print(f"error in flow {id}!")
+ del self.flows[id]
+
+ def handle_packet(self, pkg):
+ flow = self.flows.get(pkg.flowid, None)
+
+ # flow ACKed
+ if pkg.ptype == bPacketType.ACK:
+ if flow and flow.flow:
+ flow.flow.resume()
+ else:
+ raise ValueError("unknown flow")
+
+ # flow killed
+ elif pkg.ptype == bPacketType.KILL:
+ if flow and flow.flow:
+ flow.flow.kill()
+ else:
+ raise ValueError("unknown flow")
+
+ if flow:
+ del self.flows[pkg.flowid]
+
+ else:
+ print(f"got unexpected message {pkg.ptype}")
# handle incoming packets / update the statemachine
def handle_packets(self):
@@ -242,14 +264,7 @@ class NetworkThread(threading.Thread):
if msg:
result = json.loads(str(msg))
pkg = bPacket(json=result)
- # flow ACKed
- if pkg.ptype == bPacketType.ACK:
- continue
- # flow killed
- elif pkg.ptype == bPacketType.KILL:
- continue
- else:
- print(f"got unexpected message {pkg.ptype}")
+ self.handle_packet(pkg)
except json.JSONDecodeError:
print(f"malformed message received {msg}")
diff --git a/mitmaddon/test_bigsnitch.py b/mitmaddon/test_bigsnitch.py
index 9083fde..11ac2bc 100644
--- a/mitmaddon/test_bigsnitch.py
+++ b/mitmaddon/test_bigsnitch.py
@@ -2,8 +2,10 @@
import pdb
import queue
-
+import uuid
import pytest
+from mitmproxy.http import HTTPFlow, HTTPRequest, HTTPResponse
+from mitmproxy.connections import ClientConnection, ServerConnection
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
import os
import tempfile
@@ -13,22 +15,64 @@ import zmq
from deepdiff import DeepDiff
+@pytest.fixture
+def flowstate_clientconn():
+ return dict(
+ id=str(uuid.uuid4()),
+ address=("address", 22),
+ source_address=("address", 22),
+ ip_address=("192.168.0.1", 22),
+ timestamp_start=946681202,
+ timestamp_tcp_setup=946681203,
+ timestamp_tls_setup=946681204,
+ timestamp_end=946681205,
+ tls_established=True,
+ sni="address",
+ alpn=None,
+ tls_version="TLSv1.2",
+ via=None,
+ state=0,
+ error=None,
+ tls=False,
+ certificate_list=[],
+ alpn_offers=[],
+ cipher_name=None,
+ cipher_list=[],
+ via2=None,
+ )
+
+
+@pytest.fixture
+def flowstate_serverconn():
+ return dict(
+ id=str(uuid.uuid4()),
+ address=("address", 22),
+ source_address=("address", 22),
+ ip_address=("192.168.0.1", 22),
+ timestamp_start=946681202,
+ timestamp_tcp_setup=946681203,
+ timestamp_tls_setup=946681204,
+ timestamp_end=946681205,
+ tls_established=True,
+ sni="address",
+ alpn=None,
+ tls_version="TLSv1.2",
+ via=None,
+ state=0,
+ error=None,
+ tls=False,
+ certificate_list=[],
+ alpn_offers=[],
+ cipher_name=None,
+ cipher_list=[],
+ via2=None,
+ )
+
+
# usual flow state of the request with some big parts removed
@pytest.fixture
-def flowstate_request():
- return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0),
- 'alpn_proto_negotiated': b'http/1.1',
- 'cipher_name': 'TLS_AES_256_GCM_SHA384',
- 'clientcert': None,
- 'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea',
- 'mitmcert': '',
- 'sni': 'yolo.jetzt',
- 'timestamp_end': None,
- 'timestamp_start': 1619390481.8003347,
- 'timestamp_tls_setup': 1619390482.6879823,
- 'tls_established': True,
- 'tls_extensions': [],
- 'tls_version': 'TLSv1.3'},
+def flowstate_request(flowstate_clientconn, flowstate_serverconn):
+ return {'client_conn': flowstate_clientconn,
'error': None,
'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
'intercepted': False,
@@ -36,8 +80,7 @@ def flowstate_request():
'marked': False,
'metadata': {},
'mode': 'transparent',
- 'request': {'authority': b'',
- 'content': b'',
+ 'request': {'content': b'',
'headers': ((b'Host', b'yolo.jetzt'),
(b'User-Agent', b'curl/7.75.0'),
(b'Accept', b'*/*')),
@@ -49,156 +92,138 @@ def flowstate_request():
'scheme': b'https',
'timestamp_end': 1619390482.69,
'timestamp_start': 1619390482.6886377,
- 'trailers': None},
+ 'authority': '',
+ 'trailers': '',
+ },
'response': None,
- 'server_conn': {'address': ('yolo.jetzt', 443),
- 'alpn_proto_negotiated': b'http/1.1',
- 'cert': '',
- 'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b',
- 'ip_address': ('95.156.226.69', 443),
- 'sni': 'yolo.jetzt',
- 'source_address': ('192.168.42.182', 51514),
- 'timestamp_end': None,
- 'timestamp_start': 1619390481.8154442,
- 'timestamp_tcp_setup': 1619390481.994565,
- 'timestamp_tls_setup': 1619390482.6819758,
- 'tls_established': True,
- 'tls_version': 'TLSv1.2',
- 'via': None},
+ "server_conn": flowstate_serverconn,
'type': 'http',
'version': 9}
+@pytest.fixture
+def flow_request(flowstate_request):
+ c = ClientConnection.make_dummy("192.168.0.2")
+ s = ServerConnection.make_dummy("192.168.0.1")
+ flow = HTTPFlow(c, s)
+ flow.request = HTTPRequest.from_state(flowstate_request["request"])
+ return flow
+
@pytest.fixture()
-def flowstate_response():
- return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
- 'alpn_proto_negotiated': b'http/1.1',
- 'cipher_name': 'TLS_AES_256_GCM_SHA384',
- 'clientcert': None,
- 'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
- 'mitmcert': '',
- 'sni': 'yolo.jetzt',
- 'timestamp_end': None,
- 'timestamp_start': 1619461916.6160116,
- 'timestamp_tls_setup': 1619461916.7581937,
- 'tls_established': True,
- 'tls_extensions': [],
- 'tls_version': 'TLSv1.3'},
- 'error': None,
- 'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
- 'intercepted': True,
- 'is_replay': None,
- 'marked': False,
- 'metadata': {},
- 'mode': 'transparent',
- 'request': {'authority': b'',
- 'content': b'',
- 'headers': ((b'Host', b'yolo.jetzt'),
- (b'User-Agent', b'curl/7.75.0'),
- (b'Accept', b'*/*')),
- 'host': 'yolo.jetzt',
- 'http_version': b'HTTP/1.1',
- 'method': b'GET',
- 'path': b'/',
- 'port': 443,
- 'scheme': b'https',
- 'timestamp_end': 1619461916.7603076,
- 'timestamp_start': 1619461916.7588415,
- 'trailers': None},
- 'response': {'content': b'\n\n \n\ntodays yolo - 3026 \n\n'
+ '\n'
+ '\n'
+ '\n'
+ '\n'
+ 'the yolo for today is
\n'
+ '3026
\n'
+ '
\n'
+ '
\n'
+ '\n'
+ '\n'
+ '\n'
+ '\n',
+ 'headers': [{'Server': 'nginx'},
+ {'Date': 'Mon, 26 Apr 2021 18:31:56 GMT'},
+ {'Content-Type': 'text/html'},
+ {'Content-Length': '2460'},
+ {'Last-Modified': 'Sun, 25 Apr 2021 22:00:00 GMT'},
+ {'Connection': 'keep-alive'},
+ {'ETag': '"6085e660-99c"'},
+ {'Strict-Transport-Security': 'max-age=31536000; '
+ 'includeSubDomains; preload'},
+ {'X-Xss-Protection': '1; mode=block'},
+ {'X-Content-Type-Options': 'nosniff'},
+ {'Content-Security-Policy': "default-src 'self'; script-src "
+ "'self' 'unsafe-inline'; connect-src "
+ "'self'; img-src 'self'; style-src "
+ "'self' 'unsafe-inline';"},
+ {'X-Frame-Options': 'SAMEORIGIN'},
+ {'Referrer-Policy': 'no-referrer'},
+ {'Accept-Ranges': 'bytes'}],
+ 'http_version': 'HTTP/1.1',
+ 'reason': 'OK',
+ 'status_code': 200,
+ 'timestamp_end': 1619461916.7979567,
+ 'timestamp_start': 1619461916.7935555}
+
+ assert not DeepDiff(j, d)
class TestMitmAddon:
def test_get_new_flows_empty(self, client_server):
@@ -419,8 +626,37 @@ class TestMitmAddon:
assert queue.empty()
assert not len(client.flows)
- def test_get_new_flows_single(self, client_server):
- queue, client, server = client_server
+ def test_get_new_flows_single(self, client_no_start, flow_request):
+ queue, client = client_no_start
+
+ flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request)
+ queue.put_nowait((flow_request.id, flowitem))
+
+ client.get_new_flows()
+
+ assert queue.empty()
+ assert len(client.flows) == 1
+
+ assert flow_request.id in client.flows.keys()
+ assert not DeepDiff(flowitem, client.flows[flow_request.id])
+
+ def test_get_new_flows_double(self, client_no_start, flow_request, flow_response):
+ queue, client = client_no_start
+
+ flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request)
+ queue.put_nowait((flow_request.id, flowitem))
+ flowitem2 = FlowItem(bFlowState.UNSENT_HTTP_RESPONSE, flow_response)
+ queue.put_nowait((flow_response.id, flowitem2))
+
+ client.get_new_flows()
+
+ assert queue.empty()
+ assert len(client.flows) == 2
+
+ assert flow_request.id in client.flows.keys()
+ assert not DeepDiff(flowitem, client.flows[flow_request.id])
+ assert flow_response.id in client.flows.keys()
+ assert not DeepDiff(flowitem2, client.flows[flow_response.id])
def test_request(self, client_server):
queue, client, server = client_server