From 34f7d960c2d17e1c017eccf852a8089cefa8f607 Mon Sep 17 00:00:00 2001 From: Tim Blume Date: Wed, 28 Apr 2021 19:20:07 +0200 Subject: [PATCH] more tests --- mitmaddon/networkthread.py | 89 +++--- mitmaddon/test_bigsnitch.py | 560 +++++++++++++++++++++++++----------- 2 files changed, 450 insertions(+), 199 deletions(-) diff --git a/mitmaddon/networkthread.py b/mitmaddon/networkthread.py index b8a79c7..154259d 100644 --- a/mitmaddon/networkthread.py +++ b/mitmaddon/networkthread.py @@ -71,11 +71,13 @@ class bRequest: self.timestamp_end = flow["request"]["timestamp_end"] self.headers = [] - for k, v in flow["request"]["headers"]: + for k, v in flow["request"].get("headers", {}): self.headers.append(bHeader(str(k), str(v))) def json(self) -> dict: - return {} + state = vars(self).copy() + state["headers"] = {h.key: h.value for h in self.headers} + return state @dataclass @@ -99,11 +101,13 @@ class bResponse: self.timestamp_end = flow["response"]["timestamp_end"] self.headers = [] - for k, v in flow["response"]["headers"]: + for k, v in flow["response"].get("headers", {}): self.headers.append(bHeader(k, v)) def json(self) -> dict: - return {} + state = vars(self).copy() + state["headers"] = [{h.key: h.value} for h in self.headers] + return state @dataclass @@ -133,18 +137,13 @@ class bPacket: flowid: str data: str - def __init__(self, json: Dict): - self.ptype = json["type"] - self.flowid = str(json["id"]) - self.data = json["data"] - @dataclass class FlowItem: state: bFlowState flow: Flow time: float = 0 - retries_left: int = 5 + retries: int = 5 """ @@ -187,7 +186,7 @@ class NetworkThread(threading.Thread): # get new self.flows that may occured i, flowitem = self.q.get(block=False) if self.flows.get(i, None): - print(f"flow {i} doubled? ignoring...") + raise ValueError(f"flow {i} doubled? ignoring...") continue else: self.flows[i] = flowitem @@ -199,40 +198,63 @@ class NetworkThread(threading.Thread): msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data} self.send(msg) - def send_http_request(self, id: int, request: bRequest): - pkg = bPacket(bPacketType.HTTP_REQUEST, request.json()) - self.send_packet(pkg) - self.flows[id].state = bFlowState.SENT_HTTP_REQUEST - self.flows[id].time = time.monotonic() - - def send_http_response(self, id: int, response: bResponse): - pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json()) - self.send_packet(pkg) - self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE - self.flows[id].time = time.monotonic() - # update all current self.flows # handles the state machine for each flow def update_flows(self): - for id, flow in self.flows.items(): + # force copy of item list, so we can remove dict items in the loop + for id, flow in list(self.flows.items()): if self.flows[id].retries <= 0: - self.flows[id].flow.kill() + if self.flows[id].flow: + self.flows[id].flow.kill() print(f"http flow {id} timed out! flow killed.") + del self.flows[id] delta = time.monotonic() - self.flows[id].time if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \ flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5: - self.send_http_request(id, bRequest(flow.flow)) + pkg = bPacket(bPacketType.HTTP_REQUEST, id, bRequest(flow.flow.get_state()).json()) + self.send_packet(pkg) + + self.flows[id].time = time.monotonic() + self.flows[id].state = bFlowState.SENT_HTTP_REQUEST self.flows[id].retries -= 1 - if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \ + elif flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \ flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5: - self.send_http_response(id, bResponse(flow.flow)) + pkg = bPacket(bPacketType.HTTP_RESPONSE, id, bResponse(flow.flow.get_state()).json()) + self.send_packet(pkg) + + self.flows[id].time = time.monotonic() + self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE self.flows[id].retries -= 1 - elif flow.state == bFlowState.ERROR: + if flow.state == bFlowState.ERROR: print(f"error in flow {id}!") + del self.flows[id] + + def handle_packet(self, pkg): + flow = self.flows.get(pkg.flowid, None) + + # flow ACKed + if pkg.ptype == bPacketType.ACK: + if flow and flow.flow: + flow.flow.resume() + else: + raise ValueError("unknown flow") + + # flow killed + elif pkg.ptype == bPacketType.KILL: + if flow and flow.flow: + flow.flow.kill() + else: + raise ValueError("unknown flow") + + if flow: + del self.flows[pkg.flowid] + + else: + print(f"got unexpected message {pkg.ptype}") # handle incoming packets / update the statemachine def handle_packets(self): @@ -242,14 +264,7 @@ class NetworkThread(threading.Thread): if msg: result = json.loads(str(msg)) pkg = bPacket(json=result) - # flow ACKed - if pkg.ptype == bPacketType.ACK: - continue - # flow killed - elif pkg.ptype == bPacketType.KILL: - continue - else: - print(f"got unexpected message {pkg.ptype}") + self.handle_packet(pkg) except json.JSONDecodeError: print(f"malformed message received {msg}") diff --git a/mitmaddon/test_bigsnitch.py b/mitmaddon/test_bigsnitch.py index 9083fde..11ac2bc 100644 --- a/mitmaddon/test_bigsnitch.py +++ b/mitmaddon/test_bigsnitch.py @@ -2,8 +2,10 @@ import pdb import queue - +import uuid import pytest +from mitmproxy.http import HTTPFlow, HTTPRequest, HTTPResponse +from mitmproxy.connections import ClientConnection, ServerConnection from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState import os import tempfile @@ -13,22 +15,64 @@ import zmq from deepdiff import DeepDiff +@pytest.fixture +def flowstate_clientconn(): + return dict( + id=str(uuid.uuid4()), + address=("address", 22), + source_address=("address", 22), + ip_address=("192.168.0.1", 22), + timestamp_start=946681202, + timestamp_tcp_setup=946681203, + timestamp_tls_setup=946681204, + timestamp_end=946681205, + tls_established=True, + sni="address", + alpn=None, + tls_version="TLSv1.2", + via=None, + state=0, + error=None, + tls=False, + certificate_list=[], + alpn_offers=[], + cipher_name=None, + cipher_list=[], + via2=None, + ) + + +@pytest.fixture +def flowstate_serverconn(): + return dict( + id=str(uuid.uuid4()), + address=("address", 22), + source_address=("address", 22), + ip_address=("192.168.0.1", 22), + timestamp_start=946681202, + timestamp_tcp_setup=946681203, + timestamp_tls_setup=946681204, + timestamp_end=946681205, + tls_established=True, + sni="address", + alpn=None, + tls_version="TLSv1.2", + via=None, + state=0, + error=None, + tls=False, + certificate_list=[], + alpn_offers=[], + cipher_name=None, + cipher_list=[], + via2=None, + ) + + # usual flow state of the request with some big parts removed @pytest.fixture -def flowstate_request(): - return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0), - 'alpn_proto_negotiated': b'http/1.1', - 'cipher_name': 'TLS_AES_256_GCM_SHA384', - 'clientcert': None, - 'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea', - 'mitmcert': '', - 'sni': 'yolo.jetzt', - 'timestamp_end': None, - 'timestamp_start': 1619390481.8003347, - 'timestamp_tls_setup': 1619390482.6879823, - 'tls_established': True, - 'tls_extensions': [], - 'tls_version': 'TLSv1.3'}, +def flowstate_request(flowstate_clientconn, flowstate_serverconn): + return {'client_conn': flowstate_clientconn, 'error': None, 'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88', 'intercepted': False, @@ -36,8 +80,7 @@ def flowstate_request(): 'marked': False, 'metadata': {}, 'mode': 'transparent', - 'request': {'authority': b'', - 'content': b'', + 'request': {'content': b'', 'headers': ((b'Host', b'yolo.jetzt'), (b'User-Agent', b'curl/7.75.0'), (b'Accept', b'*/*')), @@ -49,156 +92,138 @@ def flowstate_request(): 'scheme': b'https', 'timestamp_end': 1619390482.69, 'timestamp_start': 1619390482.6886377, - 'trailers': None}, + 'authority': '', + 'trailers': '', + }, 'response': None, - 'server_conn': {'address': ('yolo.jetzt', 443), - 'alpn_proto_negotiated': b'http/1.1', - 'cert': '', - 'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b', - 'ip_address': ('95.156.226.69', 443), - 'sni': 'yolo.jetzt', - 'source_address': ('192.168.42.182', 51514), - 'timestamp_end': None, - 'timestamp_start': 1619390481.8154442, - 'timestamp_tcp_setup': 1619390481.994565, - 'timestamp_tls_setup': 1619390482.6819758, - 'tls_established': True, - 'tls_version': 'TLSv1.2', - 'via': None}, + "server_conn": flowstate_serverconn, 'type': 'http', 'version': 9} +@pytest.fixture +def flow_request(flowstate_request): + c = ClientConnection.make_dummy("192.168.0.2") + s = ServerConnection.make_dummy("192.168.0.1") + flow = HTTPFlow(c, s) + flow.request = HTTPRequest.from_state(flowstate_request["request"]) + return flow + @pytest.fixture() -def flowstate_response(): - return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0), - 'alpn_proto_negotiated': b'http/1.1', - 'cipher_name': 'TLS_AES_256_GCM_SHA384', - 'clientcert': None, - 'id': '2507e6ce-3132-4394-9432-f55fb5f55b05', - 'mitmcert': '', - 'sni': 'yolo.jetzt', - 'timestamp_end': None, - 'timestamp_start': 1619461916.6160116, - 'timestamp_tls_setup': 1619461916.7581937, - 'tls_established': True, - 'tls_extensions': [], - 'tls_version': 'TLSv1.3'}, - 'error': None, - 'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62', - 'intercepted': True, - 'is_replay': None, - 'marked': False, - 'metadata': {}, - 'mode': 'transparent', - 'request': {'authority': b'', - 'content': b'', - 'headers': ((b'Host', b'yolo.jetzt'), - (b'User-Agent', b'curl/7.75.0'), - (b'Accept', b'*/*')), - 'host': 'yolo.jetzt', - 'http_version': b'HTTP/1.1', - 'method': b'GET', - 'path': b'/', - 'port': 443, - 'scheme': b'https', - 'timestamp_end': 1619461916.7603076, - 'timestamp_start': 1619461916.7588415, - 'trailers': None}, - 'response': {'content': b'\n\n \n\ntodays yolo - 3026 \n\n' + '\n' + '\n' + '
\n' + '
\n' + 'the yolo for today is
\n' + '3026
\n' + '
\n' + '
\n' + '
\n' + '\tCat\n' + '\t
\n' + '\tRegulation (EU) 2016/679 compliant\n' + '
\n' + '\n' + '\n' + '\n', + 'headers': [{'Server': 'nginx'}, + {'Date': 'Mon, 26 Apr 2021 18:31:56 GMT'}, + {'Content-Type': 'text/html'}, + {'Content-Length': '2460'}, + {'Last-Modified': 'Sun, 25 Apr 2021 22:00:00 GMT'}, + {'Connection': 'keep-alive'}, + {'ETag': '"6085e660-99c"'}, + {'Strict-Transport-Security': 'max-age=31536000; ' + 'includeSubDomains; preload'}, + {'X-Xss-Protection': '1; mode=block'}, + {'X-Content-Type-Options': 'nosniff'}, + {'Content-Security-Policy': "default-src 'self'; script-src " + "'self' 'unsafe-inline'; connect-src " + "'self'; img-src 'self'; style-src " + "'self' 'unsafe-inline';"}, + {'X-Frame-Options': 'SAMEORIGIN'}, + {'Referrer-Policy': 'no-referrer'}, + {'Accept-Ranges': 'bytes'}], + 'http_version': 'HTTP/1.1', + 'reason': 'OK', + 'status_code': 200, + 'timestamp_end': 1619461916.7979567, + 'timestamp_start': 1619461916.7935555} + + assert not DeepDiff(j, d) class TestMitmAddon: def test_get_new_flows_empty(self, client_server): @@ -419,8 +626,37 @@ class TestMitmAddon: assert queue.empty() assert not len(client.flows) - def test_get_new_flows_single(self, client_server): - queue, client, server = client_server + def test_get_new_flows_single(self, client_no_start, flow_request): + queue, client = client_no_start + + flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request) + queue.put_nowait((flow_request.id, flowitem)) + + client.get_new_flows() + + assert queue.empty() + assert len(client.flows) == 1 + + assert flow_request.id in client.flows.keys() + assert not DeepDiff(flowitem, client.flows[flow_request.id]) + + def test_get_new_flows_double(self, client_no_start, flow_request, flow_response): + queue, client = client_no_start + + flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request) + queue.put_nowait((flow_request.id, flowitem)) + flowitem2 = FlowItem(bFlowState.UNSENT_HTTP_RESPONSE, flow_response) + queue.put_nowait((flow_response.id, flowitem2)) + + client.get_new_flows() + + assert queue.empty() + assert len(client.flows) == 2 + + assert flow_request.id in client.flows.keys() + assert not DeepDiff(flowitem, client.flows[flow_request.id]) + assert flow_response.id in client.flows.keys() + assert not DeepDiff(flowitem2, client.flows[flow_response.id]) def test_request(self, client_server): queue, client, server = client_server