more tests

This commit is contained in:
Tim Blume 2021-04-27 16:23:37 +02:00
parent 75a5ba79db
commit c20ec945a1
4 changed files with 567 additions and 264 deletions

View file

@ -3,18 +3,18 @@
import pdb
from queue import Queue, Empty
from queue import Queue
from networkthread import bFlowState, FlowItem, NetworkThread
class BigSnitchBridge:
def __init__(self):
print("BigSnitchBridge started")
self.q = Queue()
#self.thread = NetworkThread("network", self.q)
#self.thread.start()
self.thread = NetworkThread("network", self.q)
self.thread.start()
def request(self, flow):
pdb.set_trace()
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
self.q.put_nowait((flow.id, flowitem))
# intercept until ACK received
@ -27,7 +27,7 @@ class BigSnitchBridge:
flow.intercept()
def error(self, flow):
flowitem = FlowItem(bFlowState.ERROR, flow, time.monotonic())
flowitem = FlowItem(bFlowState.ERROR, flow)
self.q.put_nowait((flow.id, flowitem))

View file

@ -1,6 +1,5 @@
import pdb
from mitmproxy import ctx
from mitmproxy.flow import Flow
import threading
import time
@ -9,12 +8,12 @@ import json
import os
from enum import Enum
from dataclasses import dataclass
from typing import List, Dict
from typing import List, Dict, Any
from queue import Queue, Empty
"""
# this method is used to convert flow states (generated with get_state()) to json
def convert_to_strings(obj):
def convert_to_strings(obj: Any) -> Any:
if isinstance(obj, dict):
return {convert_to_strings(key): convert_to_strings(value)
for key, value in obj.items()}
@ -29,13 +28,14 @@ def convert_to_strings(obj):
return data
return obj
"""
@dataclass
class bHeader:
key: str
value: str
@dataclass
class bRequest:
server_ip_address: str
@ -57,6 +57,8 @@ class bRequest:
# init from flow dict
def __init__(self, flow: dict):
flow = convert_to_strings(flow)
self.server_ip_address = flow["server_conn"]["ip_address"][0]
self.tls = flow["server_conn"]["tls_established"]
self.content = flow["request"]["content"]
@ -69,8 +71,12 @@ class bRequest:
self.timestamp_end = flow["request"]["timestamp_end"]
self.headers = []
for k,v in flow["request"]["headers"]:
self.headers.append(bHeader(k,v))
for k, v in flow["request"]["headers"]:
self.headers.append(bHeader(str(k), str(v)))
def json(self) -> dict:
return {}
@dataclass
class bResponse:
@ -84,15 +90,21 @@ class bResponse:
headers: List[bHeader]
def __init__(self, flow: dict):
self.status_code = flow["status_code"]
self.http_version = flow["http_version"]
self.reason = flow["reason"]
self.content = flow["content"]
self.timestamp_start = flow["timestamp_start"]
self.timestamp_end = flow["timestamp_end"]
flow = convert_to_strings(flow)
self.status_code = flow["response"]["status_code"]
self.http_version = flow["response"]["http_version"]
self.reason = flow["response"]["reason"]
self.content = flow["response"]["content"]
self.timestamp_start = flow["response"]["timestamp_start"]
self.timestamp_end = flow["response"]["timestamp_end"]
self.headers = []
for k, v in flow["response"]["headers"]:
self.headers.append(bHeader(k, v))
def json(self) -> dict:
return {}
for k,v in flow["headers"]:
self.headers.append(bHeader(k,v))
@dataclass
class bFlowState(Enum):
@ -102,6 +114,7 @@ class bFlowState(Enum):
UNSENT_HTTP_RESPONSE = 3
SENT_HTTP_RESPONSE = 4
@dataclass
class bPacketType:
NACK = 0
@ -113,6 +126,7 @@ class bPacketType:
HTTP_REQUEST = 6
HTTP_RESPONSE = 7
@dataclass
class bPacket:
ptype: bPacketType
@ -124,6 +138,7 @@ class bPacket:
self.flowid = str(json["id"])
self.data = json["data"]
@dataclass
class FlowItem:
state: bFlowState
@ -131,6 +146,7 @@ class FlowItem:
time: float = 0
retries_left: int = 5
"""
The network thread communicates with the bigsnitch plugin using zeromq.
@ -152,7 +168,7 @@ class NetworkThread(threading.Thread):
self.q = queue
# for zmq use
self.context = zmq.Context()
# all current flows being handled by mitmproxy
# all current self.flows being handled by mitmproxy
self.flows: Dict[FlowItem] = {}
# timer for sending pings to check if the connection broke
self.timer = time.monotonic()
@ -161,20 +177,20 @@ class NetworkThread(threading.Thread):
# send a single message, no checks involved
def send(self, msg):
a = convert_to_strings(msg)
self.socket.send(str.encode(json.dumps(a)))
#a = convert_to_strings(msg)
self.socket.send(str.encode(json.dumps(msg)))
# add new flows from the queue
# add new self.flows from the queue
def get_new_flows(self):
while True:
try:
# get new flows that may occured
# get new self.flows that may occured
i, flowitem = self.q.get(block=False)
if self.flows.get(i, None):
print(f"flow {i} doubled? ignoring...")
continue
else:
self.flows.append(flowitem)
self.flows[i] = flowitem
except Empty:
break
@ -184,54 +200,38 @@ class NetworkThread(threading.Thread):
self.send(msg)
def send_http_request(self, id: int, request: bRequest):
pkg = bPacket(bPacketType.HTTP_REQUEST, id, request.json())
pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
self.send_packet(pkg)
flows[id].state = bFlowState.SENT_HTTP_REQUEST
flows[id].time = time.monotonic()
self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
self.flows[id].time = time.monotonic()
def send_http_response(self, id: int, response: bResponse):
pkg = bPacket(bPacketType.HTTP_RESPONSE, id, response.json())
pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
self.send_packet(pkg)
flows[id].state = bFlowState.SENT_HTTP_RESPONSE
flows[id].time = time.monotonic()
self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
self.flows[id].time = time.monotonic()
# update all current flows
# update all current self.flows
# handles the state machine for each flow
def update_flows(self):
for id, flow in self.flows.items():
# send the request if not sent
if state == bFlowState.UNSENT_HTTP_REQUEST:
self.send_request(id, bRequest(flow.flow))
if self.flows[id].retries <= 0:
self.flows[id].flow.kill()
print(f"http flow {id} timed out! flow killed.")
elif state == bFlowState.SENT_HTTP_REQUEST:
# check timer, try resend
delta = time.monotonic() - flows[id].time
if delta > 5:
self.send_request(id, bRequest(flow.flow))
flows[id].retries -= 1
continue
delta = time.monotonic() - self.flows[id].time
if flows[id].retries <= 0:
flows[id].flow.kill()
print(f"http request {id} timed out! flow killed.")
if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
self.send_http_request(id, bRequest(flow.flow))
self.flows[id].retries -= 1
# send the response if not sent
elif state == bFlowState.UNSENT_HTTP_RESPONSE:
self.send_response(id, bResponse(flow.flow))
if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
self.send_http_response(id, bResponse(flow.flow))
self.flows[id].retries -= 1
elif state == bFlowState.SENT_HTTP_RESPONSE:
# check timer, try resend
delta = time.monotonic() - flows[id].time
if delta > 5:
self.send_response(id, bResponse(flow.flow))
flows[id].retries -= 1
continue
if flows[id].retries <= 0:
flows[id].flow.kill()
print(f"http response {id} timedout! flow killed.")
elif state == bFlowState.ERROR:
elif flow.state == bFlowState.ERROR:
print(f"error in flow {id}!")
# handle incoming packets / update the statemachine
@ -240,7 +240,7 @@ class NetworkThread(threading.Thread):
msg = self.socket.recv()
try:
if msg:
result = json.loads(msg)
result = json.loads(str(msg))
pkg = bPacket(json=result)
# flow ACKed
if pkg.ptype == bPacketType.ACK:

View file

@ -3,3 +3,4 @@ mitmdump
pyzmq
deepdiff
pytest
tox

View file

@ -1,9 +1,10 @@
#!/usr/bin/env python3
import pdb
import queue
import pytest
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
import os
import tempfile
from queue import Queue
@ -11,6 +12,7 @@ import zmq
from deepdiff import DeepDiff
# usual flow state of the request with some big parts removed
@pytest.fixture
def flowstate_request():
@ -66,6 +68,137 @@ def flowstate_request():
'type': 'http',
'version': 9}
@pytest.fixture()
def flowstate_response():
return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
'alpn_proto_negotiated': b'http/1.1',
'cipher_name': 'TLS_AES_256_GCM_SHA384',
'clientcert': None,
'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
'mitmcert': '',
'sni': 'yolo.jetzt',
'timestamp_end': None,
'timestamp_start': 1619461916.6160116,
'timestamp_tls_setup': 1619461916.7581937,
'tls_established': True,
'tls_extensions': [],
'tls_version': 'TLSv1.3'},
'error': None,
'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
'intercepted': True,
'is_replay': None,
'marked': False,
'metadata': {},
'mode': 'transparent',
'request': {'authority': b'',
'content': b'',
'headers': ((b'Host', b'yolo.jetzt'),
(b'User-Agent', b'curl/7.75.0'),
(b'Accept', b'*/*')),
'host': 'yolo.jetzt',
'http_version': b'HTTP/1.1',
'method': b'GET',
'path': b'/',
'port': 443,
'scheme': b'https',
'timestamp_end': 1619461916.7603076,
'timestamp_start': 1619461916.7588415,
'trailers': None},
'response': {'content': b'<!DOCTYPE html">\n<html>\n<head> \n<meta http-equi'
b'v="content-type" content="text/html;charset=ISO-8859'
b'-1">\n<title>todays yolo - 3026</title> \n<style '
b'type="text/css" media="screen">\n<!--\nbody \n{\ncol'
b'or: white;\nbackground-color: #003;\nmargin: 0px\n}'
b'\n\n.yolo\n{\nfont-family: Verdana, Geneva, Arial, s'
b'ans-serif;\nfont-weight: bold;\nfont-size: 3em;\n}\n'
b'\na:visited {\n\tcolor: blue\n}\n\n#content '
b' \n{\nfont-family: Verdana, Geneva, Arial, sans-se'
b'rif;\ncolor: white;\nbackground-color: transparent'
b';\ntext-align: center;\nfont-size: 2em;\nposition: '
b'absolute;\ntop: 50%;\nleft: 50%;\n-webkit-transform'
b': translateX(-50%) translateY(-50%);\n-moz-transf'
b'orm: translateX(-50%) translateY(-50%);\n-ms-tran'
b'sform: translateX(-50%) translateY(-50%);\ntransf'
b'orm: translateX(-50%) translateY(-50%);\noverflow'
b': visible;\nvisibility: visible;\ndisplay: block\n}'
b'\n\n#sponsors \n{\nfont-family: Verdana, Gene'
b'va, Arial, sans-serif;\ncolor: white;\nbackground-'
b'color: transparent;\ntext-align: center;\nfont-siz'
b'e: 1em;\nposition: absolute;\ntop: 90%;\nleft: 50%;'
b'\n-webkit-transform: translateX(-50%) translateY('
b'-50%);\n-moz-transform: translateX(-50%) translat'
b'eY(-50%);\n-ms-transform: translateX(-50%) transl'
b'ateY(-50%);\ntransform: translateX(-50%) translat'
b'eY(-50%);\noverflow: visible;\nvisibility: visible'
b';\ndisplay: block\n}\n\n#progress\n{\n\tbackground-'
b'color: #f08;\n\tposition: absolute;\n\tleft: 0px'
b';\n\ttop: 0px;\n\theight: 10px;\n}\n\n-->\n</sty'
b'le>\n</head>\n<body>\n<div id="progress" class="pro'
b'gressBarDiv"></div>\n<div id="content">\nthe yolo '
b'for today is<br>\n<span class="yolo" id="yoloValu'
b'e" title="YOLOVALUE">3026</span><br>\n<br>\n</div>'
b'\n<div id="sponsors">\n\t<a href="https://cat.yolo.'
b'jetzt">Cat</a>\n\t<br>\n\tRegulation (EU) 2016/679 c'
b'ompliant\n</div>\n<script type="text/javascript">\n'
b'var currentYoloTS = 1619388000000; //Multiply by 100'
b'0 the YOLO way\nvar tDiff = Date.now() - currentY'
b'oloTS;\n\nvar dayMS = 24 * 60 * 60 * 1000;\n\nconsol'
b'e.debug(currentYoloTS);\nconsole.debug(Date.now()'
b' - currentYoloTS);\nfunction checkTimestamps()\n{\n'
b'\ttDiff = Date.now() - currentYoloTS;\n\tupdateUi()'
b';\n\tif(tDiff > dayMS)\n\t{\n\t\tclearInterval(upda'
b'teInterval);\n\t\tlocation.reload();\t\n\t}\n}\n'
b'\nfunction updateUi()\n{\n\tvalPercent = tDiff / day'
b'MS * 100;\n\tif(valPercent > 100) valPercent = 100'
b';\n\tdocument.getElementById("progress").style = "'
b'width: " + valPercent + "%;";\n\tif(tDiff > dayMS)'
b'\n\t{\n\t\tdocument.getElementById("yoloValue").i'
b'nnerHTML = "&#xfffd;";\n\t}\n}\n\nupdateUi();\nvar'
b' updateInterval = setInterval(checkTimestamps, 5000)'
b';\n\n//YOLODATE\n</script>\n</body>\n</html>\n',
'headers': ((b'Server', b'nginx'),
(b'Date', b'Mon, 26 Apr 2021 18:31:56 GMT'),
(b'Content-Type', b'text/html'),
(b'Content-Length', b'2460'),
(b'Last-Modified', b'Sun, 25 Apr 2021 22:00:00 GMT'),
(b'Connection', b'keep-alive'),
(b'ETag', b'"6085e660-99c"'),
(b'Strict-Transport-Security',
b'max-age=31536000; includeSubDomains; preload'),
(b'X-Xss-Protection', b'1; mode=block'),
(b'X-Content-Type-Options', b'nosniff'),
(b'Content-Security-Policy',
b"default-src 'self'; script-src 'self' 'unsafe-in"
b"line'; connect-src 'self'; img-src 'self'; style"
b"-src 'self' 'unsafe-inline';"),
(b'X-Frame-Options', b'SAMEORIGIN'),
(b'Referrer-Policy', b'no-referrer'),
(b'Accept-Ranges', b'bytes')),
'http_version': b'HTTP/1.1',
'reason': b'OK',
'status_code': 200,
'timestamp_end': 1619461916.7979567,
'timestamp_start': 1619461916.7935555,
'trailers': None},
'server_conn': {'address': ('yolo.jetzt', 443),
'alpn_proto_negotiated': b'http/1.1',
'cert': '',
'id': '64c838e2-ab74-49c4-9ebe-70e8d17e9081',
'ip_address': ('95.156.226.69', 443),
'sni': 'yolo.jetzt',
'source_address': ('10.23.42.102', 43399),
'timestamp_end': None,
'timestamp_start': 1619461916.63117,
'timestamp_tcp_setup': 1619461916.6659565,
'timestamp_tls_setup': 1619461916.7522914,
'tls_established': True,
'tls_version': 'TLSv1.2',
'via': None},
'type': 'http',
'version': 9}
class MitmAddonTestServer:
def __init__(self, queue, path: str):
self.queue = queue
@ -86,6 +219,7 @@ class MitmAddonTestServer:
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
self.send(msg)
@pytest.fixture
def client_server():
queue = Queue()
@ -96,32 +230,200 @@ def client_server():
client.start()
server = MitmAddonTestServer(queue, sock)
server.connect()
yield client, server
yield queue, client, server
client.join(1)
server.disconnect()
class TestBigSnitchWrapper:
def test_request_convert(self, flowstate_request):
req = bRequest(flow=flowstate_request)
d = {'content': b'',
'headers': [bHeader(key=b'Host', value=b'yolo.jetzt'),
bHeader(key=b'User-Agent', value=b'curl/7.75.0'),
bHeader(key=b'Accept', value=b'*/*')],
d = {'content': '',
'headers': [bHeader(key='Host', value='yolo.jetzt'),
bHeader(key='User-Agent', value='curl/7.75.0'),
bHeader(key='Accept', value='*/*')],
'host': 'yolo.jetzt',
'http_version': b'HTTP/1.1',
'method': b'GET',
'http_version': 'HTTP/1.1',
'method': 'GET',
'port': 443,
'scheme': b'https',
'scheme': 'https',
'server_ip_address': '95.156.226.69',
'timestamp_end': 1619390482.69,
'timestamp_start': 1619390482.6886377,
'tls': True}
assert not DeepDiff(req.__dict__, d)
"""
def test_response_convert(self, flowstate_response):
res = bResponse(flow=flowstate_response)
d = {'content': '<!DOCTYPE html">\n'
'<html>\n'
'<head> \n'
'<meta http-equiv="content-type" '
'content="text/html;charset=ISO-8859-1">\n'
'<title>todays yolo - 3026</title> \n'
'<style type="text/css" media="screen">\n'
'<!--\n'
'body \n'
'{\n'
'color: white;\n'
'background-color: #003;\n'
'margin: 0px\n'
'}\n'
'\n'
'.yolo\n'
'{\n'
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
'font-weight: bold;\n'
'font-size: 3em;\n'
'}\n'
'\n'
'a:visited {\n'
'\tcolor: blue\n'
'}\n'
'\n'
'#content \n'
'{\n'
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
'color: white;\n'
'background-color: transparent;\n'
'text-align: center;\n'
'font-size: 2em;\n'
'position: absolute;\n'
'top: 50%;\n'
'left: 50%;\n'
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
'-moz-transform: translateX(-50%) translateY(-50%);\n'
'-ms-transform: translateX(-50%) translateY(-50%);\n'
'transform: translateX(-50%) translateY(-50%);\n'
'overflow: visible;\n'
'visibility: visible;\n'
'display: block\n'
'}\n'
'\n'
'#sponsors \n'
'{\n'
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
'color: white;\n'
'background-color: transparent;\n'
'text-align: center;\n'
'font-size: 1em;\n'
'position: absolute;\n'
'top: 90%;\n'
'left: 50%;\n'
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
'-moz-transform: translateX(-50%) translateY(-50%);\n'
'-ms-transform: translateX(-50%) translateY(-50%);\n'
'transform: translateX(-50%) translateY(-50%);\n'
'overflow: visible;\n'
'visibility: visible;\n'
'display: block\n'
'}\n'
'\n'
'#progress\n'
'{\n'
'\tbackground-color: #f08;\n'
'\tposition: absolute;\n'
'\tleft: 0px;\n'
'\ttop: 0px;\n'
'\theight: 10px;\n'
'}\n'
'\n'
'-->\n'
'</style>\n'
'</head>\n'
'<body>\n'
'<div id="progress" class="progressBarDiv"></div>\n'
'<div id="content">\n'
'the yolo for today is<br>\n'
'<span class="yolo" id="yoloValue" '
'title="YOLOVALUE">3026</span><br>\n'
'<br>\n'
'</div>\n'
'<div id="sponsors">\n'
'\t<a href="https://cat.yolo.jetzt">Cat</a>\n'
'\t<br>\n'
'\tRegulation (EU) 2016/679 compliant\n'
'</div>\n'
'<script type="text/javascript">\n'
'var currentYoloTS = 1619388000000; //Multiply by 1000 the YOLO '
'way\n'
'var tDiff = Date.now() - currentYoloTS;\n'
'\n'
'var dayMS = 24 * 60 * 60 * 1000;\n'
'\n'
'console.debug(currentYoloTS);\n'
'console.debug(Date.now() - currentYoloTS);\n'
'function checkTimestamps()\n'
'{\n'
'\ttDiff = Date.now() - currentYoloTS;\n'
'\tupdateUi();\n'
'\tif(tDiff > dayMS)\n'
'\t{\n'
'\t\tclearInterval(updateInterval);\n'
'\t\tlocation.reload();\t\n'
'\t}\n'
'}\n'
'\n'
'function updateUi()\n'
'{\n'
'\tvalPercent = tDiff / dayMS * 100;\n'
'\tif(valPercent > 100) valPercent = 100;\n'
'\tdocument.getElementById("progress").style = "width: " + '
'valPercent + "%;";\n'
'\tif(tDiff > dayMS)\n'
'\t{\n'
'\t\tdocument.getElementById("yoloValue").innerHTML = "&#xfffd;";\n'
'\t}\n'
'}\n'
'\n'
'updateUi();\n'
'var updateInterval = setInterval(checkTimestamps, 5000);\n'
'\n'
'//YOLODATE\n'
'</script>\n'
'</body>\n'
'</html>\n',
'headers': [bHeader(key='Server', value='nginx'),
bHeader(key='Date', value='Mon, 26 Apr 2021 18:31:56 GMT'),
bHeader(key='Content-Type', value='text/html'),
bHeader(key='Content-Length', value='2460'),
bHeader(key='Last-Modified', value='Sun, 25 Apr 2021 22:00:00 GMT'),
bHeader(key='Connection', value='keep-alive'),
bHeader(key='ETag', value='"6085e660-99c"'),
bHeader(key='Strict-Transport-Security', value='max-age=31536000; includeSubDomains; preload'),
bHeader(key='X-Xss-Protection', value='1; mode=block'),
bHeader(key='X-Content-Type-Options', value='nosniff'),
bHeader(key='Content-Security-Policy', value="default-src 'self'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self'; style-src 'self' 'unsafe-inline';"),
bHeader(key='X-Frame-Options', value='SAMEORIGIN'),
bHeader(key='Referrer-Policy', value='no-referrer'),
bHeader(key='Accept-Ranges', value='bytes')],
'http_version': 'HTTP/1.1',
'reason': 'OK',
'status_code': 200,
'timestamp_end': 1619461916.7979567,
'timestamp_start': 1619461916.7935555}
assert not DeepDiff(res.__dict__, d)
class TestMitmAddon:
def test_get_new_flows_empty(self, client_server):
queue, client, server = client_server
# queue empty, flows empty
assert queue.empty()
assert not len(client.flows)
client.get_new_flows()
# afterwards too
assert queue.empty()
assert not len(client.flows)
def test_get_new_flows_single(self, client_server):
queue, client, server = client_server
def test_request(self, client_server):
self.client, self.server = client_server
queue, client, server = client_server
# create request
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
self.q.put_nowait((flow.id, flowitem))
"""
#flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flowstate_request)
#self.q.put_nowait(('51215b69-c76f-4ac2-afcb-da3b823d9f88', flowitem))