End/bigsnitch
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

more tests

Browse source
This commit is contained in:
Tim Blume 2021-04-27 16:23:37 +02:00
parent 75a5ba79db
commit c20ec945a1
4 changed files with 567 additions and 264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
mitmaddon/bigsnitch.py
View file

@ -3,18 +3,18 @@
import pdb import pdb
from queue import Queue, Empty from queue import Queue
from networkthread import bFlowState, FlowItem, NetworkThread from networkthread import bFlowState, FlowItem, NetworkThread
class BigSnitchBridge: class BigSnitchBridge:
def __init__(self): def __init__(self):
print("BigSnitchBridge started") print("BigSnitchBridge started")
self.q = Queue() self.q = Queue()
#self.thread = NetworkThread("network", self.q) self.thread = NetworkThread("network", self.q)
#self.thread.start() self.thread.start()
def request(self, flow): def request(self, flow):
pdb.set_trace()
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow) flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
self.q.put_nowait((flow.id, flowitem)) self.q.put_nowait((flow.id, flowitem))
# intercept until ACK received # intercept until ACK received
@ -27,7 +27,7 @@ class BigSnitchBridge:
flow.intercept() flow.intercept()
def error(self, flow): def error(self, flow):
flowitem = FlowItem(bFlowState.ERROR, flow, time.monotonic()) flowitem = FlowItem(bFlowState.ERROR, flow)
self.q.put_nowait((flow.id, flowitem)) self.q.put_nowait((flow.id, flowitem))

378
mitmaddon/networkthread.py
View file

@ -1,6 +1,5 @@
import pdb import pdb
from mitmproxy import ctx
from mitmproxy.flow import Flow from mitmproxy.flow import Flow
import threading import threading
import time import time
@ -9,33 +8,34 @@ import json
import os import os
from enum import Enum from enum import Enum
from dataclasses import dataclass from dataclasses import dataclass
from typing import List, Dict from typing import List, Dict, Any
from queue import Queue, Empty from queue import Queue, Empty
"""
# this method is used to convert flow states (generated with get_state()) to json
def convert_to_strings(obj):
if isinstance(obj, dict):
return {convert_to_strings(key): convert_to_strings(value)
for key, value in obj.items()}
elif isinstance(obj, list) or isinstance(obj, tuple):
return [convert_to_strings(element) for element in obj]
elif isinstance(obj, bytes):
try:
data = obj.decode('unicode-escape').encode('latin1').decode('utf-8')
except:
print(obj)
data = str(obj)[2:-1]
return data
return obj # this method is used to convert flow states (generated with get_state()) to json
""" def convert_to_strings(obj: Any) -> Any:
if isinstance(obj, dict):
return {convert_to_strings(key): convert_to_strings(value)
for key, value in obj.items()}
elif isinstance(obj, list) or isinstance(obj, tuple):
return [convert_to_strings(element) for element in obj]
elif isinstance(obj, bytes):
try:
data = obj.decode('unicode-escape').encode('latin1').decode('utf-8')
except:
print(obj)
data = str(obj)[2:-1]
return data
return obj
@dataclass @dataclass
class bHeader: class bHeader:
key: str key: str
value: str value: str
@dataclass @dataclass
class bRequest: class bRequest:
server_ip_address: str server_ip_address: str
@ -57,6 +57,8 @@ class bRequest:
# init from flow dict # init from flow dict
def __init__(self, flow: dict): def __init__(self, flow: dict):
flow = convert_to_strings(flow)
self.server_ip_address = flow["server_conn"]["ip_address"][0] self.server_ip_address = flow["server_conn"]["ip_address"][0]
self.tls = flow["server_conn"]["tls_established"] self.tls = flow["server_conn"]["tls_established"]
self.content = flow["request"]["content"] self.content = flow["request"]["content"]
@ -69,8 +71,12 @@ class bRequest:
self.timestamp_end = flow["request"]["timestamp_end"] self.timestamp_end = flow["request"]["timestamp_end"]
self.headers = [] self.headers = []
for k,v in flow["request"]["headers"]: for k, v in flow["request"]["headers"]:
self.headers.append(bHeader(k,v)) self.headers.append(bHeader(str(k), str(v)))
def json(self) -> dict:
return {}
@dataclass @dataclass
class bResponse: class bResponse:
@ -84,15 +90,21 @@ class bResponse:
headers: List[bHeader] headers: List[bHeader]
def __init__(self, flow: dict): def __init__(self, flow: dict):
self.status_code = flow["status_code"] flow = convert_to_strings(flow)
self.http_version = flow["http_version"] self.status_code = flow["response"]["status_code"]
self.reason = flow["reason"] self.http_version = flow["response"]["http_version"]
self.content = flow["content"] self.reason = flow["response"]["reason"]
self.timestamp_start = flow["timestamp_start"] self.content = flow["response"]["content"]
self.timestamp_end = flow["timestamp_end"] self.timestamp_start = flow["response"]["timestamp_start"]
self.timestamp_end = flow["response"]["timestamp_end"]
self.headers = []
for k, v in flow["response"]["headers"]:
self.headers.append(bHeader(k, v))
def json(self) -> dict:
return {}
for k,v in flow["headers"]:
self.headers.append(bHeader(k,v))
@dataclass @dataclass
class bFlowState(Enum): class bFlowState(Enum):
@ -102,6 +114,7 @@ class bFlowState(Enum):
UNSENT_HTTP_RESPONSE = 3 UNSENT_HTTP_RESPONSE = 3
SENT_HTTP_RESPONSE = 4 SENT_HTTP_RESPONSE = 4
@dataclass @dataclass
class bPacketType: class bPacketType:
NACK = 0 NACK = 0
@ -113,6 +126,7 @@ class bPacketType:
HTTP_REQUEST = 6 HTTP_REQUEST = 6
HTTP_RESPONSE = 7 HTTP_RESPONSE = 7
@dataclass @dataclass
class bPacket: class bPacket:
ptype: bPacketType ptype: bPacketType
@ -124,6 +138,7 @@ class bPacket:
self.flowid = str(json["id"]) self.flowid = str(json["id"])
self.data = json["data"] self.data = json["data"]
@dataclass @dataclass
class FlowItem: class FlowItem:
state: bFlowState state: bFlowState
@ -131,178 +146,163 @@ class FlowItem:
time: float = 0 time: float = 0
retries_left: int = 5 retries_left: int = 5
""" """
The network thread communicates with the bigsnitch plugin using zeromq. The network thread communicates with the bigsnitch plugin using zeromq.
""" """
class NetworkThread(threading.Thread): class NetworkThread(threading.Thread):
def __init__(self, name: str, queue: Queue, path: str = None): def __init__(self, name: str, queue: Queue, path: str = None):
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.name = name self.name = name
# path # path
self.path = path self.path = path
if not self.path: if not self.path:
self.path = os.environ.get("BIGSNITCH_PATH", None) self.path = os.environ.get("BIGSNITCH_PATH", None)
if not self.path: if not self.path:
self.path = "tcp://127.0.0.1:12345" self.path = "tcp://127.0.0.1:12345"
# queue for communicating with the main mitmproxy thread # queue for communicating with the main mitmproxy thread
# contains tuples of (id, FlowItem) # contains tuples of (id, FlowItem)
self.q = queue self.q = queue
# for zmq use # for zmq use
self.context = zmq.Context() self.context = zmq.Context()
# all current flows being handled by mitmproxy # all current self.flows being handled by mitmproxy
self.flows: Dict[FlowItem] = {} self.flows: Dict[FlowItem] = {}
# timer for sending pings to check if the connection broke # timer for sending pings to check if the connection broke
self.timer = time.monotonic() self.timer = time.monotonic()
# retries left for reconnecting # retries left for reconnecting
self.retries = 5 self.retries = 5
# send a single message, no checks involved # send a single message, no checks involved
def send(self, msg): def send(self, msg):
a = convert_to_strings(msg) #a = convert_to_strings(msg)
self.socket.send(str.encode(json.dumps(a))) self.socket.send(str.encode(json.dumps(msg)))
# add new flows from the queue # add new self.flows from the queue
def get_new_flows(self): def get_new_flows(self):
while True: while True:
try: try:
# get new flows that may occured # get new self.flows that may occured
i, flowitem = self.q.get(block=False) i, flowitem = self.q.get(block=False)
if self.flows.get(i, None): if self.flows.get(i, None):
print(f"flow {i} doubled? ignoring...") print(f"flow {i} doubled? ignoring...")
continue
else:
self.flows[i] = flowitem
except Empty:
break
def send_packet(self, pkg: bPacket):
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
self.send(msg)
def send_http_request(self, id: int, request: bRequest):
pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
self.send_packet(pkg)
self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
self.flows[id].time = time.monotonic()
def send_http_response(self, id: int, response: bResponse):
pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
self.send_packet(pkg)
self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
self.flows[id].time = time.monotonic()
# update all current self.flows
# handles the state machine for each flow
def update_flows(self):
for id, flow in self.flows.items():
if self.flows[id].retries <= 0:
self.flows[id].flow.kill()
print(f"http flow {id} timed out! flow killed.")
delta = time.monotonic() - self.flows[id].time
if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
self.send_http_request(id, bRequest(flow.flow))
self.flows[id].retries -= 1
if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
self.send_http_response(id, bResponse(flow.flow))
self.flows[id].retries -= 1
elif flow.state == bFlowState.ERROR:
print(f"error in flow {id}!")
# handle incoming packets / update the statemachine
def handle_packets(self):
while((self.socket.poll(50) & zmq.POLLIN) != 0):
msg = self.socket.recv()
try:
if msg:
result = json.loads(str(msg))
pkg = bPacket(json=result)
# flow ACKed
if pkg.ptype == bPacketType.ACK:
continue
# flow killed
elif pkg.ptype == bPacketType.KILL:
continue continue
else: else:
self.flows.append(flowitem) print(f"got unexpected message {pkg.ptype}")
except json.JSONDecodeError:
print(f"malformed message received {msg}")
except Empty: def run(self):
break print("thread started")
self.connect()
while True:
self.timer = time.monotonic()
self.get_new_flows()
self.handle_packets()
self.update_flows()
def send_packet(self, pkg: bPacket): if self.timer - time.monotonic() < -5:
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data} pass
self.send(msg)
def send_http_request(self, id: int, request: bRequest):
pkg = bPacket(bPacketType.HTTP_REQUEST, id, request.json())
self.send_packet(pkg)
flows[id].state = bFlowState.SENT_HTTP_REQUEST
flows[id].time = time.monotonic()
def send_http_response(self, id: int, response: bResponse):
pkg = bPacket(bPacketType.HTTP_RESPONSE, id, response.json())
self.send_packet(pkg)
flows[id].state = bFlowState.SENT_HTTP_RESPONSE
flows[id].time = time.monotonic()
# update all current flows
# handles the state machine for each flow
def update_flows(self):
for id, flow in self.flows.items():
# send the request if not sent
if state == bFlowState.UNSENT_HTTP_REQUEST:
self.send_request(id, bRequest(flow.flow))
elif state == bFlowState.SENT_HTTP_REQUEST:
# check timer, try resend
delta = time.monotonic() - flows[id].time
if delta > 5:
self.send_request(id, bRequest(flow.flow))
flows[id].retries -= 1
continue
if flows[id].retries <= 0:
flows[id].flow.kill()
print(f"http request {id} timed out! flow killed.")
# send the response if not sent
elif state == bFlowState.UNSENT_HTTP_RESPONSE:
self.send_response(id, bResponse(flow.flow))
elif state == bFlowState.SENT_HTTP_RESPONSE:
# check timer, try resend
delta = time.monotonic() - flows[id].time
if delta > 5:
self.send_response(id, bResponse(flow.flow))
flows[id].retries -= 1
continue
if flows[id].retries <= 0:
flows[id].flow.kill()
print(f"http response {id} timedout! flow killed.")
elif state == bFlowState.ERROR:
print(f"error in flow {id}!")
# handle incoming packets / update the statemachine
def handle_packets(self):
while((self.socket.poll(50) & zmq.POLLIN) != 0):
msg = self.socket.recv()
try:
if msg:
result = json.loads(msg)
pkg = bPacket(json=result)
# flow ACKed
if pkg.ptype == bPacketType.ACK:
continue
# flow killed
elif pkg.ptype == bPacketType.KILL:
continue
else:
print(f"got unexpected message {pkg.ptype}")
except json.JSONDecodeError:
print(f"malformed message received {msg}")
def run(self):
print("thread started")
self.connect()
while True:
self.timer = time.monotonic()
self.get_new_flows()
self.handle_packets()
self.update_flows()
if self.timer - time.monotonic() < -5:
pass
#self.send_msg_and_ack({"msg": "ping"})
def disconnect(self):
self.socket.setsockopt(zmq.LINGER,0)
self.socket.close()
print("disconnected")
def reconnect(self):
print("reconnecting")
self.disconnect()
time.sleep(1)
self.connect()
def connect(self):
self.socket = self.context.socket(zmq.PAIR)
self.socket.connect(self.path)
#self.send_msg_and_ack({"msg": "ping"}) #self.send_msg_and_ack({"msg": "ping"})
print("connected")
""" def disconnect(self):
def send_msg_and_ack(self, msg): self.socket.setsockopt(zmq.LINGER,0)
self.timer = time.monotonic() self.socket.close()
while True: print("disconnected")
#print("m sending")
self.send(msg) def reconnect(self):
if (self.socket.poll(50) & zmq.POLLIN) != 0: print("reconnecting")
msg = self.socket.recv() self.disconnect()
try: time.sleep(1)
if msg: self.connect()
result = json.loads(msg)
if result["msg"] == "ack": def connect(self):
print("m ack received") self.socket = self.context.socket(zmq.PAIR)
return result self.socket.connect(self.path)
else: #self.send_msg_and_ack({"msg": "ping"})
print("got unexpected message {result}") print("connected")
except json.JSONDecodeError: """
print(f"malformed message received {msg}") def send_msg_and_ack(self, msg):
print("no ack received, reconnecting...") self.timer = time.monotonic()
self.reconnect() while True:
return NO_MSG #print("m sending")
""" self.send(msg)
if (self.socket.poll(50) & zmq.POLLIN) != 0:
msg = self.socket.recv()
try:
if msg:
result = json.loads(msg)
if result["msg"] == "ack":
print("m ack received")
return result
else:
print("got unexpected message {result}")
except json.JSONDecodeError:
print(f"malformed message received {msg}")
print("no ack received, reconnecting...")
self.reconnect()
return NO_MSG
"""

1
mitmaddon/requirements.txt
View file

@ -3,3 +3,4 @@ mitmdump
pyzmq pyzmq
deepdiff deepdiff
pytest pytest
tox

442
mitmaddon/test_bigsnitch.py
View file

@ -1,9 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import pdb import pdb
import queue
import pytest import pytest
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
import os import os
import tempfile import tempfile
from queue import Queue from queue import Queue
@ -11,60 +12,192 @@ import zmq
from deepdiff import DeepDiff from deepdiff import DeepDiff
# usual flow state of the request with some big parts removed # usual flow state of the request with some big parts removed
@pytest.fixture @pytest.fixture
def flowstate_request(): def flowstate_request():
return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0), return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0),
'alpn_proto_negotiated': b'http/1.1', 'alpn_proto_negotiated': b'http/1.1',
'cipher_name': 'TLS_AES_256_GCM_SHA384', 'cipher_name': 'TLS_AES_256_GCM_SHA384',
'clientcert': None, 'clientcert': None,
'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea', 'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea',
'mitmcert': '', 'mitmcert': '',
'sni': 'yolo.jetzt', 'sni': 'yolo.jetzt',
'timestamp_end': None, 'timestamp_end': None,
'timestamp_start': 1619390481.8003347, 'timestamp_start': 1619390481.8003347,
'timestamp_tls_setup': 1619390482.6879823, 'timestamp_tls_setup': 1619390482.6879823,
'tls_established': True, 'tls_established': True,
'tls_extensions': [], 'tls_extensions': [],
'tls_version': 'TLSv1.3'}, 'tls_version': 'TLSv1.3'},
'error': None, 'error': None,
'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88', 'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
'intercepted': False, 'intercepted': False,
'is_replay': None, 'is_replay': None,
'marked': False, 'marked': False,
'metadata': {}, 'metadata': {},
'mode': 'transparent', 'mode': 'transparent',
'request': {'authority': b'', 'request': {'authority': b'',
'content': b'', 'content': b'',
'headers': ((b'Host', b'yolo.jetzt'), 'headers': ((b'Host', b'yolo.jetzt'),
(b'User-Agent', b'curl/7.75.0'), (b'User-Agent', b'curl/7.75.0'),
(b'Accept', b'*/*')), (b'Accept', b'*/*')),
'host': 'yolo.jetzt', 'host': 'yolo.jetzt',
'http_version': b'HTTP/1.1', 'http_version': b'HTTP/1.1',
'method': b'GET', 'method': b'GET',
'path': b'/', 'path': b'/',
'port': 443, 'port': 443,
'scheme': b'https', 'scheme': b'https',
'timestamp_end': 1619390482.69, 'timestamp_end': 1619390482.69,
'timestamp_start': 1619390482.6886377, 'timestamp_start': 1619390482.6886377,
'trailers': None}, 'trailers': None},
'response': None, 'response': None,
'server_conn': {'address': ('yolo.jetzt', 443), 'server_conn': {'address': ('yolo.jetzt', 443),
'alpn_proto_negotiated': b'http/1.1', 'alpn_proto_negotiated': b'http/1.1',
'cert': '', 'cert': '',
'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b', 'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b',
'ip_address': ('95.156.226.69', 443), 'ip_address': ('95.156.226.69', 443),
'sni': 'yolo.jetzt', 'sni': 'yolo.jetzt',
'source_address': ('192.168.42.182', 51514), 'source_address': ('192.168.42.182', 51514),
'timestamp_end': None, 'timestamp_end': None,
'timestamp_start': 1619390481.8154442, 'timestamp_start': 1619390481.8154442,
'timestamp_tcp_setup': 1619390481.994565, 'timestamp_tcp_setup': 1619390481.994565,
'timestamp_tls_setup': 1619390482.6819758, 'timestamp_tls_setup': 1619390482.6819758,
'tls_established': True, 'tls_established': True,
'tls_version': 'TLSv1.2', 'tls_version': 'TLSv1.2',
'via': None}, 'via': None},
'type': 'http', 'type': 'http',
'version': 9} 'version': 9}
@pytest.fixture()
def flowstate_response():
return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
'alpn_proto_negotiated': b'http/1.1',
'cipher_name': 'TLS_AES_256_GCM_SHA384',
'clientcert': None,
'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
'mitmcert': '',
'sni': 'yolo.jetzt',
'timestamp_end': None,
'timestamp_start': 1619461916.6160116,
'timestamp_tls_setup': 1619461916.7581937,
'tls_established': True,
'tls_extensions': [],
'tls_version': 'TLSv1.3'},
'error': None,
'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
'intercepted': True,
'is_replay': None,
'marked': False,
'metadata': {},
'mode': 'transparent',
'request': {'authority': b'',
'content': b'',
'headers': ((b'Host', b'yolo.jetzt'),
(b'User-Agent', b'curl/7.75.0'),
(b'Accept', b'*/*')),
'host': 'yolo.jetzt',
'http_version': b'HTTP/1.1',
'method': b'GET',
'path': b'/',
'port': 443,
'scheme': b'https',
'timestamp_end': 1619461916.7603076,
'timestamp_start': 1619461916.7588415,
'trailers': None},
'response': {'content': b'<!DOCTYPE html">\n<html>\n<head> \n<meta http-equi'
b'v="content-type" content="text/html;charset=ISO-8859'
b'-1">\n<title>todays yolo - 3026</title> \n<style '
b'type="text/css" media="screen">\n<!--\nbody \n{\ncol'
b'or: white;\nbackground-color: #003;\nmargin: 0px\n}'
b'\n\n.yolo\n{\nfont-family: Verdana, Geneva, Arial, s'
b'ans-serif;\nfont-weight: bold;\nfont-size: 3em;\n}\n'
b'\na:visited {\n\tcolor: blue\n}\n\n#content '
b' \n{\nfont-family: Verdana, Geneva, Arial, sans-se'
b'rif;\ncolor: white;\nbackground-color: transparent'
b';\ntext-align: center;\nfont-size: 2em;\nposition: '
b'absolute;\ntop: 50%;\nleft: 50%;\n-webkit-transform'
b': translateX(-50%) translateY(-50%);\n-moz-transf'
b'orm: translateX(-50%) translateY(-50%);\n-ms-tran'
b'sform: translateX(-50%) translateY(-50%);\ntransf'
b'orm: translateX(-50%) translateY(-50%);\noverflow'
b': visible;\nvisibility: visible;\ndisplay: block\n}'
b'\n\n#sponsors \n{\nfont-family: Verdana, Gene'
b'va, Arial, sans-serif;\ncolor: white;\nbackground-'
b'color: transparent;\ntext-align: center;\nfont-siz'
b'e: 1em;\nposition: absolute;\ntop: 90%;\nleft: 50%;'
b'\n-webkit-transform: translateX(-50%) translateY('
b'-50%);\n-moz-transform: translateX(-50%) translat'
b'eY(-50%);\n-ms-transform: translateX(-50%) transl'
b'ateY(-50%);\ntransform: translateX(-50%) translat'
b'eY(-50%);\noverflow: visible;\nvisibility: visible'
b';\ndisplay: block\n}\n\n#progress\n{\n\tbackground-'
b'color: #f08;\n\tposition: absolute;\n\tleft: 0px'
b';\n\ttop: 0px;\n\theight: 10px;\n}\n\n-->\n</sty'
b'le>\n</head>\n<body>\n<div id="progress" class="pro'
b'gressBarDiv"></div>\n<div id="content">\nthe yolo '
b'for today is<br>\n<span class="yolo" id="yoloValu'
b'e" title="YOLOVALUE">3026</span><br>\n<br>\n</div>'
b'\n<div id="sponsors">\n\t<a href="https://cat.yolo.'
b'jetzt">Cat</a>\n\t<br>\n\tRegulation (EU) 2016/679 c'
b'ompliant\n</div>\n<script type="text/javascript">\n'
b'var currentYoloTS = 1619388000000; //Multiply by 100'
b'0 the YOLO way\nvar tDiff = Date.now() - currentY'
b'oloTS;\n\nvar dayMS = 24 * 60 * 60 * 1000;\n\nconsol'
b'e.debug(currentYoloTS);\nconsole.debug(Date.now()'
b' - currentYoloTS);\nfunction checkTimestamps()\n{\n'
b'\ttDiff = Date.now() - currentYoloTS;\n\tupdateUi()'
b';\n\tif(tDiff > dayMS)\n\t{\n\t\tclearInterval(upda'
b'teInterval);\n\t\tlocation.reload();\t\n\t}\n}\n'
b'\nfunction updateUi()\n{\n\tvalPercent = tDiff / day'
b'MS * 100;\n\tif(valPercent > 100) valPercent = 100'
b';\n\tdocument.getElementById("progress").style = "'
b'width: " + valPercent + "%;";\n\tif(tDiff > dayMS)'
b'\n\t{\n\t\tdocument.getElementById("yoloValue").i'
b'nnerHTML = "&#xfffd;";\n\t}\n}\n\nupdateUi();\nvar'
b' updateInterval = setInterval(checkTimestamps, 5000)'
b';\n\n//YOLODATE\n</script>\n</body>\n</html>\n',
'headers': ((b'Server', b'nginx'),
(b'Date', b'Mon, 26 Apr 2021 18:31:56 GMT'),
(b'Content-Type', b'text/html'),
(b'Content-Length', b'2460'),
(b'Last-Modified', b'Sun, 25 Apr 2021 22:00:00 GMT'),
(b'Connection', b'keep-alive'),
(b'ETag', b'"6085e660-99c"'),
(b'Strict-Transport-Security',
b'max-age=31536000; includeSubDomains; preload'),
(b'X-Xss-Protection', b'1; mode=block'),
(b'X-Content-Type-Options', b'nosniff'),
(b'Content-Security-Policy',
b"default-src 'self'; script-src 'self' 'unsafe-in"
b"line'; connect-src 'self'; img-src 'self'; style"
b"-src 'self' 'unsafe-inline';"),
(b'X-Frame-Options', b'SAMEORIGIN'),
(b'Referrer-Policy', b'no-referrer'),
(b'Accept-Ranges', b'bytes')),
'http_version': b'HTTP/1.1',
'reason': b'OK',
'status_code': 200,
'timestamp_end': 1619461916.7979567,
'timestamp_start': 1619461916.7935555,
'trailers': None},
'server_conn': {'address': ('yolo.jetzt', 443),
'alpn_proto_negotiated': b'http/1.1',
'cert': '',
'id': '64c838e2-ab74-49c4-9ebe-70e8d17e9081',
'ip_address': ('95.156.226.69', 443),
'sni': 'yolo.jetzt',
'source_address': ('10.23.42.102', 43399),
'timestamp_end': None,
'timestamp_start': 1619461916.63117,
'timestamp_tcp_setup': 1619461916.6659565,
'timestamp_tls_setup': 1619461916.7522914,
'tls_established': True,
'tls_version': 'TLSv1.2',
'via': None},
'type': 'http',
'version': 9}
class MitmAddonTestServer: class MitmAddonTestServer:
def __init__(self, queue, path: str): def __init__(self, queue, path: str):
@ -86,6 +219,7 @@ class MitmAddonTestServer:
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data} msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
self.send(msg) self.send(msg)
@pytest.fixture @pytest.fixture
def client_server(): def client_server():
queue = Queue() queue = Queue()
@ -96,32 +230,200 @@ def client_server():
client.start() client.start()
server = MitmAddonTestServer(queue, sock) server = MitmAddonTestServer(queue, sock)
server.connect() server.connect()
yield client, server yield queue, client, server
client.join(1) client.join(1)
server.disconnect() server.disconnect()
class TestBigSnitchWrapper: class TestBigSnitchWrapper:
def test_request_convert(self, flowstate_request): def test_request_convert(self, flowstate_request):
req = bRequest(flow=flowstate_request) req = bRequest(flow=flowstate_request)
d = {'content': b'', d = {'content': '',
'headers': [bHeader(key=b'Host', value=b'yolo.jetzt'), 'headers': [bHeader(key='Host', value='yolo.jetzt'),
bHeader(key=b'User-Agent', value=b'curl/7.75.0'), bHeader(key='User-Agent', value='curl/7.75.0'),
bHeader(key=b'Accept', value=b'*/*')], bHeader(key='Accept', value='*/*')],
'host': 'yolo.jetzt', 'host': 'yolo.jetzt',
'http_version': b'HTTP/1.1', 'http_version': 'HTTP/1.1',
'method': b'GET', 'method': 'GET',
'port': 443, 'port': 443,
'scheme': b'https', 'scheme': 'https',
'server_ip_address': '95.156.226.69', 'server_ip_address': '95.156.226.69',
'timestamp_end': 1619390482.69, 'timestamp_end': 1619390482.69,
'timestamp_start': 1619390482.6886377, 'timestamp_start': 1619390482.6886377,
'tls': True} 'tls': True}
assert not DeepDiff(req.__dict__, d) assert not DeepDiff(req.__dict__, d)
"""
def test_response_convert(self, flowstate_response):
res = bResponse(flow=flowstate_response)
d = {'content': '<!DOCTYPE html">\n'
'<html>\n'
'<head> \n'
'<meta http-equiv="content-type" '
'content="text/html;charset=ISO-8859-1">\n'
'<title>todays yolo - 3026</title> \n'
'<style type="text/css" media="screen">\n'
'<!--\n'
'body \n'
'{\n'
'color: white;\n'
'background-color: #003;\n'
'margin: 0px\n'
'}\n'
'\n'
'.yolo\n'
'{\n'
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
'font-weight: bold;\n'
'font-size: 3em;\n'
'}\n'
'\n'
'a:visited {\n'
'\tcolor: blue\n'
'}\n'
'\n'
'#content \n'
'{\n'
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
'color: white;\n'
'background-color: transparent;\n'
'text-align: center;\n'
'font-size: 2em;\n'
'position: absolute;\n'
'top: 50%;\n'
'left: 50%;\n'
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
'-moz-transform: translateX(-50%) translateY(-50%);\n'
'-ms-transform: translateX(-50%) translateY(-50%);\n'
'transform: translateX(-50%) translateY(-50%);\n'
'overflow: visible;\n'
'visibility: visible;\n'
'display: block\n'
'}\n'
'\n'
'#sponsors \n'
'{\n'
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
'color: white;\n'
'background-color: transparent;\n'
'text-align: center;\n'
'font-size: 1em;\n'
'position: absolute;\n'
'top: 90%;\n'
'left: 50%;\n'
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
'-moz-transform: translateX(-50%) translateY(-50%);\n'
'-ms-transform: translateX(-50%) translateY(-50%);\n'
'transform: translateX(-50%) translateY(-50%);\n'
'overflow: visible;\n'
'visibility: visible;\n'
'display: block\n'
'}\n'
'\n'
'#progress\n'
'{\n'
'\tbackground-color: #f08;\n'
'\tposition: absolute;\n'
'\tleft: 0px;\n'
'\ttop: 0px;\n'
'\theight: 10px;\n'
'}\n'
'\n'
'-->\n'
'</style>\n'
'</head>\n'
'<body>\n'
'<div id="progress" class="progressBarDiv"></div>\n'
'<div id="content">\n'
'the yolo for today is<br>\n'
'<span class="yolo" id="yoloValue" '
'title="YOLOVALUE">3026</span><br>\n'
'<br>\n'
'</div>\n'
'<div id="sponsors">\n'
'\t<a href="https://cat.yolo.jetzt">Cat</a>\n'
'\t<br>\n'
'\tRegulation (EU) 2016/679 compliant\n'
'</div>\n'
'<script type="text/javascript">\n'
'var currentYoloTS = 1619388000000; //Multiply by 1000 the YOLO '
'way\n'
'var tDiff = Date.now() - currentYoloTS;\n'
'\n'
'var dayMS = 24 * 60 * 60 * 1000;\n'
'\n'
'console.debug(currentYoloTS);\n'
'console.debug(Date.now() - currentYoloTS);\n'
'function checkTimestamps()\n'
'{\n'
'\ttDiff = Date.now() - currentYoloTS;\n'
'\tupdateUi();\n'
'\tif(tDiff > dayMS)\n'
'\t{\n'
'\t\tclearInterval(updateInterval);\n'
'\t\tlocation.reload();\t\n'
'\t}\n'
'}\n'
'\n'
'function updateUi()\n'
'{\n'
'\tvalPercent = tDiff / dayMS * 100;\n'
'\tif(valPercent > 100) valPercent = 100;\n'
'\tdocument.getElementById("progress").style = "width: " + '
'valPercent + "%;";\n'
'\tif(tDiff > dayMS)\n'
'\t{\n'
'\t\tdocument.getElementById("yoloValue").innerHTML = "&#xfffd;";\n'
'\t}\n'
'}\n'
'\n'
'updateUi();\n'
'var updateInterval = setInterval(checkTimestamps, 5000);\n'
'\n'
'//YOLODATE\n'
'</script>\n'
'</body>\n'
'</html>\n',
'headers': [bHeader(key='Server', value='nginx'),
bHeader(key='Date', value='Mon, 26 Apr 2021 18:31:56 GMT'),
bHeader(key='Content-Type', value='text/html'),
bHeader(key='Content-Length', value='2460'),
bHeader(key='Last-Modified', value='Sun, 25 Apr 2021 22:00:00 GMT'),
bHeader(key='Connection', value='keep-alive'),
bHeader(key='ETag', value='"6085e660-99c"'),
bHeader(key='Strict-Transport-Security', value='max-age=31536000; includeSubDomains; preload'),
bHeader(key='X-Xss-Protection', value='1; mode=block'),
bHeader(key='X-Content-Type-Options', value='nosniff'),
bHeader(key='Content-Security-Policy', value="default-src 'self'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self'; style-src 'self' 'unsafe-inline';"),
bHeader(key='X-Frame-Options', value='SAMEORIGIN'),
bHeader(key='Referrer-Policy', value='no-referrer'),
bHeader(key='Accept-Ranges', value='bytes')],
'http_version': 'HTTP/1.1',
'reason': 'OK',
'status_code': 200,
'timestamp_end': 1619461916.7979567,
'timestamp_start': 1619461916.7935555}
assert not DeepDiff(res.__dict__, d)
class TestMitmAddon: class TestMitmAddon:
def test_get_new_flows_empty(self, client_server):
queue, client, server = client_server
# queue empty, flows empty
assert queue.empty()
assert not len(client.flows)
client.get_new_flows()
# afterwards too
assert queue.empty()
assert not len(client.flows)
def test_get_new_flows_single(self, client_server):
queue, client, server = client_server
def test_request(self, client_server): def test_request(self, client_server):
self.client, self.server = client_server queue, client, server = client_server
# create request # create request
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow) #flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flowstate_request)
self.q.put_nowait((flow.id, flowitem)) #self.q.put_nowait(('51215b69-c76f-4ac2-afcb-da3b823d9f88', flowitem))
"""