more tests
This commit is contained in:
Tim Blume
parent
c20ec945a1
commit
34f7d960c2
2 changed files with 450 additions and 199 deletions
|
|
@ -71,11 +71,13 @@ class bRequest:
|
||||||
self.timestamp_end = flow["request"]["timestamp_end"]
|
self.timestamp_end = flow["request"]["timestamp_end"]
|
||||||
|
|
||||||
self.headers = []
|
self.headers = []
|
||||||
for k, v in flow["request"]["headers"]:
|
for k, v in flow["request"].get("headers", {}):
|
||||||
self.headers.append(bHeader(str(k), str(v)))
|
self.headers.append(bHeader(str(k), str(v)))
|
||||||
|
|
||||||
def json(self) -> dict:
|
def json(self) -> dict:
|
||||||
return {}
|
state = vars(self).copy()
|
||||||
|
state["headers"] = {h.key: h.value for h in self.headers}
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -99,11 +101,13 @@ class bResponse:
|
||||||
self.timestamp_end = flow["response"]["timestamp_end"]
|
self.timestamp_end = flow["response"]["timestamp_end"]
|
||||||
|
|
||||||
self.headers = []
|
self.headers = []
|
||||||
for k, v in flow["response"]["headers"]:
|
for k, v in flow["response"].get("headers", {}):
|
||||||
self.headers.append(bHeader(k, v))
|
self.headers.append(bHeader(k, v))
|
||||||
|
|
||||||
def json(self) -> dict:
|
def json(self) -> dict:
|
||||||
return {}
|
state = vars(self).copy()
|
||||||
|
state["headers"] = [{h.key: h.value} for h in self.headers]
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -133,18 +137,13 @@ class bPacket:
|
||||||
flowid: str
|
flowid: str
|
||||||
data: str
|
data: str
|
||||||
|
|
||||||
def __init__(self, json: Dict):
|
|
||||||
self.ptype = json["type"]
|
|
||||||
self.flowid = str(json["id"])
|
|
||||||
self.data = json["data"]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class FlowItem:
|
class FlowItem:
|
||||||
state: bFlowState
|
state: bFlowState
|
||||||
flow: Flow
|
flow: Flow
|
||||||
time: float = 0
|
time: float = 0
|
||||||
retries_left: int = 5
|
retries: int = 5
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
@ -187,7 +186,7 @@ class NetworkThread(threading.Thread):
|
||||||
# get new self.flows that may occured
|
# get new self.flows that may occured
|
||||||
i, flowitem = self.q.get(block=False)
|
i, flowitem = self.q.get(block=False)
|
||||||
if self.flows.get(i, None):
|
if self.flows.get(i, None):
|
||||||
print(f"flow {i} doubled? ignoring...")
|
raise ValueError(f"flow {i} doubled? ignoring...")
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
self.flows[i] = flowitem
|
self.flows[i] = flowitem
|
||||||
|
|
@ -199,40 +198,63 @@ class NetworkThread(threading.Thread):
|
||||||
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
|
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
|
||||||
self.send(msg)
|
self.send(msg)
|
||||||
|
|
||||||
def send_http_request(self, id: int, request: bRequest):
|
|
||||||
pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
|
|
||||||
self.send_packet(pkg)
|
|
||||||
self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
|
|
||||||
self.flows[id].time = time.monotonic()
|
|
||||||
|
|
||||||
def send_http_response(self, id: int, response: bResponse):
|
|
||||||
pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
|
|
||||||
self.send_packet(pkg)
|
|
||||||
self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
|
|
||||||
self.flows[id].time = time.monotonic()
|
|
||||||
|
|
||||||
# update all current self.flows
|
# update all current self.flows
|
||||||
# handles the state machine for each flow
|
# handles the state machine for each flow
|
||||||
def update_flows(self):
|
def update_flows(self):
|
||||||
for id, flow in self.flows.items():
|
# force copy of item list, so we can remove dict items in the loop
|
||||||
|
for id, flow in list(self.flows.items()):
|
||||||
if self.flows[id].retries <= 0:
|
if self.flows[id].retries <= 0:
|
||||||
self.flows[id].flow.kill()
|
if self.flows[id].flow:
|
||||||
|
self.flows[id].flow.kill()
|
||||||
print(f"http flow {id} timed out! flow killed.")
|
print(f"http flow {id} timed out! flow killed.")
|
||||||
|
del self.flows[id]
|
||||||
|
|
||||||
delta = time.monotonic() - self.flows[id].time
|
delta = time.monotonic() - self.flows[id].time
|
||||||
|
|
||||||
if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
|
if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
|
||||||
flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
|
flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
|
||||||
self.send_http_request(id, bRequest(flow.flow))
|
pkg = bPacket(bPacketType.HTTP_REQUEST, id, bRequest(flow.flow.get_state()).json())
|
||||||
|
self.send_packet(pkg)
|
||||||
|
|
||||||
|
self.flows[id].time = time.monotonic()
|
||||||
|
self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
|
||||||
self.flows[id].retries -= 1
|
self.flows[id].retries -= 1
|
||||||
|
|
||||||
if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
|
elif flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
|
||||||
flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
|
flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
|
||||||
self.send_http_response(id, bResponse(flow.flow))
|
pkg = bPacket(bPacketType.HTTP_RESPONSE, id, bResponse(flow.flow.get_state()).json())
|
||||||
|
self.send_packet(pkg)
|
||||||
|
|
||||||
|
self.flows[id].time = time.monotonic()
|
||||||
|
self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
|
||||||
self.flows[id].retries -= 1
|
self.flows[id].retries -= 1
|
||||||
|
|
||||||
elif flow.state == bFlowState.ERROR:
|
if flow.state == bFlowState.ERROR:
|
||||||
print(f"error in flow {id}!")
|
print(f"error in flow {id}!")
|
||||||
|
del self.flows[id]
|
||||||
|
|
||||||
|
def handle_packet(self, pkg):
|
||||||
|
flow = self.flows.get(pkg.flowid, None)
|
||||||
|
|
||||||
|
# flow ACKed
|
||||||
|
if pkg.ptype == bPacketType.ACK:
|
||||||
|
if flow and flow.flow:
|
||||||
|
flow.flow.resume()
|
||||||
|
else:
|
||||||
|
raise ValueError("unknown flow")
|
||||||
|
|
||||||
|
# flow killed
|
||||||
|
elif pkg.ptype == bPacketType.KILL:
|
||||||
|
if flow and flow.flow:
|
||||||
|
flow.flow.kill()
|
||||||
|
else:
|
||||||
|
raise ValueError("unknown flow")
|
||||||
|
|
||||||
|
if flow:
|
||||||
|
del self.flows[pkg.flowid]
|
||||||
|
|
||||||
|
else:
|
||||||
|
print(f"got unexpected message {pkg.ptype}")
|
||||||
|
|
||||||
# handle incoming packets / update the statemachine
|
# handle incoming packets / update the statemachine
|
||||||
def handle_packets(self):
|
def handle_packets(self):
|
||||||
|
|
@ -242,14 +264,7 @@ class NetworkThread(threading.Thread):
|
||||||
if msg:
|
if msg:
|
||||||
result = json.loads(str(msg))
|
result = json.loads(str(msg))
|
||||||
pkg = bPacket(json=result)
|
pkg = bPacket(json=result)
|
||||||
# flow ACKed
|
self.handle_packet(pkg)
|
||||||
if pkg.ptype == bPacketType.ACK:
|
|
||||||
continue
|
|
||||||
# flow killed
|
|
||||||
elif pkg.ptype == bPacketType.KILL:
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
print(f"got unexpected message {pkg.ptype}")
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
print(f"malformed message received {msg}")
|
print(f"malformed message received {msg}")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,10 @@
|
||||||
|
|
||||||
import pdb
|
import pdb
|
||||||
import queue
|
import queue
|
||||||
|
import uuid
|
||||||
import pytest
|
import pytest
|
||||||
|
from mitmproxy.http import HTTPFlow, HTTPRequest, HTTPResponse
|
||||||
|
from mitmproxy.connections import ClientConnection, ServerConnection
|
||||||
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
|
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
@ -13,22 +15,64 @@ import zmq
|
||||||
from deepdiff import DeepDiff
|
from deepdiff import DeepDiff
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def flowstate_clientconn():
|
||||||
|
return dict(
|
||||||
|
id=str(uuid.uuid4()),
|
||||||
|
address=("address", 22),
|
||||||
|
source_address=("address", 22),
|
||||||
|
ip_address=("192.168.0.1", 22),
|
||||||
|
timestamp_start=946681202,
|
||||||
|
timestamp_tcp_setup=946681203,
|
||||||
|
timestamp_tls_setup=946681204,
|
||||||
|
timestamp_end=946681205,
|
||||||
|
tls_established=True,
|
||||||
|
sni="address",
|
||||||
|
alpn=None,
|
||||||
|
tls_version="TLSv1.2",
|
||||||
|
via=None,
|
||||||
|
state=0,
|
||||||
|
error=None,
|
||||||
|
tls=False,
|
||||||
|
certificate_list=[],
|
||||||
|
alpn_offers=[],
|
||||||
|
cipher_name=None,
|
||||||
|
cipher_list=[],
|
||||||
|
via2=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def flowstate_serverconn():
|
||||||
|
return dict(
|
||||||
|
id=str(uuid.uuid4()),
|
||||||
|
address=("address", 22),
|
||||||
|
source_address=("address", 22),
|
||||||
|
ip_address=("192.168.0.1", 22),
|
||||||
|
timestamp_start=946681202,
|
||||||
|
timestamp_tcp_setup=946681203,
|
||||||
|
timestamp_tls_setup=946681204,
|
||||||
|
timestamp_end=946681205,
|
||||||
|
tls_established=True,
|
||||||
|
sni="address",
|
||||||
|
alpn=None,
|
||||||
|
tls_version="TLSv1.2",
|
||||||
|
via=None,
|
||||||
|
state=0,
|
||||||
|
error=None,
|
||||||
|
tls=False,
|
||||||
|
certificate_list=[],
|
||||||
|
alpn_offers=[],
|
||||||
|
cipher_name=None,
|
||||||
|
cipher_list=[],
|
||||||
|
via2=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# usual flow state of the request with some big parts removed
|
# usual flow state of the request with some big parts removed
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def flowstate_request():
|
def flowstate_request(flowstate_clientconn, flowstate_serverconn):
|
||||||
return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0),
|
return {'client_conn': flowstate_clientconn,
|
||||||
'alpn_proto_negotiated': b'http/1.1',
|
|
||||||
'cipher_name': 'TLS_AES_256_GCM_SHA384',
|
|
||||||
'clientcert': None,
|
|
||||||
'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea',
|
|
||||||
'mitmcert': '',
|
|
||||||
'sni': 'yolo.jetzt',
|
|
||||||
'timestamp_end': None,
|
|
||||||
'timestamp_start': 1619390481.8003347,
|
|
||||||
'timestamp_tls_setup': 1619390482.6879823,
|
|
||||||
'tls_established': True,
|
|
||||||
'tls_extensions': [],
|
|
||||||
'tls_version': 'TLSv1.3'},
|
|
||||||
'error': None,
|
'error': None,
|
||||||
'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
|
'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
|
||||||
'intercepted': False,
|
'intercepted': False,
|
||||||
|
|
@ -36,8 +80,7 @@ def flowstate_request():
|
||||||
'marked': False,
|
'marked': False,
|
||||||
'metadata': {},
|
'metadata': {},
|
||||||
'mode': 'transparent',
|
'mode': 'transparent',
|
||||||
'request': {'authority': b'',
|
'request': {'content': b'',
|
||||||
'content': b'',
|
|
||||||
'headers': ((b'Host', b'yolo.jetzt'),
|
'headers': ((b'Host', b'yolo.jetzt'),
|
||||||
(b'User-Agent', b'curl/7.75.0'),
|
(b'User-Agent', b'curl/7.75.0'),
|
||||||
(b'Accept', b'*/*')),
|
(b'Accept', b'*/*')),
|
||||||
|
|
@ -49,156 +92,138 @@ def flowstate_request():
|
||||||
'scheme': b'https',
|
'scheme': b'https',
|
||||||
'timestamp_end': 1619390482.69,
|
'timestamp_end': 1619390482.69,
|
||||||
'timestamp_start': 1619390482.6886377,
|
'timestamp_start': 1619390482.6886377,
|
||||||
'trailers': None},
|
'authority': '',
|
||||||
|
'trailers': '',
|
||||||
|
},
|
||||||
'response': None,
|
'response': None,
|
||||||
'server_conn': {'address': ('yolo.jetzt', 443),
|
"server_conn": flowstate_serverconn,
|
||||||
'alpn_proto_negotiated': b'http/1.1',
|
|
||||||
'cert': '',
|
|
||||||
'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b',
|
|
||||||
'ip_address': ('95.156.226.69', 443),
|
|
||||||
'sni': 'yolo.jetzt',
|
|
||||||
'source_address': ('192.168.42.182', 51514),
|
|
||||||
'timestamp_end': None,
|
|
||||||
'timestamp_start': 1619390481.8154442,
|
|
||||||
'timestamp_tcp_setup': 1619390481.994565,
|
|
||||||
'timestamp_tls_setup': 1619390482.6819758,
|
|
||||||
'tls_established': True,
|
|
||||||
'tls_version': 'TLSv1.2',
|
|
||||||
'via': None},
|
|
||||||
'type': 'http',
|
'type': 'http',
|
||||||
'version': 9}
|
'version': 9}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def flow_request(flowstate_request):
|
||||||
|
c = ClientConnection.make_dummy("192.168.0.2")
|
||||||
|
s = ServerConnection.make_dummy("192.168.0.1")
|
||||||
|
flow = HTTPFlow(c, s)
|
||||||
|
flow.request = HTTPRequest.from_state(flowstate_request["request"])
|
||||||
|
return flow
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def flowstate_response():
|
def flowstate_response(flowstate_clientconn, flowstate_serverconn):
|
||||||
return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
|
return {
|
||||||
'alpn_proto_negotiated': b'http/1.1',
|
"client_conn": flowstate_clientconn,
|
||||||
'cipher_name': 'TLS_AES_256_GCM_SHA384',
|
'error': None,
|
||||||
'clientcert': None,
|
'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
|
||||||
'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
|
'intercepted': True,
|
||||||
'mitmcert': '',
|
'is_replay': None,
|
||||||
'sni': 'yolo.jetzt',
|
'marked': False,
|
||||||
'timestamp_end': None,
|
'metadata': {},
|
||||||
'timestamp_start': 1619461916.6160116,
|
'mode': 'transparent',
|
||||||
'timestamp_tls_setup': 1619461916.7581937,
|
'request': {'authority': b'',
|
||||||
'tls_established': True,
|
'content': b'',
|
||||||
'tls_extensions': [],
|
'headers': ((b'Host', b'yolo.jetzt'),
|
||||||
'tls_version': 'TLSv1.3'},
|
(b'User-Agent', b'curl/7.75.0'),
|
||||||
'error': None,
|
(b'Accept', b'*/*')),
|
||||||
'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
|
'host': 'yolo.jetzt',
|
||||||
'intercepted': True,
|
'http_version': b'HTTP/1.1',
|
||||||
'is_replay': None,
|
'method': b'GET',
|
||||||
'marked': False,
|
'path': b'/',
|
||||||
'metadata': {},
|
'port': 443,
|
||||||
'mode': 'transparent',
|
'scheme': b'https',
|
||||||
'request': {'authority': b'',
|
'timestamp_end': 1619461916.7603076,
|
||||||
'content': b'',
|
'timestamp_start': 1619461916.7588415,
|
||||||
'headers': ((b'Host', b'yolo.jetzt'),
|
'trailers': None},
|
||||||
(b'User-Agent', b'curl/7.75.0'),
|
'response': {'content': b'<!DOCTYPE html">\n<html>\n<head> \n<meta http-equi'
|
||||||
(b'Accept', b'*/*')),
|
b'v="content-type" content="text/html;charset=ISO-8859'
|
||||||
'host': 'yolo.jetzt',
|
b'-1">\n<title>todays yolo - 3026</title> \n<style '
|
||||||
'http_version': b'HTTP/1.1',
|
b'type="text/css" media="screen">\n<!--\nbody \n{\ncol'
|
||||||
'method': b'GET',
|
b'or: white;\nbackground-color: #003;\nmargin: 0px\n}'
|
||||||
'path': b'/',
|
b'\n\n.yolo\n{\nfont-family: Verdana, Geneva, Arial, s'
|
||||||
'port': 443,
|
b'ans-serif;\nfont-weight: bold;\nfont-size: 3em;\n}\n'
|
||||||
'scheme': b'https',
|
b'\na:visited {\n\tcolor: blue\n}\n\n#content '
|
||||||
'timestamp_end': 1619461916.7603076,
|
b' \n{\nfont-family: Verdana, Geneva, Arial, sans-se'
|
||||||
'timestamp_start': 1619461916.7588415,
|
b'rif;\ncolor: white;\nbackground-color: transparent'
|
||||||
'trailers': None},
|
b';\ntext-align: center;\nfont-size: 2em;\nposition: '
|
||||||
'response': {'content': b'<!DOCTYPE html">\n<html>\n<head> \n<meta http-equi'
|
b'absolute;\ntop: 50%;\nleft: 50%;\n-webkit-transform'
|
||||||
b'v="content-type" content="text/html;charset=ISO-8859'
|
b': translateX(-50%) translateY(-50%);\n-moz-transf'
|
||||||
b'-1">\n<title>todays yolo - 3026</title> \n<style '
|
b'orm: translateX(-50%) translateY(-50%);\n-ms-tran'
|
||||||
b'type="text/css" media="screen">\n<!--\nbody \n{\ncol'
|
b'sform: translateX(-50%) translateY(-50%);\ntransf'
|
||||||
b'or: white;\nbackground-color: #003;\nmargin: 0px\n}'
|
b'orm: translateX(-50%) translateY(-50%);\noverflow'
|
||||||
b'\n\n.yolo\n{\nfont-family: Verdana, Geneva, Arial, s'
|
b': visible;\nvisibility: visible;\ndisplay: block\n}'
|
||||||
b'ans-serif;\nfont-weight: bold;\nfont-size: 3em;\n}\n'
|
b'\n\n#sponsors \n{\nfont-family: Verdana, Gene'
|
||||||
b'\na:visited {\n\tcolor: blue\n}\n\n#content '
|
b'va, Arial, sans-serif;\ncolor: white;\nbackground-'
|
||||||
b' \n{\nfont-family: Verdana, Geneva, Arial, sans-se'
|
b'color: transparent;\ntext-align: center;\nfont-siz'
|
||||||
b'rif;\ncolor: white;\nbackground-color: transparent'
|
b'e: 1em;\nposition: absolute;\ntop: 90%;\nleft: 50%;'
|
||||||
b';\ntext-align: center;\nfont-size: 2em;\nposition: '
|
b'\n-webkit-transform: translateX(-50%) translateY('
|
||||||
b'absolute;\ntop: 50%;\nleft: 50%;\n-webkit-transform'
|
b'-50%);\n-moz-transform: translateX(-50%) translat'
|
||||||
b': translateX(-50%) translateY(-50%);\n-moz-transf'
|
b'eY(-50%);\n-ms-transform: translateX(-50%) transl'
|
||||||
b'orm: translateX(-50%) translateY(-50%);\n-ms-tran'
|
b'ateY(-50%);\ntransform: translateX(-50%) translat'
|
||||||
b'sform: translateX(-50%) translateY(-50%);\ntransf'
|
b'eY(-50%);\noverflow: visible;\nvisibility: visible'
|
||||||
b'orm: translateX(-50%) translateY(-50%);\noverflow'
|
b';\ndisplay: block\n}\n\n#progress\n{\n\tbackground-'
|
||||||
b': visible;\nvisibility: visible;\ndisplay: block\n}'
|
b'color: #f08;\n\tposition: absolute;\n\tleft: 0px'
|
||||||
b'\n\n#sponsors \n{\nfont-family: Verdana, Gene'
|
b';\n\ttop: 0px;\n\theight: 10px;\n}\n\n-->\n</sty'
|
||||||
b'va, Arial, sans-serif;\ncolor: white;\nbackground-'
|
b'le>\n</head>\n<body>\n<div id="progress" class="pro'
|
||||||
b'color: transparent;\ntext-align: center;\nfont-siz'
|
b'gressBarDiv"></div>\n<div id="content">\nthe yolo '
|
||||||
b'e: 1em;\nposition: absolute;\ntop: 90%;\nleft: 50%;'
|
b'for today is<br>\n<span class="yolo" id="yoloValu'
|
||||||
b'\n-webkit-transform: translateX(-50%) translateY('
|
b'e" title="YOLOVALUE">3026</span><br>\n<br>\n</div>'
|
||||||
b'-50%);\n-moz-transform: translateX(-50%) translat'
|
b'\n<div id="sponsors">\n\t<a href="https://cat.yolo.'
|
||||||
b'eY(-50%);\n-ms-transform: translateX(-50%) transl'
|
b'jetzt">Cat</a>\n\t<br>\n\tRegulation (EU) 2016/679 c'
|
||||||
b'ateY(-50%);\ntransform: translateX(-50%) translat'
|
b'ompliant\n</div>\n<script type="text/javascript">\n'
|
||||||
b'eY(-50%);\noverflow: visible;\nvisibility: visible'
|
b'var currentYoloTS = 1619388000000; //Multiply by 100'
|
||||||
b';\ndisplay: block\n}\n\n#progress\n{\n\tbackground-'
|
b'0 the YOLO way\nvar tDiff = Date.now() - currentY'
|
||||||
b'color: #f08;\n\tposition: absolute;\n\tleft: 0px'
|
b'oloTS;\n\nvar dayMS = 24 * 60 * 60 * 1000;\n\nconsol'
|
||||||
b';\n\ttop: 0px;\n\theight: 10px;\n}\n\n-->\n</sty'
|
b'e.debug(currentYoloTS);\nconsole.debug(Date.now()'
|
||||||
b'le>\n</head>\n<body>\n<div id="progress" class="pro'
|
b' - currentYoloTS);\nfunction checkTimestamps()\n{\n'
|
||||||
b'gressBarDiv"></div>\n<div id="content">\nthe yolo '
|
b'\ttDiff = Date.now() - currentYoloTS;\n\tupdateUi()'
|
||||||
b'for today is<br>\n<span class="yolo" id="yoloValu'
|
b';\n\tif(tDiff > dayMS)\n\t{\n\t\tclearInterval(upda'
|
||||||
b'e" title="YOLOVALUE">3026</span><br>\n<br>\n</div>'
|
b'teInterval);\n\t\tlocation.reload();\t\n\t}\n}\n'
|
||||||
b'\n<div id="sponsors">\n\t<a href="https://cat.yolo.'
|
b'\nfunction updateUi()\n{\n\tvalPercent = tDiff / day'
|
||||||
b'jetzt">Cat</a>\n\t<br>\n\tRegulation (EU) 2016/679 c'
|
b'MS * 100;\n\tif(valPercent > 100) valPercent = 100'
|
||||||
b'ompliant\n</div>\n<script type="text/javascript">\n'
|
b';\n\tdocument.getElementById("progress").style = "'
|
||||||
b'var currentYoloTS = 1619388000000; //Multiply by 100'
|
b'width: " + valPercent + "%;";\n\tif(tDiff > dayMS)'
|
||||||
b'0 the YOLO way\nvar tDiff = Date.now() - currentY'
|
b'\n\t{\n\t\tdocument.getElementById("yoloValue").i'
|
||||||
b'oloTS;\n\nvar dayMS = 24 * 60 * 60 * 1000;\n\nconsol'
|
b'nnerHTML = "�";\n\t}\n}\n\nupdateUi();\nvar'
|
||||||
b'e.debug(currentYoloTS);\nconsole.debug(Date.now()'
|
b' updateInterval = setInterval(checkTimestamps, 5000)'
|
||||||
b' - currentYoloTS);\nfunction checkTimestamps()\n{\n'
|
b';\n\n//YOLODATE\n</script>\n</body>\n</html>\n',
|
||||||
b'\ttDiff = Date.now() - currentYoloTS;\n\tupdateUi()'
|
'headers': ((b'Server', b'nginx'),
|
||||||
b';\n\tif(tDiff > dayMS)\n\t{\n\t\tclearInterval(upda'
|
(b'Date', b'Mon, 26 Apr 2021 18:31:56 GMT'),
|
||||||
b'teInterval);\n\t\tlocation.reload();\t\n\t}\n}\n'
|
(b'Content-Type', b'text/html'),
|
||||||
b'\nfunction updateUi()\n{\n\tvalPercent = tDiff / day'
|
(b'Content-Length', b'2460'),
|
||||||
b'MS * 100;\n\tif(valPercent > 100) valPercent = 100'
|
(b'Last-Modified', b'Sun, 25 Apr 2021 22:00:00 GMT'),
|
||||||
b';\n\tdocument.getElementById("progress").style = "'
|
(b'Connection', b'keep-alive'),
|
||||||
b'width: " + valPercent + "%;";\n\tif(tDiff > dayMS)'
|
(b'ETag', b'"6085e660-99c"'),
|
||||||
b'\n\t{\n\t\tdocument.getElementById("yoloValue").i'
|
(b'Strict-Transport-Security',
|
||||||
b'nnerHTML = "�";\n\t}\n}\n\nupdateUi();\nvar'
|
b'max-age=31536000; includeSubDomains; preload'),
|
||||||
b' updateInterval = setInterval(checkTimestamps, 5000)'
|
(b'X-Xss-Protection', b'1; mode=block'),
|
||||||
b';\n\n//YOLODATE\n</script>\n</body>\n</html>\n',
|
(b'X-Content-Type-Options', b'nosniff'),
|
||||||
'headers': ((b'Server', b'nginx'),
|
(b'Content-Security-Policy',
|
||||||
(b'Date', b'Mon, 26 Apr 2021 18:31:56 GMT'),
|
b"default-src 'self'; script-src 'self' 'unsafe-in"
|
||||||
(b'Content-Type', b'text/html'),
|
b"line'; connect-src 'self'; img-src 'self'; style"
|
||||||
(b'Content-Length', b'2460'),
|
b"-src 'self' 'unsafe-inline';"),
|
||||||
(b'Last-Modified', b'Sun, 25 Apr 2021 22:00:00 GMT'),
|
(b'X-Frame-Options', b'SAMEORIGIN'),
|
||||||
(b'Connection', b'keep-alive'),
|
(b'Referrer-Policy', b'no-referrer'),
|
||||||
(b'ETag', b'"6085e660-99c"'),
|
(b'Accept-Ranges', b'bytes')),
|
||||||
(b'Strict-Transport-Security',
|
'http_version': b'HTTP/1.1',
|
||||||
b'max-age=31536000; includeSubDomains; preload'),
|
'reason': b'OK',
|
||||||
(b'X-Xss-Protection', b'1; mode=block'),
|
'status_code': 200,
|
||||||
(b'X-Content-Type-Options', b'nosniff'),
|
'timestamp_end': 1619461916.7979567,
|
||||||
(b'Content-Security-Policy',
|
'timestamp_start': 1619461916.7935555,
|
||||||
b"default-src 'self'; script-src 'self' 'unsafe-in"
|
'trailers': None},
|
||||||
b"line'; connect-src 'self'; img-src 'self'; style"
|
'server_conn': flowstate_serverconn,
|
||||||
b"-src 'self' 'unsafe-inline';"),
|
'type': 'http',
|
||||||
(b'X-Frame-Options', b'SAMEORIGIN'),
|
'version': 9}
|
||||||
(b'Referrer-Policy', b'no-referrer'),
|
|
||||||
(b'Accept-Ranges', b'bytes')),
|
|
||||||
'http_version': b'HTTP/1.1',
|
|
||||||
'reason': b'OK',
|
|
||||||
'status_code': 200,
|
|
||||||
'timestamp_end': 1619461916.7979567,
|
|
||||||
'timestamp_start': 1619461916.7935555,
|
|
||||||
'trailers': None},
|
|
||||||
'server_conn': {'address': ('yolo.jetzt', 443),
|
|
||||||
'alpn_proto_negotiated': b'http/1.1',
|
|
||||||
'cert': '',
|
|
||||||
'id': '64c838e2-ab74-49c4-9ebe-70e8d17e9081',
|
|
||||||
'ip_address': ('95.156.226.69', 443),
|
|
||||||
'sni': 'yolo.jetzt',
|
|
||||||
'source_address': ('10.23.42.102', 43399),
|
|
||||||
'timestamp_end': None,
|
|
||||||
'timestamp_start': 1619461916.63117,
|
|
||||||
'timestamp_tcp_setup': 1619461916.6659565,
|
|
||||||
'timestamp_tls_setup': 1619461916.7522914,
|
|
||||||
'tls_established': True,
|
|
||||||
'tls_version': 'TLSv1.2',
|
|
||||||
'via': None},
|
|
||||||
'type': 'http',
|
|
||||||
'version': 9}
|
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def flow_response(flowstate_request, flowstate_response):
|
||||||
|
c = ClientConnection.make_dummy("192.168.0.2")
|
||||||
|
s = ServerConnection.make_dummy("192.168.0.1")
|
||||||
|
flow = HTTPFlow(c, s)
|
||||||
|
flow.request = HTTPRequest.from_state(flowstate_request["request"])
|
||||||
|
flow.response = HTTPResponse.from_state(flowstate_response["response"])
|
||||||
|
return flow
|
||||||
|
|
||||||
class MitmAddonTestServer:
|
class MitmAddonTestServer:
|
||||||
def __init__(self, queue, path: str):
|
def __init__(self, queue, path: str):
|
||||||
self.queue = queue
|
self.queue = queue
|
||||||
|
|
@ -235,6 +260,15 @@ def client_server():
|
||||||
server.disconnect()
|
server.disconnect()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client_no_start():
|
||||||
|
queue = Queue()
|
||||||
|
sock = os.path.join(tempfile.mkdtemp(), "bigsnitchtest.sock")
|
||||||
|
sock = f"ipc://{sock}"
|
||||||
|
client = NetworkThread(name="testthread",queue=queue,path=sock)
|
||||||
|
return queue, client
|
||||||
|
|
||||||
|
|
||||||
class TestBigSnitchWrapper:
|
class TestBigSnitchWrapper:
|
||||||
def test_request_convert(self, flowstate_request):
|
def test_request_convert(self, flowstate_request):
|
||||||
req = bRequest(flow=flowstate_request)
|
req = bRequest(flow=flowstate_request)
|
||||||
|
|
@ -247,7 +281,7 @@ class TestBigSnitchWrapper:
|
||||||
'method': 'GET',
|
'method': 'GET',
|
||||||
'port': 443,
|
'port': 443,
|
||||||
'scheme': 'https',
|
'scheme': 'https',
|
||||||
'server_ip_address': '95.156.226.69',
|
'server_ip_address': '192.168.0.1',
|
||||||
'timestamp_end': 1619390482.69,
|
'timestamp_end': 1619390482.69,
|
||||||
'timestamp_start': 1619390482.6886377,
|
'timestamp_start': 1619390482.6886377,
|
||||||
'tls': True}
|
'tls': True}
|
||||||
|
|
@ -405,6 +439,179 @@ class TestBigSnitchWrapper:
|
||||||
|
|
||||||
assert not DeepDiff(res.__dict__, d)
|
assert not DeepDiff(res.__dict__, d)
|
||||||
|
|
||||||
|
def test_convert_request_to_json(self, flowstate_request):
|
||||||
|
j = bRequest(flowstate_request).json()
|
||||||
|
d = {'content': '',
|
||||||
|
'headers': {'Accept': '*/*',
|
||||||
|
'Host': 'yolo.jetzt',
|
||||||
|
'User-Agent': 'curl/7.75.0'},
|
||||||
|
'host': 'yolo.jetzt',
|
||||||
|
'http_version': 'HTTP/1.1',
|
||||||
|
'method': 'GET',
|
||||||
|
'port': 443,
|
||||||
|
'scheme': 'https',
|
||||||
|
'server_ip_address': '192.168.0.1',
|
||||||
|
'timestamp_end': 1619390482.69,
|
||||||
|
'timestamp_start': 1619390482.6886377,
|
||||||
|
'tls': True}
|
||||||
|
|
||||||
|
assert not DeepDiff(j, d)
|
||||||
|
|
||||||
|
def test_convert_response_to_json(self, flowstate_response):
|
||||||
|
j = bResponse(flowstate_response).json()
|
||||||
|
d = {'content': '<!DOCTYPE html">\n'
|
||||||
|
'<html>\n'
|
||||||
|
'<head> \n'
|
||||||
|
'<meta http-equiv="content-type" '
|
||||||
|
'content="text/html;charset=ISO-8859-1">\n'
|
||||||
|
'<title>todays yolo - 3026</title> \n'
|
||||||
|
'<style type="text/css" media="screen">\n'
|
||||||
|
'<!--\n'
|
||||||
|
'body \n'
|
||||||
|
'{\n'
|
||||||
|
'color: white;\n'
|
||||||
|
'background-color: #003;\n'
|
||||||
|
'margin: 0px\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'.yolo\n'
|
||||||
|
'{\n'
|
||||||
|
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
|
||||||
|
'font-weight: bold;\n'
|
||||||
|
'font-size: 3em;\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'a:visited {\n'
|
||||||
|
'\tcolor: blue\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'#content \n'
|
||||||
|
'{\n'
|
||||||
|
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
|
||||||
|
'color: white;\n'
|
||||||
|
'background-color: transparent;\n'
|
||||||
|
'text-align: center;\n'
|
||||||
|
'font-size: 2em;\n'
|
||||||
|
'position: absolute;\n'
|
||||||
|
'top: 50%;\n'
|
||||||
|
'left: 50%;\n'
|
||||||
|
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'-moz-transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'-ms-transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'overflow: visible;\n'
|
||||||
|
'visibility: visible;\n'
|
||||||
|
'display: block\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'#sponsors \n'
|
||||||
|
'{\n'
|
||||||
|
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
|
||||||
|
'color: white;\n'
|
||||||
|
'background-color: transparent;\n'
|
||||||
|
'text-align: center;\n'
|
||||||
|
'font-size: 1em;\n'
|
||||||
|
'position: absolute;\n'
|
||||||
|
'top: 90%;\n'
|
||||||
|
'left: 50%;\n'
|
||||||
|
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'-moz-transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'-ms-transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'transform: translateX(-50%) translateY(-50%);\n'
|
||||||
|
'overflow: visible;\n'
|
||||||
|
'visibility: visible;\n'
|
||||||
|
'display: block\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'#progress\n'
|
||||||
|
'{\n'
|
||||||
|
'\tbackground-color: #f08;\n'
|
||||||
|
'\tposition: absolute;\n'
|
||||||
|
'\tleft: 0px;\n'
|
||||||
|
'\ttop: 0px;\n'
|
||||||
|
'\theight: 10px;\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'-->\n'
|
||||||
|
'</style>\n'
|
||||||
|
'</head>\n'
|
||||||
|
'<body>\n'
|
||||||
|
'<div id="progress" class="progressBarDiv"></div>\n'
|
||||||
|
'<div id="content">\n'
|
||||||
|
'the yolo for today is<br>\n'
|
||||||
|
'<span class="yolo" id="yoloValue" '
|
||||||
|
'title="YOLOVALUE">3026</span><br>\n'
|
||||||
|
'<br>\n'
|
||||||
|
'</div>\n'
|
||||||
|
'<div id="sponsors">\n'
|
||||||
|
'\t<a href="https://cat.yolo.jetzt">Cat</a>\n'
|
||||||
|
'\t<br>\n'
|
||||||
|
'\tRegulation (EU) 2016/679 compliant\n'
|
||||||
|
'</div>\n'
|
||||||
|
'<script type="text/javascript">\n'
|
||||||
|
'var currentYoloTS = 1619388000000; //Multiply by 1000 the YOLO '
|
||||||
|
'way\n'
|
||||||
|
'var tDiff = Date.now() - currentYoloTS;\n'
|
||||||
|
'\n'
|
||||||
|
'var dayMS = 24 * 60 * 60 * 1000;\n'
|
||||||
|
'\n'
|
||||||
|
'console.debug(currentYoloTS);\n'
|
||||||
|
'console.debug(Date.now() - currentYoloTS);\n'
|
||||||
|
'function checkTimestamps()\n'
|
||||||
|
'{\n'
|
||||||
|
'\ttDiff = Date.now() - currentYoloTS;\n'
|
||||||
|
'\tupdateUi();\n'
|
||||||
|
'\tif(tDiff > dayMS)\n'
|
||||||
|
'\t{\n'
|
||||||
|
'\t\tclearInterval(updateInterval);\n'
|
||||||
|
'\t\tlocation.reload();\t\n'
|
||||||
|
'\t}\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'function updateUi()\n'
|
||||||
|
'{\n'
|
||||||
|
'\tvalPercent = tDiff / dayMS * 100;\n'
|
||||||
|
'\tif(valPercent > 100) valPercent = 100;\n'
|
||||||
|
'\tdocument.getElementById("progress").style = "width: " + '
|
||||||
|
'valPercent + "%;";\n'
|
||||||
|
'\tif(tDiff > dayMS)\n'
|
||||||
|
'\t{\n'
|
||||||
|
'\t\tdocument.getElementById("yoloValue").innerHTML = "�";\n'
|
||||||
|
'\t}\n'
|
||||||
|
'}\n'
|
||||||
|
'\n'
|
||||||
|
'updateUi();\n'
|
||||||
|
'var updateInterval = setInterval(checkTimestamps, 5000);\n'
|
||||||
|
'\n'
|
||||||
|
'//YOLODATE\n'
|
||||||
|
'</script>\n'
|
||||||
|
'</body>\n'
|
||||||
|
'</html>\n',
|
||||||
|
'headers': [{'Server': 'nginx'},
|
||||||
|
{'Date': 'Mon, 26 Apr 2021 18:31:56 GMT'},
|
||||||
|
{'Content-Type': 'text/html'},
|
||||||
|
{'Content-Length': '2460'},
|
||||||
|
{'Last-Modified': 'Sun, 25 Apr 2021 22:00:00 GMT'},
|
||||||
|
{'Connection': 'keep-alive'},
|
||||||
|
{'ETag': '"6085e660-99c"'},
|
||||||
|
{'Strict-Transport-Security': 'max-age=31536000; '
|
||||||
|
'includeSubDomains; preload'},
|
||||||
|
{'X-Xss-Protection': '1; mode=block'},
|
||||||
|
{'X-Content-Type-Options': 'nosniff'},
|
||||||
|
{'Content-Security-Policy': "default-src 'self'; script-src "
|
||||||
|
"'self' 'unsafe-inline'; connect-src "
|
||||||
|
"'self'; img-src 'self'; style-src "
|
||||||
|
"'self' 'unsafe-inline';"},
|
||||||
|
{'X-Frame-Options': 'SAMEORIGIN'},
|
||||||
|
{'Referrer-Policy': 'no-referrer'},
|
||||||
|
{'Accept-Ranges': 'bytes'}],
|
||||||
|
'http_version': 'HTTP/1.1',
|
||||||
|
'reason': 'OK',
|
||||||
|
'status_code': 200,
|
||||||
|
'timestamp_end': 1619461916.7979567,
|
||||||
|
'timestamp_start': 1619461916.7935555}
|
||||||
|
|
||||||
|
assert not DeepDiff(j, d)
|
||||||
|
|
||||||
class TestMitmAddon:
|
class TestMitmAddon:
|
||||||
def test_get_new_flows_empty(self, client_server):
|
def test_get_new_flows_empty(self, client_server):
|
||||||
|
|
@ -419,8 +626,37 @@ class TestMitmAddon:
|
||||||
assert queue.empty()
|
assert queue.empty()
|
||||||
assert not len(client.flows)
|
assert not len(client.flows)
|
||||||
|
|
||||||
def test_get_new_flows_single(self, client_server):
|
def test_get_new_flows_single(self, client_no_start, flow_request):
|
||||||
queue, client, server = client_server
|
queue, client = client_no_start
|
||||||
|
|
||||||
|
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request)
|
||||||
|
queue.put_nowait((flow_request.id, flowitem))
|
||||||
|
|
||||||
|
client.get_new_flows()
|
||||||
|
|
||||||
|
assert queue.empty()
|
||||||
|
assert len(client.flows) == 1
|
||||||
|
|
||||||
|
assert flow_request.id in client.flows.keys()
|
||||||
|
assert not DeepDiff(flowitem, client.flows[flow_request.id])
|
||||||
|
|
||||||
|
def test_get_new_flows_double(self, client_no_start, flow_request, flow_response):
|
||||||
|
queue, client = client_no_start
|
||||||
|
|
||||||
|
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow_request)
|
||||||
|
queue.put_nowait((flow_request.id, flowitem))
|
||||||
|
flowitem2 = FlowItem(bFlowState.UNSENT_HTTP_RESPONSE, flow_response)
|
||||||
|
queue.put_nowait((flow_response.id, flowitem2))
|
||||||
|
|
||||||
|
client.get_new_flows()
|
||||||
|
|
||||||
|
assert queue.empty()
|
||||||
|
assert len(client.flows) == 2
|
||||||
|
|
||||||
|
assert flow_request.id in client.flows.keys()
|
||||||
|
assert not DeepDiff(flowitem, client.flows[flow_request.id])
|
||||||
|
assert flow_response.id in client.flows.keys()
|
||||||
|
assert not DeepDiff(flowitem2, client.flows[flow_response.id])
|
||||||
|
|
||||||
def test_request(self, client_server):
|
def test_request(self, client_server):
|
||||||
queue, client, server = client_server
|
queue, client, server = client_server
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue