more tests
This commit is contained in:
Tim Blume
parent
75a5ba79db
commit
c20ec945a1
4 changed files with 567 additions and 264 deletions
|
|
@ -3,18 +3,18 @@
|
|||
|
||||
import pdb
|
||||
|
||||
from queue import Queue, Empty
|
||||
from queue import Queue
|
||||
from networkthread import bFlowState, FlowItem, NetworkThread
|
||||
|
||||
|
||||
class BigSnitchBridge:
|
||||
def __init__(self):
|
||||
print("BigSnitchBridge started")
|
||||
self.q = Queue()
|
||||
#self.thread = NetworkThread("network", self.q)
|
||||
#self.thread.start()
|
||||
self.thread = NetworkThread("network", self.q)
|
||||
self.thread.start()
|
||||
|
||||
def request(self, flow):
|
||||
pdb.set_trace()
|
||||
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
|
||||
self.q.put_nowait((flow.id, flowitem))
|
||||
# intercept until ACK received
|
||||
|
|
@ -27,7 +27,7 @@ class BigSnitchBridge:
|
|||
flow.intercept()
|
||||
|
||||
def error(self, flow):
|
||||
flowitem = FlowItem(bFlowState.ERROR, flow, time.monotonic())
|
||||
flowitem = FlowItem(bFlowState.ERROR, flow)
|
||||
self.q.put_nowait((flow.id, flowitem))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import pdb
|
||||
|
||||
from mitmproxy import ctx
|
||||
from mitmproxy.flow import Flow
|
||||
import threading
|
||||
import time
|
||||
|
|
@ -9,33 +8,34 @@ import json
|
|||
import os
|
||||
from enum import Enum
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Dict
|
||||
from typing import List, Dict, Any
|
||||
from queue import Queue, Empty
|
||||
|
||||
"""
|
||||
# this method is used to convert flow states (generated with get_state()) to json
|
||||
def convert_to_strings(obj):
|
||||
if isinstance(obj, dict):
|
||||
return {convert_to_strings(key): convert_to_strings(value)
|
||||
for key, value in obj.items()}
|
||||
elif isinstance(obj, list) or isinstance(obj, tuple):
|
||||
return [convert_to_strings(element) for element in obj]
|
||||
elif isinstance(obj, bytes):
|
||||
try:
|
||||
data = obj.decode('unicode-escape').encode('latin1').decode('utf-8')
|
||||
except:
|
||||
print(obj)
|
||||
data = str(obj)[2:-1]
|
||||
return data
|
||||
|
||||
return obj
|
||||
"""
|
||||
# this method is used to convert flow states (generated with get_state()) to json
|
||||
def convert_to_strings(obj: Any) -> Any:
|
||||
if isinstance(obj, dict):
|
||||
return {convert_to_strings(key): convert_to_strings(value)
|
||||
for key, value in obj.items()}
|
||||
elif isinstance(obj, list) or isinstance(obj, tuple):
|
||||
return [convert_to_strings(element) for element in obj]
|
||||
elif isinstance(obj, bytes):
|
||||
try:
|
||||
data = obj.decode('unicode-escape').encode('latin1').decode('utf-8')
|
||||
except:
|
||||
print(obj)
|
||||
data = str(obj)[2:-1]
|
||||
return data
|
||||
|
||||
return obj
|
||||
|
||||
|
||||
@dataclass
|
||||
class bHeader:
|
||||
key: str
|
||||
value: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class bRequest:
|
||||
server_ip_address: str
|
||||
|
|
@ -57,6 +57,8 @@ class bRequest:
|
|||
|
||||
# init from flow dict
|
||||
def __init__(self, flow: dict):
|
||||
flow = convert_to_strings(flow)
|
||||
|
||||
self.server_ip_address = flow["server_conn"]["ip_address"][0]
|
||||
self.tls = flow["server_conn"]["tls_established"]
|
||||
self.content = flow["request"]["content"]
|
||||
|
|
@ -69,8 +71,12 @@ class bRequest:
|
|||
self.timestamp_end = flow["request"]["timestamp_end"]
|
||||
|
||||
self.headers = []
|
||||
for k,v in flow["request"]["headers"]:
|
||||
self.headers.append(bHeader(k,v))
|
||||
for k, v in flow["request"]["headers"]:
|
||||
self.headers.append(bHeader(str(k), str(v)))
|
||||
|
||||
def json(self) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class bResponse:
|
||||
|
|
@ -84,15 +90,21 @@ class bResponse:
|
|||
headers: List[bHeader]
|
||||
|
||||
def __init__(self, flow: dict):
|
||||
self.status_code = flow["status_code"]
|
||||
self.http_version = flow["http_version"]
|
||||
self.reason = flow["reason"]
|
||||
self.content = flow["content"]
|
||||
self.timestamp_start = flow["timestamp_start"]
|
||||
self.timestamp_end = flow["timestamp_end"]
|
||||
flow = convert_to_strings(flow)
|
||||
self.status_code = flow["response"]["status_code"]
|
||||
self.http_version = flow["response"]["http_version"]
|
||||
self.reason = flow["response"]["reason"]
|
||||
self.content = flow["response"]["content"]
|
||||
self.timestamp_start = flow["response"]["timestamp_start"]
|
||||
self.timestamp_end = flow["response"]["timestamp_end"]
|
||||
|
||||
self.headers = []
|
||||
for k, v in flow["response"]["headers"]:
|
||||
self.headers.append(bHeader(k, v))
|
||||
|
||||
def json(self) -> dict:
|
||||
return {}
|
||||
|
||||
for k,v in flow["headers"]:
|
||||
self.headers.append(bHeader(k,v))
|
||||
|
||||
@dataclass
|
||||
class bFlowState(Enum):
|
||||
|
|
@ -102,6 +114,7 @@ class bFlowState(Enum):
|
|||
UNSENT_HTTP_RESPONSE = 3
|
||||
SENT_HTTP_RESPONSE = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class bPacketType:
|
||||
NACK = 0
|
||||
|
|
@ -113,6 +126,7 @@ class bPacketType:
|
|||
HTTP_REQUEST = 6
|
||||
HTTP_RESPONSE = 7
|
||||
|
||||
|
||||
@dataclass
|
||||
class bPacket:
|
||||
ptype: bPacketType
|
||||
|
|
@ -124,6 +138,7 @@ class bPacket:
|
|||
self.flowid = str(json["id"])
|
||||
self.data = json["data"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlowItem:
|
||||
state: bFlowState
|
||||
|
|
@ -131,178 +146,163 @@ class FlowItem:
|
|||
time: float = 0
|
||||
retries_left: int = 5
|
||||
|
||||
|
||||
"""
|
||||
|
||||
The network thread communicates with the bigsnitch plugin using zeromq.
|
||||
|
||||
"""
|
||||
class NetworkThread(threading.Thread):
|
||||
def __init__(self, name: str, queue: Queue, path: str = None):
|
||||
threading.Thread.__init__(self)
|
||||
self.name = name
|
||||
# path
|
||||
self.path = path
|
||||
if not self.path:
|
||||
self.path = os.environ.get("BIGSNITCH_PATH", None)
|
||||
if not self.path:
|
||||
self.path = "tcp://127.0.0.1:12345"
|
||||
def __init__(self, name: str, queue: Queue, path: str = None):
|
||||
threading.Thread.__init__(self)
|
||||
self.name = name
|
||||
# path
|
||||
self.path = path
|
||||
if not self.path:
|
||||
self.path = os.environ.get("BIGSNITCH_PATH", None)
|
||||
if not self.path:
|
||||
self.path = "tcp://127.0.0.1:12345"
|
||||
|
||||
# queue for communicating with the main mitmproxy thread
|
||||
# contains tuples of (id, FlowItem)
|
||||
self.q = queue
|
||||
# for zmq use
|
||||
self.context = zmq.Context()
|
||||
# all current flows being handled by mitmproxy
|
||||
self.flows: Dict[FlowItem] = {}
|
||||
# timer for sending pings to check if the connection broke
|
||||
self.timer = time.monotonic()
|
||||
# retries left for reconnecting
|
||||
self.retries = 5
|
||||
# queue for communicating with the main mitmproxy thread
|
||||
# contains tuples of (id, FlowItem)
|
||||
self.q = queue
|
||||
# for zmq use
|
||||
self.context = zmq.Context()
|
||||
# all current self.flows being handled by mitmproxy
|
||||
self.flows: Dict[FlowItem] = {}
|
||||
# timer for sending pings to check if the connection broke
|
||||
self.timer = time.monotonic()
|
||||
# retries left for reconnecting
|
||||
self.retries = 5
|
||||
|
||||
# send a single message, no checks involved
|
||||
def send(self, msg):
|
||||
a = convert_to_strings(msg)
|
||||
self.socket.send(str.encode(json.dumps(a)))
|
||||
# send a single message, no checks involved
|
||||
def send(self, msg):
|
||||
#a = convert_to_strings(msg)
|
||||
self.socket.send(str.encode(json.dumps(msg)))
|
||||
|
||||
# add new flows from the queue
|
||||
def get_new_flows(self):
|
||||
while True:
|
||||
try:
|
||||
# get new flows that may occured
|
||||
i, flowitem = self.q.get(block=False)
|
||||
if self.flows.get(i, None):
|
||||
print(f"flow {i} doubled? ignoring...")
|
||||
# add new self.flows from the queue
|
||||
def get_new_flows(self):
|
||||
while True:
|
||||
try:
|
||||
# get new self.flows that may occured
|
||||
i, flowitem = self.q.get(block=False)
|
||||
if self.flows.get(i, None):
|
||||
print(f"flow {i} doubled? ignoring...")
|
||||
continue
|
||||
else:
|
||||
self.flows[i] = flowitem
|
||||
|
||||
except Empty:
|
||||
break
|
||||
|
||||
def send_packet(self, pkg: bPacket):
|
||||
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
|
||||
self.send(msg)
|
||||
|
||||
def send_http_request(self, id: int, request: bRequest):
|
||||
pkg = bPacket(bPacketType.HTTP_REQUEST, request.json())
|
||||
self.send_packet(pkg)
|
||||
self.flows[id].state = bFlowState.SENT_HTTP_REQUEST
|
||||
self.flows[id].time = time.monotonic()
|
||||
|
||||
def send_http_response(self, id: int, response: bResponse):
|
||||
pkg = bPacket(bPacketType.HTTP_RESPONSE, response.json())
|
||||
self.send_packet(pkg)
|
||||
self.flows[id].state = bFlowState.SENT_HTTP_RESPONSE
|
||||
self.flows[id].time = time.monotonic()
|
||||
|
||||
# update all current self.flows
|
||||
# handles the state machine for each flow
|
||||
def update_flows(self):
|
||||
for id, flow in self.flows.items():
|
||||
if self.flows[id].retries <= 0:
|
||||
self.flows[id].flow.kill()
|
||||
print(f"http flow {id} timed out! flow killed.")
|
||||
|
||||
delta = time.monotonic() - self.flows[id].time
|
||||
|
||||
if flow.state == bFlowState.UNSENT_HTTP_REQUEST or \
|
||||
flow.state == bFlowState.SENT_HTTP_REQUEST and delta > 5:
|
||||
self.send_http_request(id, bRequest(flow.flow))
|
||||
self.flows[id].retries -= 1
|
||||
|
||||
if flow.state == bFlowState.UNSENT_HTTP_RESPONSE or \
|
||||
flow.state == bFlowState.SENT_HTTP_RESPONSE and delta > 5:
|
||||
self.send_http_response(id, bResponse(flow.flow))
|
||||
self.flows[id].retries -= 1
|
||||
|
||||
elif flow.state == bFlowState.ERROR:
|
||||
print(f"error in flow {id}!")
|
||||
|
||||
# handle incoming packets / update the statemachine
|
||||
def handle_packets(self):
|
||||
while((self.socket.poll(50) & zmq.POLLIN) != 0):
|
||||
msg = self.socket.recv()
|
||||
try:
|
||||
if msg:
|
||||
result = json.loads(str(msg))
|
||||
pkg = bPacket(json=result)
|
||||
# flow ACKed
|
||||
if pkg.ptype == bPacketType.ACK:
|
||||
continue
|
||||
# flow killed
|
||||
elif pkg.ptype == bPacketType.KILL:
|
||||
continue
|
||||
else:
|
||||
self.flows.append(flowitem)
|
||||
print(f"got unexpected message {pkg.ptype}")
|
||||
except json.JSONDecodeError:
|
||||
print(f"malformed message received {msg}")
|
||||
|
||||
except Empty:
|
||||
break
|
||||
def run(self):
|
||||
print("thread started")
|
||||
self.connect()
|
||||
while True:
|
||||
self.timer = time.monotonic()
|
||||
self.get_new_flows()
|
||||
self.handle_packets()
|
||||
self.update_flows()
|
||||
|
||||
def send_packet(self, pkg: bPacket):
|
||||
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
|
||||
self.send(msg)
|
||||
|
||||
def send_http_request(self, id: int, request: bRequest):
|
||||
pkg = bPacket(bPacketType.HTTP_REQUEST, id, request.json())
|
||||
self.send_packet(pkg)
|
||||
flows[id].state = bFlowState.SENT_HTTP_REQUEST
|
||||
flows[id].time = time.monotonic()
|
||||
|
||||
def send_http_response(self, id: int, response: bResponse):
|
||||
pkg = bPacket(bPacketType.HTTP_RESPONSE, id, response.json())
|
||||
self.send_packet(pkg)
|
||||
flows[id].state = bFlowState.SENT_HTTP_RESPONSE
|
||||
flows[id].time = time.monotonic()
|
||||
|
||||
# update all current flows
|
||||
# handles the state machine for each flow
|
||||
def update_flows(self):
|
||||
for id, flow in self.flows.items():
|
||||
# send the request if not sent
|
||||
if state == bFlowState.UNSENT_HTTP_REQUEST:
|
||||
self.send_request(id, bRequest(flow.flow))
|
||||
|
||||
elif state == bFlowState.SENT_HTTP_REQUEST:
|
||||
# check timer, try resend
|
||||
delta = time.monotonic() - flows[id].time
|
||||
if delta > 5:
|
||||
self.send_request(id, bRequest(flow.flow))
|
||||
flows[id].retries -= 1
|
||||
continue
|
||||
|
||||
if flows[id].retries <= 0:
|
||||
flows[id].flow.kill()
|
||||
print(f"http request {id} timed out! flow killed.")
|
||||
|
||||
# send the response if not sent
|
||||
elif state == bFlowState.UNSENT_HTTP_RESPONSE:
|
||||
self.send_response(id, bResponse(flow.flow))
|
||||
|
||||
elif state == bFlowState.SENT_HTTP_RESPONSE:
|
||||
# check timer, try resend
|
||||
delta = time.monotonic() - flows[id].time
|
||||
if delta > 5:
|
||||
self.send_response(id, bResponse(flow.flow))
|
||||
flows[id].retries -= 1
|
||||
continue
|
||||
|
||||
if flows[id].retries <= 0:
|
||||
flows[id].flow.kill()
|
||||
print(f"http response {id} timedout! flow killed.")
|
||||
|
||||
elif state == bFlowState.ERROR:
|
||||
print(f"error in flow {id}!")
|
||||
|
||||
# handle incoming packets / update the statemachine
|
||||
def handle_packets(self):
|
||||
while((self.socket.poll(50) & zmq.POLLIN) != 0):
|
||||
msg = self.socket.recv()
|
||||
try:
|
||||
if msg:
|
||||
result = json.loads(msg)
|
||||
pkg = bPacket(json=result)
|
||||
# flow ACKed
|
||||
if pkg.ptype == bPacketType.ACK:
|
||||
continue
|
||||
# flow killed
|
||||
elif pkg.ptype == bPacketType.KILL:
|
||||
continue
|
||||
else:
|
||||
print(f"got unexpected message {pkg.ptype}")
|
||||
except json.JSONDecodeError:
|
||||
print(f"malformed message received {msg}")
|
||||
|
||||
def run(self):
|
||||
print("thread started")
|
||||
self.connect()
|
||||
while True:
|
||||
self.timer = time.monotonic()
|
||||
self.get_new_flows()
|
||||
self.handle_packets()
|
||||
self.update_flows()
|
||||
|
||||
if self.timer - time.monotonic() < -5:
|
||||
pass
|
||||
#self.send_msg_and_ack({"msg": "ping"})
|
||||
|
||||
def disconnect(self):
|
||||
self.socket.setsockopt(zmq.LINGER,0)
|
||||
self.socket.close()
|
||||
print("disconnected")
|
||||
|
||||
def reconnect(self):
|
||||
print("reconnecting")
|
||||
self.disconnect()
|
||||
time.sleep(1)
|
||||
self.connect()
|
||||
|
||||
def connect(self):
|
||||
self.socket = self.context.socket(zmq.PAIR)
|
||||
self.socket.connect(self.path)
|
||||
if self.timer - time.monotonic() < -5:
|
||||
pass
|
||||
#self.send_msg_and_ack({"msg": "ping"})
|
||||
print("connected")
|
||||
|
||||
"""
|
||||
def send_msg_and_ack(self, msg):
|
||||
self.timer = time.monotonic()
|
||||
while True:
|
||||
#print("m sending")
|
||||
self.send(msg)
|
||||
if (self.socket.poll(50) & zmq.POLLIN) != 0:
|
||||
msg = self.socket.recv()
|
||||
try:
|
||||
if msg:
|
||||
result = json.loads(msg)
|
||||
if result["msg"] == "ack":
|
||||
print("m ack received")
|
||||
return result
|
||||
else:
|
||||
print("got unexpected message {result}")
|
||||
|
||||
except json.JSONDecodeError:
|
||||
print(f"malformed message received {msg}")
|
||||
print("no ack received, reconnecting...")
|
||||
self.reconnect()
|
||||
return NO_MSG
|
||||
"""
|
||||
def disconnect(self):
|
||||
self.socket.setsockopt(zmq.LINGER,0)
|
||||
self.socket.close()
|
||||
print("disconnected")
|
||||
|
||||
def reconnect(self):
|
||||
print("reconnecting")
|
||||
self.disconnect()
|
||||
time.sleep(1)
|
||||
self.connect()
|
||||
|
||||
def connect(self):
|
||||
self.socket = self.context.socket(zmq.PAIR)
|
||||
self.socket.connect(self.path)
|
||||
#self.send_msg_and_ack({"msg": "ping"})
|
||||
print("connected")
|
||||
|
||||
"""
|
||||
def send_msg_and_ack(self, msg):
|
||||
self.timer = time.monotonic()
|
||||
while True:
|
||||
#print("m sending")
|
||||
self.send(msg)
|
||||
if (self.socket.poll(50) & zmq.POLLIN) != 0:
|
||||
msg = self.socket.recv()
|
||||
try:
|
||||
if msg:
|
||||
result = json.loads(msg)
|
||||
if result["msg"] == "ack":
|
||||
print("m ack received")
|
||||
return result
|
||||
else:
|
||||
print("got unexpected message {result}")
|
||||
|
||||
except json.JSONDecodeError:
|
||||
print(f"malformed message received {msg}")
|
||||
print("no ack received, reconnecting...")
|
||||
self.reconnect()
|
||||
return NO_MSG
|
||||
"""
|
||||
|
|
@ -3,3 +3,4 @@ mitmdump
|
|||
pyzmq
|
||||
deepdiff
|
||||
pytest
|
||||
tox
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import pdb
|
||||
import queue
|
||||
|
||||
import pytest
|
||||
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread
|
||||
from networkthread import bPacket, bRequest, bResponse, bHeader, NetworkThread, FlowItem, bFlowState
|
||||
import os
|
||||
import tempfile
|
||||
from queue import Queue
|
||||
|
|
@ -11,60 +12,192 @@ import zmq
|
|||
|
||||
from deepdiff import DeepDiff
|
||||
|
||||
|
||||
# usual flow state of the request with some big parts removed
|
||||
@pytest.fixture
|
||||
def flowstate_request():
|
||||
return {'client_conn': {'address': ('::ffff:127.0.0.1', 60630, 0, 0),
|
||||
'alpn_proto_negotiated': b'http/1.1',
|
||||
'cipher_name': 'TLS_AES_256_GCM_SHA384',
|
||||
'clientcert': None,
|
||||
'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea',
|
||||
'mitmcert': '',
|
||||
'sni': 'yolo.jetzt',
|
||||
'timestamp_end': None,
|
||||
'timestamp_start': 1619390481.8003347,
|
||||
'timestamp_tls_setup': 1619390482.6879823,
|
||||
'tls_established': True,
|
||||
'tls_extensions': [],
|
||||
'tls_version': 'TLSv1.3'},
|
||||
'error': None,
|
||||
'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
|
||||
'intercepted': False,
|
||||
'is_replay': None,
|
||||
'marked': False,
|
||||
'metadata': {},
|
||||
'mode': 'transparent',
|
||||
'request': {'authority': b'',
|
||||
'content': b'',
|
||||
'headers': ((b'Host', b'yolo.jetzt'),
|
||||
(b'User-Agent', b'curl/7.75.0'),
|
||||
(b'Accept', b'*/*')),
|
||||
'host': 'yolo.jetzt',
|
||||
'http_version': b'HTTP/1.1',
|
||||
'method': b'GET',
|
||||
'path': b'/',
|
||||
'port': 443,
|
||||
'scheme': b'https',
|
||||
'timestamp_end': 1619390482.69,
|
||||
'timestamp_start': 1619390482.6886377,
|
||||
'trailers': None},
|
||||
'response': None,
|
||||
'server_conn': {'address': ('yolo.jetzt', 443),
|
||||
'alpn_proto_negotiated': b'http/1.1',
|
||||
'cert': '',
|
||||
'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b',
|
||||
'ip_address': ('95.156.226.69', 443),
|
||||
'sni': 'yolo.jetzt',
|
||||
'source_address': ('192.168.42.182', 51514),
|
||||
'timestamp_end': None,
|
||||
'timestamp_start': 1619390481.8154442,
|
||||
'timestamp_tcp_setup': 1619390481.994565,
|
||||
'timestamp_tls_setup': 1619390482.6819758,
|
||||
'tls_established': True,
|
||||
'tls_version': 'TLSv1.2',
|
||||
'via': None},
|
||||
'type': 'http',
|
||||
'version': 9}
|
||||
'alpn_proto_negotiated': b'http/1.1',
|
||||
'cipher_name': 'TLS_AES_256_GCM_SHA384',
|
||||
'clientcert': None,
|
||||
'id': '5dde7ef8-9b1a-4b60-9d15-d308442a27ea',
|
||||
'mitmcert': '',
|
||||
'sni': 'yolo.jetzt',
|
||||
'timestamp_end': None,
|
||||
'timestamp_start': 1619390481.8003347,
|
||||
'timestamp_tls_setup': 1619390482.6879823,
|
||||
'tls_established': True,
|
||||
'tls_extensions': [],
|
||||
'tls_version': 'TLSv1.3'},
|
||||
'error': None,
|
||||
'id': '51215b69-c76f-4ac2-afcb-da3b823d9f88',
|
||||
'intercepted': False,
|
||||
'is_replay': None,
|
||||
'marked': False,
|
||||
'metadata': {},
|
||||
'mode': 'transparent',
|
||||
'request': {'authority': b'',
|
||||
'content': b'',
|
||||
'headers': ((b'Host', b'yolo.jetzt'),
|
||||
(b'User-Agent', b'curl/7.75.0'),
|
||||
(b'Accept', b'*/*')),
|
||||
'host': 'yolo.jetzt',
|
||||
'http_version': b'HTTP/1.1',
|
||||
'method': b'GET',
|
||||
'path': b'/',
|
||||
'port': 443,
|
||||
'scheme': b'https',
|
||||
'timestamp_end': 1619390482.69,
|
||||
'timestamp_start': 1619390482.6886377,
|
||||
'trailers': None},
|
||||
'response': None,
|
||||
'server_conn': {'address': ('yolo.jetzt', 443),
|
||||
'alpn_proto_negotiated': b'http/1.1',
|
||||
'cert': '',
|
||||
'id': 'ecc4cd3b-7e35-4815-b618-5931fe64729b',
|
||||
'ip_address': ('95.156.226.69', 443),
|
||||
'sni': 'yolo.jetzt',
|
||||
'source_address': ('192.168.42.182', 51514),
|
||||
'timestamp_end': None,
|
||||
'timestamp_start': 1619390481.8154442,
|
||||
'timestamp_tcp_setup': 1619390481.994565,
|
||||
'timestamp_tls_setup': 1619390482.6819758,
|
||||
'tls_established': True,
|
||||
'tls_version': 'TLSv1.2',
|
||||
'via': None},
|
||||
'type': 'http',
|
||||
'version': 9}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def flowstate_response():
|
||||
return {'client_conn': {'address': ('::ffff:127.0.0.1', 30190, 0, 0),
|
||||
'alpn_proto_negotiated': b'http/1.1',
|
||||
'cipher_name': 'TLS_AES_256_GCM_SHA384',
|
||||
'clientcert': None,
|
||||
'id': '2507e6ce-3132-4394-9432-f55fb5f55b05',
|
||||
'mitmcert': '',
|
||||
'sni': 'yolo.jetzt',
|
||||
'timestamp_end': None,
|
||||
'timestamp_start': 1619461916.6160116,
|
||||
'timestamp_tls_setup': 1619461916.7581937,
|
||||
'tls_established': True,
|
||||
'tls_extensions': [],
|
||||
'tls_version': 'TLSv1.3'},
|
||||
'error': None,
|
||||
'id': '449d1a87-744f-4a18-9a5d-f085f99a5c62',
|
||||
'intercepted': True,
|
||||
'is_replay': None,
|
||||
'marked': False,
|
||||
'metadata': {},
|
||||
'mode': 'transparent',
|
||||
'request': {'authority': b'',
|
||||
'content': b'',
|
||||
'headers': ((b'Host', b'yolo.jetzt'),
|
||||
(b'User-Agent', b'curl/7.75.0'),
|
||||
(b'Accept', b'*/*')),
|
||||
'host': 'yolo.jetzt',
|
||||
'http_version': b'HTTP/1.1',
|
||||
'method': b'GET',
|
||||
'path': b'/',
|
||||
'port': 443,
|
||||
'scheme': b'https',
|
||||
'timestamp_end': 1619461916.7603076,
|
||||
'timestamp_start': 1619461916.7588415,
|
||||
'trailers': None},
|
||||
'response': {'content': b'<!DOCTYPE html">\n<html>\n<head> \n<meta http-equi'
|
||||
b'v="content-type" content="text/html;charset=ISO-8859'
|
||||
b'-1">\n<title>todays yolo - 3026</title> \n<style '
|
||||
b'type="text/css" media="screen">\n<!--\nbody \n{\ncol'
|
||||
b'or: white;\nbackground-color: #003;\nmargin: 0px\n}'
|
||||
b'\n\n.yolo\n{\nfont-family: Verdana, Geneva, Arial, s'
|
||||
b'ans-serif;\nfont-weight: bold;\nfont-size: 3em;\n}\n'
|
||||
b'\na:visited {\n\tcolor: blue\n}\n\n#content '
|
||||
b' \n{\nfont-family: Verdana, Geneva, Arial, sans-se'
|
||||
b'rif;\ncolor: white;\nbackground-color: transparent'
|
||||
b';\ntext-align: center;\nfont-size: 2em;\nposition: '
|
||||
b'absolute;\ntop: 50%;\nleft: 50%;\n-webkit-transform'
|
||||
b': translateX(-50%) translateY(-50%);\n-moz-transf'
|
||||
b'orm: translateX(-50%) translateY(-50%);\n-ms-tran'
|
||||
b'sform: translateX(-50%) translateY(-50%);\ntransf'
|
||||
b'orm: translateX(-50%) translateY(-50%);\noverflow'
|
||||
b': visible;\nvisibility: visible;\ndisplay: block\n}'
|
||||
b'\n\n#sponsors \n{\nfont-family: Verdana, Gene'
|
||||
b'va, Arial, sans-serif;\ncolor: white;\nbackground-'
|
||||
b'color: transparent;\ntext-align: center;\nfont-siz'
|
||||
b'e: 1em;\nposition: absolute;\ntop: 90%;\nleft: 50%;'
|
||||
b'\n-webkit-transform: translateX(-50%) translateY('
|
||||
b'-50%);\n-moz-transform: translateX(-50%) translat'
|
||||
b'eY(-50%);\n-ms-transform: translateX(-50%) transl'
|
||||
b'ateY(-50%);\ntransform: translateX(-50%) translat'
|
||||
b'eY(-50%);\noverflow: visible;\nvisibility: visible'
|
||||
b';\ndisplay: block\n}\n\n#progress\n{\n\tbackground-'
|
||||
b'color: #f08;\n\tposition: absolute;\n\tleft: 0px'
|
||||
b';\n\ttop: 0px;\n\theight: 10px;\n}\n\n-->\n</sty'
|
||||
b'le>\n</head>\n<body>\n<div id="progress" class="pro'
|
||||
b'gressBarDiv"></div>\n<div id="content">\nthe yolo '
|
||||
b'for today is<br>\n<span class="yolo" id="yoloValu'
|
||||
b'e" title="YOLOVALUE">3026</span><br>\n<br>\n</div>'
|
||||
b'\n<div id="sponsors">\n\t<a href="https://cat.yolo.'
|
||||
b'jetzt">Cat</a>\n\t<br>\n\tRegulation (EU) 2016/679 c'
|
||||
b'ompliant\n</div>\n<script type="text/javascript">\n'
|
||||
b'var currentYoloTS = 1619388000000; //Multiply by 100'
|
||||
b'0 the YOLO way\nvar tDiff = Date.now() - currentY'
|
||||
b'oloTS;\n\nvar dayMS = 24 * 60 * 60 * 1000;\n\nconsol'
|
||||
b'e.debug(currentYoloTS);\nconsole.debug(Date.now()'
|
||||
b' - currentYoloTS);\nfunction checkTimestamps()\n{\n'
|
||||
b'\ttDiff = Date.now() - currentYoloTS;\n\tupdateUi()'
|
||||
b';\n\tif(tDiff > dayMS)\n\t{\n\t\tclearInterval(upda'
|
||||
b'teInterval);\n\t\tlocation.reload();\t\n\t}\n}\n'
|
||||
b'\nfunction updateUi()\n{\n\tvalPercent = tDiff / day'
|
||||
b'MS * 100;\n\tif(valPercent > 100) valPercent = 100'
|
||||
b';\n\tdocument.getElementById("progress").style = "'
|
||||
b'width: " + valPercent + "%;";\n\tif(tDiff > dayMS)'
|
||||
b'\n\t{\n\t\tdocument.getElementById("yoloValue").i'
|
||||
b'nnerHTML = "�";\n\t}\n}\n\nupdateUi();\nvar'
|
||||
b' updateInterval = setInterval(checkTimestamps, 5000)'
|
||||
b';\n\n//YOLODATE\n</script>\n</body>\n</html>\n',
|
||||
'headers': ((b'Server', b'nginx'),
|
||||
(b'Date', b'Mon, 26 Apr 2021 18:31:56 GMT'),
|
||||
(b'Content-Type', b'text/html'),
|
||||
(b'Content-Length', b'2460'),
|
||||
(b'Last-Modified', b'Sun, 25 Apr 2021 22:00:00 GMT'),
|
||||
(b'Connection', b'keep-alive'),
|
||||
(b'ETag', b'"6085e660-99c"'),
|
||||
(b'Strict-Transport-Security',
|
||||
b'max-age=31536000; includeSubDomains; preload'),
|
||||
(b'X-Xss-Protection', b'1; mode=block'),
|
||||
(b'X-Content-Type-Options', b'nosniff'),
|
||||
(b'Content-Security-Policy',
|
||||
b"default-src 'self'; script-src 'self' 'unsafe-in"
|
||||
b"line'; connect-src 'self'; img-src 'self'; style"
|
||||
b"-src 'self' 'unsafe-inline';"),
|
||||
(b'X-Frame-Options', b'SAMEORIGIN'),
|
||||
(b'Referrer-Policy', b'no-referrer'),
|
||||
(b'Accept-Ranges', b'bytes')),
|
||||
'http_version': b'HTTP/1.1',
|
||||
'reason': b'OK',
|
||||
'status_code': 200,
|
||||
'timestamp_end': 1619461916.7979567,
|
||||
'timestamp_start': 1619461916.7935555,
|
||||
'trailers': None},
|
||||
'server_conn': {'address': ('yolo.jetzt', 443),
|
||||
'alpn_proto_negotiated': b'http/1.1',
|
||||
'cert': '',
|
||||
'id': '64c838e2-ab74-49c4-9ebe-70e8d17e9081',
|
||||
'ip_address': ('95.156.226.69', 443),
|
||||
'sni': 'yolo.jetzt',
|
||||
'source_address': ('10.23.42.102', 43399),
|
||||
'timestamp_end': None,
|
||||
'timestamp_start': 1619461916.63117,
|
||||
'timestamp_tcp_setup': 1619461916.6659565,
|
||||
'timestamp_tls_setup': 1619461916.7522914,
|
||||
'tls_established': True,
|
||||
'tls_version': 'TLSv1.2',
|
||||
'via': None},
|
||||
'type': 'http',
|
||||
'version': 9}
|
||||
|
||||
|
||||
class MitmAddonTestServer:
|
||||
def __init__(self, queue, path: str):
|
||||
|
|
@ -86,6 +219,7 @@ class MitmAddonTestServer:
|
|||
msg = {"type": pkg.ptype, "id": pkg.flowid, "data": pkg.data}
|
||||
self.send(msg)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client_server():
|
||||
queue = Queue()
|
||||
|
|
@ -96,32 +230,200 @@ def client_server():
|
|||
client.start()
|
||||
server = MitmAddonTestServer(queue, sock)
|
||||
server.connect()
|
||||
yield client, server
|
||||
yield queue, client, server
|
||||
client.join(1)
|
||||
server.disconnect()
|
||||
|
||||
|
||||
class TestBigSnitchWrapper:
|
||||
def test_request_convert(self, flowstate_request):
|
||||
req = bRequest(flow=flowstate_request)
|
||||
d = {'content': b'',
|
||||
'headers': [bHeader(key=b'Host', value=b'yolo.jetzt'),
|
||||
bHeader(key=b'User-Agent', value=b'curl/7.75.0'),
|
||||
bHeader(key=b'Accept', value=b'*/*')],
|
||||
'host': 'yolo.jetzt',
|
||||
'http_version': b'HTTP/1.1',
|
||||
'method': b'GET',
|
||||
'port': 443,
|
||||
'scheme': b'https',
|
||||
'server_ip_address': '95.156.226.69',
|
||||
'timestamp_end': 1619390482.69,
|
||||
'timestamp_start': 1619390482.6886377,
|
||||
'tls': True}
|
||||
d = {'content': '',
|
||||
'headers': [bHeader(key='Host', value='yolo.jetzt'),
|
||||
bHeader(key='User-Agent', value='curl/7.75.0'),
|
||||
bHeader(key='Accept', value='*/*')],
|
||||
'host': 'yolo.jetzt',
|
||||
'http_version': 'HTTP/1.1',
|
||||
'method': 'GET',
|
||||
'port': 443,
|
||||
'scheme': 'https',
|
||||
'server_ip_address': '95.156.226.69',
|
||||
'timestamp_end': 1619390482.69,
|
||||
'timestamp_start': 1619390482.6886377,
|
||||
'tls': True}
|
||||
assert not DeepDiff(req.__dict__, d)
|
||||
"""
|
||||
|
||||
def test_response_convert(self, flowstate_response):
|
||||
res = bResponse(flow=flowstate_response)
|
||||
d = {'content': '<!DOCTYPE html">\n'
|
||||
'<html>\n'
|
||||
'<head> \n'
|
||||
'<meta http-equiv="content-type" '
|
||||
'content="text/html;charset=ISO-8859-1">\n'
|
||||
'<title>todays yolo - 3026</title> \n'
|
||||
'<style type="text/css" media="screen">\n'
|
||||
'<!--\n'
|
||||
'body \n'
|
||||
'{\n'
|
||||
'color: white;\n'
|
||||
'background-color: #003;\n'
|
||||
'margin: 0px\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'.yolo\n'
|
||||
'{\n'
|
||||
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
|
||||
'font-weight: bold;\n'
|
||||
'font-size: 3em;\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'a:visited {\n'
|
||||
'\tcolor: blue\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'#content \n'
|
||||
'{\n'
|
||||
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
|
||||
'color: white;\n'
|
||||
'background-color: transparent;\n'
|
||||
'text-align: center;\n'
|
||||
'font-size: 2em;\n'
|
||||
'position: absolute;\n'
|
||||
'top: 50%;\n'
|
||||
'left: 50%;\n'
|
||||
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
|
||||
'-moz-transform: translateX(-50%) translateY(-50%);\n'
|
||||
'-ms-transform: translateX(-50%) translateY(-50%);\n'
|
||||
'transform: translateX(-50%) translateY(-50%);\n'
|
||||
'overflow: visible;\n'
|
||||
'visibility: visible;\n'
|
||||
'display: block\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'#sponsors \n'
|
||||
'{\n'
|
||||
'font-family: Verdana, Geneva, Arial, sans-serif;\n'
|
||||
'color: white;\n'
|
||||
'background-color: transparent;\n'
|
||||
'text-align: center;\n'
|
||||
'font-size: 1em;\n'
|
||||
'position: absolute;\n'
|
||||
'top: 90%;\n'
|
||||
'left: 50%;\n'
|
||||
'-webkit-transform: translateX(-50%) translateY(-50%);\n'
|
||||
'-moz-transform: translateX(-50%) translateY(-50%);\n'
|
||||
'-ms-transform: translateX(-50%) translateY(-50%);\n'
|
||||
'transform: translateX(-50%) translateY(-50%);\n'
|
||||
'overflow: visible;\n'
|
||||
'visibility: visible;\n'
|
||||
'display: block\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'#progress\n'
|
||||
'{\n'
|
||||
'\tbackground-color: #f08;\n'
|
||||
'\tposition: absolute;\n'
|
||||
'\tleft: 0px;\n'
|
||||
'\ttop: 0px;\n'
|
||||
'\theight: 10px;\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'-->\n'
|
||||
'</style>\n'
|
||||
'</head>\n'
|
||||
'<body>\n'
|
||||
'<div id="progress" class="progressBarDiv"></div>\n'
|
||||
'<div id="content">\n'
|
||||
'the yolo for today is<br>\n'
|
||||
'<span class="yolo" id="yoloValue" '
|
||||
'title="YOLOVALUE">3026</span><br>\n'
|
||||
'<br>\n'
|
||||
'</div>\n'
|
||||
'<div id="sponsors">\n'
|
||||
'\t<a href="https://cat.yolo.jetzt">Cat</a>\n'
|
||||
'\t<br>\n'
|
||||
'\tRegulation (EU) 2016/679 compliant\n'
|
||||
'</div>\n'
|
||||
'<script type="text/javascript">\n'
|
||||
'var currentYoloTS = 1619388000000; //Multiply by 1000 the YOLO '
|
||||
'way\n'
|
||||
'var tDiff = Date.now() - currentYoloTS;\n'
|
||||
'\n'
|
||||
'var dayMS = 24 * 60 * 60 * 1000;\n'
|
||||
'\n'
|
||||
'console.debug(currentYoloTS);\n'
|
||||
'console.debug(Date.now() - currentYoloTS);\n'
|
||||
'function checkTimestamps()\n'
|
||||
'{\n'
|
||||
'\ttDiff = Date.now() - currentYoloTS;\n'
|
||||
'\tupdateUi();\n'
|
||||
'\tif(tDiff > dayMS)\n'
|
||||
'\t{\n'
|
||||
'\t\tclearInterval(updateInterval);\n'
|
||||
'\t\tlocation.reload();\t\n'
|
||||
'\t}\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'function updateUi()\n'
|
||||
'{\n'
|
||||
'\tvalPercent = tDiff / dayMS * 100;\n'
|
||||
'\tif(valPercent > 100) valPercent = 100;\n'
|
||||
'\tdocument.getElementById("progress").style = "width: " + '
|
||||
'valPercent + "%;";\n'
|
||||
'\tif(tDiff > dayMS)\n'
|
||||
'\t{\n'
|
||||
'\t\tdocument.getElementById("yoloValue").innerHTML = "�";\n'
|
||||
'\t}\n'
|
||||
'}\n'
|
||||
'\n'
|
||||
'updateUi();\n'
|
||||
'var updateInterval = setInterval(checkTimestamps, 5000);\n'
|
||||
'\n'
|
||||
'//YOLODATE\n'
|
||||
'</script>\n'
|
||||
'</body>\n'
|
||||
'</html>\n',
|
||||
'headers': [bHeader(key='Server', value='nginx'),
|
||||
bHeader(key='Date', value='Mon, 26 Apr 2021 18:31:56 GMT'),
|
||||
bHeader(key='Content-Type', value='text/html'),
|
||||
bHeader(key='Content-Length', value='2460'),
|
||||
bHeader(key='Last-Modified', value='Sun, 25 Apr 2021 22:00:00 GMT'),
|
||||
bHeader(key='Connection', value='keep-alive'),
|
||||
bHeader(key='ETag', value='"6085e660-99c"'),
|
||||
bHeader(key='Strict-Transport-Security', value='max-age=31536000; includeSubDomains; preload'),
|
||||
bHeader(key='X-Xss-Protection', value='1; mode=block'),
|
||||
bHeader(key='X-Content-Type-Options', value='nosniff'),
|
||||
bHeader(key='Content-Security-Policy', value="default-src 'self'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self'; style-src 'self' 'unsafe-inline';"),
|
||||
bHeader(key='X-Frame-Options', value='SAMEORIGIN'),
|
||||
bHeader(key='Referrer-Policy', value='no-referrer'),
|
||||
bHeader(key='Accept-Ranges', value='bytes')],
|
||||
'http_version': 'HTTP/1.1',
|
||||
'reason': 'OK',
|
||||
'status_code': 200,
|
||||
'timestamp_end': 1619461916.7979567,
|
||||
'timestamp_start': 1619461916.7935555}
|
||||
|
||||
assert not DeepDiff(res.__dict__, d)
|
||||
|
||||
|
||||
class TestMitmAddon:
|
||||
def test_get_new_flows_empty(self, client_server):
|
||||
queue, client, server = client_server
|
||||
# queue empty, flows empty
|
||||
assert queue.empty()
|
||||
assert not len(client.flows)
|
||||
|
||||
client.get_new_flows()
|
||||
|
||||
# afterwards too
|
||||
assert queue.empty()
|
||||
assert not len(client.flows)
|
||||
|
||||
def test_get_new_flows_single(self, client_server):
|
||||
queue, client, server = client_server
|
||||
|
||||
def test_request(self, client_server):
|
||||
self.client, self.server = client_server
|
||||
queue, client, server = client_server
|
||||
# create request
|
||||
flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flow)
|
||||
self.q.put_nowait((flow.id, flowitem))
|
||||
"""
|
||||
#flowitem = FlowItem(bFlowState.UNSENT_HTTP_REQUEST, flowstate_request)
|
||||
#self.q.put_nowait(('51215b69-c76f-4ac2-afcb-da3b823d9f88', flowitem))
|
||||
Loading…
Reference in a new issue