mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-10 04:06:47 +00:00
a87fbf739f
Each 32 MiB chunk went out as 1024 SFTP writes, each waiting for its answer: 19 MiB/s on a gigabit LAN, ~1.9 s per chunk with the SSH session. The writes now go out without waiting and their answers are read as they come and at the end: ~0.53 s per chunk, the same file (checked by sha256). paramiko's own pipelined writes (set_pipelined) drop the errors in their answers, so a full disk would leave a short ISO behind a successful upload. Each write is registered to its file instead, the way paramiko does for prefetched reads: every answer, in any order, reaches the file and an error is raised as soon as one is read.
94 lines
3.9 KiB
Python
94 lines
3.9 KiB
Python
"""ISO chunks go to an SSH compute as pipelined SFTP writes. paramiko's own
|
|
pipelined writes drop their errors, so the upload registers each write to
|
|
the file: every answer, in whatever order it comes, reaches the file and an
|
|
error is raised. Checked with paramiko's real client and file, on a fake
|
|
transport."""
|
|
|
|
import logging
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import MagicMock
|
|
|
|
import paramiko
|
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
|
from paramiko.message import Message
|
|
from paramiko.sftp import CMD_STATUS, CMD_WRITE, SFTP_FAILURE, SFTP_OK
|
|
|
|
from storages.upload import _UploadTarget
|
|
from vrtManager.connection import CONN_SSH
|
|
|
|
|
|
class Client(paramiko.SFTPClient):
|
|
"""paramiko's SFTPClient without a channel. The server answers once it is
|
|
asked for an answer, all requests so far in the given order, and fails
|
|
the request numbered fail_at."""
|
|
|
|
def __init__(self, order=None, fail_at=None, answer_early=False):
|
|
self._lock = threading.Lock()
|
|
self._expecting = {}
|
|
self.request_number = 1
|
|
self.logger = logging.getLogger("test_sftp_pipelined")
|
|
self.ultra_debug = False
|
|
self.order, self.fail_at = order, fail_at
|
|
self.written = {}
|
|
self.answers = []
|
|
# answer_early: an answer is ready as soon as its request is sent
|
|
self.sock = MagicMock(recv_ready=lambda: answer_early and bool(self.answers))
|
|
|
|
def _send_packet(self, kind, packet):
|
|
message = Message(packet.asbytes())
|
|
number = message.get_int()
|
|
if kind == CMD_WRITE:
|
|
message.get_string() # handle
|
|
offset = message.get_int64()
|
|
self.written[offset] = message.get_string()
|
|
self.answers.append(number) # the close too
|
|
|
|
def _read_packet(self):
|
|
if self.order is not None:
|
|
self.answers = [self.answers[i] for i in self.order(len(self.answers))]
|
|
self.order = None
|
|
number = self.answers.pop(0)
|
|
answer = Message()
|
|
answer.add_int(number)
|
|
answer.add_int(SFTP_FAILURE if number == self.fail_at else SFTP_OK)
|
|
answer.add_string("Failure" if number == self.fail_at else "")
|
|
answer.add_string("")
|
|
return CMD_STATUS, answer.asbytes()
|
|
|
|
|
|
SIZE = 10 * paramiko.SFTPFile.MAX_REQUEST_SIZE
|
|
|
|
|
|
def upload(client):
|
|
target = _UploadTarget(MagicMock(conn=CONN_SSH, host="192.0.2.1"), "/pool", "x.iso", "id")
|
|
target.sftp = MagicMock(open=lambda path, mode: paramiko.SFTPFile(client, b"handle", "w"))
|
|
data = bytes(range(256)) * (SIZE // 256)
|
|
return target.write_chunk(SimpleUploadedFile("x.iso", data), 0, new=True), data
|
|
|
|
|
|
class PipelinedWriteTestCase(unittest.TestCase):
|
|
def test_all_writes_are_sent_then_answered_in_any_order(self):
|
|
for name, order in (("in order", None), ("reversed", lambda n: reversed(range(n)))):
|
|
with self.subTest(name):
|
|
client = Client(order=order)
|
|
written, data = upload(client)
|
|
self.assertEqual(written, SIZE)
|
|
self.assertEqual(b"".join(client.written[k] for k in sorted(client.written)), data)
|
|
self.assertEqual(client._expecting, {}) # every answer was taken
|
|
|
|
def test_answers_read_while_sending(self):
|
|
client = Client(answer_early=True)
|
|
written, data = upload(client)
|
|
self.assertEqual(b"".join(client.written[k] for k in sorted(client.written)), data)
|
|
self.assertEqual(client._expecting, {})
|
|
|
|
def test_an_error_of_any_write_is_raised(self):
|
|
for name, client in (
|
|
("in order", lambda: Client(fail_at=4)),
|
|
("reversed", lambda: Client(order=lambda n: reversed(range(n)), fail_at=4)),
|
|
("read while sending", lambda: Client(fail_at=4, answer_early=True)),
|
|
):
|
|
with self.subTest(name):
|
|
with self.assertRaisesRegex(IOError, "Failure"):
|
|
upload(client())
|