mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-09 19:56:48 +00:00
56b4b03500
The upload session lives in a job instead of flocks and .meta/.done files; the job panel shows and cancels it; an upload idle for an hour is ended and its partial file deleted; the page no longer reloads after an upload.
94 lines
3.9 KiB
Python
94 lines
3.9 KiB
Python
"""ISO chunks go to an SSH compute as pipelined SFTP writes. paramiko's own
|
|
pipelined writes drop their errors, so the upload registers each write to
|
|
the file: every answer, in whatever order it comes, reaches the file and an
|
|
error is raised. Checked with paramiko's real client and file, on a fake
|
|
transport."""
|
|
|
|
import logging
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import MagicMock
|
|
|
|
import paramiko
|
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
|
from paramiko.message import Message
|
|
from paramiko.sftp import CMD_STATUS, CMD_WRITE, SFTP_FAILURE, SFTP_OK
|
|
|
|
from storages.upload import _UploadTarget
|
|
from vrtManager.connection import CONN_SSH
|
|
|
|
|
|
class Client(paramiko.SFTPClient):
|
|
"""paramiko's SFTPClient without a channel. The server answers once it is
|
|
asked for an answer, all requests so far in the given order, and fails
|
|
the request numbered fail_at."""
|
|
|
|
def __init__(self, order=None, fail_at=None, answer_early=False):
|
|
self._lock = threading.Lock()
|
|
self._expecting = {}
|
|
self.request_number = 1
|
|
self.logger = logging.getLogger("test_sftp_pipelined")
|
|
self.ultra_debug = False
|
|
self.order, self.fail_at = order, fail_at
|
|
self.written = {}
|
|
self.answers = []
|
|
# answer_early: an answer is ready as soon as its request is sent
|
|
self.sock = MagicMock(recv_ready=lambda: answer_early and bool(self.answers))
|
|
|
|
def _send_packet(self, kind, packet):
|
|
message = Message(packet.asbytes())
|
|
number = message.get_int()
|
|
if kind == CMD_WRITE:
|
|
message.get_string() # handle
|
|
offset = message.get_int64()
|
|
self.written[offset] = message.get_string()
|
|
self.answers.append(number) # the close too
|
|
|
|
def _read_packet(self):
|
|
if self.order is not None:
|
|
self.answers = [self.answers[i] for i in self.order(len(self.answers))]
|
|
self.order = None
|
|
number = self.answers.pop(0)
|
|
answer = Message()
|
|
answer.add_int(number)
|
|
answer.add_int(SFTP_FAILURE if number == self.fail_at else SFTP_OK)
|
|
answer.add_string("Failure" if number == self.fail_at else "")
|
|
answer.add_string("")
|
|
return CMD_STATUS, answer.asbytes()
|
|
|
|
|
|
SIZE = 10 * paramiko.SFTPFile.MAX_REQUEST_SIZE
|
|
|
|
|
|
def upload(client):
|
|
target = _UploadTarget(MagicMock(conn=CONN_SSH, host="192.0.2.1"), "/pool", "x.iso", "id")
|
|
target.sftp = MagicMock(open=lambda path, mode: paramiko.SFTPFile(client, b"handle", "w"))
|
|
data = bytes(range(256)) * (SIZE // 256)
|
|
return target.write(SimpleUploadedFile("x.iso", data), 0, new=True), data
|
|
|
|
|
|
class PipelinedWriteTestCase(unittest.TestCase):
|
|
def test_all_writes_are_sent_then_answered_in_any_order(self):
|
|
for name, order in (("in order", None), ("reversed", lambda n: reversed(range(n)))):
|
|
with self.subTest(name):
|
|
client = Client(order=order)
|
|
written, data = upload(client)
|
|
self.assertEqual(written, SIZE)
|
|
self.assertEqual(b"".join(client.written[k] for k in sorted(client.written)), data)
|
|
self.assertEqual(client._expecting, {}) # every answer was taken
|
|
|
|
def test_answers_read_while_sending(self):
|
|
client = Client(answer_early=True)
|
|
written, data = upload(client)
|
|
self.assertEqual(b"".join(client.written[k] for k in sorted(client.written)), data)
|
|
self.assertEqual(client._expecting, {})
|
|
|
|
def test_an_error_of_any_write_is_raised(self):
|
|
for name, client in (
|
|
("in order", lambda: Client(fail_at=4)),
|
|
("reversed", lambda: Client(order=lambda n: reversed(range(n)), fail_at=4)),
|
|
("read while sending", lambda: Client(fail_at=4, answer_early=True)),
|
|
):
|
|
with self.subTest(name):
|
|
with self.assertRaisesRegex(IOError, "Failure"):
|
|
upload(client())
|