1
0
Fork 0
mirror of https://github.com/retspen/webvirtcloud synced 2026-11-11 12:46:49 +00:00

perf(storages): an ISO uploads to an SSH compute without waiting for each write

Each 32 MiB chunk went out as 1024 SFTP writes, each waiting for its
answer: 19 MiB/s on a gigabit LAN, ~1.9 s per chunk with the SSH session.
The writes now go out without waiting and their answers are read as they
come and at the end: ~0.53 s per chunk, the same file (checked by sha256).

paramiko's own pipelined writes (set_pipelined) drop the errors in their
answers, so a full disk would leave a short ISO behind a successful
upload. Each write is registered to its file instead, the way paramiko
does for prefetched reads: every answer, in any order, reaches the file
and an error is raised as soon as one is read.
This commit is contained in:
catborise 2026-10-09 13:15:55 +03:00
parent 24231813d7
commit a87fbf739f
4 changed files with 136 additions and 5 deletions

View file

@ -113,6 +113,26 @@ def _sftp_replace_file(sftp, src, dst, ssh=None):
raise OSError(_("Remote server cannot safely replace upload metadata"))
def _write_pipelined(stream, chunk, offset):
"""Write chunk to an SFTP file at offset without waiting for the answer to
each 32 KiB write, raising an error as soon as one is read. Each write is
registered to the file, as paramiko does for prefetched reads, so its
answer reaches the file in whatever order it comes. (paramiko's own
pipelined writes drop their errors.)"""
sftp = stream.sftp
for data in chunk.chunks():
for start in range(0, len(data), stream.MAX_REQUEST_SIZE):
piece = data[start:start + stream.MAX_REQUEST_SIZE]
sftp._async_request(stream, paramiko.sftp.CMD_WRITE, stream.handle, paramiko.sftp.int64(offset), piece)
offset += len(piece)
# answers are taken as they come, so the server never waits for us to read
while sftp.sock.recv_ready():
sftp._read_response()
stream._check_exception()
sftp._finish_responses(stream)
return offset
def _cleanup_stale_local_uploads(base_dir, max_age_seconds=86400):
try:
now = time.time()
@ -404,11 +424,8 @@ class _UploadTarget:
if not new:
stream.seek(offset)
stream.truncate(offset)
for data in chunk.chunks():
stream.write(data)
offset += len(data)
if callable(getattr(stream, "flush", None)):
stream.flush()
# ~5x faster on a LAN than waiting for each 32 KiB write
offset = _write_pipelined(stream, chunk, offset)
return offset
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL) if new else os.O_RDWR
flags |= getattr(os, "O_NOFOLLOW", 0)