mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-11 12:46:49 +00:00
perf(storages): an ISO uploads to an SSH compute without waiting for each write
Each 32 MiB chunk went out as 1024 SFTP writes, each waiting for its answer: 19 MiB/s on a gigabit LAN, ~1.9 s per chunk with the SSH session. The writes now go out without waiting and their answers are read as they come and at the end: ~0.53 s per chunk, the same file (checked by sha256). paramiko's own pipelined writes (set_pipelined) drop the errors in their answers, so a full disk would leave a short ISO behind a successful upload. Each write is registered to its file instead, the way paramiko does for prefetched reads: every answer, in any order, reaches the file and an error is raised as soon as one is read.
This commit is contained in:
parent
24231813d7
commit
a87fbf739f
4 changed files with 136 additions and 5 deletions
|
|
@ -113,6 +113,26 @@ def _sftp_replace_file(sftp, src, dst, ssh=None):
|
|||
raise OSError(_("Remote server cannot safely replace upload metadata"))
|
||||
|
||||
|
||||
def _write_pipelined(stream, chunk, offset):
|
||||
"""Write chunk to an SFTP file at offset without waiting for the answer to
|
||||
each 32 KiB write, raising an error as soon as one is read. Each write is
|
||||
registered to the file, as paramiko does for prefetched reads, so its
|
||||
answer reaches the file in whatever order it comes. (paramiko's own
|
||||
pipelined writes drop their errors.)"""
|
||||
sftp = stream.sftp
|
||||
for data in chunk.chunks():
|
||||
for start in range(0, len(data), stream.MAX_REQUEST_SIZE):
|
||||
piece = data[start:start + stream.MAX_REQUEST_SIZE]
|
||||
sftp._async_request(stream, paramiko.sftp.CMD_WRITE, stream.handle, paramiko.sftp.int64(offset), piece)
|
||||
offset += len(piece)
|
||||
# answers are taken as they come, so the server never waits for us to read
|
||||
while sftp.sock.recv_ready():
|
||||
sftp._read_response()
|
||||
stream._check_exception()
|
||||
sftp._finish_responses(stream)
|
||||
return offset
|
||||
|
||||
|
||||
def _cleanup_stale_local_uploads(base_dir, max_age_seconds=86400):
|
||||
try:
|
||||
now = time.time()
|
||||
|
|
@ -404,11 +424,8 @@ class _UploadTarget:
|
|||
if not new:
|
||||
stream.seek(offset)
|
||||
stream.truncate(offset)
|
||||
for data in chunk.chunks():
|
||||
stream.write(data)
|
||||
offset += len(data)
|
||||
if callable(getattr(stream, "flush", None)):
|
||||
stream.flush()
|
||||
# ~5x faster on a LAN than waiting for each 32 KiB write
|
||||
offset = _write_pipelined(stream, chunk, offset)
|
||||
return offset
|
||||
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL) if new else os.O_RDWR
|
||||
flags |= getattr(os, "O_NOFOLLOW", 0)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue