SFTP_USER = 'root'
ZPOOL = 'base'
GPGUSER = '9FED5C6CE206B70A585770CA965522B9D49AE731'
-BUFFER = 1024 * 1024
+BUFFER = 4 * 1024 * 1024
#
####################
agent = paramiko.Agent()
transport = paramiko.Transport((SFTP_HOST, 22),
- default_window_size=1024*BUFFER,
- default_max_packet_size=64*BUFFER)
+ default_window_size=128*BUFFER,
+ default_max_packet_size=BUFFER)
# transport.max_packet_size = BUFFER
# transport.window_size = BUFFER * 64
transport.connect(hostkey=hostkey)
zfs = subprocess.Popen(zfscommand, stdout=subprocess.PIPE, bufsize=2*BUFFER)
print(zfscommand)
- gpgcommand = [ 'gpg', '--batch', '--compress-algo', 'ZLIB',
+ gpgcommand = [ 'gpg2', '--batch', '--compress-algo', 'ZLIB',
'--sign', '--encrypt', '--recipient', GPGUSER ]
gpg = subprocess.Popen(gpgcommand, bufsize=2*BUFFER,
stdout=subprocess.PIPE,
print_colored("Error:\n\n" + gpg.stderr, 'red')
return
+ lastflush = 0
with open('/tmp/SHA256SUM', 'a') as digestfile:
with sftp.open(filename, 'xw', BUFFER) as remotefile:
+ remotefile.set_pipelined()
digest = hashlib.sha256()
sys.stdout.write(term['purple'])
done = 0
done = done + len(junk)
# sys.stdout.write('#')
# sys.stdout.flush()
- sys.stdout.write("\r%s %.3f GB (%.3f MB/s) " % (term['green'], (1.0 * done) / (1024 ** 3), (done / (1024 ** 2 *(time.time() - startt)))))
- sys.stdout.flush()
+ if done != len(junk):
+ sys.stdout.write(" \r%s %.3f GB (%.3f MB/s)" % (term['green'], (1.0 * done) / (1024 ** 3), (done / (1024 ** 2 *(time.time() - startt)))))
+ sys.stdout.flush()
remotefile.write(junk)
+
+ if done - lastflush > 128 * 1024**2:
+ remotefile.flush()
+ lastflush = done
+
digest.update(junk)
junk = gpg.stdout.read(BUFFER)
sys.stdout.write('\r')
- print_colored(" %.3f GB DONE (%.3f MB/s)" % ((1.0 * done) / (1024 ** 3), (done / (1024 ** 2 *(time.time() - startt)))), 'green')
+ timedelta = time.time() - startt
+ print_colored(" %.3f GB DONE (%.3f MB/s) Total: %02d:%02d:%02d" % ((1.0 * done) / (1024 ** 3), (done / (1024 ** 2 *timedelta)), timedelta/3600, (timedelta/60)%60, timedelta%60), 'green')
digestfile.write("%s %s\n" % (digest.hexdigest(), filename))
def syncronize(local_datasets, remote_datasets):