backup-zfs

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import print_function
   5
   6 ####################
   7 # Config
   8 SFTP_HOST = 'botero.siccegge.de'
   9 SFTP_DIR  = '/srv/backup/mitoraj'
  10 SFTP_USER = 'root'
  11 ZPOOL     = 'base'
  12 GPGUSER   = '9FED5C6CE206B70A585770CA965522B9D49AE731'
  13 BUFFER    = 4 * 1024 * 1024
  14 #
  15 ####################
  16
  17 import subprocess
  18 import os.path
  19 import sys
  20 import time
  21 import hashlib
  22 import paramiko
  23
  24 term = {
  25     'green':  "\033[0;32m",
  26     'red':    "\033[0;31m",
  27     'yellow': "\033[0;33m",
  28     'purple': "\033[0;35m",
  29     'none':   "\033[0m",
  30     }
  31
  32 sftp = None
  33
  34 def print_colored(data, color):
  35     sys.stdout.write(term[color])
  36     sys.stdout.write(data)
  37     sys.stdout.write(term['none'])
  38     sys.stdout.write('\n')
  39     sys.stdout.flush()
  40
  41 def postprocess_datasets(datasets):
  42     devices = set([entry.split('@')[0] for entry in datasets])
  43
  44     result = dict()
  45     for device in devices:
  46         result[device] = sorted([ entry.split('@')[1] for entry in datasets
  47                                     if entry.startswith(device) ])
  48
  49     return result
  50
  51 def sftp_connect():
  52     global sftp
  53     paramiko.util.log_to_file("/tmp/paramiko.log")
  54
  55     host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
  56     hostkeytype = host_keys[SFTP_HOST].keys()[0]
  57     hostkey = host_keys[SFTP_HOST][hostkeytype]
  58
  59     agent = paramiko.Agent()
  60     transport = paramiko.Transport((SFTP_HOST, 22),
  61                                    default_window_size=128*BUFFER,
  62                                    default_max_packet_size=BUFFER)
  63     #  transport.max_packet_size = BUFFER
  64     #  transport.window_size = BUFFER * 64
  65     transport.connect(hostkey=hostkey)
  66
  67     for key in agent.get_keys():
  68         try:
  69             transport.auth_publickey(SFTP_USER, key)
  70             break
  71         except paramiko.SSHException:
  72             continue
  73
  74     sftp = transport.open_sftp_client()
  75     sftp.chdir(SFTP_DIR)
  76
  77 def sftp_send(dataset, reference=None):
  78     if reference is None:
  79         filename = '%s.full.zfs.gpg' % dataset
  80     else:
  81         filename = '%s.from.%s.zfs.gpg' % (dataset, reference)
  82
  83     try:
  84         sftp.stat(filename)
  85         return
  86     except:
  87         pass
  88
  89     zfscommand = ['sudo', 'zfs', 'send', '-D', '%s/%s' % (ZPOOL, dataset)]
  90     if reference is not None:
  91         zfscommand = zfscommand + ['-i', reference]
  92
  93     zfs = subprocess.Popen(zfscommand, stdout=subprocess.PIPE, bufsize=2*BUFFER)
  94     print(zfscommand)
  95
  96     gpgcommand = [ 'gpg2', '--batch', '--compress-algo', 'ZLIB',
  97                    '--sign', '--encrypt', '--recipient', GPGUSER ]
  98     gpg = subprocess.Popen(gpgcommand, bufsize=2*BUFFER,
  99                                        stdout=subprocess.PIPE,
 100                                        stdin=zfs.stdout,
 101                                        stderr=subprocess.PIPE)
 102     print(gpgcommand)
 103
 104     junk = gpg.stdout.read(BUFFER)
 105     gpg.poll()
 106     if gpg.returncode not in [None, 0]:
 107         print_colored("Error:\n\n" + gpg.stderr, 'red')
 108         return
 109
 110     lastflush = 0
 111     with open('/tmp/SHA256SUM', 'a') as digestfile:
 112         with sftp.open(filename, 'xw', BUFFER) as remotefile:
 113             remotefile.set_pipelined()
 114             digest = hashlib.sha256()
 115             sys.stdout.write(term['purple'])
 116             done = 0
 117             startt = time.time()
 118             while True:
 119                 if len(junk) == 0:
 120                     break
 121                 #if len(junk) < BUFFER:
 122                 #    print_colored("short read: %d" % len(junk), 'yellow')
 123                 done = done + len(junk)
 124 #                sys.stdout.write('#')
 125 #                sys.stdout.flush()
 126                 if done != len(junk):
 127                     sys.stdout.write("     \r%s %.3f GB      (%.3f MB/s)" % (term['green'], (1.0 * done) / (1024 ** 3), (done / (1024 ** 2 *(time.time() - startt)))))
 128                     sys.stdout.flush()
 129                 remotefile.write(junk)
 130
 131                 if done - lastflush > 128 * 1024**2:
 132                     remotefile.flush()
 133                     lastflush = done
 134
 135                 digest.update(junk)
 136                 junk = gpg.stdout.read(BUFFER)
 137
 138             sys.stdout.write('\r')
 139             timedelta = time.time() - startt
 140             print_colored(" %.3f GB DONE (%.3f MB/s) Total: %02d:%02d:%02d" % ((1.0 * done) / (1024 ** 3), (done / (1024 ** 2 *timedelta)), timedelta/3600, (timedelta/60)%60, timedelta%60), 'green')
 141             digestfile.write("%s  %s\n" % (digest.hexdigest(), filename))
 142
 143 def syncronize(local_datasets, remote_datasets):
 144     for device in local_datasets.keys():
 145         current = ""
 146         for dataset in local_datasets[device]:
 147             last = current
 148             current = dataset
 149
 150             if device in remote_datasets:
 151                 if dataset in remote_datasets[device]:
 152                     print_colored("%s@%s -- found on remote server" % (device, dataset), 'yellow')
 153                     continue
 154
 155             if last == '':
 156                 print_colored("Initial syncronization for device %s" % device, 'green')
 157                 sftp_send("%s@%s" % (device, dataset))
 158                 lastmonth = dataset
 159                 continue
 160
 161             if last[:7] == dataset[:7]:
 162                 print_colored("%s@%s -- incremental backup (reference: %s)" %
 163                               (device, dataset, last), 'green')
 164                 sftp_send("%s@%s" % (device, dataset), last)
 165             else:
 166                 print_colored("%s@%s -- full backup" % (device, dataset), 'green')
 167                 sftp_send("%s@%s" % (device, dataset))
 168                 #print_colored("%s@%s -- doing incremental backup" % (device, dataset), 'green')
 169                 #sftp_send("%s@%s" % (device, dataset), lastmonth)
 170                 #lastmonth = dataset
 171
 172 def get_remote_datasets():
 173     datasets = sftp.listdir()
 174     datasets = filter(lambda x: '@' in x, datasets)
 175
 176     datasets = [ entry.split('.')[0] for entry in datasets ]
 177
 178     return postprocess_datasets(datasets)
 179
 180 def get_local_datasets():
 181     datasets = subprocess.check_output(['sudo', 'zfs', 'list', '-t', 'snapshot', '-H', '-o', 'name'])
 182     datasets = datasets.strip().split('\n')
 183
 184     datasets = [ entry[5:] for entry in datasets ]
 185
 186     return postprocess_datasets(datasets)
 187
 188 def main():
 189     sftp_connect()
 190     syncronize(get_local_datasets(), get_remote_datasets())
 191
 192 if __name__ == '__main__':
 193     main()