vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011, 2012  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import shutil
  24 import subprocess
  25 import tempfile
  26
  27
  28 class VmDebootstrap(cliapp.Application):
  29
  30     def add_settings(self):
  31         default_arch = 'amd64'
  32
  33         self.settings.boolean(['verbose'], 'report what is going on')
  34         self.settings.string(['image'], 'put created disk image in FILE',
  35                              metavar='FILE')
  36         self.settings.bytesize(['size'],
  37                                'create a disk image of size SIZE (%default)',
  38                                metavar='SIZE',
  39                                default='1G')
  40         self.settings.bytesize(['bootsize'],
  41                                'create boot partition of size SIZE (%default)',
  42                                metavar='BOOTSIZE',
  43                                default='0%')
  44         self.settings.string(['boottype'],
  45                              'specify file system type for /boot/',
  46                              default='ext2')
  47         self.settings.string(['foreign'],
  48                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  49         self.settings.string(['variant'],
  50                              'select debootstrap variant it not using the default')
  51         self.settings.boolean(['no-extlinux'], 'do not install extlinux')
  52         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  53                              metavar='FILE')
  54         self.settings.string(['mirror'],
  55                              'use MIRROR as package source (%default)',
  56                              metavar='URL',
  57                              default='http://cdn.debian.net/debian/')
  58         self.settings.string(['arch'], 'architecture to use (%default)',
  59                              metavar='ARCH',
  60                              default=default_arch)
  61         self.settings.string(['distribution'],
  62                              'release to use (%default)',
  63                              metavar='NAME',
  64                              default='stable')
  65         self.settings.string_list(['package'], 'install PACKAGE onto system')
  66         self.settings.string_list(['custom-package'],
  67                                   'install package in DEB file onto system '
  68                                     '(not from mirror)',
  69                                   metavar='DEB')
  70         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  71         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  72         self.settings.string(['root-password'], 'set root password',
  73                              metavar='PASSWORD')
  74         self.settings.boolean(['lock-root-password'],
  75                               'lock root account so they cannot login?')
  76         self.settings.string(['customize'],
  77                              'run SCRIPT after setting up system',
  78                              metavar='SCRIPT')
  79         self.settings.string(['hostname'],
  80                              'set name to HOSTNAME (%default)',
  81                              metavar='HOSTNAME',
  82                              default='debian')
  83         self.settings.string_list(['user'],
  84                                   'create USER with PASSWORD',
  85                                   metavar='USER/PASSWORD')
  86         self.settings.boolean(['serial-console'],
  87                               'configure image to use a serial console')
  88         self.settings.string(['serial-console-command'],
  89                              'command to manage the serial console, appended '
  90                                'to /etc/inittab (%default)',
  91                              metavar='COMMAND',
  92                              default='/sbin/getty -L ttyS0 115200 vt100')
  93         self.settings.boolean(['sudo'],
  94                               'install sudo, and if user is created, add them '
  95                                 'to sudo group')
  96
  97     def process_args(self, args):
  98         if not self.settings['image'] and not self.settings['tarball']:
  99             raise cliapp.AppException('You must give disk image filename, '
 100                                       'or tarball filename')
 101         if self.settings['image'] and not self.settings['size']:
 102             raise cliapp.AppException('If disk image is specified, '
 103                                       'You must give image size.')
 104
 105         self.remove_dirs = []
 106         self.mount_points = []
 107
 108         try:
 109             if self.settings['image']:
 110                 self.create_empty_image()
 111                 self.partition_image()
 112                 self.install_mbr()
 113                 (rootdev,bootdev) = self.setup_kpartx()
 114                 self.mkfs(rootdev)
 115                 rootdir = self.mount(rootdev)
 116                 if bootdev:
 117                     if self.settings['boottype']:
 118                         fstype = self.settings['boottype']
 119                     else:
 120                         fstype = 'ext2'
 121                     self.mkfs(bootdev, type=fstype)
 122                     bootdir = '%s/%s' % (rootdir, 'boot/')
 123                     os.mkdir(bootdir)
 124                     bootdir = self.mount(bootdev, bootdir)
 125             else:
 126                 rootdir = self.mkdtemp()
 127             self.debootstrap(rootdir)
 128             self.set_hostname(rootdir)
 129             self.create_fstab(rootdir)
 130             self.install_debs(rootdir)
 131             self.cleanup_apt_cache(rootdir)
 132             self.set_root_password(rootdir)
 133             self.create_users(rootdir)
 134             self.remove_udev_persistent_rules(rootdir)
 135             self.setup_networking(rootdir)
 136             self.customize(rootdir)
 137             if self.settings['image']:
 138                 if not self.settings['no-extlinux']:
 139                     self.install_extlinux(rootdev, rootdir)
 140                 self.optimize_image(rootdir)
 141
 142             if self.settings['foreign']:
 143                 os.unlink('%s/usr/bin/%s' %
 144                           (rootdir, os.path.basename(self.settings['foreign'])))
 145
 146             if self.settings['tarball']:
 147                 self.create_tarball(rootdir)
 148         except BaseException, e:
 149             self.message('EEEK! Something bad happened...')
 150             self.cleanup_system()
 151             raise
 152         else:
 153             self.cleanup_system()
 154
 155     def message(self, msg):
 156         logging.info(msg)
 157         if self.settings['verbose']:
 158             print msg
 159
 160     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 161         logging.debug('runcmd: %s %s' % (argv, kwargs))
 162         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 163                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 164                              **kwargs)
 165         out, err = p.communicate(stdin)
 166         if p.returncode != 0:
 167             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 168             logging.error(msg)
 169             if not ignore_fail:
 170                 raise cliapp.AppException(msg)
 171         return out
 172
 173     def mkdtemp(self):
 174         dirname = tempfile.mkdtemp()
 175         self.remove_dirs.append(dirname)
 176         logging.debug('mkdir %s' % dirname)
 177         return dirname
 178
 179     def mount(self, device, path=None):
 180         self.message('Mounting %s' % device)
 181         if not path:
 182             mount_point = self.mkdtemp()
 183         else:
 184             mount_point = path
 185         self.runcmd(['mount', device, mount_point])
 186         self.mount_points.append(mount_point)
 187         logging.debug('mounted %s on %s' % (device, mount_point))
 188         return mount_point
 189
 190     def create_empty_image(self):
 191         self.message('Creating disk image')
 192         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 193                      self.settings['image'],
 194                      str(self.settings['size'])])
 195
 196     def partition_image(self):
 197         self.message('Creating partitions')
 198         self.runcmd(['parted', '-s', self.settings['image'],
 199                      'mklabel', 'msdos'])
 200         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 201             bootsize=str(self.settings['bootsize']/(1024*1024))
 202             self.runcmd(['parted', '-s', self.settings['image'],
 203                          'mkpart', 'primary', 'fat16', '0', bootsize])
 204         else:
 205             bootsize='0%'
 206         self.runcmd(['parted', '-s', self.settings['image'],
 207                      'mkpart', 'primary', bootsize, '100%'])
 208         self.runcmd(['parted', '-s', self.settings['image'],
 209                      'set', '1', 'boot', 'on'])
 210
 211     def install_mbr(self):
 212         self.message('Installing MBR')
 213         self.runcmd(['install-mbr', self.settings['image']])
 214
 215     def setup_kpartx(self):
 216         out = self.runcmd(['kpartx', '-av', self.settings['image']])
 217         if self.settings['bootsize']:
 218             bootindex = 0
 219             rootindex = 1
 220             parts = 2
 221         else:
 222             rootindex = 0
 223             parts = 1
 224             boot = None
 225         devices = [line.split()[2]
 226                    for line in out.splitlines()
 227                    if line.startswith('add map ')]
 228         if len(devices) != parts:
 229             raise cliapp.AppException('Surprising number of partitions')
 230         root = '/dev/mapper/%s' % devices[rootindex]
 231         if self.settings['bootsize']:
 232             boot = '/dev/mapper/%s' % devices[bootindex]
 233         return (root,boot)
 234
 235     def mkfs(self, device, type='ext2'):
 236         self.message('Creating filesystem')
 237         self.runcmd(['mkfs', '-t', type, device])
 238
 239     def debootstrap(self, rootdir):
 240         self.message('Debootstrapping')
 241
 242         if self.settings['foreign']:
 243             necessary_packages = []
 244         else:
 245             necessary_packages = ['acpid']
 246
 247         include = self.settings['package']
 248
 249         if not self.settings['no-kernel']:
 250             if self.settings['arch'] == 'i386':
 251                 kernel_arch = '486'
 252             else:
 253                 kernel_arch = self.settings['arch']
 254             kernel_image = 'linux-image-%s' % kernel_arch
 255             include.append(kernel_image)
 256
 257         if self.settings['sudo'] and 'sudo' not in include:
 258             include.append('sudo')
 259
 260         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 261         args.append(
 262             '--include=%s' % ','.join(necessary_packages + include))
 263         if self.settings['foreign']:
 264             args.append('--foreign')
 265         if self.settings['variant']:
 266             args.append('--variant')
 267             args.append(self.settings['variant'])
 268         args += [self.settings['distribution'],
 269                  rootdir, self.settings['mirror']]
 270         self.runcmd(args)
 271         if self.settings['foreign']:
 272             # First copy the binfmt handler over
 273             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 274             # Next, run the package install scripts etc.
 275             self.runcmd(['chroot', rootdir,
 276                          '/debootstrap/debootstrap', '--second-stage'])
 277
 278     def set_hostname(self, rootdir):
 279         hostname = self.settings['hostname']
 280         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 281             f.write('%s\n' % hostname)
 282
 283         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 284         with open(etc_hosts, 'r') as f:
 285             data = f.read()
 286         with open(etc_hosts, 'w') as f:
 287             for line in data.splitlines():
 288                 if line.startswith('127.0.0.1'):
 289                     line += ' %s' % hostname
 290                 f.write('%s\n' % line)
 291
 292     def create_fstab(self, rootdir):
 293         fstab = os.path.join(rootdir, 'etc', 'fstab')
 294         with open(fstab, 'w') as f:
 295             f.write('proc /proc proc defaults 0 0\n')
 296             f.write('/dev/sda1 / ext4 errors=remount-ro 0 1\n')
 297
 298     def install_debs(self, rootdir):
 299         if not self.settings['custom-package']:
 300             return
 301         self.message('Installing custom packages')
 302         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 303         os.mkdir(tmp)
 304         for deb in self.settings['custom-package']:
 305             shutil.copy(deb, tmp)
 306         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 307                      for deb in self.settings['custom-package']]
 308         out, err, exit = \
 309             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 310         logging.debug('stdout:\n%s' % out)
 311         logging.debug('stderr:\n%s' % err)
 312         out = self.runcmd(['chroot', rootdir,
 313                      'apt-get', '-f', '--no-remove', 'install'])
 314         logging.debug('stdout:\n%s' % out)
 315         shutil.rmtree(tmp)
 316
 317     def cleanup_apt_cache(self, rootdir):
 318         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 319         logging.debug('stdout:\n%s' % out)
 320
 321     def set_root_password(self, rootdir):
 322         if self.settings['root-password']:
 323             self.message('Setting root password')
 324             self.set_password(rootdir, 'root', self.settings['root-password'])
 325         elif self.settings['lock-root-password']:
 326             self.message('Locking root password')
 327             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 328         else:
 329             self.message('Give root an empty password')
 330             self.delete_password(rootdir, 'root')
 331
 332     def create_users(self, rootdir):
 333         def create_user(user):
 334             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 335                          '--disabled-password', user])
 336             if self.settings['sudo']:
 337                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 338
 339         for userpass in self.settings['user']:
 340             if '/' in userpass:
 341                 user, password = userpass.split('/', 1)
 342                 create_user(user)
 343                 self.set_password(rootdir, user, password)
 344             else:
 345                 create_user(userpass)
 346                 self.delete_password(rootdir, userpass)
 347
 348     def set_password(self, rootdir, user, password):
 349         encrypted = crypt.crypt(password, '..')
 350         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 351
 352     def delete_password(self, rootdir, user):
 353         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 354
 355     def remove_udev_persistent_rules(self, rootdir):
 356         self.message('Removing udev persistent cd and net rules')
 357         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 358             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 359             if os.path.exists(pathname):
 360                 logging.debug('rm %s' % pathname)
 361                 os.remove(pathname)
 362             else:
 363                 logging.debug('not removing non-existent %s' % pathname)
 364
 365     def setup_networking(self, rootdir):
 366         self.message('Setting up networking')
 367
 368         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 369         f.write('auto lo\n')
 370         f.write('iface lo inet loopback\n')
 371
 372         if self.settings['enable-dhcp']:
 373             f.write('\n')
 374             f.write('auto eth0\n')
 375             f.write('iface eth0 inet dhcp\n')
 376
 377         f.close()
 378
 379     def install_extlinux(self, rootdev, rootdir):
 380         self.message('Installing extlinux')
 381
 382         def find(pattern):
 383             dirname = os.path.join(rootdir, 'boot')
 384             basenames = os.listdir(dirname)
 385             logging.debug('find: %s' % basenames)
 386             for basename in basenames:
 387                 if re.search(pattern, basename):
 388                     return os.path.join('boot', basename)
 389             raise cliapp.AppException('Cannot find match: %s' % pattern)
 390
 391         kernel_image = find('vmlinuz-.*')
 392         initrd_image = find('initrd.img-.*')
 393
 394         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 395                            '-s', 'UUID', rootdev])
 396         uuid = out.splitlines()[0].strip()
 397
 398         conf = os.path.join(rootdir, 'extlinux.conf')
 399         logging.debug('configure extlinux %s' % conf)
 400         f = open(conf, 'w')
 401         f.write('''
 402 default linux
 403 timeout 1
 404
 405 label linux
 406 kernel %(kernel)s
 407 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 408 %(extserial)s
 409 ''' % {
 410     'kernel': kernel_image,
 411     'initrd': initrd_image,
 412     'uuid': uuid,
 413     'kserial':
 414         'console=ttyS0,115200' if self.settings['serial-console'] else '',
 415     'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 416 })
 417         f.close()
 418
 419         if self.settings['serial-console']:
 420             serial_command = self.settings['serial-console-command']
 421             logging.debug('adding getty to serial console')
 422             inittab = os.path.join(rootdir, 'etc/inittab')
 423             with open(inittab, 'a') as f:
 424                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 425
 426         self.runcmd(['extlinux', '--install', rootdir])
 427         self.runcmd(['sync'])
 428         import time; time.sleep(2)
 429
 430     def optimize_image(self, rootdir):
 431         """
 432         Filing up the image with zeros will increase its compression rate
 433         """
 434         zeros = os.path.join(rootdir, 'ZEROS')
 435         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 436         self.runcmd(['rm', '-f', zeros])
 437
 438
 439     def cleanup_system(self):
 440         # Clean up after any errors.
 441
 442         self.message('Cleaning up')
 443
 444         # Umount in the reverse mount order
 445         if self.settings['image']:
 446             for i in xrange(len(self.mount_points) - 1, -1, -1):
 447                 mount_point = self.mount_points[i]
 448                 self.runcmd(['umount', mount_point], ignore_fail=True)
 449
 450             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 451
 452         for dirname in self.remove_dirs:
 453             shutil.rmtree(dirname)
 454
 455     def customize(self, rootdir):
 456         script = self.settings['customize']
 457         if script:
 458             self.message('Running customize script %s' % script)
 459             with open('/dev/tty', 'w') as tty:
 460                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 461
 462     def create_tarball(self, rootdir):
 463         # Create a tarball of the disk's contents
 464         # shell out to runcmd since it more easily handles rootdir
 465         self.message('Creating tarball of disk contents')
 466         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 467
 468
 469 if __name__ == '__main__':
 470     VmDebootstrap().run()
 471