vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import shutil
  24 import subprocess
  25 import tempfile
  26 import time
  27
  28
  29 __version__ = '0.3'
  30
  31
  32 class VmDebootstrap(cliapp.Application):
  33
  34     def add_settings(self):
  35         default_arch = 'amd64'
  36
  37         self.settings.boolean(['verbose'], 'report what is going on')
  38         self.settings.string(['image'], 'put created disk image in FILE',
  39                              metavar='FILE')
  40         self.settings.bytesize(['size'],
  41                                'create a disk image of size SIZE (%default)',
  42                                metavar='SIZE',
  43                                default='1G')
  44         self.settings.bytesize(['bootsize'],
  45                                'create boot partition of size SIZE (%default)',
  46                                metavar='BOOTSIZE',
  47                                default='0%')
  48         self.settings.string(['boottype'],
  49                              'specify file system type for /boot/',
  50                              default='ext2')
  51         self.settings.string(['foreign'],
  52                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  53         self.settings.string(['variant'],
  54                              'select debootstrap variant it not using the default')
  55         self.settings.boolean(
  56             ['extlinux'],
  57             'install extlinux?',
  58             default=True)
  59         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  60                              metavar='FILE')
  61         self.settings.string(['mirror'],
  62                              'use MIRROR as package source (%default)',
  63                              metavar='URL',
  64                              default='http://cdn.debian.net/debian/')
  65         self.settings.string(['arch'], 'architecture to use (%default)',
  66                              metavar='ARCH',
  67                              default=default_arch)
  68         self.settings.string(['distribution'],
  69                              'release to use (%default)',
  70                              metavar='NAME',
  71                              default='stable')
  72         self.settings.string_list(['package'], 'install PACKAGE onto system')
  73         self.settings.string_list(['custom-package'],
  74                                   'install package in DEB file onto system '
  75                                     '(not from mirror)',
  76                                   metavar='DEB')
  77         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  78         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  79         self.settings.string(['root-password'], 'set root password',
  80                              metavar='PASSWORD')
  81         self.settings.boolean(['lock-root-password'],
  82                               'lock root account so they cannot login?')
  83         self.settings.string(['customize'],
  84                              'run SCRIPT after setting up system',
  85                              metavar='SCRIPT')
  86         self.settings.string(['hostname'],
  87                              'set name to HOSTNAME (%default)',
  88                              metavar='HOSTNAME',
  89                              default='debian')
  90         self.settings.string_list(['user'],
  91                                   'create USER with PASSWORD',
  92                                   metavar='USER/PASSWORD')
  93         self.settings.boolean(['serial-console'],
  94                               'configure image to use a serial console')
  95         self.settings.string(['serial-console-command'],
  96                              'command to manage the serial console, appended '
  97                                'to /etc/inittab (%default)',
  98                              metavar='COMMAND',
  99                              default='/sbin/getty -L ttyS0 115200 vt100')
 100         self.settings.boolean(['sudo'],
 101                               'install sudo, and if user is created, add them '
 102                                 'to sudo group')
 103         self.settings.string(['owner'],
 104                              'the user who will own the image when the build '
 105                                'is complete.')
 106         self.settings.boolean(['squash'],
 107                               'use squashfs on the final image.')
 108
 109     def process_args(self, args):
 110         if not self.settings['image'] and not self.settings['tarball']:
 111             raise cliapp.AppException('You must give disk image filename, '
 112                                       'or tarball filename')
 113         if self.settings['image'] and not self.settings['size']:
 114             raise cliapp.AppException('If disk image is specified, '
 115                                       'You must give image size.')
 116
 117         self.remove_dirs = []
 118         self.mount_points = []
 119
 120         try:
 121             rootdev = None
 122             roottype = 'ext4'
 123             bootdev = None
 124             boottype = None
 125             if self.settings['image']:
 126                 self.create_empty_image()
 127                 self.partition_image()
 128                 self.install_mbr()
 129                 (rootdev,bootdev) = self.setup_kpartx()
 130                 self.mkfs(rootdev, type=roottype)
 131                 rootdir = self.mount(rootdev)
 132                 if bootdev:
 133                     if self.settings['boottype']:
 134                         boottype = self.settings['boottype']
 135                     else:
 136                         boottype = 'ext2'
 137                     self.mkfs(bootdev, type=boottype)
 138                     bootdir = '%s/%s' % (rootdir, 'boot/')
 139                     os.mkdir(bootdir)
 140                     bootdir = self.mount(bootdev, bootdir)
 141             else:
 142                 rootdir = self.mkdtemp()
 143             self.debootstrap(rootdir)
 144             self.set_hostname(rootdir)
 145             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 146             self.install_debs(rootdir)
 147             self.cleanup_apt_cache(rootdir)
 148             self.set_root_password(rootdir)
 149             self.create_users(rootdir)
 150             self.remove_udev_persistent_rules(rootdir)
 151             self.setup_networking(rootdir)
 152             self.customize(rootdir)
 153             if self.settings['image']:
 154                 if self.settings['extlinux']:
 155                     self.install_extlinux(rootdev, rootdir)
 156                 self.append_serial_console(rootdir)
 157                 self.optimize_image(rootdir)
 158                 if self.settings['squash']:
 159                     self.squash()
 160
 161             if self.settings['foreign']:
 162                 os.unlink('%s/usr/bin/%s' %
 163                           (rootdir, os.path.basename(self.settings['foreign'])))
 164
 165             if self.settings['tarball']:
 166                 self.create_tarball(rootdir)
 167
 168             if self.settings['owner']:
 169                 self.chown(rootdir)
 170         except BaseException, e:
 171             self.message('EEEK! Something bad happened...')
 172             self.message(e)
 173             self.cleanup_system()
 174             raise
 175         else:
 176             self.cleanup_system()
 177
 178     def message(self, msg):
 179         logging.info(msg)
 180         if self.settings['verbose']:
 181             print msg
 182
 183     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 184         logging.debug('runcmd: %s %s' % (argv, kwargs))
 185         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 186                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 187                              **kwargs)
 188         out, err = p.communicate(stdin)
 189         if p.returncode != 0:
 190             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 191             logging.error(msg)
 192             if not ignore_fail:
 193                 raise cliapp.AppException(msg)
 194         return out
 195
 196     def mkdtemp(self):
 197         dirname = tempfile.mkdtemp()
 198         self.remove_dirs.append(dirname)
 199         logging.debug('mkdir %s' % dirname)
 200         return dirname
 201
 202     def mount(self, device, path=None):
 203         if not path:
 204             mount_point = self.mkdtemp()
 205         else:
 206             mount_point = path
 207         self.message('Mounting %s on %s' % (device,mount_point))
 208         self.runcmd(['mount', device, mount_point])
 209         self.mount_points.append(mount_point)
 210         logging.debug('mounted %s on %s' % (device, mount_point))
 211         return mount_point
 212
 213     def create_empty_image(self):
 214         self.message('Creating disk image')
 215         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 216                      self.settings['image'],
 217                      str(self.settings['size'])])
 218
 219     def partition_image(self):
 220         self.message('Creating partitions')
 221         self.runcmd(['parted', '-s', self.settings['image'],
 222                      'mklabel', 'msdos'])
 223         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 224             bootsize=str(self.settings['bootsize']/(1024*1024))
 225             self.runcmd(['parted', '-s', self.settings['image'],
 226                          'mkpart', 'primary', 'fat16', '0', bootsize])
 227         else:
 228             bootsize='0%'
 229         self.runcmd(['parted', '-s', self.settings['image'],
 230                      'mkpart', 'primary', bootsize, '100%'])
 231         self.runcmd(['parted', '-s', self.settings['image'],
 232                      'set', '1', 'boot', 'on'])
 233
 234     def install_mbr(self):
 235         self.message('Installing MBR')
 236         self.runcmd(['install-mbr', self.settings['image']])
 237
 238     def setup_kpartx(self):
 239         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 240         if self.settings['bootsize']:
 241             bootindex = 0
 242             rootindex = 1
 243             parts = 2
 244         else:
 245             rootindex = 0
 246             parts = 1
 247             boot = None
 248         devices = [line.split()[2]
 249                    for line in out.splitlines()
 250                    if line.startswith('add map ')]
 251         if len(devices) != parts:
 252             raise cliapp.AppException('Surprising number of partitions')
 253         root = '/dev/mapper/%s' % devices[rootindex]
 254         if self.settings['bootsize']:
 255             boot = '/dev/mapper/%s' % devices[bootindex]
 256         return (root,boot)
 257
 258     def mkfs(self, device, type):
 259         self.message('Creating filesystem %s' % type)
 260         self.runcmd(['mkfs', '-t', type, device])
 261
 262     def debootstrap(self, rootdir):
 263         self.message('Debootstrapping')
 264
 265         if self.settings['foreign']:
 266             necessary_packages = []
 267         else:
 268             necessary_packages = ['acpid']
 269
 270         include = self.settings['package']
 271
 272         if not self.settings['no-kernel']:
 273             if self.settings['arch'] == 'i386':
 274                 kernel_arch = '486'
 275             else:
 276                 kernel_arch = self.settings['arch']
 277             kernel_image = 'linux-image-%s' % kernel_arch
 278             include.append(kernel_image)
 279
 280         if self.settings['sudo'] and 'sudo' not in include:
 281             include.append('sudo')
 282
 283         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 284         args.append(
 285             '--include=%s' % ','.join(necessary_packages + include))
 286         if self.settings['foreign']:
 287             args.append('--foreign')
 288         if self.settings['variant']:
 289             args.append('--variant')
 290             args.append(self.settings['variant'])
 291         args += [self.settings['distribution'],
 292                  rootdir, self.settings['mirror']]
 293         self.runcmd(args)
 294         if self.settings['foreign']:
 295             # First copy the binfmt handler over
 296             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 297             # Next, run the package install scripts etc.
 298             self.runcmd(['chroot', rootdir,
 299                          '/debootstrap/debootstrap', '--second-stage'])
 300
 301     def set_hostname(self, rootdir):
 302         hostname = self.settings['hostname']
 303         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 304             f.write('%s\n' % hostname)
 305
 306         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 307         try:
 308             with open(etc_hosts, 'r') as f:
 309                 data = f.read()
 310             with open(etc_hosts, 'w') as f:
 311                 for line in data.splitlines():
 312                     if line.startswith('127.0.0.1'):
 313                         line += ' %s' % hostname
 314                     f.write('%s\n' % line)
 315         except IOError, e:
 316             pass
 317
 318     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
 319         def fsuuid(device):
 320             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 321                                '-s', 'UUID', device])
 322             return out.splitlines()[0].strip()
 323
 324         if rootdev:
 325             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 326         else:
 327             rootdevstr = '/dev/sda1'
 328
 329         if bootdev:
 330             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 331         else:
 332             bootdevstr = None
 333
 334         fstab = os.path.join(rootdir, 'etc', 'fstab')
 335         with open(fstab, 'w') as f:
 336             f.write('proc /proc proc defaults 0 0\n')
 337             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 338             if bootdevstr:
 339                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 340
 341     def install_debs(self, rootdir):
 342         if not self.settings['custom-package']:
 343             return
 344         self.message('Installing custom packages')
 345         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 346         os.mkdir(tmp)
 347         for deb in self.settings['custom-package']:
 348             shutil.copy(deb, tmp)
 349         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 350                      for deb in self.settings['custom-package']]
 351         out, err, exit = \
 352             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 353         logging.debug('stdout:\n%s' % out)
 354         logging.debug('stderr:\n%s' % err)
 355         out = self.runcmd(['chroot', rootdir,
 356                      'apt-get', '-f', '--no-remove', 'install'])
 357         logging.debug('stdout:\n%s' % out)
 358         shutil.rmtree(tmp)
 359
 360     def cleanup_apt_cache(self, rootdir):
 361         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 362         logging.debug('stdout:\n%s' % out)
 363
 364     def set_root_password(self, rootdir):
 365         if self.settings['root-password']:
 366             self.message('Setting root password')
 367             self.set_password(rootdir, 'root', self.settings['root-password'])
 368         elif self.settings['lock-root-password']:
 369             self.message('Locking root password')
 370             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 371         else:
 372             self.message('Give root an empty password')
 373             self.delete_password(rootdir, 'root')
 374
 375     def create_users(self, rootdir):
 376         def create_user(user):
 377             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 378                          '--disabled-password', user])
 379             if self.settings['sudo']:
 380                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 381
 382         for userpass in self.settings['user']:
 383             if '/' in userpass:
 384                 user, password = userpass.split('/', 1)
 385                 create_user(user)
 386                 self.set_password(rootdir, user, password)
 387             else:
 388                 create_user(userpass)
 389                 self.delete_password(rootdir, userpass)
 390
 391     def set_password(self, rootdir, user, password):
 392         encrypted = crypt.crypt(password, '..')
 393         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 394
 395     def delete_password(self, rootdir, user):
 396         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 397
 398     def remove_udev_persistent_rules(self, rootdir):
 399         self.message('Removing udev persistent cd and net rules')
 400         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 401             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 402             if os.path.exists(pathname):
 403                 logging.debug('rm %s' % pathname)
 404                 os.remove(pathname)
 405             else:
 406                 logging.debug('not removing non-existent %s' % pathname)
 407
 408     def setup_networking(self, rootdir):
 409         self.message('Setting up networking')
 410
 411         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 412         f.write('auto lo\n')
 413         f.write('iface lo inet loopback\n')
 414
 415         if self.settings['enable-dhcp']:
 416             f.write('\n')
 417             f.write('auto eth0\n')
 418             f.write('iface eth0 inet dhcp\n')
 419
 420         f.close()
 421
 422     def append_serial_console(self, rootdir):
 423         if self.settings['serial-console']:
 424             serial_command = self.settings['serial-console-command']
 425             logging.debug('adding getty to serial console')
 426             inittab = os.path.join(rootdir, 'etc/inittab')
 427             with open(inittab, 'a') as f:
 428                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 429
 430     def install_extlinux(self, rootdev, rootdir):
 431         self.message('Installing extlinux')
 432
 433         def find(pattern):
 434             dirname = os.path.join(rootdir, 'boot')
 435             basenames = os.listdir(dirname)
 436             logging.debug('find: %s' % basenames)
 437             for basename in basenames:
 438                 if re.search(pattern, basename):
 439                     return os.path.join('boot', basename)
 440             raise cliapp.AppException('Cannot find match: %s' % pattern)
 441
 442         kernel_image = find('vmlinuz-.*')
 443         initrd_image = find('initrd.img-.*')
 444
 445         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 446                            '-s', 'UUID', rootdev])
 447         uuid = out.splitlines()[0].strip()
 448
 449         conf = os.path.join(rootdir, 'extlinux.conf')
 450         logging.debug('configure extlinux %s' % conf)
 451         f = open(conf, 'w')
 452         f.write('''
 453 default linux
 454 timeout 1
 455
 456 label linux
 457 kernel %(kernel)s
 458 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 459 %(extserial)s
 460 ''' % {
 461     'kernel': kernel_image,
 462     'initrd': initrd_image,
 463     'uuid': uuid,
 464     'kserial':
 465         'console=ttyS0,115200' if self.settings['serial-console'] else '',
 466     'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 467 })
 468         f.close()
 469
 470         self.runcmd(['extlinux', '--install', rootdir])
 471         self.runcmd(['sync'])
 472         import time; time.sleep(2)
 473
 474     def optimize_image(self, rootdir):
 475         """
 476         Filing up the image with zeros will increase its compression rate
 477         """
 478         zeros = os.path.join(rootdir, 'ZEROS')
 479         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 480         self.runcmd(['rm', '-f', zeros])
 481
 482     def squash(self):
 483         """
 484         Run squashfs on the image.
 485         """
 486         if not os.path.exists('/usr/bin/mksquashfs'):
 487             logging.warning("Squash selected but mksquashfs not found!")
 488             return
 489         self.message("Running mksquashfs")
 490         suffixed = "%s.squashfs" % self.settings['image']
 491         self.runcmd(['mksquashfs', self.settings['image'],
 492             suffixed,
 493             '-no-progress', '-comp', 'xz'], ignore_fail=False)
 494         os.unlink(self.settings['image'])
 495         self.settings['image'] = suffixed
 496
 497     def cleanup_system(self):
 498         # Clean up after any errors.
 499
 500         self.message('Cleaning up')
 501
 502         # Umount in the reverse mount order
 503         if self.settings['image']:
 504             for i in xrange(len(self.mount_points) - 1, -1, -1):
 505                 mount_point = self.mount_points[i]
 506                 try:
 507                     self.runcmd(['umount', mount_point], ignore_fail=False)
 508                 except cliapp.AppException:
 509                     logging.debug("umount failed, sleeping and trying again")
 510                     time.sleep(5)
 511                     self.runcmd(['umount', mount_point], ignore_fail=False)
 512
 513             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 514
 515         for dirname in self.remove_dirs:
 516             shutil.rmtree(dirname)
 517
 518     def customize(self, rootdir):
 519         script = self.settings['customize']
 520         if script:
 521             self.message('Running customize script %s' % script)
 522             with open('/dev/tty', 'w') as tty:
 523                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 524
 525     def create_tarball(self, rootdir):
 526         # Create a tarball of the disk's contents
 527         # shell out to runcmd since it more easily handles rootdir
 528         self.message('Creating tarball of disk contents')
 529         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 530
 531     def chown(self, rootdir):
 532         # Change image owner after completed build
 533         self.message("Changing owner to %s" % self.settings["owner"])
 534         subprocess.call(["chown",
 535                          self.settings["owner"],
 536                          self.settings["image"]])
 537
 538
 539 if __name__ == '__main__':
 540     VmDebootstrap(version=__version__).run()
 541