vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import time
  24 import shutil
  25 import subprocess
  26 import tempfile
  27 import time
  28
  29
  30 __version__ = '0.3'
  31
  32
  33 class VmDebootstrap(cliapp.Application):
  34
  35     def add_settings(self):
  36         default_arch = 'amd64'
  37
  38         self.settings.boolean(['verbose'], 'report what is going on')
  39         self.settings.string(['image'], 'put created disk image in FILE',
  40                              metavar='FILE')
  41         self.settings.bytesize(['size'],
  42                                'create a disk image of size SIZE (%default)',
  43                                metavar='SIZE',
  44                                default='1G')
  45         self.settings.bytesize(['bootsize'],
  46                                'create boot partition of size SIZE (%default)',
  47                                metavar='BOOTSIZE',
  48                                default='0%')
  49         self.settings.string(['boottype'],
  50                              'specify file system type for /boot/',
  51                              default='ext2')
  52         self.settings.string(['foreign'],
  53                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  54         self.settings.string(['variant'],
  55                              'select debootstrap variant it not using the default')
  56         self.settings.boolean(
  57             ['extlinux'],
  58             'install extlinux?',
  59             default=True)
  60         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  61                              metavar='FILE')
  62         self.settings.string(['mirror'],
  63                              'use MIRROR as package source (%default)',
  64                              metavar='URL',
  65                              default='http://cdn.debian.net/debian/')
  66         self.settings.string(['arch'], 'architecture to use (%default)',
  67                              metavar='ARCH',
  68                              default=default_arch)
  69         self.settings.string(['distribution'],
  70                              'release to use (%default)',
  71                              metavar='NAME',
  72                              default='stable')
  73         self.settings.string_list(['package'], 'install PACKAGE onto system')
  74         self.settings.string_list(['custom-package'],
  75                                   'install package in DEB file onto system '
  76                                   '(not from mirror)',
  77                                   metavar='DEB')
  78         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  79         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  80         self.settings.string(['root-password'], 'set root password',
  81                              metavar='PASSWORD')
  82         self.settings.boolean(['lock-root-password'],
  83                               'lock root account so they cannot login?')
  84         self.settings.string(['customize'],
  85                              'run SCRIPT after setting up system',
  86                              metavar='SCRIPT')
  87         self.settings.string(['hostname'],
  88                              'set name to HOSTNAME (%default)',
  89                              metavar='HOSTNAME',
  90                              default='debian')
  91         self.settings.string_list(['user'],
  92                                   'create USER with PASSWORD',
  93                                   metavar='USER/PASSWORD')
  94         self.settings.boolean(['serial-console'],
  95                               'configure image to use a serial console')
  96         self.settings.string(['serial-console-command'],
  97                              'command to manage the serial console, appended '
  98                              'to /etc/inittab (%default)',
  99                              metavar='COMMAND',
 100                              default='/sbin/getty -L ttyS0 115200 vt100')
 101         self.settings.boolean(['sudo'],
 102                               'install sudo, and if user is created, add them '
 103                               'to sudo group')
 104         self.settings.string(['owner'],
 105                              'the user who will own the image when the build '
 106                              'is complete.')
 107         self.settings.boolean(['squash'],
 108                               'use squashfs on the final image.')
 109
 110     def process_args(self, args):
 111         if not self.settings['image'] and not self.settings['tarball']:
 112             raise cliapp.AppException('You must give disk image filename, '
 113                                       'or tarball filename')
 114         if self.settings['image'] and not self.settings['size']:
 115             raise cliapp.AppException('If disk image is specified, '
 116                                       'You must give image size.')
 117
 118         self.remove_dirs = []
 119         self.mount_points = []
 120
 121         try:
 122             rootdev = None
 123             roottype = 'ext4'
 124             bootdev = None
 125             boottype = None
 126             if self.settings['image']:
 127                 self.create_empty_image()
 128                 self.partition_image()
 129                 self.install_mbr()
 130                 (rootdev, bootdev) = self.setup_kpartx()
 131                 self.mkfs(rootdev, type=roottype)
 132                 rootdir = self.mount(rootdev)
 133                 if bootdev:
 134                     if self.settings['boottype']:
 135                         boottype = self.settings['boottype']
 136                     else:
 137                         boottype = 'ext2'
 138                     self.mkfs(bootdev, type=boottype)
 139                     bootdir = '%s/%s' % (rootdir, 'boot/')
 140                     os.mkdir(bootdir)
 141                     bootdir = self.mount(bootdev, bootdir)
 142             else:
 143                 rootdir = self.mkdtemp()
 144             self.debootstrap(rootdir)
 145             self.set_hostname(rootdir)
 146             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 147             self.install_debs(rootdir)
 148             self.cleanup_apt_cache(rootdir)
 149             self.set_root_password(rootdir)
 150             self.create_users(rootdir)
 151             self.remove_udev_persistent_rules(rootdir)
 152             self.setup_networking(rootdir)
 153             self.customize(rootdir)
 154             if self.settings['image']:
 155                 if self.settings['extlinux']:
 156                     self.install_extlinux(rootdev, rootdir)
 157                 self.append_serial_console(rootdir)
 158                 self.optimize_image(rootdir)
 159                 if self.settings['squash']:
 160                     self.squash()
 161
 162             if self.settings['foreign']:
 163                 os.unlink('%s/usr/bin/%s' %
 164                           (rootdir, os.path.basename(self.settings['foreign'])))
 165
 166             if self.settings['tarball']:
 167                 self.create_tarball(rootdir)
 168
 169             if self.settings['owner']:
 170                 self.chown(rootdir)
 171         except BaseException, e:
 172             self.message('EEEK! Something bad happened...')
 173             self.message(e)
 174             self.cleanup_system()
 175             raise
 176         else:
 177             self.cleanup_system()
 178
 179     def message(self, msg):
 180         logging.info(msg)
 181         if self.settings['verbose']:
 182             print msg
 183
 184     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 185         logging.debug('runcmd: %s %s' % (argv, kwargs))
 186         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 187                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 188                              **kwargs)
 189         out, err = p.communicate(stdin)
 190         if p.returncode != 0:
 191             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 192             logging.error(msg)
 193             if not ignore_fail:
 194                 raise cliapp.AppException(msg)
 195         return out
 196
 197     def mkdtemp(self):
 198         dirname = tempfile.mkdtemp()
 199         self.remove_dirs.append(dirname)
 200         logging.debug('mkdir %s' % dirname)
 201         return dirname
 202
 203     def mount(self, device, path=None):
 204         if not path:
 205             mount_point = self.mkdtemp()
 206         else:
 207             mount_point = path
 208         self.message('Mounting %s on %s' % (device, mount_point))
 209         self.runcmd(['mount', device, mount_point])
 210         self.mount_points.append(mount_point)
 211         logging.debug('mounted %s on %s' % (device, mount_point))
 212         return mount_point
 213
 214     def create_empty_image(self):
 215         self.message('Creating disk image')
 216         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 217                      self.settings['image'],
 218                      str(self.settings['size'])])
 219
 220     def partition_image(self):
 221         self.message('Creating partitions')
 222         self.runcmd(['parted', '-s', self.settings['image'],
 223                      'mklabel', 'msdos'])
 224         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 225             bootsize = str(self.settings['bootsize'] / (1024 * 1024))
 226             self.runcmd(['parted', '-s', self.settings['image'],
 227                          'mkpart', 'primary', 'fat16', '0', bootsize])
 228         else:
 229             bootsize = '0%'
 230         self.runcmd(['parted', '-s', self.settings['image'],
 231                      'mkpart', 'primary', bootsize, '100%'])
 232         self.runcmd(['parted', '-s', self.settings['image'],
 233                      'set', '1', 'boot', 'on'])
 234
 235     def install_mbr(self):
 236         self.message('Installing MBR')
 237         self.runcmd(['install-mbr', self.settings['image']])
 238
 239     def setup_kpartx(self):
 240         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 241         if self.settings['bootsize']:
 242             bootindex = 0
 243             rootindex = 1
 244             parts = 2
 245         else:
 246             rootindex = 0
 247             parts = 1
 248             boot = None
 249         devices = [line.split()[2]
 250                    for line in out.splitlines()
 251                    if line.startswith('add map ')]
 252         if len(devices) != parts:
 253             raise cliapp.AppException('Surprising number of partitions')
 254         root = '/dev/mapper/%s' % devices[rootindex]
 255         if self.settings['bootsize']:
 256             boot = '/dev/mapper/%s' % devices[bootindex]
 257         return (root, boot)
 258
 259     def mkfs(self, device, type):
 260         self.message('Creating filesystem %s' % type)
 261         self.runcmd(['mkfs', '-t', type, device])
 262
 263     def debootstrap(self, rootdir):
 264         self.message('Debootstrapping')
 265
 266         if self.settings['foreign']:
 267             necessary_packages = []
 268         else:
 269             necessary_packages = ['acpid']
 270
 271         include = self.settings['package']
 272
 273         if not self.settings['no-kernel']:
 274             if self.settings['arch'] == 'i386':
 275                 kernel_arch = '486'
 276             else:
 277                 kernel_arch = self.settings['arch']
 278             kernel_image = 'linux-image-%s' % kernel_arch
 279             include.append(kernel_image)
 280
 281         if self.settings['sudo'] and 'sudo' not in include:
 282             include.append('sudo')
 283
 284         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 285         args.append(
 286             '--include=%s' % ','.join(necessary_packages + include))
 287         if self.settings['foreign']:
 288             args.append('--foreign')
 289         if self.settings['variant']:
 290             args.append('--variant')
 291             args.append(self.settings['variant'])
 292         args += [self.settings['distribution'],
 293                  rootdir, self.settings['mirror']]
 294         self.runcmd(args)
 295         if self.settings['foreign']:
 296             # First copy the binfmt handler over
 297             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 298             # Next, run the package install scripts etc.
 299             self.runcmd(['chroot', rootdir,
 300                          '/debootstrap/debootstrap', '--second-stage'])
 301
 302     def set_hostname(self, rootdir):
 303         hostname = self.settings['hostname']
 304         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 305             f.write('%s\n' % hostname)
 306
 307         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 308         try:
 309             with open(etc_hosts, 'r') as f:
 310                 data = f.read()
 311             with open(etc_hosts, 'w') as f:
 312                 for line in data.splitlines():
 313                     if line.startswith('127.0.0.1'):
 314                         line += ' %s' % hostname
 315                     f.write('%s\n' % line)
 316         except IOError, e:
 317             pass
 318
 319     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
 320         def fsuuid(device):
 321             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 322                                '-s', 'UUID', device])
 323             return out.splitlines()[0].strip()
 324
 325         if rootdev:
 326             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 327         else:
 328             rootdevstr = '/dev/sda1'
 329
 330         if bootdev:
 331             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 332         else:
 333             bootdevstr = None
 334
 335         fstab = os.path.join(rootdir, 'etc', 'fstab')
 336         with open(fstab, 'w') as f:
 337             f.write('proc /proc proc defaults 0 0\n')
 338             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 339             if bootdevstr:
 340                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 341
 342     def install_debs(self, rootdir):
 343         if not self.settings['custom-package']:
 344             return
 345         self.message('Installing custom packages')
 346         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 347         os.mkdir(tmp)
 348         for deb in self.settings['custom-package']:
 349             shutil.copy(deb, tmp)
 350         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 351                      for deb in self.settings['custom-package']]
 352         out, err, exit = \
 353             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 354         logging.debug('stdout:\n%s' % out)
 355         logging.debug('stderr:\n%s' % err)
 356         out = self.runcmd(['chroot', rootdir,
 357                           'apt-get', '-f', '--no-remove', 'install'])
 358         logging.debug('stdout:\n%s' % out)
 359         shutil.rmtree(tmp)
 360
 361     def cleanup_apt_cache(self, rootdir):
 362         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 363         logging.debug('stdout:\n%s' % out)
 364
 365     def set_root_password(self, rootdir):
 366         if self.settings['root-password']:
 367             self.message('Setting root password')
 368             self.set_password(rootdir, 'root', self.settings['root-password'])
 369         elif self.settings['lock-root-password']:
 370             self.message('Locking root password')
 371             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 372         else:
 373             self.message('Give root an empty password')
 374             self.delete_password(rootdir, 'root')
 375
 376     def create_users(self, rootdir):
 377         def create_user(user):
 378             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 379                          '--disabled-password', user])
 380             if self.settings['sudo']:
 381                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 382
 383         for userpass in self.settings['user']:
 384             if '/' in userpass:
 385                 user, password = userpass.split('/', 1)
 386                 create_user(user)
 387                 self.set_password(rootdir, user, password)
 388             else:
 389                 create_user(userpass)
 390                 self.delete_password(rootdir, userpass)
 391
 392     def set_password(self, rootdir, user, password):
 393         encrypted = crypt.crypt(password, '..')
 394         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 395
 396     def delete_password(self, rootdir, user):
 397         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 398
 399     def remove_udev_persistent_rules(self, rootdir):
 400         self.message('Removing udev persistent cd and net rules')
 401         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 402             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 403             if os.path.exists(pathname):
 404                 logging.debug('rm %s' % pathname)
 405                 os.remove(pathname)
 406             else:
 407                 logging.debug('not removing non-existent %s' % pathname)
 408
 409     def setup_networking(self, rootdir):
 410         self.message('Setting up networking')
 411
 412         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 413         f.write('auto lo\n')
 414         f.write('iface lo inet loopback\n')
 415
 416         if self.settings['enable-dhcp']:
 417             f.write('\n')
 418             f.write('auto eth0\n')
 419             f.write('iface eth0 inet dhcp\n')
 420
 421         f.close()
 422
 423     def append_serial_console(self, rootdir):
 424         if self.settings['serial-console']:
 425             serial_command = self.settings['serial-console-command']
 426             logging.debug('adding getty to serial console')
 427             inittab = os.path.join(rootdir, 'etc/inittab')
 428             with open(inittab, 'a') as f:
 429                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 430
 431     def install_extlinux(self, rootdev, rootdir):
 432         self.message('Installing extlinux')
 433
 434         def find(pattern):
 435             dirname = os.path.join(rootdir, 'boot')
 436             basenames = os.listdir(dirname)
 437             logging.debug('find: %s' % basenames)
 438             for basename in basenames:
 439                 if re.search(pattern, basename):
 440                     return os.path.join('boot', basename)
 441             raise cliapp.AppException('Cannot find match: %s' % pattern)
 442
 443         kernel_image = find('vmlinuz-.*')
 444         initrd_image = find('initrd.img-.*')
 445
 446         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 447                            '-s', 'UUID', rootdev])
 448         uuid = out.splitlines()[0].strip()
 449
 450         conf = os.path.join(rootdir, 'extlinux.conf')
 451         logging.debug('configure extlinux %s' % conf)
 452         f = open(conf, 'w')
 453         f.write('''
 454 default linux
 455 timeout 1
 456
 457 label linux
 458 kernel %(kernel)s
 459 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 460 %(extserial)s
 461 ''' % {
 462             'kernel': kernel_image,
 463             'initrd': initrd_image,
 464             'uuid': uuid,
 465             'kserial':
 466             'console=ttyS0,115200' if self.settings['serial-console'] else '',
 467             'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 468         })
 469         f.close()
 470
 471         self.runcmd(['extlinux', '--install', rootdir])
 472         self.runcmd(['sync'])
 473         time.sleep(2)
 474
 475     def optimize_image(self, rootdir):
 476         """
 477         Filing up the image with zeros will increase its compression rate
 478         """
 479         zeros = os.path.join(rootdir, 'ZEROS')
 480         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 481         self.runcmd(['rm', '-f', zeros])
 482
 483     def squash(self):
 484         """
 485         Run squashfs on the image.
 486         """
 487         if not os.path.exists('/usr/bin/mksquashfs'):
 488             logging.warning("Squash selected but mksquashfs not found!")
 489             return
 490         self.message("Running mksquashfs")
 491         suffixed = "%s.squashfs" % self.settings['image']
 492         self.runcmd(['mksquashfs', self.settings['image'],
 493                      suffixed,
 494                      '-no-progress', '-comp', 'xz'], ignore_fail=False)
 495         os.unlink(self.settings['image'])
 496         self.settings['image'] = suffixed
 497
 498     def cleanup_system(self):
 499         # Clean up after any errors.
 500
 501         self.message('Cleaning up')
 502
 503         # Umount in the reverse mount order
 504         if self.settings['image']:
 505             for i in xrange(len(self.mount_points) - 1, -1, -1):
 506                 mount_point = self.mount_points[i]
 507                 try:
 508                     self.runcmd(['umount', mount_point], ignore_fail=False)
 509                 except cliapp.AppException:
 510                     logging.debug("umount failed, sleeping and trying again")
 511                     time.sleep(5)
 512                     self.runcmd(['umount', mount_point], ignore_fail=False)
 513
 514             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 515
 516         for dirname in self.remove_dirs:
 517             shutil.rmtree(dirname)
 518
 519     def customize(self, rootdir):
 520         script = self.settings['customize']
 521         if script:
 522             self.message('Running customize script %s' % script)
 523             with open('/dev/tty', 'w') as tty:
 524                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 525
 526     def create_tarball(self, rootdir):
 527         # Create a tarball of the disk's contents
 528         # shell out to runcmd since it more easily handles rootdir
 529         self.message('Creating tarball of disk contents')
 530         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 531
 532     def chown(self, rootdir):
 533         # Change image owner after completed build
 534         self.message("Changing owner to %s" % self.settings["owner"])
 535         subprocess.call(["chown",
 536                          self.settings["owner"],
 537                          self.settings["image"]])
 538
 539 if __name__ == '__main__':
 540     VmDebootstrap(version=__version__).run()