vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import time
  24 import shutil
  25 import subprocess
  26 import tempfile
  27 import time
  28
  29
  30 __version__ = '0.3'
  31
  32
  33 class VmDebootstrap(cliapp.Application):
  34
  35     def add_settings(self):
  36         default_arch = 'amd64'
  37
  38         self.settings.boolean(['verbose'], 'report what is going on')
  39         self.settings.string(['image'], 'put created disk image in FILE',
  40                              metavar='FILE')
  41         self.settings.bytesize(['size'],
  42                                'create a disk image of size SIZE (%default)',
  43                                metavar='SIZE',
  44                                default='1G')
  45         self.settings.bytesize(['bootsize'],
  46                                'create boot partition of size SIZE (%default)',
  47                                metavar='BOOTSIZE',
  48                                default='0%')
  49         self.settings.string(['boottype'],
  50                              'specify file system type for /boot/',
  51                              default='ext2')
  52         self.settings.string(['foreign'],
  53                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  54         self.settings.string(['variant'],
  55                              'select debootstrap variant it not using the default')
  56         self.settings.boolean(
  57             ['extlinux'],
  58             'install extlinux?',
  59             default=True)
  60         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  61                              metavar='FILE')
  62         self.settings.string(['mirror'],
  63                              'use MIRROR as package source (%default)',
  64                              metavar='URL',
  65                              default='http://cdn.debian.net/debian/')
  66         self.settings.string(['arch'], 'architecture to use (%default)',
  67                              metavar='ARCH',
  68                              default=default_arch)
  69         self.settings.string(['distribution'],
  70                              'release to use (%default)',
  71                              metavar='NAME',
  72                              default='stable')
  73         self.settings.string_list(['package'], 'install PACKAGE onto system')
  74         self.settings.string_list(['custom-package'],
  75                                   'install package in DEB file onto system '
  76                                   '(not from mirror)',
  77                                   metavar='DEB')
  78         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  79         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  80         self.settings.string(['root-password'], 'set root password',
  81                              metavar='PASSWORD')
  82         self.settings.boolean(['lock-root-password'],
  83                               'lock root account so they cannot login?')
  84         self.settings.string(['customize'],
  85                              'run SCRIPT after setting up system',
  86                              metavar='SCRIPT')
  87         self.settings.string(['hostname'],
  88                              'set name to HOSTNAME (%default)',
  89                              metavar='HOSTNAME',
  90                              default='debian')
  91         self.settings.string_list(['user'],
  92                                   'create USER with PASSWORD',
  93                                   metavar='USER/PASSWORD')
  94         self.settings.boolean(['serial-console'],
  95                               'configure image to use a serial console')
  96         self.settings.string(['serial-console-command'],
  97                              'command to manage the serial console, appended '
  98                              'to /etc/inittab (%default)',
  99                              metavar='COMMAND',
 100                              default='/sbin/getty -L ttyS0 115200 vt100')
 101         self.settings.boolean(['sudo'],
 102                               'install sudo, and if user is created, add them '
 103                               'to sudo group')
 104         self.settings.string(['owner'],
 105                              'the user who will own the image when the build '
 106                              'is complete.')
 107         self.settings.boolean(['squash'],
 108                               'use squashfs on the final image.')
 109         self.settings.boolean(['configure-apt'],
 110                               'Create an apt source based on the distribution '
 111                               'and mirror selected.')
 112
 113     def process_args(self, args):
 114         if not self.settings['image'] and not self.settings['tarball']:
 115             raise cliapp.AppException('You must give disk image filename, '
 116                                       'or tarball filename')
 117         if self.settings['image'] and not self.settings['size']:
 118             raise cliapp.AppException('If disk image is specified, '
 119                                       'You must give image size.')
 120
 121         self.remove_dirs = []
 122         self.mount_points = []
 123
 124         try:
 125             rootdev = None
 126             roottype = 'ext4'
 127             bootdev = None
 128             boottype = None
 129             rootdir = None
 130             if self.settings['image']:
 131                 self.create_empty_image()
 132                 self.partition_image()
 133                 self.install_mbr()
 134                 (rootdev, bootdev) = self.setup_kpartx()
 135                 self.mkfs(rootdev, type=roottype)
 136                 rootdir = self.mount(rootdev)
 137                 if bootdev:
 138                     if self.settings['boottype']:
 139                         boottype = self.settings['boottype']
 140                     else:
 141                         boottype = 'ext2'
 142                     self.mkfs(bootdev, type=boottype)
 143                     bootdir = '%s/%s' % (rootdir, 'boot/')
 144                     os.mkdir(bootdir)
 145                     bootdir = self.mount(bootdev, bootdir)
 146             else:
 147                 rootdir = self.mkdtemp()
 148             self.debootstrap(rootdir)
 149             self.set_hostname(rootdir)
 150             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 151             self.install_debs(rootdir)
 152             self.cleanup_apt_cache(rootdir)
 153             self.set_root_password(rootdir)
 154             self.create_users(rootdir)
 155             self.remove_udev_persistent_rules(rootdir)
 156             self.setup_networking(rootdir)
 157             self.configure_apt(rootdir)
 158             self.customize(rootdir)
 159             if self.settings['image']:
 160                 if self.settings['extlinux']:
 161                     self.install_extlinux(rootdev, rootdir)
 162                 self.append_serial_console(rootdir)
 163                 self.optimize_image(rootdir)
 164                 if self.settings['squash']:
 165                     self.squash()
 166
 167             if self.settings['foreign']:
 168                 os.unlink('%s/usr/bin/%s' %
 169                           (rootdir, os.path.basename(self.settings['foreign'])))
 170
 171             if self.settings['tarball']:
 172                 self.create_tarball(rootdir)
 173
 174             if self.settings['owner']:
 175                 self.chown(rootdir)
 176         except BaseException, e:
 177             self.message('EEEK! Something bad happened...')
 178             if rootdir:
 179                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 180                 if os.path.exists(db_log):
 181                     shutil.copy(db_log, os.getcwd())
 182             self.message(e)
 183             self.cleanup_system()
 184             raise
 185         else:
 186             self.cleanup_system()
 187
 188     def message(self, msg):
 189         logging.info(msg)
 190         if self.settings['verbose']:
 191             print msg
 192
 193     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 194         logging.debug('runcmd: %s %s' % (argv, kwargs))
 195         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 196                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 197                              **kwargs)
 198         out, err = p.communicate(stdin)
 199         if p.returncode != 0:
 200             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 201             logging.error(msg)
 202             if not ignore_fail:
 203                 raise cliapp.AppException(msg)
 204         return out
 205
 206     def mkdtemp(self):
 207         dirname = tempfile.mkdtemp()
 208         self.remove_dirs.append(dirname)
 209         logging.debug('mkdir %s' % dirname)
 210         return dirname
 211
 212     def mount(self, device, path=None):
 213         if not path:
 214             mount_point = self.mkdtemp()
 215         else:
 216             mount_point = path
 217         self.message('Mounting %s on %s' % (device, mount_point))
 218         self.runcmd(['mount', device, mount_point])
 219         self.mount_points.append(mount_point)
 220         logging.debug('mounted %s on %s' % (device, mount_point))
 221         return mount_point
 222
 223     def create_empty_image(self):
 224         self.message('Creating disk image')
 225         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 226                      self.settings['image'],
 227                      str(self.settings['size'])])
 228
 229     def partition_image(self):
 230         self.message('Creating partitions')
 231         self.runcmd(['parted', '-s', self.settings['image'],
 232                      'mklabel', 'msdos'])
 233         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 234             bootsize = str(self.settings['bootsize'] / (1024 * 1024))
 235             self.runcmd(['parted', '-s', self.settings['image'],
 236                          'mkpart', 'primary', 'fat16', '0', bootsize])
 237         else:
 238             bootsize = '0%'
 239         self.runcmd(['parted', '-s', self.settings['image'],
 240                      'mkpart', 'primary', bootsize, '100%'])
 241         self.runcmd(['parted', '-s', self.settings['image'],
 242                      'set', '1', 'boot', 'on'])
 243
 244     def install_mbr(self):
 245         self.message('Installing MBR')
 246         self.runcmd(['install-mbr', self.settings['image']])
 247
 248     def setup_kpartx(self):
 249         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 250         if self.settings['bootsize']:
 251             bootindex = 0
 252             rootindex = 1
 253             parts = 2
 254         else:
 255             rootindex = 0
 256             parts = 1
 257             boot = None
 258         devices = [line.split()[2]
 259                    for line in out.splitlines()
 260                    if line.startswith('add map ')]
 261         if len(devices) != parts:
 262             raise cliapp.AppException('Surprising number of partitions')
 263         root = '/dev/mapper/%s' % devices[rootindex]
 264         if self.settings['bootsize']:
 265             boot = '/dev/mapper/%s' % devices[bootindex]
 266         return (root, boot)
 267
 268     def mkfs(self, device, type):
 269         self.message('Creating filesystem %s' % type)
 270         self.runcmd(['mkfs', '-t', type, device])
 271
 272     def debootstrap(self, rootdir):
 273         self.message('Debootstrapping')
 274
 275         if self.settings['foreign']:
 276             necessary_packages = []
 277         else:
 278             necessary_packages = ['acpid']
 279
 280         include = self.settings['package']
 281
 282         if not self.settings['no-kernel']:
 283             if self.settings['arch'] == 'i386':
 284                 kernel_arch = '486'
 285             else:
 286                 kernel_arch = self.settings['arch']
 287             kernel_image = 'linux-image-%s' % kernel_arch
 288             include.append(kernel_image)
 289
 290         if self.settings['sudo'] and 'sudo' not in include:
 291             include.append('sudo')
 292
 293         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 294         args.append(
 295             '--include=%s' % ','.join(necessary_packages + include))
 296         if self.settings['foreign']:
 297             args.append('--foreign')
 298         if self.settings['variant']:
 299             args.append('--variant')
 300             args.append(self.settings['variant'])
 301         args += [self.settings['distribution'],
 302                  rootdir, self.settings['mirror']]
 303         logging.debug(" ".join(args))
 304         self.runcmd(args)
 305         if self.settings['foreign']:
 306             # First copy the binfmt handler over
 307             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 308             # Next, run the package install scripts etc.
 309             self.message('Running debootstrap second stage')
 310             self.runcmd(['chroot', rootdir,
 311                          '/debootstrap/debootstrap', '--second-stage'])
 312
 313     def set_hostname(self, rootdir):
 314         hostname = self.settings['hostname']
 315         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 316             f.write('%s\n' % hostname)
 317
 318         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 319         try:
 320             with open(etc_hosts, 'r') as f:
 321                 data = f.read()
 322             with open(etc_hosts, 'w') as f:
 323                 for line in data.splitlines():
 324                     if line.startswith('127.0.0.1'):
 325                         line += ' %s' % hostname
 326                     f.write('%s\n' % line)
 327         except IOError, e:
 328             pass
 329
 330     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
 331         def fsuuid(device):
 332             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 333                                '-s', 'UUID', device])
 334             return out.splitlines()[0].strip()
 335
 336         if rootdev:
 337             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 338         else:
 339             rootdevstr = '/dev/sda1'
 340
 341         if bootdev:
 342             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 343         else:
 344             bootdevstr = None
 345
 346         fstab = os.path.join(rootdir, 'etc', 'fstab')
 347         with open(fstab, 'w') as f:
 348             f.write('proc /proc proc defaults 0 0\n')
 349             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 350             if bootdevstr:
 351                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 352
 353     def install_debs(self, rootdir):
 354         if not self.settings['custom-package']:
 355             return
 356         self.message('Installing custom packages')
 357         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 358         os.mkdir(tmp)
 359         for deb in self.settings['custom-package']:
 360             shutil.copy(deb, tmp)
 361         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 362                      for deb in self.settings['custom-package']]
 363         out, err, exit = \
 364             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 365         logging.debug('stdout:\n%s' % out)
 366         logging.debug('stderr:\n%s' % err)
 367         out = self.runcmd(['chroot', rootdir,
 368                           'apt-get', '-f', '--no-remove', 'install'])
 369         logging.debug('stdout:\n%s' % out)
 370         shutil.rmtree(tmp)
 371
 372     def cleanup_apt_cache(self, rootdir):
 373         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 374         logging.debug('stdout:\n%s' % out)
 375
 376     def set_root_password(self, rootdir):
 377         if self.settings['root-password']:
 378             self.message('Setting root password')
 379             self.set_password(rootdir, 'root', self.settings['root-password'])
 380         elif self.settings['lock-root-password']:
 381             self.message('Locking root password')
 382             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 383         else:
 384             self.message('Give root an empty password')
 385             self.delete_password(rootdir, 'root')
 386
 387     def create_users(self, rootdir):
 388         def create_user(user):
 389             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 390                          '--disabled-password', user])
 391             if self.settings['sudo']:
 392                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 393
 394         for userpass in self.settings['user']:
 395             if '/' in userpass:
 396                 user, password = userpass.split('/', 1)
 397                 create_user(user)
 398                 self.set_password(rootdir, user, password)
 399             else:
 400                 create_user(userpass)
 401                 self.delete_password(rootdir, userpass)
 402
 403     def set_password(self, rootdir, user, password):
 404         encrypted = crypt.crypt(password, '..')
 405         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 406
 407     def delete_password(self, rootdir, user):
 408         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 409
 410     def remove_udev_persistent_rules(self, rootdir):
 411         self.message('Removing udev persistent cd and net rules')
 412         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 413             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 414             if os.path.exists(pathname):
 415                 logging.debug('rm %s' % pathname)
 416                 os.remove(pathname)
 417             else:
 418                 logging.debug('not removing non-existent %s' % pathname)
 419
 420     def setup_networking(self, rootdir):
 421         self.message('Setting up networking')
 422
 423         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 424         f.write('auto lo\n')
 425         f.write('iface lo inet loopback\n')
 426
 427         if self.settings['enable-dhcp']:
 428             f.write('\n')
 429             f.write('auto eth0\n')
 430             f.write('iface eth0 inet dhcp\n')
 431
 432         f.close()
 433
 434     def append_serial_console(self, rootdir):
 435         if self.settings['serial-console']:
 436             serial_command = self.settings['serial-console-command']
 437             logging.debug('adding getty to serial console')
 438             inittab = os.path.join(rootdir, 'etc/inittab')
 439             with open(inittab, 'a') as f:
 440                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 441
 442     def install_extlinux(self, rootdev, rootdir):
 443         self.message('Installing extlinux')
 444
 445         def find(pattern):
 446             dirname = os.path.join(rootdir, 'boot')
 447             basenames = os.listdir(dirname)
 448             logging.debug('find: %s' % basenames)
 449             for basename in basenames:
 450                 if re.search(pattern, basename):
 451                     return os.path.join('boot', basename)
 452             raise cliapp.AppException('Cannot find match: %s' % pattern)
 453
 454         kernel_image = find('vmlinuz-.*')
 455         initrd_image = find('initrd.img-.*')
 456
 457         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 458                            '-s', 'UUID', rootdev])
 459         uuid = out.splitlines()[0].strip()
 460
 461         conf = os.path.join(rootdir, 'extlinux.conf')
 462         logging.debug('configure extlinux %s' % conf)
 463         f = open(conf, 'w')
 464         f.write('''
 465 default linux
 466 timeout 1
 467
 468 label linux
 469 kernel %(kernel)s
 470 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 471 %(extserial)s
 472 ''' % {
 473             'kernel': kernel_image,
 474             'initrd': initrd_image,
 475             'uuid': uuid,
 476             'kserial':
 477             'console=ttyS0,115200' if self.settings['serial-console'] else '',
 478             'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 479         })
 480         f.close()
 481
 482         self.runcmd(['extlinux', '--install', rootdir])
 483         self.runcmd(['sync'])
 484         time.sleep(2)
 485
 486     def optimize_image(self, rootdir):
 487         """
 488         Filing up the image with zeros will increase its compression rate
 489         """
 490         zeros = os.path.join(rootdir, 'ZEROS')
 491         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 492         self.runcmd(['rm', '-f', zeros])
 493
 494     def squash(self):
 495         """
 496         Run squashfs on the image.
 497         """
 498         if not os.path.exists('/usr/bin/mksquashfs'):
 499             logging.warning("Squash selected but mksquashfs not found!")
 500             return
 501         self.message("Running mksquashfs")
 502         suffixed = "%s.squashfs" % self.settings['image']
 503         self.runcmd(['mksquashfs', self.settings['image'],
 504                      suffixed,
 505                      '-no-progress', '-comp', 'xz'], ignore_fail=False)
 506         os.unlink(self.settings['image'])
 507         self.settings['image'] = suffixed
 508
 509     def cleanup_system(self):
 510         # Clean up after any errors.
 511
 512         self.message('Cleaning up')
 513
 514         # Umount in the reverse mount order
 515         if self.settings['image']:
 516             for i in xrange(len(self.mount_points) - 1, -1, -1):
 517                 mount_point = self.mount_points[i]
 518                 try:
 519                     self.runcmd(['umount', mount_point], ignore_fail=False)
 520                 except cliapp.AppException:
 521                     logging.debug("umount failed, sleeping and trying again")
 522                     time.sleep(5)
 523                     self.runcmd(['umount', mount_point], ignore_fail=False)
 524
 525             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 526
 527         for dirname in self.remove_dirs:
 528             shutil.rmtree(dirname)
 529
 530     def customize(self, rootdir):
 531         script = self.settings['customize']
 532         if script:
 533             self.message('Running customize script %s' % script)
 534             with open('/dev/tty', 'w') as tty:
 535                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 536
 537     def create_tarball(self, rootdir):
 538         # Create a tarball of the disk's contents
 539         # shell out to runcmd since it more easily handles rootdir
 540         self.message('Creating tarball of disk contents')
 541         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 542
 543     def chown(self, rootdir):
 544         # Change image owner after completed build
 545         self.message("Changing owner to %s" % self.settings["owner"])
 546         subprocess.call(["chown",
 547                          self.settings["owner"],
 548                          self.settings["image"]])
 549
 550     def configure_apt(self, rootdir):
 551         # use the distribution and mirror to create an apt source
 552         self.message("Configuring apt to use distribution and mirror")
 553         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
 554         logging.debug('configure apt %s' % conf)
 555         f = open(conf, 'w')
 556         f.write('''
 557         deb %(mirror)s %(distribution)s main
 558         #deb-src %(mirror)s %(distribution)s main
 559         ''' % {
 560             'mirror': self.settings['mirror'],
 561             'distribution': self.settings['distribution']
 562         })
 563         f.close()
 564
 565 if __name__ == '__main__':
 566     VmDebootstrap(version=__version__).run()