vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import time
  24 import shutil
  25 import subprocess
  26 import tempfile
  27 import time
  28
  29
  30 __version__ = '0.3'
  31
  32
  33 class VmDebootstrap(cliapp.Application):
  34
  35     def add_settings(self):
  36         default_arch = 'amd64'
  37
  38         self.settings.boolean(['verbose'], 'report what is going on')
  39         self.settings.string(['image'], 'put created disk image in FILE',
  40                              metavar='FILE')
  41         self.settings.bytesize(['size'],
  42                                'create a disk image of size SIZE (%default)',
  43                                metavar='SIZE',
  44                                default='1G')
  45         self.settings.bytesize(['bootsize'],
  46                                'create boot partition of size SIZE (%default)',
  47                                metavar='BOOTSIZE',
  48                                default='0%')
  49         self.settings.string(['boottype'],
  50                              'specify file system type for /boot/',
  51                              default='ext2')
  52         self.settings.string(['foreign'],
  53                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  54         self.settings.string(['variant'],
  55                              'select debootstrap variant it not using the default')
  56         self.settings.boolean(
  57             ['extlinux'],
  58             'install extlinux?',
  59             default=True)
  60         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  61                              metavar='FILE')
  62         self.settings.string(['mirror'],
  63                              'use MIRROR as package source (%default)',
  64                              metavar='URL',
  65                              default='http://cdn.debian.net/debian/')
  66         self.settings.string(['arch'], 'architecture to use (%default)',
  67                              metavar='ARCH',
  68                              default=default_arch)
  69         self.settings.string(['distribution'],
  70                              'release to use (%default)',
  71                              metavar='NAME',
  72                              default='stable')
  73         self.settings.string_list(['package'], 'install PACKAGE onto system')
  74         self.settings.string_list(['custom-package'],
  75                                   'install package in DEB file onto system '
  76                                   '(not from mirror)',
  77                                   metavar='DEB')
  78         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  79         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  80         self.settings.string(['root-password'], 'set root password',
  81                              metavar='PASSWORD')
  82         self.settings.boolean(['lock-root-password'],
  83                               'lock root account so they cannot login?')
  84         self.settings.string(['customize'],
  85                              'run SCRIPT after setting up system',
  86                              metavar='SCRIPT')
  87         self.settings.string(['hostname'],
  88                              'set name to HOSTNAME (%default)',
  89                              metavar='HOSTNAME',
  90                              default='debian')
  91         self.settings.string_list(['user'],
  92                                   'create USER with PASSWORD',
  93                                   metavar='USER/PASSWORD')
  94         self.settings.boolean(['serial-console'],
  95                               'configure image to use a serial console')
  96         self.settings.string(['serial-console-command'],
  97                              'command to manage the serial console, appended '
  98                              'to /etc/inittab (%default)',
  99                              metavar='COMMAND',
 100                              default='/sbin/getty -L ttyS0 115200 vt100')
 101         self.settings.boolean(['sudo'],
 102                               'install sudo, and if user is created, add them '
 103                               'to sudo group')
 104         self.settings.string(['owner'],
 105                              'the user who will own the image when the build '
 106                              'is complete.')
 107         self.settings.boolean(['squash'],
 108                               'use squashfs on the final image.')
 109         self.settings.boolean(['configure-apt'],
 110                               'Create an apt source based on the distribution '
 111                               'and mirror selected.')
 112
 113     def process_args(self, args):
 114         if not self.settings['image'] and not self.settings['tarball']:
 115             raise cliapp.AppException('You must give disk image filename, '
 116                                       'or tarball filename')
 117         if self.settings['image'] and not self.settings['size']:
 118             raise cliapp.AppException('If disk image is specified, '
 119                                       'You must give image size.')
 120
 121         self.remove_dirs = []
 122         self.mount_points = []
 123
 124         try:
 125             rootdev = None
 126             roottype = 'ext4'
 127             bootdev = None
 128             boottype = None
 129             rootdir = None
 130             if self.settings['image']:
 131                 self.create_empty_image()
 132                 self.partition_image()
 133                 self.install_mbr()
 134                 (rootdev, bootdev) = self.setup_kpartx()
 135                 self.mkfs(rootdev, type=roottype)
 136                 rootdir = self.mount(rootdev)
 137                 if bootdev:
 138                     if self.settings['boottype']:
 139                         boottype = self.settings['boottype']
 140                     else:
 141                         boottype = 'ext2'
 142                     self.mkfs(bootdev, type=boottype)
 143                     bootdir = '%s/%s' % (rootdir, 'boot/')
 144                     os.mkdir(bootdir)
 145                     bootdir = self.mount(bootdev, bootdir)
 146             else:
 147                 rootdir = self.mkdtemp()
 148             self.debootstrap(rootdir)
 149             self.set_hostname(rootdir)
 150             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 151             self.install_debs(rootdir)
 152             self.cleanup_apt_cache(rootdir)
 153             self.set_root_password(rootdir)
 154             self.create_users(rootdir)
 155             self.remove_udev_persistent_rules(rootdir)
 156             self.setup_networking(rootdir)
 157             self.configure_apt(rootdir)
 158             self.customize(rootdir)
 159             if self.settings['image']:
 160                 if self.settings['extlinux']:
 161                     self.install_extlinux(rootdev, rootdir)
 162                 self.append_serial_console(rootdir)
 163                 self.optimize_image(rootdir)
 164                 if self.settings['squash']:
 165                     self.squash()
 166
 167             if self.settings['foreign']:
 168                 os.unlink('%s/usr/bin/%s' %
 169                           (rootdir, os.path.basename(self.settings['foreign'])))
 170
 171             if self.settings['tarball']:
 172                 self.create_tarball(rootdir)
 173
 174             if self.settings['owner']:
 175                 self.chown(rootdir)
 176         except BaseException, e:
 177             self.message('EEEK! Something bad happened...')
 178             if rootdir:
 179                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 180                 if os.path.exists(db_log):
 181                     shutil.copy(db_log, os.getcwd())
 182             self.message(e)
 183             self.cleanup_system()
 184             raise
 185         else:
 186             self.cleanup_system()
 187
 188     def message(self, msg):
 189         logging.info(msg)
 190         if self.settings['verbose']:
 191             print msg
 192
 193     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 194         logging.debug('runcmd: %s %s' % (argv, kwargs))
 195         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 196                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 197                              **kwargs)
 198         out, err = p.communicate(stdin)
 199         if p.returncode != 0:
 200             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 201             logging.error(msg)
 202             if not ignore_fail:
 203                 raise cliapp.AppException(msg)
 204         return out
 205
 206     def mkdtemp(self):
 207         dirname = tempfile.mkdtemp()
 208         self.remove_dirs.append(dirname)
 209         logging.debug('mkdir %s' % dirname)
 210         return dirname
 211
 212     def mount(self, device, path=None):
 213         if not path:
 214             mount_point = self.mkdtemp()
 215         else:
 216             mount_point = path
 217         self.message('Mounting %s on %s' % (device, mount_point))
 218         self.runcmd(['mount', device, mount_point])
 219         self.mount_points.append(mount_point)
 220         logging.debug('mounted %s on %s' % (device, mount_point))
 221         return mount_point
 222
 223     def create_empty_image(self):
 224         self.message('Creating disk image')
 225         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 226                      self.settings['image'],
 227                      str(self.settings['size'])])
 228
 229     def partition_image(self):
 230         self.message('Creating partitions')
 231         self.runcmd(['parted', '-s', self.settings['image'],
 232                      'mklabel', 'msdos'])
 233         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 234             bootsize = str(self.settings['bootsize'] / (1024 * 1024))
 235             self.runcmd(['parted', '-s', self.settings['image'],
 236                          'mkpart', 'primary', 'fat16', '0', bootsize])
 237         else:
 238             bootsize = '0%'
 239         self.runcmd(['parted', '-s', self.settings['image'],
 240                      'mkpart', 'primary', bootsize, '100%'])
 241         self.runcmd(['parted', '-s', self.settings['image'],
 242                      'set', '1', 'boot', 'on'])
 243
 244     def install_mbr(self):
 245         if os.path.exists("/sbin/install-mbr"):
 246             self.message('Installing MBR')
 247             self.runcmd(['install-mbr', self.settings['image']])
 248
 249     def setup_kpartx(self):
 250         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 251         if self.settings['bootsize']:
 252             bootindex = 0
 253             rootindex = 1
 254             parts = 2
 255         else:
 256             rootindex = 0
 257             parts = 1
 258             boot = None
 259         devices = [line.split()[2]
 260                    for line in out.splitlines()
 261                    if line.startswith('add map ')]
 262         if len(devices) != parts:
 263             raise cliapp.AppException('Surprising number of partitions')
 264         root = '/dev/mapper/%s' % devices[rootindex]
 265         if self.settings['bootsize']:
 266             boot = '/dev/mapper/%s' % devices[bootindex]
 267         return (root, boot)
 268
 269     def mkfs(self, device, type):
 270         self.message('Creating filesystem %s' % type)
 271         self.runcmd(['mkfs', '-t', type, device])
 272
 273     def debootstrap(self, rootdir):
 274         self.message('Debootstrapping')
 275
 276         if self.settings['foreign']:
 277             necessary_packages = []
 278         else:
 279             necessary_packages = ['acpid']
 280
 281         include = self.settings['package']
 282
 283         if not self.settings['no-kernel']:
 284             if self.settings['arch'] == 'i386':
 285                 kernel_arch = '486'
 286             else:
 287                 kernel_arch = self.settings['arch']
 288             kernel_image = 'linux-image-%s' % kernel_arch
 289             include.append(kernel_image)
 290
 291         if self.settings['sudo'] and 'sudo' not in include:
 292             include.append('sudo')
 293
 294         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 295         args.append(
 296             '--include=%s' % ','.join(necessary_packages + include))
 297         if self.settings['foreign']:
 298             args.append('--foreign')
 299         if self.settings['variant']:
 300             args.append('--variant')
 301             args.append(self.settings['variant'])
 302         args += [self.settings['distribution'],
 303                  rootdir, self.settings['mirror']]
 304         logging.debug(" ".join(args))
 305         self.runcmd(args)
 306         if self.settings['foreign']:
 307             # First copy the binfmt handler over
 308             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 309             # Next, run the package install scripts etc.
 310             self.message('Running debootstrap second stage')
 311             self.runcmd(['chroot', rootdir,
 312                          '/debootstrap/debootstrap', '--second-stage'])
 313
 314     def set_hostname(self, rootdir):
 315         hostname = self.settings['hostname']
 316         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 317             f.write('%s\n' % hostname)
 318
 319         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 320         try:
 321             with open(etc_hosts, 'r') as f:
 322                 data = f.read()
 323             with open(etc_hosts, 'w') as f:
 324                 for line in data.splitlines():
 325                     if line.startswith('127.0.0.1'):
 326                         line += ' %s' % hostname
 327                     f.write('%s\n' % line)
 328         except IOError, e:
 329             pass
 330
 331     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
 332         def fsuuid(device):
 333             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 334                                '-s', 'UUID', device])
 335             return out.splitlines()[0].strip()
 336
 337         if rootdev:
 338             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 339         else:
 340             rootdevstr = '/dev/sda1'
 341
 342         if bootdev:
 343             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 344         else:
 345             bootdevstr = None
 346
 347         fstab = os.path.join(rootdir, 'etc', 'fstab')
 348         with open(fstab, 'w') as f:
 349             f.write('proc /proc proc defaults 0 0\n')
 350             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 351             if bootdevstr:
 352                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 353
 354     def install_debs(self, rootdir):
 355         if not self.settings['custom-package']:
 356             return
 357         self.message('Installing custom packages')
 358         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 359         os.mkdir(tmp)
 360         for deb in self.settings['custom-package']:
 361             shutil.copy(deb, tmp)
 362         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 363                      for deb in self.settings['custom-package']]
 364         out, err, exit = \
 365             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 366         logging.debug('stdout:\n%s' % out)
 367         logging.debug('stderr:\n%s' % err)
 368         out = self.runcmd(['chroot', rootdir,
 369                           'apt-get', '-f', '--no-remove', 'install'])
 370         logging.debug('stdout:\n%s' % out)
 371         shutil.rmtree(tmp)
 372
 373     def cleanup_apt_cache(self, rootdir):
 374         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 375         logging.debug('stdout:\n%s' % out)
 376
 377     def set_root_password(self, rootdir):
 378         if self.settings['root-password']:
 379             self.message('Setting root password')
 380             self.set_password(rootdir, 'root', self.settings['root-password'])
 381         elif self.settings['lock-root-password']:
 382             self.message('Locking root password')
 383             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 384         else:
 385             self.message('Give root an empty password')
 386             self.delete_password(rootdir, 'root')
 387
 388     def create_users(self, rootdir):
 389         def create_user(user):
 390             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 391                          '--disabled-password', user])
 392             if self.settings['sudo']:
 393                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 394
 395         for userpass in self.settings['user']:
 396             if '/' in userpass:
 397                 user, password = userpass.split('/', 1)
 398                 create_user(user)
 399                 self.set_password(rootdir, user, password)
 400             else:
 401                 create_user(userpass)
 402                 self.delete_password(rootdir, userpass)
 403
 404     def set_password(self, rootdir, user, password):
 405         encrypted = crypt.crypt(password, '..')
 406         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 407
 408     def delete_password(self, rootdir, user):
 409         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 410
 411     def remove_udev_persistent_rules(self, rootdir):
 412         self.message('Removing udev persistent cd and net rules')
 413         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 414             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 415             if os.path.exists(pathname):
 416                 logging.debug('rm %s' % pathname)
 417                 os.remove(pathname)
 418             else:
 419                 logging.debug('not removing non-existent %s' % pathname)
 420
 421     def setup_networking(self, rootdir):
 422         self.message('Setting up networking')
 423
 424         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 425         f.write('auto lo\n')
 426         f.write('iface lo inet loopback\n')
 427
 428         if self.settings['enable-dhcp']:
 429             f.write('\n')
 430             f.write('auto eth0\n')
 431             f.write('iface eth0 inet dhcp\n')
 432
 433         f.close()
 434
 435     def append_serial_console(self, rootdir):
 436         if self.settings['serial-console']:
 437             serial_command = self.settings['serial-console-command']
 438             logging.debug('adding getty to serial console')
 439             inittab = os.path.join(rootdir, 'etc/inittab')
 440             with open(inittab, 'a') as f:
 441                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 442
 443     def install_extlinux(self, rootdev, rootdir):
 444         if not os.path.exists("/usr/bin/extlinux"):
 445             self.message("extlinux not installed, skipping.")
 446             return
 447         self.message('Installing extlinux')
 448
 449         def find(pattern):
 450             dirname = os.path.join(rootdir, 'boot')
 451             basenames = os.listdir(dirname)
 452             logging.debug('find: %s' % basenames)
 453             for basename in basenames:
 454                 if re.search(pattern, basename):
 455                     return os.path.join('boot', basename)
 456             raise cliapp.AppException('Cannot find match: %s' % pattern)
 457
 458         kernel_image = find('vmlinuz-.*')
 459         initrd_image = find('initrd.img-.*')
 460
 461         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 462                            '-s', 'UUID', rootdev])
 463         uuid = out.splitlines()[0].strip()
 464
 465         conf = os.path.join(rootdir, 'extlinux.conf')
 466         logging.debug('configure extlinux %s' % conf)
 467         f = open(conf, 'w')
 468         f.write('''
 469 default linux
 470 timeout 1
 471
 472 label linux
 473 kernel %(kernel)s
 474 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 475 %(extserial)s
 476 ''' % {
 477             'kernel': kernel_image,
 478             'initrd': initrd_image,
 479             'uuid': uuid,
 480             'kserial':
 481             'console=ttyS0,115200' if self.settings['serial-console'] else '',
 482             'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 483         })
 484         f.close()
 485
 486         self.runcmd(['extlinux', '--install', rootdir])
 487         self.runcmd(['sync'])
 488         time.sleep(2)
 489
 490     def optimize_image(self, rootdir):
 491         """
 492         Filing up the image with zeros will increase its compression rate
 493         """
 494         zeros = os.path.join(rootdir, 'ZEROS')
 495         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 496         self.runcmd(['rm', '-f', zeros])
 497
 498     def squash(self):
 499         """
 500         Run squashfs on the image.
 501         """
 502         if not os.path.exists('/usr/bin/mksquashfs'):
 503             logging.warning("Squash selected but mksquashfs not found!")
 504             return
 505         self.message("Running mksquashfs")
 506         suffixed = "%s.squashfs" % self.settings['image']
 507         self.runcmd(['mksquashfs', self.settings['image'],
 508                      suffixed,
 509                      '-no-progress', '-comp', 'xz'], ignore_fail=False)
 510         os.unlink(self.settings['image'])
 511         self.settings['image'] = suffixed
 512
 513     def cleanup_system(self):
 514         # Clean up after any errors.
 515
 516         self.message('Cleaning up')
 517
 518         # Umount in the reverse mount order
 519         if self.settings['image']:
 520             for i in xrange(len(self.mount_points) - 1, -1, -1):
 521                 mount_point = self.mount_points[i]
 522                 try:
 523                     self.runcmd(['umount', mount_point], ignore_fail=False)
 524                 except cliapp.AppException:
 525                     logging.debug("umount failed, sleeping and trying again")
 526                     time.sleep(5)
 527                     self.runcmd(['umount', mount_point], ignore_fail=False)
 528
 529             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 530
 531         for dirname in self.remove_dirs:
 532             shutil.rmtree(dirname)
 533
 534     def customize(self, rootdir):
 535         script = self.settings['customize']
 536         if script:
 537             self.message('Running customize script %s' % script)
 538             with open('/dev/tty', 'w') as tty:
 539                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 540
 541     def create_tarball(self, rootdir):
 542         # Create a tarball of the disk's contents
 543         # shell out to runcmd since it more easily handles rootdir
 544         self.message('Creating tarball of disk contents')
 545         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 546
 547     def chown(self, rootdir):
 548         # Change image owner after completed build
 549         self.message("Changing owner to %s" % self.settings["owner"])
 550         subprocess.call(["chown",
 551                          self.settings["owner"],
 552                          self.settings["image"]])
 553
 554     def configure_apt(self, rootdir):
 555         # use the distribution and mirror to create an apt source
 556         self.message("Configuring apt to use distribution and mirror")
 557         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
 558         logging.debug('configure apt %s' % conf)
 559         f = open(conf, 'w')
 560         f.write('''
 561         deb %(mirror)s %(distribution)s main
 562         #deb-src %(mirror)s %(distribution)s main
 563         ''' % {
 564             'mirror': self.settings['mirror'],
 565             'distribution': self.settings['distribution']
 566         })
 567         f.close()
 568
 569 if __name__ == '__main__':
 570     VmDebootstrap(version=__version__).run()