vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import time
  24 import shutil
  25 import subprocess
  26 import tempfile
  27 import time
  28
  29
  30 __version__ = '0.3'
  31
  32
  33 class VmDebootstrap(cliapp.Application):
  34
  35     def add_settings(self):
  36         default_arch = self.runcmd(
  37             ["dpkg", "--print-architecture"],
  38             ignore_fail=False).strip()
  39
  40         self.settings.boolean(['verbose'], 'report what is going on')
  41         self.settings.string(['image'], 'put created disk image in FILE',
  42                              metavar='FILE')
  43         self.settings.bytesize(['size'],
  44                                'create a disk image of size SIZE (%default)',
  45                                metavar='SIZE',
  46                                default='1G')
  47         self.settings.bytesize(['bootsize'],
  48                                'create boot partition of size SIZE (%default)',
  49                                metavar='BOOTSIZE',
  50                                default='0%')
  51         self.settings.string(['boottype'],
  52                              'specify file system type for /boot/',
  53                              default='ext2')
  54         self.settings.string(['foreign'],
  55                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  56         self.settings.string(['variant'],
  57                              'select debootstrap variant it not using the default')
  58         self.settings.boolean(
  59             ['extlinux'],
  60             'install extlinux?',
  61             default=True)
  62         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  63                              metavar='FILE')
  64         self.settings.string(['mirror'],
  65                              'use MIRROR as package source (%default)',
  66                              metavar='URL',
  67                              default='http://cdn.debian.net/debian/')
  68         self.settings.string(['arch'], 'architecture to use (%default)',
  69                              metavar='ARCH',
  70                              default=default_arch)
  71         self.settings.string(['distribution'],
  72                              'release to use (%default)',
  73                              metavar='NAME',
  74                              default='stable')
  75         self.settings.string_list(['package'], 'install PACKAGE onto system')
  76         self.settings.string_list(['custom-package'],
  77                                   'install package in DEB file onto system '
  78                                   '(not from mirror)',
  79                                   metavar='DEB')
  80         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  81         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  82         self.settings.string(['root-password'], 'set root password',
  83                              metavar='PASSWORD')
  84         self.settings.boolean(['lock-root-password'],
  85                               'lock root account so they cannot login?')
  86         self.settings.string(['customize'],
  87                              'run SCRIPT after setting up system',
  88                              metavar='SCRIPT')
  89         self.settings.string(['hostname'],
  90                              'set name to HOSTNAME (%default)',
  91                              metavar='HOSTNAME',
  92                              default='debian')
  93         self.settings.string_list(['user'],
  94                                   'create USER with PASSWORD',
  95                                   metavar='USER/PASSWORD')
  96         self.settings.boolean(['serial-console'],
  97                               'configure image to use a serial console')
  98         self.settings.string(['serial-console-command'],
  99                              'command to manage the serial console, appended '
 100                              'to /etc/inittab (%default)',
 101                              metavar='COMMAND',
 102                              default='/sbin/getty -L ttyS0 115200 vt100')
 103         self.settings.boolean(['sudo'],
 104                               'install sudo, and if user is created, add them '
 105                               'to sudo group')
 106         self.settings.string(['owner'],
 107                              'the user who will own the image when the build '
 108                              'is complete.')
 109         self.settings.boolean(['squash'],
 110                               'use squashfs on the final image.')
 111         self.settings.boolean(['configure-apt'],
 112                               'Create an apt source based on the distribution '
 113                               'and mirror selected.')
 114
 115     def process_args(self, args):
 116         if not self.settings['image'] and not self.settings['tarball']:
 117             raise cliapp.AppException('You must give disk image filename, '
 118                                       'or tarball filename')
 119         if self.settings['image'] and not self.settings['size']:
 120             raise cliapp.AppException('If disk image is specified, '
 121                                       'You must give image size.')
 122
 123         self.remove_dirs = []
 124         self.mount_points = []
 125
 126         try:
 127             rootdev = None
 128             roottype = 'ext4'
 129             bootdev = None
 130             boottype = None
 131             rootdir = None
 132             if self.settings['image']:
 133                 self.create_empty_image()
 134                 self.partition_image()
 135                 self.install_mbr()
 136                 (rootdev, bootdev) = self.setup_kpartx()
 137                 self.mkfs(rootdev, type=roottype)
 138                 rootdir = self.mount(rootdev)
 139                 if bootdev:
 140                     if self.settings['boottype']:
 141                         boottype = self.settings['boottype']
 142                     else:
 143                         boottype = 'ext2'
 144                     self.mkfs(bootdev, type=boottype)
 145                     bootdir = '%s/%s' % (rootdir, 'boot/')
 146                     os.mkdir(bootdir)
 147                     bootdir = self.mount(bootdev, bootdir)
 148             else:
 149                 rootdir = self.mkdtemp()
 150             self.debootstrap(rootdir)
 151             self.set_hostname(rootdir)
 152             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 153             self.install_debs(rootdir)
 154             self.cleanup_apt_cache(rootdir)
 155             self.set_root_password(rootdir)
 156             self.create_users(rootdir)
 157             self.remove_udev_persistent_rules(rootdir)
 158             self.setup_networking(rootdir)
 159             self.configure_apt(rootdir)
 160             self.customize(rootdir)
 161             if self.settings['image']:
 162                 if self.settings['extlinux']:
 163                     self.install_extlinux(rootdev, rootdir)
 164                 self.append_serial_console(rootdir)
 165                 self.optimize_image(rootdir)
 166                 if self.settings['squash']:
 167                     self.squash()
 168
 169             if self.settings['foreign']:
 170                 os.unlink('%s/usr/bin/%s' %
 171                           (rootdir, os.path.basename(self.settings['foreign'])))
 172
 173             if self.settings['tarball']:
 174                 self.create_tarball(rootdir)
 175
 176             if self.settings['owner']:
 177                 self.chown(rootdir)
 178         except BaseException, e:
 179             self.message('EEEK! Something bad happened...')
 180             if rootdir:
 181                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 182                 if os.path.exists(db_log):
 183                     shutil.copy(db_log, os.getcwd())
 184             self.message(e)
 185             self.cleanup_system()
 186             raise
 187         else:
 188             self.cleanup_system()
 189
 190     def message(self, msg):
 191         logging.info(msg)
 192         if self.settings['verbose']:
 193             print msg
 194
 195     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 196         logging.debug('runcmd: %s %s' % (argv, kwargs))
 197         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 198                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 199                              **kwargs)
 200         out, err = p.communicate(stdin)
 201         if p.returncode != 0:
 202             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 203             logging.error(msg)
 204             if not ignore_fail:
 205                 raise cliapp.AppException(msg)
 206         return out
 207
 208     def mkdtemp(self):
 209         dirname = tempfile.mkdtemp()
 210         self.remove_dirs.append(dirname)
 211         logging.debug('mkdir %s' % dirname)
 212         return dirname
 213
 214     def mount(self, device, path=None):
 215         if not path:
 216             mount_point = self.mkdtemp()
 217         else:
 218             mount_point = path
 219         self.message('Mounting %s on %s' % (device, mount_point))
 220         self.runcmd(['mount', device, mount_point])
 221         self.mount_points.append(mount_point)
 222         logging.debug('mounted %s on %s' % (device, mount_point))
 223         return mount_point
 224
 225     def create_empty_image(self):
 226         self.message('Creating disk image')
 227         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 228                      self.settings['image'],
 229                      str(self.settings['size'])])
 230
 231     def partition_image(self):
 232         self.message('Creating partitions')
 233         self.runcmd(['parted', '-s', self.settings['image'],
 234                      'mklabel', 'msdos'])
 235         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 236             bootsize = str(self.settings['bootsize'] / (1024 * 1024))
 237             self.runcmd(['parted', '-s', self.settings['image'],
 238                          'mkpart', 'primary', 'fat16', '0', bootsize])
 239         else:
 240             bootsize = '0%'
 241         self.runcmd(['parted', '-s', self.settings['image'],
 242                      'mkpart', 'primary', bootsize, '100%'])
 243         self.runcmd(['parted', '-s', self.settings['image'],
 244                      'set', '1', 'boot', 'on'])
 245
 246     def install_mbr(self):
 247         if os.path.exists("/sbin/install-mbr"):
 248             self.message('Installing MBR')
 249             self.runcmd(['install-mbr', self.settings['image']])
 250
 251     def setup_kpartx(self):
 252         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 253         if self.settings['bootsize']:
 254             bootindex = 0
 255             rootindex = 1
 256             parts = 2
 257         else:
 258             rootindex = 0
 259             parts = 1
 260             boot = None
 261         devices = [line.split()[2]
 262                    for line in out.splitlines()
 263                    if line.startswith('add map ')]
 264         if len(devices) != parts:
 265             raise cliapp.AppException('Surprising number of partitions')
 266         root = '/dev/mapper/%s' % devices[rootindex]
 267         if self.settings['bootsize']:
 268             boot = '/dev/mapper/%s' % devices[bootindex]
 269         return (root, boot)
 270
 271     def mkfs(self, device, type):
 272         self.message('Creating filesystem %s' % type)
 273         self.runcmd(['mkfs', '-t', type, device])
 274
 275     def debootstrap(self, rootdir):
 276         self.message('Debootstrapping')
 277
 278         if self.settings['foreign']:
 279             necessary_packages = []
 280         else:
 281             necessary_packages = ['acpid']
 282
 283         include = self.settings['package']
 284
 285         if not self.settings['no-kernel']:
 286             if self.settings['arch'] == 'i386':
 287                 kernel_arch = '486'
 288             else:
 289                 kernel_arch = self.settings['arch']
 290             kernel_image = 'linux-image-%s' % kernel_arch
 291             include.append(kernel_image)
 292
 293         if self.settings['sudo'] and 'sudo' not in include:
 294             include.append('sudo')
 295
 296         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 297         args.append(
 298             '--include=%s' % ','.join(necessary_packages + include))
 299         if self.settings['foreign']:
 300             args.append('--foreign')
 301         if self.settings['variant']:
 302             args.append('--variant')
 303             args.append(self.settings['variant'])
 304         args += [self.settings['distribution'],
 305                  rootdir, self.settings['mirror']]
 306         logging.debug(" ".join(args))
 307         self.runcmd(args)
 308         if self.settings['foreign']:
 309             # First copy the binfmt handler over
 310             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 311             # Next, run the package install scripts etc.
 312             self.message('Running debootstrap second stage')
 313             self.runcmd(['chroot', rootdir,
 314                          '/debootstrap/debootstrap', '--second-stage'])
 315
 316     def set_hostname(self, rootdir):
 317         hostname = self.settings['hostname']
 318         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 319             f.write('%s\n' % hostname)
 320
 321         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 322         try:
 323             with open(etc_hosts, 'r') as f:
 324                 data = f.read()
 325             with open(etc_hosts, 'w') as f:
 326                 for line in data.splitlines():
 327                     if line.startswith('127.0.0.1'):
 328                         line += ' %s' % hostname
 329                     f.write('%s\n' % line)
 330         except IOError, e:
 331             pass
 332
 333     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
 334         def fsuuid(device):
 335             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 336                                '-s', 'UUID', device])
 337             return out.splitlines()[0].strip()
 338
 339         if rootdev:
 340             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 341         else:
 342             rootdevstr = '/dev/sda1'
 343
 344         if bootdev:
 345             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 346         else:
 347             bootdevstr = None
 348
 349         fstab = os.path.join(rootdir, 'etc', 'fstab')
 350         with open(fstab, 'w') as f:
 351             f.write('proc /proc proc defaults 0 0\n')
 352             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 353             if bootdevstr:
 354                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 355
 356     def install_debs(self, rootdir):
 357         if not self.settings['custom-package']:
 358             return
 359         self.message('Installing custom packages')
 360         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 361         os.mkdir(tmp)
 362         for deb in self.settings['custom-package']:
 363             shutil.copy(deb, tmp)
 364         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 365                      for deb in self.settings['custom-package']]
 366         out, err, exit = \
 367             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 368         logging.debug('stdout:\n%s' % out)
 369         logging.debug('stderr:\n%s' % err)
 370         out = self.runcmd(['chroot', rootdir,
 371                           'apt-get', '-f', '--no-remove', 'install'])
 372         logging.debug('stdout:\n%s' % out)
 373         shutil.rmtree(tmp)
 374
 375     def cleanup_apt_cache(self, rootdir):
 376         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 377         logging.debug('stdout:\n%s' % out)
 378
 379     def set_root_password(self, rootdir):
 380         if self.settings['root-password']:
 381             self.message('Setting root password')
 382             self.set_password(rootdir, 'root', self.settings['root-password'])
 383         elif self.settings['lock-root-password']:
 384             self.message('Locking root password')
 385             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 386         else:
 387             self.message('Give root an empty password')
 388             self.delete_password(rootdir, 'root')
 389
 390     def create_users(self, rootdir):
 391         def create_user(user):
 392             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 393                          '--disabled-password', user])
 394             if self.settings['sudo']:
 395                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 396
 397         for userpass in self.settings['user']:
 398             if '/' in userpass:
 399                 user, password = userpass.split('/', 1)
 400                 create_user(user)
 401                 self.set_password(rootdir, user, password)
 402             else:
 403                 create_user(userpass)
 404                 self.delete_password(rootdir, userpass)
 405
 406     def set_password(self, rootdir, user, password):
 407         encrypted = crypt.crypt(password, '..')
 408         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 409
 410     def delete_password(self, rootdir, user):
 411         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 412
 413     def remove_udev_persistent_rules(self, rootdir):
 414         self.message('Removing udev persistent cd and net rules')
 415         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 416             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 417             if os.path.exists(pathname):
 418                 logging.debug('rm %s' % pathname)
 419                 os.remove(pathname)
 420             else:
 421                 logging.debug('not removing non-existent %s' % pathname)
 422
 423     def setup_networking(self, rootdir):
 424         self.message('Setting up networking')
 425
 426         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 427         f.write('auto lo\n')
 428         f.write('iface lo inet loopback\n')
 429
 430         if self.settings['enable-dhcp']:
 431             f.write('\n')
 432             f.write('auto eth0\n')
 433             f.write('iface eth0 inet dhcp\n')
 434
 435         f.close()
 436
 437     def append_serial_console(self, rootdir):
 438         if self.settings['serial-console']:
 439             serial_command = self.settings['serial-console-command']
 440             logging.debug('adding getty to serial console')
 441             inittab = os.path.join(rootdir, 'etc/inittab')
 442             with open(inittab, 'a') as f:
 443                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 444
 445     def install_extlinux(self, rootdev, rootdir):
 446         if not os.path.exists("/usr/bin/extlinux"):
 447             self.message("extlinux not installed, skipping.")
 448             return
 449         self.message('Installing extlinux')
 450
 451         def find(pattern):
 452             dirname = os.path.join(rootdir, 'boot')
 453             basenames = os.listdir(dirname)
 454             logging.debug('find: %s' % basenames)
 455             for basename in basenames:
 456                 if re.search(pattern, basename):
 457                     return os.path.join('boot', basename)
 458             raise cliapp.AppException('Cannot find match: %s' % pattern)
 459
 460         kernel_image = find('vmlinuz-.*')
 461         initrd_image = find('initrd.img-.*')
 462
 463         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 464                            '-s', 'UUID', rootdev])
 465         uuid = out.splitlines()[0].strip()
 466
 467         conf = os.path.join(rootdir, 'extlinux.conf')
 468         logging.debug('configure extlinux %s' % conf)
 469         f = open(conf, 'w')
 470         f.write('''
 471 default linux
 472 timeout 1
 473
 474 label linux
 475 kernel %(kernel)s
 476 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 477 %(extserial)s
 478 ''' % {
 479             'kernel': kernel_image,
 480             'initrd': initrd_image,
 481             'uuid': uuid,
 482             'kserial':
 483             'console=ttyS0,115200' if self.settings['serial-console'] else '',
 484             'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 485         })
 486         f.close()
 487
 488         self.runcmd(['extlinux', '--install', rootdir])
 489         self.runcmd(['sync'])
 490         time.sleep(2)
 491
 492     def optimize_image(self, rootdir):
 493         """
 494         Filing up the image with zeros will increase its compression rate
 495         """
 496         zeros = os.path.join(rootdir, 'ZEROS')
 497         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 498         self.runcmd(['rm', '-f', zeros])
 499
 500     def squash(self):
 501         """
 502         Run squashfs on the image.
 503         """
 504         if not os.path.exists('/usr/bin/mksquashfs'):
 505             logging.warning("Squash selected but mksquashfs not found!")
 506             return
 507         self.message("Running mksquashfs")
 508         suffixed = "%s.squashfs" % self.settings['image']
 509         self.runcmd(['mksquashfs', self.settings['image'],
 510                      suffixed,
 511                      '-no-progress', '-comp', 'xz'], ignore_fail=False)
 512         os.unlink(self.settings['image'])
 513         self.settings['image'] = suffixed
 514
 515     def cleanup_system(self):
 516         # Clean up after any errors.
 517
 518         self.message('Cleaning up')
 519
 520         # Umount in the reverse mount order
 521         if self.settings['image']:
 522             for i in xrange(len(self.mount_points) - 1, -1, -1):
 523                 mount_point = self.mount_points[i]
 524                 try:
 525                     self.runcmd(['umount', mount_point], ignore_fail=False)
 526                 except cliapp.AppException:
 527                     logging.debug("umount failed, sleeping and trying again")
 528                     time.sleep(5)
 529                     self.runcmd(['umount', mount_point], ignore_fail=False)
 530
 531             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 532
 533         for dirname in self.remove_dirs:
 534             shutil.rmtree(dirname)
 535
 536     def customize(self, rootdir):
 537         script = self.settings['customize']
 538         if script:
 539             self.message('Running customize script %s' % script)
 540             with open('/dev/tty', 'w') as tty:
 541                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 542
 543     def create_tarball(self, rootdir):
 544         # Create a tarball of the disk's contents
 545         # shell out to runcmd since it more easily handles rootdir
 546         self.message('Creating tarball of disk contents')
 547         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 548
 549     def chown(self, rootdir):
 550         # Change image owner after completed build
 551         self.message("Changing owner to %s" % self.settings["owner"])
 552         subprocess.call(["chown",
 553                          self.settings["owner"],
 554                          self.settings["image"]])
 555
 556     def configure_apt(self, rootdir):
 557         # use the distribution and mirror to create an apt source
 558         self.message("Configuring apt to use distribution and mirror")
 559         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
 560         logging.debug('configure apt %s' % conf)
 561         f = open(conf, 'w')
 562         f.write('''
 563         deb %(mirror)s %(distribution)s main
 564         #deb-src %(mirror)s %(distribution)s main
 565         ''' % {
 566             'mirror': self.settings['mirror'],
 567             'distribution': self.settings['distribution']
 568         })
 569         f.close()
 570
 571 if __name__ == '__main__':
 572     VmDebootstrap(version=__version__).run()