vmdebootstrap

   1 #!/usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 import cliapp
  19 import crypt
  20 import logging
  21 import os
  22 import re
  23 import time
  24 import shutil
  25 import subprocess
  26 import tempfile
  27 import time
  28
  29
  30 __version__ = '0.3'
  31
  32
  33 class VmDebootstrap(cliapp.Application):
  34
  35     def add_settings(self):
  36         default_arch = self.runcmd(
  37             ["dpkg", "--print-architecture"],
  38             ignore_fail=False).strip()
  39
  40         self.settings.boolean(['verbose'], 'report what is going on')
  41         self.settings.string(['image'], 'put created disk image in FILE',
  42                              metavar='FILE')
  43         self.settings.bytesize(['size'],
  44                                'create a disk image of size SIZE (%default)',
  45                                metavar='SIZE',
  46                                default='1G')
  47         self.settings.bytesize(['bootsize'],
  48                                'create boot partition of size SIZE (%default)',
  49                                metavar='BOOTSIZE',
  50                                default='0%')
  51         self.settings.string(['boottype'],
  52                              'specify file system type for /boot/',
  53                              default='ext2')
  54         self.settings.string(['foreign'],
  55                              'set up foreign debootstrap environment using provided program (ie binfmt handler)')
  56         self.settings.string(['variant'],
  57                              'select debootstrap variant it not using the default')
  58         self.settings.boolean(
  59             ['extlinux'],
  60             'install extlinux?',
  61             default=True)
  62         self.settings.string(['tarball'], "tar up the disk's contents in FILE",
  63                              metavar='FILE')
  64         self.settings.string(['mirror'],
  65                              'use MIRROR as package source (%default)',
  66                              metavar='URL',
  67                              default='http://cdn.debian.net/debian/')
  68         self.settings.string(['arch'], 'architecture to use (%default)',
  69                              metavar='ARCH',
  70                              default=default_arch)
  71         self.settings.string(['distribution'],
  72                              'release to use (%default)',
  73                              metavar='NAME',
  74                              default='stable')
  75         self.settings.string_list(['package'], 'install PACKAGE onto system')
  76         self.settings.string_list(['custom-package'],
  77                                   'install package in DEB file onto system '
  78                                   '(not from mirror)',
  79                                   metavar='DEB')
  80         self.settings.boolean(['no-kernel'], 'do not install a linux package')
  81         self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
  82         self.settings.string(['root-password'], 'set root password',
  83                              metavar='PASSWORD')
  84         self.settings.boolean(['lock-root-password'],
  85                               'lock root account so they cannot login?')
  86         self.settings.string(['customize'],
  87                              'run SCRIPT after setting up system',
  88                              metavar='SCRIPT')
  89         self.settings.string(['hostname'],
  90                              'set name to HOSTNAME (%default)',
  91                              metavar='HOSTNAME',
  92                              default='debian')
  93         self.settings.string_list(['user'],
  94                                   'create USER with PASSWORD',
  95                                   metavar='USER/PASSWORD')
  96         self.settings.boolean(['serial-console'],
  97                               'configure image to use a serial console')
  98         self.settings.string(['serial-console-command'],
  99                              'command to manage the serial console, appended '
 100                              'to /etc/inittab (%default)',
 101                              metavar='COMMAND',
 102                              default='/sbin/getty -L ttyS0 115200 vt100')
 103         self.settings.boolean(['sudo'],
 104                               'install sudo, and if user is created, add them '
 105                               'to sudo group')
 106         self.settings.string(['owner'],
 107                              'the user who will own the image when the build '
 108                              'is complete.')
 109         self.settings.boolean(['squash'],
 110                               'use squashfs on the final image.')
 111         self.settings.boolean(['configure-apt'],
 112                               'Create an apt source based on the distribution '
 113                               'and mirror selected.')
 114
 115     def process_args(self, args):
 116         if not self.settings['image'] and not self.settings['tarball']:
 117             raise cliapp.AppException('You must give disk image filename, '
 118                                       'or tarball filename')
 119         if self.settings['image'] and not self.settings['size']:
 120             raise cliapp.AppException('If disk image is specified, '
 121                                       'You must give image size.')
 122
 123         self.remove_dirs = []
 124         self.mount_points = []
 125
 126         try:
 127             rootdev = None
 128             roottype = 'ext4'
 129             bootdev = None
 130             boottype = None
 131             rootdir = None
 132             if self.settings['image']:
 133                 self.create_empty_image()
 134                 self.partition_image()
 135                 self.install_mbr()
 136                 (rootdev, bootdev) = self.setup_kpartx()
 137                 self.mkfs(rootdev, type=roottype)
 138                 rootdir = self.mount(rootdev)
 139                 if bootdev:
 140                     if self.settings['boottype']:
 141                         boottype = self.settings['boottype']
 142                     else:
 143                         boottype = 'ext2'
 144                     self.mkfs(bootdev, type=boottype)
 145                     bootdir = '%s/%s' % (rootdir, 'boot/')
 146                     os.mkdir(bootdir)
 147                     bootdir = self.mount(bootdev, bootdir)
 148             else:
 149                 rootdir = self.mkdtemp()
 150             self.debootstrap(rootdir)
 151             self.set_hostname(rootdir)
 152             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 153             self.install_debs(rootdir)
 154             self.cleanup_apt_cache(rootdir)
 155             self.set_root_password(rootdir)
 156             self.create_users(rootdir)
 157             self.remove_udev_persistent_rules(rootdir)
 158             self.setup_networking(rootdir)
 159             if self.settings['configure-apt']:
 160                 self.configure_apt(rootdir)
 161             self.customize(rootdir)
 162             if self.settings['image']:
 163                 if self.settings['extlinux']:
 164                     self.install_extlinux(rootdev, rootdir)
 165                 self.append_serial_console(rootdir)
 166                 self.optimize_image(rootdir)
 167                 if self.settings['squash']:
 168                     self.squash()
 169
 170             if self.settings['foreign']:
 171                 os.unlink('%s/usr/bin/%s' %
 172                           (rootdir, os.path.basename(self.settings['foreign'])))
 173
 174             if self.settings['tarball']:
 175                 self.create_tarball(rootdir)
 176
 177             if self.settings['owner']:
 178                 self.chown(rootdir)
 179         except BaseException, e:
 180             self.message('EEEK! Something bad happened...')
 181             if rootdir:
 182                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 183                 if os.path.exists(db_log):
 184                     shutil.copy(db_log, os.getcwd())
 185             self.message(e)
 186             self.cleanup_system()
 187             raise
 188         else:
 189             self.cleanup_system()
 190
 191     def message(self, msg):
 192         logging.info(msg)
 193         if self.settings['verbose']:
 194             print msg
 195
 196     def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs):
 197         logging.debug('runcmd: %s %s' % (argv, kwargs))
 198         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 199                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 200                              **kwargs)
 201         out, err = p.communicate(stdin)
 202         if p.returncode != 0:
 203             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 204             logging.error(msg)
 205             if not ignore_fail:
 206                 raise cliapp.AppException(msg)
 207         return out
 208
 209     def mkdtemp(self):
 210         dirname = tempfile.mkdtemp()
 211         self.remove_dirs.append(dirname)
 212         logging.debug('mkdir %s' % dirname)
 213         return dirname
 214
 215     def mount(self, device, path=None):
 216         if not path:
 217             mount_point = self.mkdtemp()
 218         else:
 219             mount_point = path
 220         self.message('Mounting %s on %s' % (device, mount_point))
 221         self.runcmd(['mount', device, mount_point])
 222         self.mount_points.append(mount_point)
 223         logging.debug('mounted %s on %s' % (device, mount_point))
 224         return mount_point
 225
 226     def create_empty_image(self):
 227         self.message('Creating disk image')
 228         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 229                      self.settings['image'],
 230                      str(self.settings['size'])])
 231
 232     def partition_image(self):
 233         self.message('Creating partitions')
 234         self.runcmd(['parted', '-s', self.settings['image'],
 235                      'mklabel', 'msdos'])
 236         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 237             bootsize = str(self.settings['bootsize'] / (1024 * 1024))
 238             self.runcmd(['parted', '-s', self.settings['image'],
 239                          'mkpart', 'primary', 'fat16', '0', bootsize])
 240         else:
 241             bootsize = '0%'
 242         self.runcmd(['parted', '-s', self.settings['image'],
 243                      'mkpart', 'primary', bootsize, '100%'])
 244         self.runcmd(['parted', '-s', self.settings['image'],
 245                      'set', '1', 'boot', 'on'])
 246
 247     def install_mbr(self):
 248         if os.path.exists("/sbin/install-mbr"):
 249             self.message('Installing MBR')
 250             self.runcmd(['install-mbr', self.settings['image']])
 251
 252     def setup_kpartx(self):
 253         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 254         if self.settings['bootsize']:
 255             bootindex = 0
 256             rootindex = 1
 257             parts = 2
 258         else:
 259             rootindex = 0
 260             parts = 1
 261             boot = None
 262         devices = [line.split()[2]
 263                    for line in out.splitlines()
 264                    if line.startswith('add map ')]
 265         if len(devices) != parts:
 266             raise cliapp.AppException('Surprising number of partitions')
 267         root = '/dev/mapper/%s' % devices[rootindex]
 268         if self.settings['bootsize']:
 269             boot = '/dev/mapper/%s' % devices[bootindex]
 270         return (root, boot)
 271
 272     def mkfs(self, device, type):
 273         self.message('Creating filesystem %s' % type)
 274         self.runcmd(['mkfs', '-t', type, device])
 275
 276     def debootstrap(self, rootdir):
 277         self.message('Debootstrapping')
 278
 279         if self.settings['foreign']:
 280             necessary_packages = []
 281         else:
 282             necessary_packages = ['acpid']
 283
 284         include = self.settings['package']
 285
 286         if not self.settings['no-kernel']:
 287             if self.settings['arch'] == 'i386':
 288                 kernel_arch = '486'
 289             else:
 290                 kernel_arch = self.settings['arch']
 291             kernel_image = 'linux-image-%s' % kernel_arch
 292             include.append(kernel_image)
 293
 294         if self.settings['sudo'] and 'sudo' not in include:
 295             include.append('sudo')
 296
 297         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 298         args.append(
 299             '--include=%s' % ','.join(necessary_packages + include))
 300         if self.settings['foreign']:
 301             args.append('--foreign')
 302         if self.settings['variant']:
 303             args.append('--variant')
 304             args.append(self.settings['variant'])
 305         args += [self.settings['distribution'],
 306                  rootdir, self.settings['mirror']]
 307         logging.debug(" ".join(args))
 308         self.runcmd(args)
 309         if self.settings['foreign']:
 310             # First copy the binfmt handler over
 311             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 312             # Next, run the package install scripts etc.
 313             self.message('Running debootstrap second stage')
 314             self.runcmd(['chroot', rootdir,
 315                          '/debootstrap/debootstrap', '--second-stage'])
 316
 317     def set_hostname(self, rootdir):
 318         hostname = self.settings['hostname']
 319         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 320             f.write('%s\n' % hostname)
 321
 322         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 323         try:
 324             with open(etc_hosts, 'r') as f:
 325                 data = f.read()
 326             with open(etc_hosts, 'w') as f:
 327                 for line in data.splitlines():
 328                     if line.startswith('127.0.0.1'):
 329                         line += ' %s' % hostname
 330                     f.write('%s\n' % line)
 331         except IOError, e:
 332             pass
 333
 334     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
 335         def fsuuid(device):
 336             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 337                                '-s', 'UUID', device])
 338             return out.splitlines()[0].strip()
 339
 340         if rootdev:
 341             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 342         else:
 343             rootdevstr = '/dev/sda1'
 344
 345         if bootdev:
 346             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 347         else:
 348             bootdevstr = None
 349
 350         fstab = os.path.join(rootdir, 'etc', 'fstab')
 351         with open(fstab, 'w') as f:
 352             f.write('proc /proc proc defaults 0 0\n')
 353             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 354             if bootdevstr:
 355                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 356
 357     def install_debs(self, rootdir):
 358         if not self.settings['custom-package']:
 359             return
 360         self.message('Installing custom packages')
 361         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 362         os.mkdir(tmp)
 363         for deb in self.settings['custom-package']:
 364             shutil.copy(deb, tmp)
 365         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 366                      for deb in self.settings['custom-package']]
 367         out, err, exit = \
 368             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 369         logging.debug('stdout:\n%s' % out)
 370         logging.debug('stderr:\n%s' % err)
 371         out = self.runcmd(['chroot', rootdir,
 372                           'apt-get', '-f', '--no-remove', 'install'])
 373         logging.debug('stdout:\n%s' % out)
 374         shutil.rmtree(tmp)
 375
 376     def cleanup_apt_cache(self, rootdir):
 377         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 378         logging.debug('stdout:\n%s' % out)
 379
 380     def set_root_password(self, rootdir):
 381         if self.settings['root-password']:
 382             self.message('Setting root password')
 383             self.set_password(rootdir, 'root', self.settings['root-password'])
 384         elif self.settings['lock-root-password']:
 385             self.message('Locking root password')
 386             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 387         else:
 388             self.message('Give root an empty password')
 389             self.delete_password(rootdir, 'root')
 390
 391     def create_users(self, rootdir):
 392         def create_user(user):
 393             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
 394                          '--disabled-password', user])
 395             if self.settings['sudo']:
 396                 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
 397
 398         for userpass in self.settings['user']:
 399             if '/' in userpass:
 400                 user, password = userpass.split('/', 1)
 401                 create_user(user)
 402                 self.set_password(rootdir, user, password)
 403             else:
 404                 create_user(userpass)
 405                 self.delete_password(rootdir, userpass)
 406
 407     def set_password(self, rootdir, user, password):
 408         encrypted = crypt.crypt(password, '..')
 409         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 410
 411     def delete_password(self, rootdir, user):
 412         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 413
 414     def remove_udev_persistent_rules(self, rootdir):
 415         self.message('Removing udev persistent cd and net rules')
 416         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 417             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 418             if os.path.exists(pathname):
 419                 logging.debug('rm %s' % pathname)
 420                 os.remove(pathname)
 421             else:
 422                 logging.debug('not removing non-existent %s' % pathname)
 423
 424     def setup_networking(self, rootdir):
 425         self.message('Setting up networking')
 426
 427         f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
 428         f.write('auto lo\n')
 429         f.write('iface lo inet loopback\n')
 430
 431         if self.settings['enable-dhcp']:
 432             f.write('\n')
 433             f.write('auto eth0\n')
 434             f.write('iface eth0 inet dhcp\n')
 435
 436         f.close()
 437
 438     def append_serial_console(self, rootdir):
 439         if self.settings['serial-console']:
 440             serial_command = self.settings['serial-console-command']
 441             logging.debug('adding getty to serial console')
 442             inittab = os.path.join(rootdir, 'etc/inittab')
 443             with open(inittab, 'a') as f:
 444                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 445
 446     def install_extlinux(self, rootdev, rootdir):
 447         if not os.path.exists("/usr/bin/extlinux"):
 448             self.message("extlinux not installed, skipping.")
 449             return
 450         self.message('Installing extlinux')
 451
 452         def find(pattern):
 453             dirname = os.path.join(rootdir, 'boot')
 454             basenames = os.listdir(dirname)
 455             logging.debug('find: %s' % basenames)
 456             for basename in basenames:
 457                 if re.search(pattern, basename):
 458                     return os.path.join('boot', basename)
 459             raise cliapp.AppException('Cannot find match: %s' % pattern)
 460
 461         kernel_image = find('vmlinuz-.*')
 462         initrd_image = find('initrd.img-.*')
 463
 464         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 465                            '-s', 'UUID', rootdev])
 466         uuid = out.splitlines()[0].strip()
 467
 468         conf = os.path.join(rootdir, 'extlinux.conf')
 469         logging.debug('configure extlinux %s' % conf)
 470         f = open(conf, 'w')
 471         f.write('''
 472 default linux
 473 timeout 1
 474
 475 label linux
 476 kernel %(kernel)s
 477 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 478 %(extserial)s
 479 ''' % {
 480             'kernel': kernel_image,
 481             'initrd': initrd_image,
 482             'uuid': uuid,
 483             'kserial':
 484             'console=ttyS0,115200' if self.settings['serial-console'] else '',
 485             'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
 486         })
 487         f.close()
 488
 489         self.runcmd(['extlinux', '--install', rootdir])
 490         self.runcmd(['sync'])
 491         time.sleep(2)
 492
 493     def optimize_image(self, rootdir):
 494         """
 495         Filing up the image with zeros will increase its compression rate
 496         """
 497         zeros = os.path.join(rootdir, 'ZEROS')
 498         self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 499         self.runcmd(['rm', '-f', zeros])
 500
 501     def squash(self):
 502         """
 503         Run squashfs on the image.
 504         """
 505         if not os.path.exists('/usr/bin/mksquashfs'):
 506             logging.warning("Squash selected but mksquashfs not found!")
 507             return
 508         self.message("Running mksquashfs")
 509         suffixed = "%s.squashfs" % self.settings['image']
 510         self.runcmd(['mksquashfs', self.settings['image'],
 511                      suffixed,
 512                      '-no-progress', '-comp', 'xz'], ignore_fail=False)
 513         os.unlink(self.settings['image'])
 514         self.settings['image'] = suffixed
 515
 516     def cleanup_system(self):
 517         # Clean up after any errors.
 518
 519         self.message('Cleaning up')
 520
 521         # Umount in the reverse mount order
 522         if self.settings['image']:
 523             for i in xrange(len(self.mount_points) - 1, -1, -1):
 524                 mount_point = self.mount_points[i]
 525                 try:
 526                     self.runcmd(['umount', mount_point], ignore_fail=False)
 527                 except cliapp.AppException:
 528                     logging.debug("umount failed, sleeping and trying again")
 529                     time.sleep(5)
 530                     self.runcmd(['umount', mount_point], ignore_fail=False)
 531
 532             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
 533
 534         for dirname in self.remove_dirs:
 535             shutil.rmtree(dirname)
 536
 537     def customize(self, rootdir):
 538         script = self.settings['customize']
 539         if script:
 540             self.message('Running customize script %s' % script)
 541             with open('/dev/tty', 'w') as tty:
 542                 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
 543
 544     def create_tarball(self, rootdir):
 545         # Create a tarball of the disk's contents
 546         # shell out to runcmd since it more easily handles rootdir
 547         self.message('Creating tarball of disk contents')
 548         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
 549
 550     def chown(self, rootdir):
 551         # Change image owner after completed build
 552         self.message("Changing owner to %s" % self.settings["owner"])
 553         subprocess.call(["chown",
 554                          self.settings["owner"],
 555                          self.settings["image"]])
 556
 557     def configure_apt(self, rootdir):
 558         # use the distribution and mirror to create an apt source
 559         self.message("Configuring apt to use distribution and mirror")
 560         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
 561         logging.debug('configure apt %s' % conf)
 562         f = open(conf, 'w')
 563         f.write('''
 564 deb %(mirror)s %(distribution)s main
 565 #deb-src %(mirror)s %(distribution)s main
 566 ''' % {
 567             'mirror': self.settings['mirror'],
 568             'distribution': self.settings['distribution']
 569         })
 570         f.close()
 571
 572 if __name__ == '__main__':
 573     VmDebootstrap(version=__version__).run()