vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import re
  24 import sys
  25 import shutil
  26 import datetime
  27 import subprocess
  28 import tempfile
  29 import time
  30 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  31
  32
  33 __version__ = '0.10'
  34
  35 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  36
  37
  38 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  39
  40     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  41         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  42         self.remove_dirs = []
  43         self.mount_points = []
  44         self.debian_info = DebianDistroInfo()
  45         self.ubuntu_info = UbuntuDistroInfo()
  46         self.bootdir = None
  47         self.efi_arch_table = {
  48             'amd64': {
  49                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  50                 'install': '/EFI/debian/grubx64.efi',  # package location
  51                 'package': 'grub-efi-amd64',  # bootstrap package
  52                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  53                 'extra': 'i386',  # architecture to add binary package
  54                 'exclusive': False,  # only EFI supported for this arch.
  55                 'target': 'x86_64-efi',  # grub target name
  56             },
  57             'i386': {
  58                 'removable': '/EFI/boot/bootia32.efi',
  59                 'install': '/EFI/debian/grubia32.efi',
  60                 'package': 'grub-efi-ia32',
  61                 'bin_package': 'grub-efi-ia32-bin',
  62                 'extra': None,
  63                 'exclusive': False,
  64                 'target': 'i386-efi',
  65             },
  66             'arm64': {
  67                 'removable': '/EFI/boot/bootaa64.efi',
  68                 'install': '/EFI/debian/grubaa64.efi',
  69                 'package': 'grub-efi-arm64',
  70                 'bin_package': 'grub-efi-arm64-bin',
  71                 'extra': None,
  72                 'exclusive': True,
  73                 'target': 'arm64-efi',
  74             }
  75         }
  76
  77     def add_settings(self):
  78         default_arch = subprocess.check_output(
  79             ["dpkg", "--print-architecture"]).strip()
  80
  81         self.settings.boolean(
  82             ['verbose'], 'report what is going on')
  83         self.settings.string(
  84             ['image'], 'put created disk image in FILE',
  85             metavar='FILE')
  86         self.settings.bytesize(
  87             ['size'],
  88             'create a disk image of size SIZE (%default)',
  89             metavar='SIZE',
  90             default='1G')
  91         self.settings.bytesize(
  92             ['bootsize'],
  93             'create boot partition of size SIZE (%default)',
  94             metavar='BOOTSIZE',
  95             default='0%')
  96         self.settings.string(
  97             ['boottype'],
  98             'specify file system type for /boot/',
  99             default='ext2')
 100         self.settings.bytesize(
 101             ['bootoffset'],
 102             'Space to leave at start of the image for bootloader',
 103             default='0')
 104         self.settings.boolean(
 105             ['use-uefi'],
 106             'Setup image for UEFI boot',
 107             default=False)
 108         self.settings.bytesize(
 109             ['esp-size'],
 110             'Size of EFI System Partition - requires use-uefi',
 111             default='5mib')
 112         self.settings.string(
 113             ['part-type'],
 114             'Partition type to use for this image',
 115             default='msdos')
 116         self.settings.string(
 117             ['roottype'],
 118             'specify file system type for /',
 119             default='ext4')
 120         self.settings.bytesize(
 121             ['swap'],
 122             'create swap space of size SIZE (min 256Mb)')
 123         self.settings.string(
 124             ['foreign'],
 125             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 126         self.settings.string(
 127             ['variant'],
 128             'select debootstrap variant if not using the default [deprecated]')
 129         self.settings.string_list(
 130             ['debootstrapopts'],
 131             'pass additional options to debootstrap'),
 132         self.settings.boolean(
 133             ['extlinux'],
 134             'install extlinux?',
 135             default=True)
 136         self.settings.string(
 137             ['tarball'],
 138             "tar up the disk's contents in FILE",
 139             metavar='FILE')
 140         self.settings.string(
 141             ['apt-mirror'],
 142             'configure apt to use MIRROR',
 143             metavar='URL')
 144         self.settings.string(
 145             ['mirror'],
 146             'use MIRROR as package source (%default)',
 147             metavar='URL',
 148             default='http://http.debian.net/debian/')
 149         self.settings.string(
 150             ['arch'],
 151             'architecture to use (%default)',
 152             metavar='ARCH',
 153             default=default_arch)
 154         self.settings.string(
 155             ['distribution'],
 156             'release to use (%default)',
 157             metavar='NAME',
 158             default='stable')
 159         self.settings.string_list(
 160             ['package'],
 161             'install PACKAGE onto system')
 162         self.settings.string_list(
 163             ['custom-package'],
 164             'install package in DEB file onto system (not from mirror)',
 165             metavar='DEB')
 166         self.settings.boolean(
 167             ['no-kernel'],
 168             'do not install a linux package')
 169         self.settings.string(
 170             ['kernel-package'],
 171             'install PACKAGE instead of the default kernel package',
 172             metavar='PACKAGE')
 173         self.settings.boolean(
 174             ['enable-dhcp'],
 175             'enable DHCP on eth0')
 176         self.settings.string(
 177             ['root-password'],
 178             'set root password',
 179             metavar='PASSWORD')
 180         self.settings.boolean(
 181             ['lock-root-password'],
 182             'lock root account so they cannot login?')
 183         self.settings.string(
 184             ['customize'],
 185             'run SCRIPT after setting up system',
 186             metavar='SCRIPT')
 187         self.settings.string(
 188             ['hostname'],
 189             'set name to HOSTNAME (%default)',
 190             metavar='HOSTNAME',
 191             default='debian')
 192         self.settings.string_list(
 193             ['user'],
 194             'create USER with PASSWORD',
 195             metavar='USER/PASSWORD')
 196         self.settings.boolean(
 197             ['serial-console'],
 198             'configure image to use a serial console')
 199         self.settings.string(
 200             ['serial-console-command'],
 201             'command to manage the serial console, appended to /etc/inittab (%default)',
 202             metavar='COMMAND',
 203             default='/sbin/getty -L ttyS0 115200 vt100')
 204         self.settings.boolean(
 205             ['sudo'],
 206             'install sudo, and if user is created, add them to sudo group')
 207         self.settings.string(
 208             ['owner'],
 209             'the user who will own the image when the build is complete.')
 210         self.settings.boolean(
 211             ['squash'],
 212             'use squashfs on the final image.')
 213         self.settings.boolean(
 214             ['configure-apt'],
 215             'Create an apt source based on the distribution and mirror selected.')
 216         self.settings.boolean(
 217             ['mbr'],
 218             'Run install-mbr (default if extlinux used)')
 219         self.settings.boolean(
 220             ['grub'],
 221             'Install and configure grub2 - disables extlinux.')
 222         self.settings.boolean(
 223             ['sparse'],
 224             'Do not fill the image with zeros to keep a sparse disk image',
 225             default=False)
 226         self.settings.boolean(
 227             ['pkglist'],
 228             'Create a list of package names included in the image.')
 229         self.settings.boolean(
 230             ['no-acpid'],
 231             'do not install the acpid package',
 232             default=False)
 233
 234     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 235         if not self.settings['image'] and not self.settings['tarball']:
 236             raise cliapp.AppException(
 237                 'You must give disk image filename, or tarball filename')
 238         if self.settings['image'] and not self.settings['size']:
 239             raise cliapp.AppException(
 240                 'If disk image is specified, you must give image size.')
 241         if not self.debian_info.valid(self.settings['distribution']):
 242             if not self.ubuntu_info.valid(self.settings['distribution']):
 243                 raise cliapp.AppException(
 244                     '%s is not a valid Debian or Ubuntu suite or codename.'
 245                     % self.settings['distribution'])
 246         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 247             raise cliapp.AppException(
 248                 'You must specify use-uefi for esp-size to have effect')
 249         if self.settings['arch'] in self.efi_arch_table and\
 250                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 251                 not self.settings['use-uefi']:
 252             raise cliapp.AppException(
 253                 'Only UEFI is supported on %s' % self.settings['arch'])
 254         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 255             raise cliapp.AppException(
 256                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 257         if self.settings['use-uefi'] and (
 258                 self.settings['bootsize'] or
 259                 self.settings['bootoffset']):
 260             raise cliapp.AppException(
 261                 'A separate boot partition is not supported with UEFI')
 262
 263         if self.settings['use-uefi'] and not self.settings['grub']:
 264             raise cliapp.AppException(
 265                 'UEFI without Grub is not supported.')
 266
 267         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 268         if self.was_oldstable(datetime.date(2015, 4, 26)):
 269             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 270                 raise cliapp.AppException(
 271                     'Only amd64 supports UEFI in Wheezy')
 272
 273         if os.geteuid() != 0:
 274             sys.exit("You need to have root privileges to run this script.")
 275         rootdir = None
 276         try:
 277             rootdev = None
 278             roottype = self.settings['roottype']
 279             bootdev = None
 280             boottype = None
 281             if self.settings['image']:
 282                 self.create_empty_image()
 283                 self.partition_image()
 284                 if self.settings['mbr'] or self.settings['extlinux']:
 285                     self.install_mbr()
 286                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 287                 if self.settings['swap'] > 0:
 288                     self.message("Creating swap space")
 289                     self.runcmd(['mkswap', swapdev])
 290                 self.mkfs(rootdev, fstype=roottype)
 291                 rootdir = self.mount(rootdev)
 292                 if self.settings['use-uefi']:
 293                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 294                     logging.debug("bootdir:%s", self.bootdir)
 295                     self.mkfs(bootdev, fstype='vfat')
 296                     os.makedirs(self.bootdir)
 297                     self.mount(bootdev, self.bootdir)
 298                 elif bootdev:
 299                     if self.settings['boottype']:
 300                         boottype = self.settings['boottype']
 301                     else:
 302                         boottype = 'ext2'
 303                     self.mkfs(bootdev, fstype=boottype)
 304                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 305                     os.mkdir(self.bootdir)
 306                     self.mount(bootdev, self.bootdir)
 307             else:
 308                 rootdir = self.mkdtemp()
 309             self.debootstrap(rootdir)
 310             self.set_hostname(rootdir)
 311             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 312             self.install_debs(rootdir)
 313             self.set_root_password(rootdir)
 314             self.create_users(rootdir)
 315             self.remove_udev_persistent_rules(rootdir)
 316             self.setup_networking(rootdir)
 317             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 318                 self.configure_apt(rootdir)
 319             self.customize(rootdir)
 320             self.cleanup_apt_cache(rootdir)
 321             self.update_initramfs(rootdir)
 322
 323             if self.settings['image']:
 324                 if self.settings['use-uefi']:
 325                     self.install_grub_uefi(rootdir)
 326                 elif self.settings['grub']:
 327                     self.install_grub2(rootdev, rootdir)
 328                 elif self.settings['extlinux']:
 329                     self.install_extlinux(rootdev, rootdir)
 330                 self.append_serial_console(rootdir)
 331                 self.optimize_image(rootdir)
 332                 if self.settings['squash']:
 333                     self.squash()
 334             if self.settings['pkglist']:
 335                 self.list_installed_pkgs(rootdir)
 336
 337             if self.settings['foreign']:
 338                 os.unlink('%s/usr/bin/%s' %
 339                           (rootdir, os.path.basename(self.settings['foreign'])))
 340
 341             if self.settings['tarball']:
 342                 self.create_tarball(rootdir)
 343
 344             if self.settings['owner']:
 345                 self.chown()
 346         except BaseException as e:
 347             self.message('EEEK! Something bad happened...')
 348             if rootdir:
 349                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 350                 if os.path.exists(db_log):
 351                     shutil.copy(db_log, os.getcwd())
 352             self.message(e)
 353             self.cleanup_system()
 354             raise
 355         else:
 356             self.cleanup_system()
 357
 358     def message(self, msg):
 359         logging.info(msg)
 360         if self.settings['verbose']:
 361             print msg
 362
 363     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 364         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 365         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 366                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 367                              env=env, **kwargs)
 368         out, err = p.communicate(stdin)
 369         if p.returncode != 0:
 370             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 371             logging.error(msg)
 372             if not ignore_fail:
 373                 raise cliapp.AppException(msg)
 374         return out
 375
 376     def mkdtemp(self):
 377         dirname = tempfile.mkdtemp()
 378         self.remove_dirs.append(dirname)
 379         logging.debug('mkdir %s', dirname)
 380         return dirname
 381
 382     def mount(self, device, path=None):
 383         if not path:
 384             mount_point = self.mkdtemp()
 385         else:
 386             mount_point = path
 387         self.message('Mounting %s on %s' % (device, mount_point))
 388         self.runcmd(['mount', device, mount_point])
 389         self.mount_points.append(mount_point)
 390         logging.debug('mounted %s on %s', device, mount_point)
 391         return mount_point
 392
 393     def create_empty_image(self):
 394         self.message('Creating disk image')
 395         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 396                      self.settings['image'],
 397                      str(self.settings['size'])])
 398
 399     def partition_image(self):
 400         """
 401         Uses fat16 (msdos) partitioning by default, use part-type to change.
 402         If bootoffset is specified, the first actual partition
 403         starts at that offset to allow customisation scripts to
 404         put bootloader images into the space, e.g. u-boot.
 405         """
 406         self.message('Creating partitions')
 407         self.runcmd(['parted', '-s', self.settings['image'],
 408                      'mklabel', self.settings['part-type']])
 409         partoffset = 0
 410         extent = '100%'
 411         swap = 256 * 1024 * 1024
 412         if self.settings['swap'] > 0:
 413             if self.settings['swap'] > swap:
 414                 swap = self.settings['swap']
 415             else:
 416                 # minimum 256Mb as default qemu ram is 128Mb
 417                 logging.debug("Setting minimum 256Mb swap space")
 418             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 419
 420         if self.settings['use-uefi']:
 421             espsize = self.settings['esp-size'] / (1024 * 1024)
 422             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 423             self.runcmd(['parted', '-s', self.settings['image'],
 424                          'mkpart', 'primary', 'fat32',
 425                          '1', str(espsize)])
 426             self.runcmd(['parted', '-s', self.settings['image'],
 427                          'set', '1', 'boot', 'on'])
 428             self.runcmd(['parted', '-s', self.settings['image'],
 429                          'set', '1', 'esp', 'on'])
 430
 431         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 432             # turn v.small offsets into something at least possible to create.
 433             if self.settings['bootoffset'] < 1048576:
 434                 partoffset = 1
 435                 logging.info(
 436                     "Setting bootoffset %smib to allow for %s bytes",
 437                     partoffset, self.settings['bootoffset'])
 438             else:
 439                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 440                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 441         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 442             if self.settings['grub'] and not partoffset:
 443                 partoffset = 1
 444             bootsize = self.settings['bootsize'] / (1024 * 1024)
 445             bootsize += partoffset
 446             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 447             logging.debug("Starting boot partition at %sMb", bootsize)
 448             self.runcmd(['parted', '-s', self.settings['image'],
 449                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 450             logging.debug("Starting root partition at %sMb", partoffset)
 451             self.runcmd(['parted', '-s', self.settings['image'],
 452                          'mkpart', 'primary', str(bootsize), extent])
 453         elif self.settings['use-uefi']:
 454             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 455             self.runcmd(['parted', '-s', self.settings['image'],
 456                          'mkpart', 'primary', str(bootsize), extent])
 457         else:
 458             self.runcmd(['parted', '-s', self.settings['image'],
 459                          'mkpart', 'primary', '0%', extent])
 460         self.runcmd(['parted', '-s', self.settings['image'],
 461                      'set', '1', 'boot', 'on'])
 462         if self.settings['swap'] > 0:
 463             logging.debug("Creating swap partition")
 464             self.runcmd(['parted', '-s', self.settings['image'],
 465                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 466
 467     def update_initramfs(self, rootdir):
 468         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 469         if os.path.exists(os.path.join(rootdir, cmd)):
 470             self.message("Updating the initramfs")
 471             self.runcmd(['chroot', rootdir, cmd, '-u'])
 472
 473     def install_mbr(self):
 474         if os.path.exists("/sbin/install-mbr"):
 475             self.message('Installing MBR')
 476             self.runcmd(['install-mbr', self.settings['image']])
 477         else:
 478             msg = "mbr enabled but /sbin/install-mbr not found" \
 479                   " - please install the mbr package."
 480             raise cliapp.AppException(msg)
 481
 482     def setup_kpartx(self):
 483         bootindex = None
 484         swapindex = None
 485         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 486         if self.settings['bootsize'] and self.settings['swap'] > 0:
 487             bootindex = 0
 488             rootindex = 1
 489             swapindex = 2
 490             parts = 3
 491         elif self.settings['use-uefi']:
 492             bootindex = 0
 493             rootindex = 1
 494             parts = 2
 495         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 496             bootindex = 0
 497             rootindex = 1
 498             swapindex = 2
 499             parts = 3
 500         elif self.settings['bootsize']:
 501             bootindex = 0
 502             rootindex = 1
 503             parts = 2
 504         elif self.settings['swap'] > 0:
 505             rootindex = 0
 506             swapindex = 1
 507             parts = 2
 508         else:
 509             rootindex = 0
 510             parts = 1
 511         boot = None
 512         swap = None
 513         devices = [line.split()[2]
 514                    for line in out.splitlines()
 515                    if line.startswith('add map ')]
 516         if len(devices) != parts:
 517             msg = 'Surprising number of partitions - check output of losetup -a'
 518             logging.debug("%s", self.runcmd(['losetup', '-a']))
 519             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 520             raise cliapp.AppException(msg)
 521         root = '/dev/mapper/%s' % devices[rootindex]
 522         if self.settings['bootsize'] or self.settings['use-uefi']:
 523             boot = '/dev/mapper/%s' % devices[bootindex]
 524         if self.settings['swap'] > 0:
 525             swap = '/dev/mapper/%s' % devices[swapindex]
 526         return root, boot, swap
 527
 528     def _efi_packages(self):
 529         packages = []
 530         pkg = self.efi_arch_table[self.settings['arch']]['package']
 531         self.message("Adding %s" % pkg)
 532         packages.append(pkg)
 533         extra = self.efi_arch_table[self.settings['arch']]['extra']
 534         if extra and isinstance(extra, str):
 535             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 536             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 537             packages.append(bin_pkg)
 538         return packages
 539
 540     def _copy_efi_binary(self, efi_removable, efi_install):
 541         logging.debug("using bootdir=%s", self.bootdir)
 542         logging.debug("moving %s to %s", efi_removable, efi_install)
 543         if efi_removable.startswith('/'):
 544             efi_removable = efi_removable[1:]
 545         if efi_install.startswith('/'):
 546             efi_install = efi_install[1:]
 547         efi_output = os.path.join(self.bootdir, efi_removable)
 548         efi_input = os.path.join(self.bootdir, efi_install)
 549         if not os.path.exists(efi_input):
 550             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 551             raise cliapp.AppException("Missing %s" % efi_install)
 552         if not os.path.exists(os.path.dirname(efi_output)):
 553             os.makedirs(os.path.dirname(efi_output))
 554         logging.debug(
 555             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 556         if os.path.exists(efi_output):
 557             os.unlink(efi_output)
 558         os.rename(efi_input, efi_output)
 559
 560     def configure_efi(self):
 561         """
 562         Copy the bootloader file from the package into the location
 563         so needs to be after grub and kernel already installed.
 564         """
 565         self.message('Configuring EFI')
 566         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 567         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 568         self.message('Installing UEFI support binary')
 569         self._copy_efi_binary(efi_removable, efi_install)
 570
 571     def configure_extra_efi(self):
 572         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 573         if extra:
 574             efi_removable = str(self.efi_arch_table[extra]['removable'])
 575             efi_install = str(self.efi_arch_table[extra]['install'])
 576             self.message('Copying UEFI support binary for %s' % extra)
 577             self._copy_efi_binary(efi_removable, efi_install)
 578
 579     def mkfs(self, device, fstype):
 580         self.message('Creating filesystem %s' % fstype)
 581         self.runcmd(['mkfs', '-t', fstype, device])
 582
 583     def suite_to_codename(self, distro):
 584         suite = self.debian_info.codename(distro, datetime.date.today())
 585         if not suite:
 586             return distro
 587         return suite
 588
 589     def was_oldstable(self, limit):
 590         suite = self.suite_to_codename(self.settings['distribution'])
 591         # this check is only for debian
 592         if not self.debian_info.valid(suite):
 593             return False
 594         return suite == self.debian_info.old(limit)
 595
 596     def was_stable(self, limit):
 597         suite = self.suite_to_codename(self.settings['distribution'])
 598         # this check is only for debian
 599         if not self.debian_info.valid(suite):
 600             return False
 601         return suite == self.debian_info.stable(limit)
 602
 603     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 604         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 605         self.message(
 606             'Debootstrapping %s [%s] %s' % (
 607                 self.settings['distribution'], self.settings['arch'], msg))
 608
 609         include = self.settings['package']
 610
 611         if not self.settings['foreign'] and not self.settings['no-acpid']:
 612             include.append('acpid')
 613
 614         if self.settings['grub']:
 615             if self.settings['use-uefi']:
 616                 include.extend(self._efi_packages())
 617             else:
 618                 include.append('grub-pc')
 619
 620         if not self.settings['no-kernel']:
 621             if self.settings['kernel-package']:
 622                 kernel_image = self.settings['kernel-package']
 623             else:
 624                 if self.settings['arch'] == 'i386':
 625                     # wheezy (which became oldstable on 04/25/2015) used '486'
 626                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 627                         kernel_arch = '486'
 628                     else:
 629                         kernel_arch = '586'
 630                 elif self.settings['arch'] == 'armhf':
 631                     kernel_arch = 'armmp'
 632                 else:
 633                     kernel_arch = self.settings['arch']
 634                 kernel_image = 'linux-image-%s' % kernel_arch
 635             include.append(kernel_image)
 636
 637         if self.settings['sudo'] and 'sudo' not in include:
 638             include.append('sudo')
 639
 640         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 641
 642         if self.settings['package']:
 643             args.append(
 644                 '--include=%s' % ','.join(include))
 645         if self.settings['foreign']:
 646             args.append('--foreign')
 647         if self.settings['debootstrapopts']:
 648             for opt in self.settings['debootstrapopts']:
 649                 for part in opt.split(' '):
 650                     args.append('--%s' % part)
 651         elif self.settings['variant']:
 652             args.append('--variant')
 653             args.append(self.settings['variant'])
 654         args += [self.settings['distribution'],
 655                  rootdir, self.settings['mirror']]
 656         logging.debug(" ".join(args))
 657         self.runcmd(args)
 658         if self.settings['foreign']:
 659             # set a noninteractive debconf environment for secondstage
 660             env = {
 661                 "DEBIAN_FRONTEND": "noninteractive",
 662                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 663                 "LC_ALL": "C"
 664             }
 665             # add the mapping to the complete environment.
 666             env.update(os.environ)
 667             # First copy the binfmt handler over
 668             self.message('Setting up binfmt handler')
 669             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 670             # Next, run the package install scripts etc.
 671             self.message('Running debootstrap second stage')
 672             self.runcmd(['chroot', rootdir,
 673                          '/debootstrap/debootstrap', '--second-stage'],
 674                         env=env)
 675
 676     def set_hostname(self, rootdir):
 677         hostname = self.settings['hostname']
 678         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 679             f.write('%s\n' % hostname)
 680
 681         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 682         try:
 683             with open(etc_hosts, 'r') as f:
 684                 data = f.read()
 685             with open(etc_hosts, 'w') as f:
 686                 for line in data.splitlines():
 687                     if line.startswith('127.0.0.1'):
 688                         line += ' %s' % hostname
 689                     f.write('%s\n' % line)
 690         except IOError:
 691             pass
 692
 693     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 694         def fsuuid(device):
 695             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 696                                '-s', 'UUID', device])
 697             return out.splitlines()[0].strip()
 698
 699         if rootdev:
 700             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 701         else:
 702             rootdevstr = '/dev/sda1'
 703
 704         if bootdev and not self.settings['use-uefi']:
 705             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 706         else:
 707             bootdevstr = None
 708
 709         fstab = os.path.join(rootdir, 'etc', 'fstab')
 710         with open(fstab, 'w') as f:
 711             f.write('proc /proc proc defaults 0 0\n')
 712             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 713             if bootdevstr:
 714                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 715                 if self.settings['swap'] > 0:
 716                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 717             elif self.settings['swap'] > 0:
 718                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 719
 720     def install_debs(self, rootdir):
 721         if not self.settings['custom-package']:
 722             return
 723         self.message('Installing custom packages')
 724         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 725         os.mkdir(tmp)
 726         for deb in self.settings['custom-package']:
 727             shutil.copy(deb, tmp)
 728         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 729                      for deb in self.settings['custom-package']]
 730         out, err, _ = \
 731             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 732         logging.debug('stdout:\n%s', out)
 733         logging.debug('stderr:\n%s', err)
 734         out = self.runcmd(['chroot', rootdir,
 735                            'apt-get', '-f', '--no-remove', 'install'])
 736         logging.debug('stdout:\n%s', out)
 737         shutil.rmtree(tmp)
 738
 739     def cleanup_apt_cache(self, rootdir):
 740         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 741         logging.debug('stdout:\n%s', out)
 742
 743     def set_root_password(self, rootdir):
 744         if self.settings['root-password']:
 745             self.message('Setting root password')
 746             self.set_password(rootdir, 'root', self.settings['root-password'])
 747         elif self.settings['lock-root-password']:
 748             self.message('Locking root password')
 749             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 750         else:
 751             self.message('Give root an empty password')
 752             self.delete_password(rootdir, 'root')
 753
 754     def create_users(self, rootdir):
 755         def create_user(vmuser):
 756             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 757                          '--disabled-password', vmuser])
 758             if self.settings['sudo']:
 759                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 760
 761         for userpass in self.settings['user']:
 762             if '/' in userpass:
 763                 user, password = userpass.split('/', 1)
 764                 create_user(user)
 765                 self.set_password(rootdir, user, password)
 766             else:
 767                 create_user(userpass)
 768                 self.delete_password(rootdir, userpass)
 769
 770     def set_password(self, rootdir, user, password):
 771         encrypted = crypt.crypt(password, '..')
 772         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 773
 774     def delete_password(self, rootdir, user):
 775         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 776
 777     def remove_udev_persistent_rules(self, rootdir):
 778         self.message('Removing udev persistent cd and net rules')
 779         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 780             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 781             if os.path.exists(pathname):
 782                 logging.debug('rm %s', pathname)
 783                 os.remove(pathname)
 784             else:
 785                 logging.debug('not removing non-existent %s', pathname)
 786
 787     def setup_networking(self, rootdir):
 788         self.message('Setting up networking')
 789
 790         # unconditionally write for wheezy (which became oldstable on 04/25/2015)
 791         if self.was_oldstable(datetime.date(2015, 4, 26)):
 792             with open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w') as netfile:
 793                 netfile.write('source /etc/network/interfaces.d/*\n')
 794             os.mkdir(os.path.join(rootdir, 'etc', 'network', 'interfaces.d'))
 795
 796         elif not os.path.exists(os.path.join(rootdir, 'etc', 'network', 'interfaces')):
 797             iface_path = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 798             with open(iface_path, 'w') as netfile:
 799                 netfile.write('source-directory /etc/network/interfaces.d\n')
 800         ethpath = os.path.join(rootdir, 'etc', 'network', 'interfaces.d', 'setup')
 801         with open(ethpath, 'w') as eth:
 802             eth.write('auto lo\n')
 803             eth.write('iface lo inet loopback\n')
 804
 805             if self.settings['enable-dhcp']:
 806                 eth.write('\n')
 807                 eth.write('auto eth0\n')
 808                 eth.write('iface eth0 inet dhcp\n')
 809
 810     def append_serial_console(self, rootdir):
 811         if self.settings['serial-console']:
 812             serial_command = self.settings['serial-console-command']
 813             logging.debug('adding getty to serial console')
 814             inittab = os.path.join(rootdir, 'etc/inittab')
 815             # to autologin, serial_command can contain '-a root'
 816             with open(inittab, 'a') as f:
 817                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 818
 819     # pylint: disable=no-self-use
 820     def _grub_serial_console(self, rootdir):
 821         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 822         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 823         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 824         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 825         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 826         with open(grub_cfg, 'a+') as cfg:
 827             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 828             cfg.write("%s\n" % cmdline)
 829             cfg.write("%s\n" % terminal)
 830             cfg.write("%s\n" % command)
 831
 832     def _mount_wrapper(self, rootdir):
 833         self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 834                      '%s' % os.path.join(rootdir, 'dev')])
 835         self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 836                      '%s' % os.path.join(rootdir, 'proc')])
 837         self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 838                      '%s' % os.path.join(rootdir, 'sys')])
 839
 840     def _umount_wrapper(self, rootdir):
 841         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 842         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 843         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 844
 845     def install_grub_uefi(self, rootdir):
 846         self.message("Configuring grub-uefi")
 847         target = self.efi_arch_table[self.settings['arch']]['target']
 848         grub_opts = "--target=%s" % target
 849         logging.debug("Running grub-install with options: %s", grub_opts)
 850         self._mount_wrapper(rootdir)
 851         try:
 852             self.runcmd(['chroot', rootdir, 'update-grub'])
 853             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 854         except cliapp.AppException as exc:
 855             logging.warning(exc)
 856             self.message(
 857                 "Failed to configure grub-uefi for %s" %
 858                 self.settings['arch'])
 859             self._umount_wrapper(rootdir)
 860         self.configure_efi()
 861         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 862         if extra:
 863             target = self.efi_arch_table[extra]['target']
 864             grub_opts = "--target=%s" % target
 865             try:
 866                 self.runcmd(['chroot', rootdir, 'update-grub'])
 867                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 868             except cliapp.AppException as exc:
 869                 logging.warning(exc)
 870                 self.message(
 871                     "Failed to configure grub-uefi for %s" % extra)
 872             self.configure_extra_efi()
 873         self._umount_wrapper(rootdir)
 874
 875     def install_grub2(self, rootdev, rootdir):
 876         self.message("Configuring grub2")
 877         # rely on kpartx using consistent naming to map loop0p1 to loop0
 878         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 879         if self.settings['serial-console']:
 880             self._grub_serial_console(rootdir)
 881         logging.debug("Running grub-install with options: %s", grub_opts)
 882         self._mount_wrapper(rootdir)
 883         try:
 884             self.runcmd(['chroot', rootdir, 'update-grub'])
 885             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 886         except cliapp.AppException as exc:
 887             logging.warning(exc)
 888             self.message("Failed. Is grub2-common installed? Using extlinux.")
 889             self.install_extlinux(rootdev, rootdir)
 890         self._umount_wrapper(rootdir)
 891
 892     def install_extlinux(self, rootdev, rootdir):
 893         if not os.path.exists("/usr/bin/extlinux"):
 894             self.message("extlinux not installed, skipping.")
 895             return
 896         self.message('Installing extlinux')
 897
 898         def find(pattern):
 899             dirname = os.path.join(rootdir, 'boot')
 900             basenames = os.listdir(dirname)
 901             logging.debug('find: %s', basenames)
 902             for basename in basenames:
 903                 if re.search(pattern, basename):
 904                     return os.path.join('boot', basename)
 905             raise cliapp.AppException('Cannot find match: %s' % pattern)
 906
 907         try:
 908             kernel_image = find('vmlinuz-.*')
 909             initrd_image = find('initrd.img-.*')
 910         except cliapp.AppException as e:
 911             self.message("Unable to find kernel. Not installing extlinux.")
 912             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 913             return
 914
 915         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 916                            '-s', 'UUID', rootdev])
 917         uuid = out.splitlines()[0].strip()
 918
 919         conf = os.path.join(rootdir, 'extlinux.conf')
 920         logging.debug('configure extlinux %s', conf)
 921         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 922         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 923         msg = '''
 924 default linux
 925 timeout 1
 926
 927 label linux
 928 kernel %(kernel)s
 929 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 930 %(extserial)s
 931 ''' % {
 932             'kernel': kernel_image,  # pylint: disable=bad-continuation
 933             'initrd': initrd_image,  # pylint: disable=bad-continuation
 934             'uuid': uuid,  # pylint: disable=bad-continuation
 935             'kserial': kserial,  # pylint: disable=bad-continuation
 936             'extserial': extserial,  # pylint: disable=bad-continuation
 937         }  # pylint: disable=bad-continuation
 938         logging.debug("extlinux config:\n%s", msg)
 939
 940         # python multiline string substitution is just ugly.
 941         # use an external file or live with the mangling, no point in
 942         # mangling the string to remove spaces just to keep it pretty in source.
 943         f = open(conf, 'w')
 944         f.write(msg)
 945
 946         self.runcmd(['extlinux', '--install', rootdir])
 947         self.runcmd(['sync'])
 948         time.sleep(2)
 949
 950     def optimize_image(self, rootdir):
 951         """
 952         Filing up the image with zeros will increase its compression rate
 953         """
 954         if not self.settings['sparse']:
 955             zeros = os.path.join(rootdir, 'ZEROS')
 956             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 957             self.runcmd(['rm', '-f', zeros])
 958
 959     def squash(self):
 960         """
 961         Run squashfs on the image.
 962         """
 963         if not os.path.exists('/usr/bin/mksquashfs'):
 964             logging.warning("Squash selected but mksquashfs not found!")
 965             return
 966         logging.debug(
 967             "%s usage: %s", self.settings['image'],
 968             self.runcmd(['du', self.settings['image']]))
 969         self.message("Running mksquashfs")
 970         suffixed = "%s.squashfs" % self.settings['image']
 971         if os.path.exists(suffixed):
 972             os.unlink(suffixed)
 973         msg = self.runcmd(
 974             ['mksquashfs', self.settings['image'],
 975              suffixed,
 976              '-no-progress', '-comp', 'xz'], ignore_fail=False)
 977         logging.debug(msg)
 978         check_size = os.path.getsize(suffixed)
 979         if check_size < (1024 * 1024):
 980             logging.warning(
 981                 "%s appears to be too small! %s bytes",
 982                 suffixed, check_size)
 983         else:
 984             logging.debug("squashed size: %s", check_size)
 985             os.unlink(self.settings['image'])
 986             self.settings['image'] = suffixed
 987             logging.debug(
 988                 "%s usage: %s", self.settings['image'],
 989                 self.runcmd(['du', self.settings['image']]))
 990
 991     def cleanup_system(self):
 992         # Clean up after any errors.
 993
 994         self.message('Cleaning up')
 995
 996         # Umount in the reverse mount order
 997         if self.settings['image']:
 998             for i in range(len(self.mount_points) - 1, -1, -1):
 999                 mount_point = self.mount_points[i]
1000                 try:
1001                     self.runcmd(['umount', mount_point], ignore_fail=False)
1002                 except cliapp.AppException:
1003                     logging.debug("umount failed, sleeping and trying again")
1004                     time.sleep(5)
1005                     self.runcmd(['umount', mount_point], ignore_fail=False)
1006
1007             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1008
1009         for dirname in self.remove_dirs:
1010             shutil.rmtree(dirname)
1011
1012     def customize(self, rootdir):
1013         script = self.settings['customize']
1014         if not script:
1015             return
1016         if not os.path.exists(script):
1017             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1018             if not os.path.exists(example):
1019                 self.message("Unable to find %s" % script)
1020                 return
1021             script = example
1022         self.message('Running customize script %s' % script)
1023         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1024         logging.debug(
1025             "%s usage: %s", self.settings['image'],
1026             self.runcmd(['du', self.settings['image']]))
1027         with open('/dev/tty', 'w') as tty:
1028             try:
1029                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1030             except IOError:
1031                 subprocess.call([script, rootdir, self.settings['image']])
1032         logging.debug(
1033             "%s usage: %s", self.settings['image'],
1034             self.runcmd(['du', self.settings['image']]))
1035
1036     def create_tarball(self, rootdir):
1037         # Create a tarball of the disk's contents
1038         # shell out to runcmd since it more easily handles rootdir
1039         self.message('Creating tarball of disk contents')
1040         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1041
1042     def chown(self):
1043         # Change image owner after completed build
1044         if self.settings['image']:
1045             filename = self.settings['image']
1046         elif self.settings['tarball']:
1047             filename = self.settings['tarball']
1048         else:
1049             return
1050         self.message("Changing owner to %s" % self.settings["owner"])
1051         subprocess.call(["chown", self.settings["owner"], filename])
1052
1053     def list_installed_pkgs(self, rootdir):
1054         # output the list of installed packages for sources identification
1055         self.message("Creating a list of installed binary package names")
1056         out = self.runcmd(['chroot', rootdir,
1057                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1058         with open('dpkg.list', 'w') as dpkg:
1059             dpkg.write(out)
1060
1061     def configure_apt(self, rootdir):
1062         # use the distribution and mirror to create an apt source
1063         self.message("Configuring apt to use distribution and mirror")
1064         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1065         logging.debug('configure apt %s', conf)
1066         mirror = self.settings['mirror']
1067         if self.settings['apt-mirror']:
1068             mirror = self.settings['apt-mirror']
1069             self.message("Setting apt mirror to %s" % mirror)
1070         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1071         f = open(conf, 'w')
1072         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1073         f.write(line)
1074         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1075         f.write(line)
1076         f.close()
1077         # ensure the apt sources have valid lists
1078         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1079
1080 if __name__ == '__main__':
1081     VmDebootstrap(version=__version__).run()