vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import re
  24 import sys
  25 import shutil
  26 import datetime
  27 import subprocess
  28 import tempfile
  29 import time
  30 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  31
  32
  33 __version__ = '0.11'
  34
  35 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  36
  37
  38 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  39
  40     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  41         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  42         self.remove_dirs = []
  43         self.mount_points = []
  44         self.debian_info = DebianDistroInfo()
  45         self.ubuntu_info = UbuntuDistroInfo()
  46         self.bootdir = None
  47         self.efi_arch_table = {
  48             'amd64': {
  49                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  50                 'install': '/EFI/debian/grubx64.efi',  # package location
  51                 'package': 'grub-efi-amd64',  # bootstrap package
  52                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  53                 'extra': 'i386',  # architecture to add binary package
  54                 'exclusive': False,  # only EFI supported for this arch.
  55                 'target': 'x86_64-efi',  # grub target name
  56             },
  57             'i386': {
  58                 'removable': '/EFI/boot/bootia32.efi',
  59                 'install': '/EFI/debian/grubia32.efi',
  60                 'package': 'grub-efi-ia32',
  61                 'bin_package': 'grub-efi-ia32-bin',
  62                 'extra': None,
  63                 'exclusive': False,
  64                 'target': 'i386-efi',
  65             },
  66             'arm64': {
  67                 'removable': '/EFI/boot/bootaa64.efi',
  68                 'install': '/EFI/debian/grubaa64.efi',
  69                 'package': 'grub-efi-arm64',
  70                 'bin_package': 'grub-efi-arm64-bin',
  71                 'extra': None,
  72                 'exclusive': True,
  73                 'target': 'arm64-efi',
  74             }
  75         }
  76
  77     def add_settings(self):
  78         default_arch = subprocess.check_output(
  79             ["dpkg", "--print-architecture"]).strip()
  80
  81         self.settings.boolean(
  82             ['verbose'], 'report what is going on')
  83         self.settings.string(
  84             ['image'], 'put created disk image in FILE',
  85             metavar='FILE')
  86         self.settings.bytesize(
  87             ['size'],
  88             'create a disk image of size SIZE (%default)',
  89             metavar='SIZE',
  90             default='1G')
  91         self.settings.bytesize(
  92             ['bootsize'],
  93             'create boot partition of size SIZE (%default)',
  94             metavar='BOOTSIZE',
  95             default='0%')
  96         self.settings.string(
  97             ['boottype'],
  98             'specify file system type for /boot/',
  99             default='ext2')
 100         self.settings.bytesize(
 101             ['bootoffset'],
 102             'Space to leave at start of the image for bootloader',
 103             default='0')
 104         self.settings.boolean(
 105             ['use-uefi'],
 106             'Setup image for UEFI boot',
 107             default=False)
 108         self.settings.bytesize(
 109             ['esp-size'],
 110             'Size of EFI System Partition - requires use-uefi',
 111             default='5mib')
 112         self.settings.string(
 113             ['part-type'],
 114             'Partition type to use for this image',
 115             default='msdos')
 116         self.settings.string(
 117             ['roottype'],
 118             'specify file system type for /',
 119             default='ext4')
 120         self.settings.bytesize(
 121             ['swap'],
 122             'create swap space of size SIZE (min 256Mb)')
 123         self.settings.string(
 124             ['foreign'],
 125             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 126         self.settings.string(
 127             ['variant'],
 128             'select debootstrap variant if not using the default [deprecated]')
 129         self.settings.string_list(
 130             ['debootstrapopts'],
 131             'pass additional options to debootstrap'),
 132         self.settings.boolean(
 133             ['extlinux'],
 134             'install extlinux?',
 135             default=True)
 136         self.settings.string(
 137             ['tarball'],
 138             "tar up the disk's contents in FILE",
 139             metavar='FILE')
 140         self.settings.string(
 141             ['apt-mirror'],
 142             'configure apt to use MIRROR',
 143             metavar='URL')
 144         self.settings.string(
 145             ['mirror'],
 146             'use MIRROR as package source (%default)',
 147             metavar='URL',
 148             default='http://http.debian.net/debian/')
 149         self.settings.string(
 150             ['arch'],
 151             'architecture to use (%default)',
 152             metavar='ARCH',
 153             default=default_arch)
 154         self.settings.string(
 155             ['distribution'],
 156             'release to use (%default)',
 157             metavar='NAME',
 158             default='stable')
 159         self.settings.string_list(
 160             ['package'],
 161             'install PACKAGE onto system')
 162         self.settings.string_list(
 163             ['custom-package'],
 164             'install package in DEB file onto system (not from mirror)',
 165             metavar='DEB')
 166         self.settings.boolean(
 167             ['no-kernel'],
 168             'do not install a linux package')
 169         self.settings.string(
 170             ['kernel-package'],
 171             'install PACKAGE instead of the default kernel package',
 172             metavar='PACKAGE')
 173         self.settings.boolean(
 174             ['enable-dhcp'],
 175             'enable DHCP on eth0')
 176         self.settings.string(
 177             ['root-password'],
 178             'set root password',
 179             metavar='PASSWORD')
 180         self.settings.boolean(
 181             ['lock-root-password'],
 182             'lock root account so they cannot login?')
 183         self.settings.string(
 184             ['customize'],
 185             'run SCRIPT after setting up system',
 186             metavar='SCRIPT')
 187         self.settings.string(
 188             ['hostname'],
 189             'set name to HOSTNAME (%default)',
 190             metavar='HOSTNAME',
 191             default='debian')
 192         self.settings.string_list(
 193             ['user'],
 194             'create USER with PASSWORD',
 195             metavar='USER/PASSWORD')
 196         self.settings.boolean(
 197             ['serial-console'],
 198             'configure image to use a serial console')
 199         self.settings.string(
 200             ['serial-console-command'],
 201             'command to manage the serial console, appended to /etc/inittab (%default)',
 202             metavar='COMMAND',
 203             default='/sbin/getty -L ttyS0 115200 vt100')
 204         self.settings.boolean(
 205             ['sudo'],
 206             'install sudo, and if user is created, add them to sudo group')
 207         self.settings.string(
 208             ['owner'],
 209             'the user who will own the image when the build is complete.')
 210         self.settings.boolean(
 211             ['squash'],
 212             'use squashfs on the final image.')
 213         self.settings.boolean(
 214             ['configure-apt'],
 215             'Create an apt source based on the distribution and mirror selected.')
 216         self.settings.boolean(
 217             ['mbr'],
 218             'Run install-mbr (default if extlinux used)')
 219         self.settings.boolean(
 220             ['grub'],
 221             'Install and configure grub2 - disables extlinux.')
 222         self.settings.boolean(
 223             ['sparse'],
 224             'Do not fill the image with zeros to keep a sparse disk image',
 225             default=False)
 226         self.settings.boolean(
 227             ['pkglist'],
 228             'Create a list of package names included in the image.')
 229         self.settings.boolean(
 230             ['no-acpid'],
 231             'do not install the acpid package',
 232             default=False)
 233
 234     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 235         if not self.settings['image'] and not self.settings['tarball']:
 236             raise cliapp.AppException(
 237                 'You must give disk image filename, or tarball filename')
 238         if self.settings['image'] and not self.settings['size']:
 239             raise cliapp.AppException(
 240                 'If disk image is specified, you must give image size.')
 241         if not self.debian_info.valid(self.settings['distribution']):
 242             if not self.ubuntu_info.valid(self.settings['distribution']):
 243                 raise cliapp.AppException(
 244                     '%s is not a valid Debian or Ubuntu suite or codename.'
 245                     % self.settings['distribution'])
 246         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 247             raise cliapp.AppException(
 248                 'You must specify use-uefi for esp-size to have effect')
 249         if self.settings['arch'] in self.efi_arch_table and\
 250                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 251                 not self.settings['use-uefi']:
 252             raise cliapp.AppException(
 253                 'Only UEFI is supported on %s' % self.settings['arch'])
 254         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 255             raise cliapp.AppException(
 256                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 257         if self.settings['use-uefi'] and (
 258                 self.settings['bootsize'] or
 259                 self.settings['bootoffset']):
 260             raise cliapp.AppException(
 261                 'A separate boot partition is not supported with UEFI')
 262
 263         if self.settings['use-uefi'] and not self.settings['grub']:
 264             raise cliapp.AppException(
 265                 'UEFI without Grub is not supported.')
 266
 267         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 268         if self.was_oldstable(datetime.date(2015, 4, 26)):
 269             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 270                 raise cliapp.AppException(
 271                     'Only amd64 supports UEFI in Wheezy')
 272
 273         if os.geteuid() != 0:
 274             sys.exit("You need to have root privileges to run this script.")
 275         rootdir = None
 276         try:
 277             rootdev = None
 278             roottype = self.settings['roottype']
 279             bootdev = None
 280             boottype = None
 281             if self.settings['image']:
 282                 self.create_empty_image()
 283                 self.partition_image()
 284                 if self.settings['mbr'] or self.settings['extlinux']:
 285                     self.install_mbr()
 286                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 287                 if self.settings['swap'] > 0:
 288                     self.message("Creating swap space")
 289                     self.runcmd(['mkswap', swapdev])
 290                 self.mkfs(rootdev, fstype=roottype)
 291                 rootdir = self.mount(rootdev)
 292                 if self.settings['use-uefi']:
 293                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 294                     logging.debug("bootdir:%s", self.bootdir)
 295                     self.mkfs(bootdev, fstype='vfat')
 296                     os.makedirs(self.bootdir)
 297                     self.mount(bootdev, self.bootdir)
 298                 elif bootdev:
 299                     if self.settings['boottype']:
 300                         boottype = self.settings['boottype']
 301                     else:
 302                         boottype = 'ext2'
 303                     self.mkfs(bootdev, fstype=boottype)
 304                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 305                     os.mkdir(self.bootdir)
 306                     self.mount(bootdev, self.bootdir)
 307             else:
 308                 rootdir = self.mkdtemp()
 309             self.debootstrap(rootdir)
 310             self.set_hostname(rootdir)
 311             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 312             self.install_debs(rootdir)
 313             self.set_root_password(rootdir)
 314             self.create_users(rootdir)
 315             self.remove_udev_persistent_rules(rootdir)
 316             self.setup_networking(rootdir)
 317             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 318                 self.configure_apt(rootdir)
 319             self.customize(rootdir)
 320             self.cleanup_apt_cache(rootdir)
 321             self.update_initramfs(rootdir)
 322
 323             if self.settings['image']:
 324                 if self.settings['use-uefi']:
 325                     self.install_grub_uefi(rootdir)
 326                 elif self.settings['grub']:
 327                     self.install_grub2(rootdev, rootdir)
 328                 elif self.settings['extlinux']:
 329                     self.install_extlinux(rootdev, rootdir)
 330                 self.append_serial_console(rootdir)
 331                 self.optimize_image(rootdir)
 332                 if self.settings['squash']:
 333                     self.squash()
 334             if self.settings['pkglist']:
 335                 self.list_installed_pkgs(rootdir)
 336
 337             if self.settings['foreign']:
 338                 os.unlink('%s/usr/bin/%s' %
 339                           (rootdir, os.path.basename(self.settings['foreign'])))
 340
 341             if self.settings['tarball']:
 342                 self.create_tarball(rootdir)
 343
 344             if self.settings['owner']:
 345                 self.chown()
 346         except BaseException as e:
 347             self.message('EEEK! Something bad happened...')
 348             if rootdir:
 349                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 350                 if os.path.exists(db_log):
 351                     shutil.copy(db_log, os.getcwd())
 352             self.message(e)
 353             self.cleanup_system()
 354             raise
 355         else:
 356             self.cleanup_system()
 357
 358     def message(self, msg):
 359         logging.info(msg)
 360         if self.settings['verbose']:
 361             print msg
 362
 363     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 364         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 365         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 366                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 367                              env=env, **kwargs)
 368         out, err = p.communicate(stdin)
 369         if p.returncode != 0:
 370             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 371             logging.error(msg)
 372             if not ignore_fail:
 373                 raise cliapp.AppException(msg)
 374         return out
 375
 376     def mkdtemp(self):
 377         dirname = tempfile.mkdtemp()
 378         self.remove_dirs.append(dirname)
 379         logging.debug('mkdir %s', dirname)
 380         return dirname
 381
 382     def mount(self, device, path=None):
 383         if not path:
 384             mount_point = self.mkdtemp()
 385         else:
 386             mount_point = path
 387         self.message('Mounting %s on %s' % (device, mount_point))
 388         self.runcmd(['mount', device, mount_point])
 389         self.mount_points.append(mount_point)
 390         logging.debug('mounted %s on %s', device, mount_point)
 391         return mount_point
 392
 393     def create_empty_image(self):
 394         self.message('Creating disk image')
 395         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 396                      self.settings['image'],
 397                      str(self.settings['size'])])
 398
 399     def partition_image(self):
 400         """
 401         Uses fat16 (msdos) partitioning by default, use part-type to change.
 402         If bootoffset is specified, the first actual partition
 403         starts at that offset to allow customisation scripts to
 404         put bootloader images into the space, e.g. u-boot.
 405         """
 406         self.message('Creating partitions')
 407         self.runcmd(['parted', '-s', self.settings['image'],
 408                      'mklabel', self.settings['part-type']])
 409         partoffset = 0
 410         extent = '100%'
 411         swap = 256 * 1024 * 1024
 412         if self.settings['swap'] > 0:
 413             if self.settings['swap'] > swap:
 414                 swap = self.settings['swap']
 415             else:
 416                 # minimum 256Mb as default qemu ram is 128Mb
 417                 logging.debug("Setting minimum 256Mb swap space")
 418             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 419
 420         if self.settings['use-uefi']:
 421             espsize = self.settings['esp-size'] / (1024 * 1024)
 422             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 423             self.runcmd(['parted', '-s', self.settings['image'],
 424                          'mkpart', 'primary', 'fat32',
 425                          '1', str(espsize)])
 426             self.runcmd(['parted', '-s', self.settings['image'],
 427                          'set', '1', 'boot', 'on'])
 428             self.runcmd(['parted', '-s', self.settings['image'],
 429                          'set', '1', 'esp', 'on'])
 430
 431         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 432             # turn v.small offsets into something at least possible to create.
 433             if self.settings['bootoffset'] < 1048576:
 434                 partoffset = 1
 435                 logging.info(
 436                     "Setting bootoffset %smib to allow for %s bytes",
 437                     partoffset, self.settings['bootoffset'])
 438             else:
 439                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 440                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 441         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 442             if self.settings['grub'] and not partoffset:
 443                 partoffset = 1
 444             bootsize = self.settings['bootsize'] / (1024 * 1024)
 445             bootsize += partoffset
 446             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 447             logging.debug("Starting boot partition at %sMb", bootsize)
 448             self.runcmd(['parted', '-s', self.settings['image'],
 449                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 450             logging.debug("Starting root partition at %sMb", partoffset)
 451             self.runcmd(['parted', '-s', self.settings['image'],
 452                          'mkpart', 'primary', str(bootsize), extent])
 453         elif self.settings['use-uefi']:
 454             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 455             self.runcmd(['parted', '-s', self.settings['image'],
 456                          'mkpart', 'primary', str(bootsize), extent])
 457         else:
 458             self.runcmd(['parted', '-s', self.settings['image'],
 459                          'mkpart', 'primary', '0%', extent])
 460         self.runcmd(['parted', '-s', self.settings['image'],
 461                      'set', '1', 'boot', 'on'])
 462         if self.settings['swap'] > 0:
 463             logging.debug("Creating swap partition")
 464             self.runcmd(['parted', '-s', self.settings['image'],
 465                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 466
 467     def update_initramfs(self, rootdir):
 468         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 469         if os.path.exists(os.path.join(rootdir, cmd)):
 470             self.message("Updating the initramfs")
 471             self.runcmd(['chroot', rootdir, cmd, '-u'])
 472
 473     def install_mbr(self):
 474         if os.path.exists("/sbin/install-mbr"):
 475             self.message('Installing MBR')
 476             self.runcmd(['install-mbr', self.settings['image']])
 477         else:
 478             msg = "mbr enabled but /sbin/install-mbr not found" \
 479                   " - please install the mbr package."
 480             raise cliapp.AppException(msg)
 481
 482     def setup_kpartx(self):
 483         bootindex = None
 484         swapindex = None
 485         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 486         if self.settings['bootsize'] and self.settings['swap'] > 0:
 487             bootindex = 0
 488             rootindex = 1
 489             swapindex = 2
 490             parts = 3
 491         elif self.settings['use-uefi']:
 492             bootindex = 0
 493             rootindex = 1
 494             parts = 2
 495         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 496             bootindex = 0
 497             rootindex = 1
 498             swapindex = 2
 499             parts = 3
 500         elif self.settings['bootsize']:
 501             bootindex = 0
 502             rootindex = 1
 503             parts = 2
 504         elif self.settings['swap'] > 0:
 505             rootindex = 0
 506             swapindex = 1
 507             parts = 2
 508         else:
 509             rootindex = 0
 510             parts = 1
 511         boot = None
 512         swap = None
 513         devices = [line.split()[2]
 514                    for line in out.splitlines()
 515                    if line.startswith('add map ')]
 516         if len(devices) != parts:
 517             msg = 'Surprising number of partitions - check output of losetup -a'
 518             logging.debug("%s", self.runcmd(['losetup', '-a']))
 519             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 520             raise cliapp.AppException(msg)
 521         root = '/dev/mapper/%s' % devices[rootindex]
 522         if self.settings['bootsize'] or self.settings['use-uefi']:
 523             boot = '/dev/mapper/%s' % devices[bootindex]
 524         if self.settings['swap'] > 0:
 525             swap = '/dev/mapper/%s' % devices[swapindex]
 526         return root, boot, swap
 527
 528     def _efi_packages(self):
 529         packages = []
 530         pkg = self.efi_arch_table[self.settings['arch']]['package']
 531         self.message("Adding %s" % pkg)
 532         packages.append(pkg)
 533         extra = self.efi_arch_table[self.settings['arch']]['extra']
 534         if extra and isinstance(extra, str):
 535             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 536             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 537             packages.append(bin_pkg)
 538         return packages
 539
 540     def _copy_efi_binary(self, efi_removable, efi_install):
 541         logging.debug("using bootdir=%s", self.bootdir)
 542         logging.debug("moving %s to %s", efi_removable, efi_install)
 543         if efi_removable.startswith('/'):
 544             efi_removable = efi_removable[1:]
 545         if efi_install.startswith('/'):
 546             efi_install = efi_install[1:]
 547         efi_output = os.path.join(self.bootdir, efi_removable)
 548         efi_input = os.path.join(self.bootdir, efi_install)
 549         if not os.path.exists(efi_input):
 550             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 551             raise cliapp.AppException("Missing %s" % efi_install)
 552         if not os.path.exists(os.path.dirname(efi_output)):
 553             os.makedirs(os.path.dirname(efi_output))
 554         logging.debug(
 555             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 556         if os.path.exists(efi_output):
 557             os.unlink(efi_output)
 558         os.rename(efi_input, efi_output)
 559
 560     def configure_efi(self):
 561         """
 562         Copy the bootloader file from the package into the location
 563         so needs to be after grub and kernel already installed.
 564         """
 565         self.message('Configuring EFI')
 566         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 567         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 568         self.message('Installing UEFI support binary')
 569         self._copy_efi_binary(efi_removable, efi_install)
 570
 571     def configure_extra_efi(self):
 572         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 573         if extra:
 574             efi_removable = str(self.efi_arch_table[extra]['removable'])
 575             efi_install = str(self.efi_arch_table[extra]['install'])
 576             self.message('Copying UEFI support binary for %s' % extra)
 577             self._copy_efi_binary(efi_removable, efi_install)
 578
 579     def mkfs(self, device, fstype):
 580         self.message('Creating filesystem %s' % fstype)
 581         self.runcmd(['mkfs', '-t', fstype, device])
 582
 583     def suite_to_codename(self, distro):
 584         suite = self.debian_info.codename(distro, datetime.date.today())
 585         if not suite:
 586             return distro
 587         return suite
 588
 589     def was_oldstable(self, limit):
 590         suite = self.suite_to_codename(self.settings['distribution'])
 591         # this check is only for debian
 592         if not self.debian_info.valid(suite):
 593             return False
 594         return suite == self.debian_info.old(limit)
 595
 596     def was_stable(self, limit):
 597         suite = self.suite_to_codename(self.settings['distribution'])
 598         # this check is only for debian
 599         if not self.debian_info.valid(suite):
 600             return False
 601         return suite == self.debian_info.stable(limit)
 602
 603     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 604         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 605         self.message(
 606             'Debootstrapping %s [%s] %s' % (
 607                 self.settings['distribution'], self.settings['arch'], msg))
 608
 609         include = self.settings['package']
 610
 611         if not self.settings['foreign'] and not self.settings['no-acpid']:
 612             include.append('acpid')
 613
 614         if self.settings['grub']:
 615             if self.settings['use-uefi']:
 616                 include.extend(self._efi_packages())
 617             else:
 618                 include.append('grub-pc')
 619
 620         if not self.settings['no-kernel']:
 621             if self.settings['kernel-package']:
 622                 kernel_image = self.settings['kernel-package']
 623             else:
 624                 if self.settings['arch'] == 'i386':
 625                     # wheezy (which became oldstable on 04/25/2015) used '486'
 626                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 627                         kernel_arch = '486'
 628                     else:
 629                         kernel_arch = '586'
 630                 elif self.settings['arch'] == 'armhf':
 631                     kernel_arch = 'armmp'
 632                 else:
 633                     kernel_arch = self.settings['arch']
 634                 kernel_image = 'linux-image-%s' % kernel_arch
 635             include.append(kernel_image)
 636
 637         if self.settings['sudo'] and 'sudo' not in include:
 638             include.append('sudo')
 639
 640         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 641
 642         if self.settings['package']:
 643             args.append(
 644                 '--include=%s' % ','.join(include))
 645         if self.settings['foreign']:
 646             args.append('--foreign')
 647         if self.settings['debootstrapopts']:
 648             for opt in self.settings['debootstrapopts']:
 649                 for part in opt.split(' '):
 650                     args.append('--%s' % part)
 651         elif self.settings['variant']:
 652             args.append('--variant')
 653             args.append(self.settings['variant'])
 654         args += [self.settings['distribution'],
 655                  rootdir, self.settings['mirror']]
 656         logging.debug(" ".join(args))
 657         self.runcmd(args)
 658         if self.settings['foreign']:
 659             # set a noninteractive debconf environment for secondstage
 660             env = {
 661                 "DEBIAN_FRONTEND": "noninteractive",
 662                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 663                 "LC_ALL": "C"
 664             }
 665             # add the mapping to the complete environment.
 666             env.update(os.environ)
 667             # First copy the binfmt handler over
 668             self.message('Setting up binfmt handler')
 669             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 670             # Next, run the package install scripts etc.
 671             self.message('Running debootstrap second stage')
 672             self.runcmd(['chroot', rootdir,
 673                          '/debootstrap/debootstrap', '--second-stage'],
 674                         env=env)
 675
 676     def set_hostname(self, rootdir):
 677         hostname = self.settings['hostname']
 678         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 679             f.write('%s\n' % hostname)
 680
 681         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 682         try:
 683             with open(etc_hosts, 'r') as f:
 684                 data = f.read()
 685             with open(etc_hosts, 'w') as f:
 686                 for line in data.splitlines():
 687                     if line.startswith('127.0.0.1'):
 688                         line += ' %s' % hostname
 689                     f.write('%s\n' % line)
 690         except IOError:
 691             pass
 692
 693     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 694         def fsuuid(device):
 695             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 696                                '-s', 'UUID', device])
 697             return out.splitlines()[0].strip()
 698
 699         if rootdev:
 700             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 701         else:
 702             rootdevstr = '/dev/sda1'
 703
 704         if bootdev and not self.settings['use-uefi']:
 705             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 706         else:
 707             bootdevstr = None
 708
 709         fstab = os.path.join(rootdir, 'etc', 'fstab')
 710         with open(fstab, 'w') as f:
 711             f.write('proc /proc proc defaults 0 0\n')
 712             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 713             if bootdevstr:
 714                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 715                 if self.settings['swap'] > 0:
 716                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 717             elif self.settings['swap'] > 0:
 718                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 719
 720     def install_debs(self, rootdir):
 721         if not self.settings['custom-package']:
 722             return
 723         self.message('Installing custom packages')
 724         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 725         os.mkdir(tmp)
 726         for deb in self.settings['custom-package']:
 727             shutil.copy(deb, tmp)
 728         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 729                      for deb in self.settings['custom-package']]
 730         out, err, _ = \
 731             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 732         logging.debug('stdout:\n%s', out)
 733         logging.debug('stderr:\n%s', err)
 734         out = self.runcmd(['chroot', rootdir,
 735                            'apt-get', '-f', '--no-remove', 'install'])
 736         logging.debug('stdout:\n%s', out)
 737         shutil.rmtree(tmp)
 738
 739     def cleanup_apt_cache(self, rootdir):
 740         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 741         logging.debug('stdout:\n%s', out)
 742
 743     def set_root_password(self, rootdir):
 744         if self.settings['root-password']:
 745             self.message('Setting root password')
 746             self.set_password(rootdir, 'root', self.settings['root-password'])
 747         elif self.settings['lock-root-password']:
 748             self.message('Locking root password')
 749             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 750         else:
 751             self.message('Give root an empty password')
 752             self.delete_password(rootdir, 'root')
 753
 754     def create_users(self, rootdir):
 755         def create_user(vmuser):
 756             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 757                          '--disabled-password', vmuser])
 758             if self.settings['sudo']:
 759                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 760
 761         for userpass in self.settings['user']:
 762             if '/' in userpass:
 763                 user, password = userpass.split('/', 1)
 764                 create_user(user)
 765                 self.set_password(rootdir, user, password)
 766             else:
 767                 create_user(userpass)
 768                 self.delete_password(rootdir, userpass)
 769
 770     def set_password(self, rootdir, user, password):
 771         encrypted = crypt.crypt(password, '..')
 772         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 773
 774     def delete_password(self, rootdir, user):
 775         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 776
 777     def remove_udev_persistent_rules(self, rootdir):
 778         self.message('Removing udev persistent cd and net rules')
 779         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 780             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 781             if os.path.exists(pathname):
 782                 logging.debug('rm %s', pathname)
 783                 os.remove(pathname)
 784             else:
 785                 logging.debug('not removing non-existent %s', pathname)
 786
 787     def setup_networking(self, rootdir):
 788         self.message('Setting up networking')
 789         ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 790         ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d')
 791
 792         # unconditionally write for wheezy (which became oldstable 2015.04.25)
 793         if self.was_oldstable(datetime.date(2015, 4, 26)):
 794             with open(ifc_file, 'w') as netfile:
 795                 netfile.write('source /etc/network/interfaces.d/*\n')
 796         elif not os.path.exists(ifc_file):
 797             with open(ifc_file, 'a') as netfile:
 798                 netfile.write('source-directory /etc/network/interfaces.d\n')
 799
 800         if not os.path.exists(ifc_d):
 801             os.mkdir(ifc_d)
 802         ethpath = os.path.join(ifc_d, 'setup')
 803         with open(ethpath, 'w') as eth:
 804             eth.write('auto lo\n')
 805             eth.write('iface lo inet loopback\n')
 806
 807             if self.settings['enable-dhcp']:
 808                 eth.write('\n')
 809                 eth.write('auto eth0\n')
 810                 eth.write('iface eth0 inet dhcp\n')
 811
 812     def append_serial_console(self, rootdir):
 813         if self.settings['serial-console']:
 814             serial_command = self.settings['serial-console-command']
 815             logging.debug('adding getty to serial console')
 816             inittab = os.path.join(rootdir, 'etc/inittab')
 817             # to autologin, serial_command can contain '-a root'
 818             with open(inittab, 'a') as f:
 819                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 820
 821     # pylint: disable=no-self-use
 822     def _grub_serial_console(self, rootdir):
 823         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 824         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 825         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 826         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 827         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 828         with open(grub_cfg, 'a+') as cfg:
 829             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 830             cfg.write("%s\n" % cmdline)
 831             cfg.write("%s\n" % terminal)
 832             cfg.write("%s\n" % command)
 833
 834     def _mount_wrapper(self, rootdir):
 835         self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 836                      '%s' % os.path.join(rootdir, 'dev')])
 837         self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 838                      '%s' % os.path.join(rootdir, 'proc')])
 839         self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 840                      '%s' % os.path.join(rootdir, 'sys')])
 841
 842     def _umount_wrapper(self, rootdir):
 843         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 844         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 845         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 846
 847     def install_grub_uefi(self, rootdir):
 848         self.message("Configuring grub-uefi")
 849         target = self.efi_arch_table[self.settings['arch']]['target']
 850         grub_opts = "--target=%s" % target
 851         logging.debug("Running grub-install with options: %s", grub_opts)
 852         self._mount_wrapper(rootdir)
 853         try:
 854             self.runcmd(['chroot', rootdir, 'update-grub'])
 855             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 856         except cliapp.AppException as exc:
 857             logging.warning(exc)
 858             self.message(
 859                 "Failed to configure grub-uefi for %s" %
 860                 self.settings['arch'])
 861             self._umount_wrapper(rootdir)
 862         self.configure_efi()
 863         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 864         if extra:
 865             target = self.efi_arch_table[extra]['target']
 866             grub_opts = "--target=%s" % target
 867             try:
 868                 self.runcmd(['chroot', rootdir, 'update-grub'])
 869                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 870             except cliapp.AppException as exc:
 871                 logging.warning(exc)
 872                 self.message(
 873                     "Failed to configure grub-uefi for %s" % extra)
 874             self.configure_extra_efi()
 875         self._umount_wrapper(rootdir)
 876
 877     def install_grub2(self, rootdev, rootdir):
 878         self.message("Configuring grub2")
 879         # rely on kpartx using consistent naming to map loop0p1 to loop0
 880         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 881         if self.settings['serial-console']:
 882             self._grub_serial_console(rootdir)
 883         logging.debug("Running grub-install with options: %s", grub_opts)
 884         self._mount_wrapper(rootdir)
 885         try:
 886             self.runcmd(['chroot', rootdir, 'update-grub'])
 887             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 888         except cliapp.AppException as exc:
 889             logging.warning(exc)
 890             self.message("Failed. Is grub2-common installed? Using extlinux.")
 891             self.install_extlinux(rootdev, rootdir)
 892         self._umount_wrapper(rootdir)
 893
 894     def install_extlinux(self, rootdev, rootdir):
 895         if not os.path.exists("/usr/bin/extlinux"):
 896             self.message("extlinux not installed, skipping.")
 897             return
 898         self.message('Installing extlinux')
 899
 900         def find(pattern):
 901             dirname = os.path.join(rootdir, 'boot')
 902             basenames = os.listdir(dirname)
 903             logging.debug('find: %s', basenames)
 904             for basename in basenames:
 905                 if re.search(pattern, basename):
 906                     return os.path.join('boot', basename)
 907             raise cliapp.AppException('Cannot find match: %s' % pattern)
 908
 909         try:
 910             kernel_image = find('vmlinuz-.*')
 911             initrd_image = find('initrd.img-.*')
 912         except cliapp.AppException as e:
 913             self.message("Unable to find kernel. Not installing extlinux.")
 914             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 915             return
 916
 917         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 918                            '-s', 'UUID', rootdev])
 919         uuid = out.splitlines()[0].strip()
 920
 921         conf = os.path.join(rootdir, 'extlinux.conf')
 922         logging.debug('configure extlinux %s', conf)
 923         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 924         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 925         msg = '''
 926 default linux
 927 timeout 1
 928
 929 label linux
 930 kernel %(kernel)s
 931 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 932 %(extserial)s
 933 ''' % {
 934             'kernel': kernel_image,  # pylint: disable=bad-continuation
 935             'initrd': initrd_image,  # pylint: disable=bad-continuation
 936             'uuid': uuid,  # pylint: disable=bad-continuation
 937             'kserial': kserial,  # pylint: disable=bad-continuation
 938             'extserial': extserial,  # pylint: disable=bad-continuation
 939         }  # pylint: disable=bad-continuation
 940         logging.debug("extlinux config:\n%s", msg)
 941
 942         # python multiline string substitution is just ugly.
 943         # use an external file or live with the mangling, no point in
 944         # mangling the string to remove spaces just to keep it pretty in source.
 945         f = open(conf, 'w')
 946         f.write(msg)
 947
 948         self.runcmd(['extlinux', '--install', rootdir])
 949         self.runcmd(['sync'])
 950         time.sleep(2)
 951
 952     def optimize_image(self, rootdir):
 953         """
 954         Filing up the image with zeros will increase its compression rate
 955         """
 956         if not self.settings['sparse']:
 957             zeros = os.path.join(rootdir, 'ZEROS')
 958             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 959             self.runcmd(['rm', '-f', zeros])
 960
 961     def squash(self):
 962         """
 963         Run squashfs on the image.
 964         """
 965         if not os.path.exists('/usr/bin/mksquashfs'):
 966             logging.warning("Squash selected but mksquashfs not found!")
 967             return
 968         logging.debug(
 969             "%s usage: %s", self.settings['image'],
 970             self.runcmd(['du', self.settings['image']]))
 971         self.message("Running mksquashfs")
 972         suffixed = "%s.squashfs" % self.settings['image']
 973         if os.path.exists(suffixed):
 974             os.unlink(suffixed)
 975         msg = self.runcmd(
 976             ['mksquashfs', self.settings['image'],
 977              suffixed,
 978              '-no-progress', '-comp', 'xz'], ignore_fail=False)
 979         logging.debug(msg)
 980         check_size = os.path.getsize(suffixed)
 981         if check_size < (1024 * 1024):
 982             logging.warning(
 983                 "%s appears to be too small! %s bytes",
 984                 suffixed, check_size)
 985         else:
 986             logging.debug("squashed size: %s", check_size)
 987             os.unlink(self.settings['image'])
 988             self.settings['image'] = suffixed
 989             logging.debug(
 990                 "%s usage: %s", self.settings['image'],
 991                 self.runcmd(['du', self.settings['image']]))
 992
 993     def cleanup_system(self):
 994         # Clean up after any errors.
 995
 996         self.message('Cleaning up')
 997
 998         # Umount in the reverse mount order
 999         if self.settings['image']:
1000             for i in range(len(self.mount_points) - 1, -1, -1):
1001                 mount_point = self.mount_points[i]
1002                 try:
1003                     self.runcmd(['umount', mount_point], ignore_fail=False)
1004                 except cliapp.AppException:
1005                     logging.debug("umount failed, sleeping and trying again")
1006                     time.sleep(5)
1007                     self.runcmd(['umount', mount_point], ignore_fail=False)
1008
1009             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1010
1011         for dirname in self.remove_dirs:
1012             shutil.rmtree(dirname)
1013
1014     def customize(self, rootdir):
1015         script = self.settings['customize']
1016         if not script:
1017             return
1018         if not os.path.exists(script):
1019             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1020             if not os.path.exists(example):
1021                 self.message("Unable to find %s" % script)
1022                 return
1023             script = example
1024         self.message('Running customize script %s' % script)
1025         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1026         logging.debug(
1027             "%s usage: %s", self.settings['image'],
1028             self.runcmd(['du', self.settings['image']]))
1029         try:
1030             with open('/dev/tty', 'w') as tty:
1031                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1032         except IOError:
1033             logging.debug('tty unavailable, trying in headless mode.')
1034             subprocess.call([script, rootdir, self.settings['image']])
1035
1036     def create_tarball(self, rootdir):
1037         # Create a tarball of the disk's contents
1038         # shell out to runcmd since it more easily handles rootdir
1039         self.message('Creating tarball of disk contents')
1040         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1041
1042     def chown(self):
1043         # Change image owner after completed build
1044         if self.settings['image']:
1045             filename = self.settings['image']
1046         elif self.settings['tarball']:
1047             filename = self.settings['tarball']
1048         else:
1049             return
1050         self.message("Changing owner to %s" % self.settings["owner"])
1051         subprocess.call(["chown", self.settings["owner"], filename])
1052
1053     def list_installed_pkgs(self, rootdir):
1054         # output the list of installed packages for sources identification
1055         self.message("Creating a list of installed binary package names")
1056         out = self.runcmd(['chroot', rootdir,
1057                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1058         with open('dpkg.list', 'w') as dpkg:
1059             dpkg.write(out)
1060
1061     def configure_apt(self, rootdir):
1062         # use the distribution and mirror to create an apt source
1063         self.message("Configuring apt to use distribution and mirror")
1064         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1065         logging.debug('configure apt %s', conf)
1066         mirror = self.settings['mirror']
1067         if self.settings['apt-mirror']:
1068             mirror = self.settings['apt-mirror']
1069             self.message("Setting apt mirror to %s" % mirror)
1070         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1071         f = open(conf, 'w')
1072         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1073         f.write(line)
1074         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1075         f.write(line)
1076         f.close()
1077         # ensure the apt sources have valid lists
1078         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1079
1080 if __name__ == '__main__':
1081     VmDebootstrap(version=__version__).run()