vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import re
  24 import sys
  25 import shutil
  26 import datetime
  27 import subprocess
  28 import tempfile
  29 import time
  30 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  31
  32
  33 __version__ = '0.10'
  34
  35 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  36
  37
  38 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  39
  40     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  41         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  42         self.remove_dirs = []
  43         self.mount_points = []
  44         self.debian_info = DebianDistroInfo()
  45         self.ubuntu_info = UbuntuDistroInfo()
  46         self.bootdir = None
  47         self.efi_arch_table = {
  48             'amd64': {
  49                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  50                 'install': '/EFI/debian/grubx64.efi',  # package location
  51                 'package': 'grub-efi-amd64',  # bootstrap package
  52                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  53                 'extra': 'i386',  # architecture to add binary package
  54                 'exclusive': False,  # only EFI supported for this arch.
  55                 'target': 'x86_64-efi',  # grub target name
  56             },
  57             'i386': {
  58                 'removable': '/EFI/boot/bootia32.efi',
  59                 'install': '/EFI/debian/grubia32.efi',
  60                 'package': 'grub-efi-ia32',
  61                 'bin_package': 'grub-efi-ia32-bin',
  62                 'extra': None,
  63                 'exclusive': False,
  64                 'target': 'i386-efi',
  65             },
  66             'arm64': {
  67                 'removable': '/EFI/boot/bootaa64.efi',
  68                 'install': '/EFI/debian/grubaa64.efi',
  69                 'package': 'grub-efi-arm64',
  70                 'bin_package': 'grub-efi-arm64-bin',
  71                 'extra': None,
  72                 'exclusive': True,
  73                 'target': 'arm64-efi',
  74             }
  75         }
  76
  77     def add_settings(self):
  78         default_arch = subprocess.check_output(
  79             ["dpkg", "--print-architecture"]).strip()
  80
  81         self.settings.boolean(
  82             ['verbose'], 'report what is going on')
  83         self.settings.string(
  84             ['image'], 'put created disk image in FILE',
  85             metavar='FILE')
  86         self.settings.bytesize(
  87             ['size'],
  88             'create a disk image of size SIZE (%default)',
  89             metavar='SIZE',
  90             default='1G')
  91         self.settings.bytesize(
  92             ['bootsize'],
  93             'create boot partition of size SIZE (%default)',
  94             metavar='BOOTSIZE',
  95             default='0%')
  96         self.settings.string(
  97             ['boottype'],
  98             'specify file system type for /boot/',
  99             default='ext2')
 100         self.settings.bytesize(
 101             ['bootoffset'],
 102             'Space to leave at start of the image for bootloader',
 103             default='0')
 104         self.settings.boolean(
 105             ['use-uefi'],
 106             'Setup image for UEFI boot',
 107             default=False)
 108         self.settings.bytesize(
 109             ['esp-size'],
 110             'Size of EFI System Partition - requires use-uefi',
 111             default='5mib')
 112         self.settings.string(
 113             ['part-type'],
 114             'Partition type to use for this image',
 115             default='msdos')
 116         self.settings.string(
 117             ['roottype'],
 118             'specify file system type for /',
 119             default='ext4')
 120         self.settings.bytesize(
 121             ['swap'],
 122             'create swap space of size SIZE (min 256Mb)')
 123         self.settings.string(
 124             ['foreign'],
 125             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 126         self.settings.string(
 127             ['variant'],
 128             'select debootstrap variant it not using the default')
 129         self.settings.boolean(
 130             ['extlinux'],
 131             'install extlinux?',
 132             default=True)
 133         self.settings.string(
 134             ['tarball'],
 135             "tar up the disk's contents in FILE",
 136             metavar='FILE')
 137         self.settings.string(
 138             ['apt-mirror'],
 139             'configure apt to use MIRROR',
 140             metavar='URL')
 141         self.settings.string(
 142             ['mirror'],
 143             'use MIRROR as package source (%default)',
 144             metavar='URL',
 145             default='http://http.debian.net/debian/')
 146         self.settings.string(
 147             ['arch'],
 148             'architecture to use (%default)',
 149             metavar='ARCH',
 150             default=default_arch)
 151         self.settings.string(
 152             ['distribution'],
 153             'release to use (%default)',
 154             metavar='NAME',
 155             default='stable')
 156         self.settings.string_list(
 157             ['package'],
 158             'install PACKAGE onto system')
 159         self.settings.string_list(
 160             ['custom-package'],
 161             'install package in DEB file onto system (not from mirror)',
 162             metavar='DEB')
 163         self.settings.boolean(
 164             ['no-kernel'],
 165             'do not install a linux package')
 166         self.settings.string(
 167             ['kernel-package'],
 168             'install PACKAGE instead of the default kernel package',
 169             metavar='PACKAGE')
 170         self.settings.boolean(
 171             ['enable-dhcp'],
 172             'enable DHCP on eth0')
 173         self.settings.string(
 174             ['root-password'],
 175             'set root password',
 176             metavar='PASSWORD')
 177         self.settings.boolean(
 178             ['lock-root-password'],
 179             'lock root account so they cannot login?')
 180         self.settings.string(
 181             ['customize'],
 182             'run SCRIPT after setting up system',
 183             metavar='SCRIPT')
 184         self.settings.string(
 185             ['hostname'],
 186             'set name to HOSTNAME (%default)',
 187             metavar='HOSTNAME',
 188             default='debian')
 189         self.settings.string_list(
 190             ['user'],
 191             'create USER with PASSWORD',
 192             metavar='USER/PASSWORD')
 193         self.settings.boolean(
 194             ['serial-console'],
 195             'configure image to use a serial console')
 196         self.settings.string(
 197             ['serial-console-command'],
 198             'command to manage the serial console, appended to /etc/inittab (%default)',
 199             metavar='COMMAND',
 200             default='/sbin/getty -L ttyS0 115200 vt100')
 201         self.settings.boolean(
 202             ['sudo'],
 203             'install sudo, and if user is created, add them to sudo group')
 204         self.settings.string(
 205             ['owner'],
 206             'the user who will own the image when the build is complete.')
 207         self.settings.boolean(
 208             ['squash'],
 209             'use squashfs on the final image.')
 210         self.settings.boolean(
 211             ['configure-apt'],
 212             'Create an apt source based on the distribution and mirror selected.')
 213         self.settings.boolean(
 214             ['mbr'],
 215             'Run install-mbr (default if extlinux used)')
 216         self.settings.boolean(
 217             ['grub'],
 218             'Install and configure grub2 - disables extlinux.')
 219         self.settings.boolean(
 220             ['sparse'],
 221             'Do not fill the image with zeros to keep a sparse disk image',
 222             default=False)
 223         self.settings.boolean(
 224             ['pkglist'],
 225             'Create a list of package names included in the image.')
 226         self.settings.boolean(
 227             ['no-acpid'],
 228             'do not install the acpid package',
 229             default=False)
 230
 231     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 232         if not self.settings['image'] and not self.settings['tarball']:
 233             raise cliapp.AppException(
 234                 'You must give disk image filename, or tarball filename')
 235         if self.settings['image'] and not self.settings['size']:
 236             raise cliapp.AppException(
 237                 'If disk image is specified, you must give image size.')
 238         if not self.debian_info.valid(self.settings['distribution']):
 239             if not self.ubuntu_info.valid(self.settings['distribution']):
 240                 raise cliapp.AppException(
 241                     '%s is not a valid Debian or Ubuntu suite or codename.'
 242                     % self.settings['distribution'])
 243         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 244             raise cliapp.AppException(
 245                 'You must specify use-uefi for esp-size to have effect')
 246         if self.settings['arch'] in self.efi_arch_table and\
 247                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 248                 not self.settings['use-uefi']:
 249             raise cliapp.AppException(
 250                 'Only UEFI is supported on %s' % self.settings['arch'])
 251         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 252             raise cliapp.AppException(
 253                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 254         if self.settings['use-uefi'] and (
 255                 self.settings['bootsize'] or
 256                 self.settings['bootoffset']):
 257             raise cliapp.AppException(
 258                 'A separate boot partition is not supported with UEFI')
 259
 260         if self.settings['use-uefi'] and not self.settings['grub']:
 261             raise cliapp.AppException(
 262                 'UEFI without Grub is not supported.')
 263
 264         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 265         if self.was_oldstable(datetime.date(2015, 4, 26)):
 266             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 267                 raise cliapp.AppException(
 268                     'Only amd64 supports UEFI in Wheezy')
 269
 270         if os.geteuid() != 0:
 271             sys.exit("You need to have root privileges to run this script.")
 272         rootdir = None
 273         try:
 274             rootdev = None
 275             roottype = self.settings['roottype']
 276             bootdev = None
 277             boottype = None
 278             if self.settings['image']:
 279                 self.create_empty_image()
 280                 self.partition_image()
 281                 if self.settings['mbr'] or self.settings['extlinux']:
 282                     self.install_mbr()
 283                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 284                 if self.settings['swap'] > 0:
 285                     self.message("Creating swap space")
 286                     self.runcmd(['mkswap', swapdev])
 287                 self.mkfs(rootdev, fstype=roottype)
 288                 rootdir = self.mount(rootdev)
 289                 if self.settings['use-uefi']:
 290                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 291                     logging.debug("bootdir:%s", self.bootdir)
 292                     self.mkfs(bootdev, fstype='vfat')
 293                     os.makedirs(self.bootdir)
 294                     self.mount(bootdev, self.bootdir)
 295                 elif bootdev:
 296                     if self.settings['boottype']:
 297                         boottype = self.settings['boottype']
 298                     else:
 299                         boottype = 'ext2'
 300                     self.mkfs(bootdev, fstype=boottype)
 301                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 302                     os.mkdir(self.bootdir)
 303                     self.mount(bootdev, self.bootdir)
 304             else:
 305                 rootdir = self.mkdtemp()
 306             self.debootstrap(rootdir)
 307             self.set_hostname(rootdir)
 308             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 309             self.install_debs(rootdir)
 310             self.set_root_password(rootdir)
 311             self.create_users(rootdir)
 312             self.remove_udev_persistent_rules(rootdir)
 313             self.setup_networking(rootdir)
 314             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 315                 self.configure_apt(rootdir)
 316             self.customize(rootdir)
 317             self.cleanup_apt_cache(rootdir)
 318             self.update_initramfs(rootdir)
 319
 320             if self.settings['image']:
 321                 if self.settings['use-uefi']:
 322                     self.install_grub_uefi(rootdir)
 323                 elif self.settings['grub']:
 324                     self.install_grub2(rootdev, rootdir)
 325                 elif self.settings['extlinux']:
 326                     self.install_extlinux(rootdev, rootdir)
 327                 self.append_serial_console(rootdir)
 328                 self.optimize_image(rootdir)
 329                 if self.settings['squash']:
 330                     self.squash()
 331             if self.settings['pkglist']:
 332                 self.list_installed_pkgs(rootdir)
 333
 334             if self.settings['foreign']:
 335                 os.unlink('%s/usr/bin/%s' %
 336                           (rootdir, os.path.basename(self.settings['foreign'])))
 337
 338             if self.settings['tarball']:
 339                 self.create_tarball(rootdir)
 340
 341             if self.settings['owner']:
 342                 self.chown()
 343         except BaseException as e:
 344             self.message('EEEK! Something bad happened...')
 345             if rootdir:
 346                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 347                 if os.path.exists(db_log):
 348                     shutil.copy(db_log, os.getcwd())
 349             self.message(e)
 350             self.cleanup_system()
 351             raise
 352         else:
 353             self.cleanup_system()
 354
 355     def message(self, msg):
 356         logging.info(msg)
 357         if self.settings['verbose']:
 358             print msg
 359
 360     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 361         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 362         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 363                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 364                              env=env, **kwargs)
 365         out, err = p.communicate(stdin)
 366         if p.returncode != 0:
 367             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 368             logging.error(msg)
 369             if not ignore_fail:
 370                 raise cliapp.AppException(msg)
 371         return out
 372
 373     def mkdtemp(self):
 374         dirname = tempfile.mkdtemp()
 375         self.remove_dirs.append(dirname)
 376         logging.debug('mkdir %s', dirname)
 377         return dirname
 378
 379     def mount(self, device, path=None):
 380         if not path:
 381             mount_point = self.mkdtemp()
 382         else:
 383             mount_point = path
 384         self.message('Mounting %s on %s' % (device, mount_point))
 385         self.runcmd(['mount', device, mount_point])
 386         self.mount_points.append(mount_point)
 387         logging.debug('mounted %s on %s', device, mount_point)
 388         return mount_point
 389
 390     def create_empty_image(self):
 391         self.message('Creating disk image')
 392         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 393                      self.settings['image'],
 394                      str(self.settings['size'])])
 395
 396     def partition_image(self):
 397         """
 398         Uses fat16 (msdos) partitioning by default, use part-type to change.
 399         If bootoffset is specified, the first actual partition
 400         starts at that offset to allow customisation scripts to
 401         put bootloader images into the space, e.g. u-boot.
 402         """
 403         self.message('Creating partitions')
 404         self.runcmd(['parted', '-s', self.settings['image'],
 405                      'mklabel', self.settings['part-type']])
 406         partoffset = 0
 407         extent = '100%'
 408         swap = 256 * 1024 * 1024
 409         if self.settings['swap'] > 0:
 410             if self.settings['swap'] > swap:
 411                 swap = self.settings['swap']
 412             else:
 413                 # minimum 256Mb as default qemu ram is 128Mb
 414                 logging.debug("Setting minimum 256Mb swap space")
 415             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 416
 417         if self.settings['use-uefi']:
 418             espsize = self.settings['esp-size'] / (1024 * 1024)
 419             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 420             self.runcmd(['parted', '-s', self.settings['image'],
 421                          'mkpart', 'primary', 'fat32',
 422                          '1', str(espsize)])
 423             self.runcmd(['parted', '-s', self.settings['image'],
 424                          'set', '1', 'boot', 'on'])
 425             self.runcmd(['parted', '-s', self.settings['image'],
 426                          'set', '1', 'esp', 'on'])
 427
 428         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 429             # turn v.small offsets into something at least possible to create.
 430             if self.settings['bootoffset'] < 1048576:
 431                 partoffset = 1
 432                 logging.info(
 433                     "Setting bootoffset %smib to allow for %s bytes",
 434                     partoffset, self.settings['bootoffset'])
 435             else:
 436                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 437                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 438         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 439             if self.settings['grub'] and not partoffset:
 440                 partoffset = 1
 441             bootsize = self.settings['bootsize'] / (1024 * 1024)
 442             bootsize += partoffset
 443             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 444             logging.debug("Starting boot partition at %sMb", bootsize)
 445             self.runcmd(['parted', '-s', self.settings['image'],
 446                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 447             logging.debug("Starting root partition at %sMb", partoffset)
 448             self.runcmd(['parted', '-s', self.settings['image'],
 449                          'mkpart', 'primary', str(bootsize), extent])
 450         elif self.settings['use-uefi']:
 451             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 452             self.runcmd(['parted', '-s', self.settings['image'],
 453                          'mkpart', 'primary', str(bootsize), extent])
 454         else:
 455             self.runcmd(['parted', '-s', self.settings['image'],
 456                          'mkpart', 'primary', '0%', extent])
 457         self.runcmd(['parted', '-s', self.settings['image'],
 458                      'set', '1', 'boot', 'on'])
 459         if self.settings['swap'] > 0:
 460             logging.debug("Creating swap partition")
 461             self.runcmd(['parted', '-s', self.settings['image'],
 462                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 463
 464     def update_initramfs(self, rootdir):
 465         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 466         if os.path.exists(os.path.join(rootdir, cmd)):
 467             self.message("Updating the initramfs")
 468             self.runcmd(['chroot', rootdir, cmd, '-u'])
 469
 470     def install_mbr(self):
 471         if os.path.exists("/sbin/install-mbr"):
 472             self.message('Installing MBR')
 473             self.runcmd(['install-mbr', self.settings['image']])
 474         else:
 475             msg = "mbr enabled but /sbin/install-mbr not found" \
 476                   " - please install the mbr package."
 477             raise cliapp.AppException(msg)
 478
 479     def setup_kpartx(self):
 480         bootindex = None
 481         swapindex = None
 482         out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 483         if self.settings['bootsize'] and self.settings['swap'] > 0:
 484             bootindex = 0
 485             rootindex = 1
 486             swapindex = 2
 487             parts = 3
 488         elif self.settings['use-uefi']:
 489             bootindex = 0
 490             rootindex = 1
 491             parts = 2
 492         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 493             bootindex = 0
 494             rootindex = 1
 495             swapindex = 2
 496             parts = 3
 497         elif self.settings['bootsize']:
 498             bootindex = 0
 499             rootindex = 1
 500             parts = 2
 501         elif self.settings['swap'] > 0:
 502             rootindex = 0
 503             swapindex = 1
 504             parts = 2
 505         else:
 506             rootindex = 0
 507             parts = 1
 508         boot = None
 509         swap = None
 510         devices = [line.split()[2]
 511                    for line in out.splitlines()
 512                    if line.startswith('add map ')]
 513         if len(devices) != parts:
 514             msg = 'Surprising number of partitions - check output of losetup -a'
 515             logging.debug("%s", self.runcmd(['losetup', '-a']))
 516             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 517             raise cliapp.AppException(msg)
 518         root = '/dev/mapper/%s' % devices[rootindex]
 519         if self.settings['bootsize'] or self.settings['use-uefi']:
 520             boot = '/dev/mapper/%s' % devices[bootindex]
 521         if self.settings['swap'] > 0:
 522             swap = '/dev/mapper/%s' % devices[swapindex]
 523         return root, boot, swap
 524
 525     def _efi_packages(self):
 526         packages = []
 527         pkg = self.efi_arch_table[self.settings['arch']]['package']
 528         self.message("Adding %s" % pkg)
 529         packages.append(pkg)
 530         extra = self.efi_arch_table[self.settings['arch']]['extra']
 531         if extra and isinstance(extra, str):
 532             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 533             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 534             packages.append(bin_pkg)
 535         return packages
 536
 537     def _copy_efi_binary(self, efi_removable, efi_install):
 538         logging.debug("using bootdir=%s", self.bootdir)
 539         logging.debug("moving %s to %s", efi_removable, efi_install)
 540         if efi_removable.startswith('/'):
 541             efi_removable = efi_removable[1:]
 542         if efi_install.startswith('/'):
 543             efi_install = efi_install[1:]
 544         efi_output = os.path.join(self.bootdir, efi_removable)
 545         efi_input = os.path.join(self.bootdir, efi_install)
 546         if not os.path.exists(efi_input):
 547             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 548             raise cliapp.AppException("Missing %s" % efi_install)
 549         if not os.path.exists(os.path.dirname(efi_output)):
 550             os.makedirs(os.path.dirname(efi_output))
 551         logging.debug(
 552             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 553         if os.path.exists(efi_output):
 554             os.unlink(efi_output)
 555         os.rename(efi_input, efi_output)
 556
 557     def configure_efi(self):
 558         """
 559         Copy the bootloader file from the package into the location
 560         so needs to be after grub and kernel already installed.
 561         """
 562         self.message('Configuring EFI')
 563         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 564         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 565         self.message('Installing UEFI support binary')
 566         self._copy_efi_binary(efi_removable, efi_install)
 567
 568     def configure_extra_efi(self):
 569         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 570         if extra:
 571             efi_removable = str(self.efi_arch_table[extra]['removable'])
 572             efi_install = str(self.efi_arch_table[extra]['install'])
 573             self.message('Copying UEFI support binary for %s' % extra)
 574             self._copy_efi_binary(efi_removable, efi_install)
 575
 576     def mkfs(self, device, fstype):
 577         self.message('Creating filesystem %s' % fstype)
 578         self.runcmd(['mkfs', '-t', fstype, device])
 579
 580     def suite_to_codename(self, distro):
 581         suite = self.debian_info.codename(distro, datetime.date.today())
 582         if not suite:
 583             return distro
 584         return suite
 585
 586     def was_oldstable(self, limit):
 587         suite = self.suite_to_codename(self.settings['distribution'])
 588         # this check is only for debian
 589         if not self.debian_info.valid(suite):
 590             return False
 591         return suite == self.debian_info.old(limit)
 592
 593     def was_stable(self, limit):
 594         suite = self.suite_to_codename(self.settings['distribution'])
 595         # this check is only for debian
 596         if not self.debian_info.valid(suite):
 597             return False
 598         return suite == self.debian_info.stable(limit)
 599
 600     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 601         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 602         self.message(
 603             'Debootstrapping %s [%s] %s' % (
 604                 self.settings['distribution'], self.settings['arch'], msg))
 605
 606         include = self.settings['package']
 607
 608         if not self.settings['foreign'] and not self.settings['no-acpid']:
 609             include.append('acpid')
 610
 611         if self.settings['grub']:
 612             if self.settings['use-uefi']:
 613                 include.extend(self._efi_packages())
 614             else:
 615                 include.append('grub-pc')
 616
 617         if not self.settings['no-kernel']:
 618             if self.settings['kernel-package']:
 619                 kernel_image = self.settings['kernel-package']
 620             else:
 621                 if self.settings['arch'] == 'i386':
 622                     # wheezy (which became oldstable on 04/25/2015) used '486'
 623                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 624                         kernel_arch = '486'
 625                     else:
 626                         kernel_arch = '586'
 627                 elif self.settings['arch'] == 'armhf':
 628                     kernel_arch = 'armmp'
 629                 else:
 630                     kernel_arch = self.settings['arch']
 631                 kernel_image = 'linux-image-%s' % kernel_arch
 632             include.append(kernel_image)
 633
 634         if self.settings['sudo'] and 'sudo' not in include:
 635             include.append('sudo')
 636
 637         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 638
 639         if self.settings['package']:
 640             args.append(
 641                 '--include=%s' % ','.join(include))
 642         if self.settings['foreign']:
 643             args.append('--foreign')
 644         if self.settings['variant']:
 645             args.append('--variant')
 646             args.append(self.settings['variant'])
 647         args += [self.settings['distribution'],
 648                  rootdir, self.settings['mirror']]
 649         logging.debug(" ".join(args))
 650         self.runcmd(args)
 651         if self.settings['foreign']:
 652             # set a noninteractive debconf environment for secondstage
 653             env = {
 654                 "DEBIAN_FRONTEND": "noninteractive",
 655                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 656                 "LC_ALL": "C"
 657             }
 658             # add the mapping to the complete environment.
 659             env.update(os.environ)
 660             # First copy the binfmt handler over
 661             self.message('Setting up binfmt handler')
 662             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 663             # Next, run the package install scripts etc.
 664             self.message('Running debootstrap second stage')
 665             self.runcmd(['chroot', rootdir,
 666                          '/debootstrap/debootstrap', '--second-stage'],
 667                         env=env)
 668
 669     def set_hostname(self, rootdir):
 670         hostname = self.settings['hostname']
 671         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 672             f.write('%s\n' % hostname)
 673
 674         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 675         try:
 676             with open(etc_hosts, 'r') as f:
 677                 data = f.read()
 678             with open(etc_hosts, 'w') as f:
 679                 for line in data.splitlines():
 680                     if line.startswith('127.0.0.1'):
 681                         line += ' %s' % hostname
 682                     f.write('%s\n' % line)
 683         except IOError:
 684             pass
 685
 686     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 687         def fsuuid(device):
 688             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 689                                '-s', 'UUID', device])
 690             return out.splitlines()[0].strip()
 691
 692         if rootdev:
 693             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 694         else:
 695             rootdevstr = '/dev/sda1'
 696
 697         if bootdev and not self.settings['use-uefi']:
 698             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 699         else:
 700             bootdevstr = None
 701
 702         fstab = os.path.join(rootdir, 'etc', 'fstab')
 703         with open(fstab, 'w') as f:
 704             f.write('proc /proc proc defaults 0 0\n')
 705             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 706             if bootdevstr:
 707                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 708                 if self.settings['swap'] > 0:
 709                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 710             elif self.settings['swap'] > 0:
 711                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 712
 713     def install_debs(self, rootdir):
 714         if not self.settings['custom-package']:
 715             return
 716         self.message('Installing custom packages')
 717         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 718         os.mkdir(tmp)
 719         for deb in self.settings['custom-package']:
 720             shutil.copy(deb, tmp)
 721         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 722                      for deb in self.settings['custom-package']]
 723         out, err, _ = \
 724             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 725         logging.debug('stdout:\n%s', out)
 726         logging.debug('stderr:\n%s', err)
 727         out = self.runcmd(['chroot', rootdir,
 728                            'apt-get', '-f', '--no-remove', 'install'])
 729         logging.debug('stdout:\n%s', out)
 730         shutil.rmtree(tmp)
 731
 732     def cleanup_apt_cache(self, rootdir):
 733         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 734         logging.debug('stdout:\n%s', out)
 735
 736     def set_root_password(self, rootdir):
 737         if self.settings['root-password']:
 738             self.message('Setting root password')
 739             self.set_password(rootdir, 'root', self.settings['root-password'])
 740         elif self.settings['lock-root-password']:
 741             self.message('Locking root password')
 742             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 743         else:
 744             self.message('Give root an empty password')
 745             self.delete_password(rootdir, 'root')
 746
 747     def create_users(self, rootdir):
 748         def create_user(vmuser):
 749             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 750                          '--disabled-password', vmuser])
 751             if self.settings['sudo']:
 752                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 753
 754         for userpass in self.settings['user']:
 755             if '/' in userpass:
 756                 user, password = userpass.split('/', 1)
 757                 create_user(user)
 758                 self.set_password(rootdir, user, password)
 759             else:
 760                 create_user(userpass)
 761                 self.delete_password(rootdir, userpass)
 762
 763     def set_password(self, rootdir, user, password):
 764         encrypted = crypt.crypt(password, '..')
 765         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 766
 767     def delete_password(self, rootdir, user):
 768         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 769
 770     def remove_udev_persistent_rules(self, rootdir):
 771         self.message('Removing udev persistent cd and net rules')
 772         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 773             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 774             if os.path.exists(pathname):
 775                 logging.debug('rm %s', pathname)
 776                 os.remove(pathname)
 777             else:
 778                 logging.debug('not removing non-existent %s', pathname)
 779
 780     def setup_networking(self, rootdir):
 781         self.message('Setting up networking')
 782
 783         # unconditionally write for wheezy (which became oldstable on 04/25/2015)
 784         if self.was_oldstable(datetime.date(2015, 4, 26)):
 785             with open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w') as netfile:
 786                 netfile.write('source /etc/network/interfaces.d/*\n')
 787             os.mkdir(os.path.join(rootdir, 'etc', 'network', 'interfaces.d'))
 788
 789         elif not os.path.exists(os.path.join(rootdir, 'etc', 'network', 'interfaces')):
 790             iface_path = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 791             with open(iface_path, 'w') as netfile:
 792                 netfile.write('source-directory /etc/network/interfaces.d\n')
 793         ethpath = os.path.join(rootdir, 'etc', 'network', 'interfaces.d', 'setup')
 794         with open(ethpath, 'w') as eth:
 795             eth.write('auto lo\n')
 796             eth.write('iface lo inet loopback\n')
 797
 798             if self.settings['enable-dhcp']:
 799                 eth.write('\n')
 800                 eth.write('auto eth0\n')
 801                 eth.write('iface eth0 inet dhcp\n')
 802
 803     def append_serial_console(self, rootdir):
 804         if self.settings['serial-console']:
 805             serial_command = self.settings['serial-console-command']
 806             logging.debug('adding getty to serial console')
 807             inittab = os.path.join(rootdir, 'etc/inittab')
 808             # to autologin, serial_command can contain '-a root'
 809             with open(inittab, 'a') as f:
 810                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 811
 812     # pylint: disable=no-self-use
 813     def _grub_serial_console(self, rootdir):
 814         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 815         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 816         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 817         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 818         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 819         with open(grub_cfg, 'a+') as cfg:
 820             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 821             cfg.write("%s\n" % cmdline)
 822             cfg.write("%s\n" % terminal)
 823             cfg.write("%s\n" % command)
 824
 825     def _mount_wrapper(self, rootdir):
 826         self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 827                      '%s' % os.path.join(rootdir, 'dev')])
 828         self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 829                      '%s' % os.path.join(rootdir, 'proc')])
 830         self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 831                      '%s' % os.path.join(rootdir, 'sys')])
 832
 833     def _umount_wrapper(self, rootdir):
 834         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 835         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 836         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 837
 838     def install_grub_uefi(self, rootdir):
 839         self.message("Configuring grub-uefi")
 840         target = self.efi_arch_table[self.settings['arch']]['target']
 841         grub_opts = "--target=%s" % target
 842         logging.debug("Running grub-install with options: %s", grub_opts)
 843         self._mount_wrapper(rootdir)
 844         try:
 845             self.runcmd(['chroot', rootdir, 'update-grub'])
 846             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 847         except cliapp.AppException as exc:
 848             logging.warning(exc)
 849             self.message(
 850                 "Failed to configure grub-uefi for %s" %
 851                 self.settings['arch'])
 852             self._umount_wrapper(rootdir)
 853         self.configure_efi()
 854         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 855         if extra:
 856             target = self.efi_arch_table[extra]['target']
 857             grub_opts = "--target=%s" % target
 858             try:
 859                 self.runcmd(['chroot', rootdir, 'update-grub'])
 860                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 861             except cliapp.AppException as exc:
 862                 logging.warning(exc)
 863                 self.message(
 864                     "Failed to configure grub-uefi for %s" % extra)
 865             self.configure_extra_efi()
 866         self._umount_wrapper(rootdir)
 867
 868     def install_grub2(self, rootdev, rootdir):
 869         self.message("Configuring grub2")
 870         # rely on kpartx using consistent naming to map loop0p1 to loop0
 871         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 872         if self.settings['serial-console']:
 873             self._grub_serial_console(rootdir)
 874         logging.debug("Running grub-install with options: %s", grub_opts)
 875         self._mount_wrapper(rootdir)
 876         try:
 877             self.runcmd(['chroot', rootdir, 'update-grub'])
 878             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 879         except cliapp.AppException as exc:
 880             logging.warning(exc)
 881             self.message("Failed. Is grub2-common installed? Using extlinux.")
 882             self.install_extlinux(rootdev, rootdir)
 883         self._umount_wrapper(rootdir)
 884
 885     def install_extlinux(self, rootdev, rootdir):
 886         if not os.path.exists("/usr/bin/extlinux"):
 887             self.message("extlinux not installed, skipping.")
 888             return
 889         self.message('Installing extlinux')
 890
 891         def find(pattern):
 892             dirname = os.path.join(rootdir, 'boot')
 893             basenames = os.listdir(dirname)
 894             logging.debug('find: %s', basenames)
 895             for basename in basenames:
 896                 if re.search(pattern, basename):
 897                     return os.path.join('boot', basename)
 898             raise cliapp.AppException('Cannot find match: %s' % pattern)
 899
 900         try:
 901             kernel_image = find('vmlinuz-.*')
 902             initrd_image = find('initrd.img-.*')
 903         except cliapp.AppException as e:
 904             self.message("Unable to find kernel. Not installing extlinux.")
 905             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 906             return
 907
 908         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 909                            '-s', 'UUID', rootdev])
 910         uuid = out.splitlines()[0].strip()
 911
 912         conf = os.path.join(rootdir, 'extlinux.conf')
 913         logging.debug('configure extlinux %s', conf)
 914         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 915         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 916         msg = '''
 917 default linux
 918 timeout 1
 919
 920 label linux
 921 kernel %(kernel)s
 922 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 923 %(extserial)s
 924 ''' % {
 925             'kernel': kernel_image,  # pylint: disable=bad-continuation
 926             'initrd': initrd_image,  # pylint: disable=bad-continuation
 927             'uuid': uuid,  # pylint: disable=bad-continuation
 928             'kserial': kserial,  # pylint: disable=bad-continuation
 929             'extserial': extserial,  # pylint: disable=bad-continuation
 930         }  # pylint: disable=bad-continuation
 931         logging.debug("extlinux config:\n%s", msg)
 932
 933         # python multiline string substitution is just ugly.
 934         # use an external file or live with the mangling, no point in
 935         # mangling the string to remove spaces just to keep it pretty in source.
 936         f = open(conf, 'w')
 937         f.write(msg)
 938
 939         self.runcmd(['extlinux', '--install', rootdir])
 940         self.runcmd(['sync'])
 941         time.sleep(2)
 942
 943     def optimize_image(self, rootdir):
 944         """
 945         Filing up the image with zeros will increase its compression rate
 946         """
 947         if not self.settings['sparse']:
 948             zeros = os.path.join(rootdir, 'ZEROS')
 949             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 950             self.runcmd(['rm', '-f', zeros])
 951
 952     def squash(self):
 953         """
 954         Run squashfs on the image.
 955         """
 956         if not os.path.exists('/usr/bin/mksquashfs'):
 957             logging.warning("Squash selected but mksquashfs not found!")
 958             return
 959         logging.debug(
 960             "%s usage: %s", self.settings['image'],
 961             self.runcmd(['du', self.settings['image']]))
 962         self.message("Running mksquashfs")
 963         suffixed = "%s.squashfs" % self.settings['image']
 964         if os.path.exists(suffixed):
 965             os.unlink(suffixed)
 966         msg = self.runcmd(
 967             ['mksquashfs', self.settings['image'],
 968              suffixed,
 969              '-no-progress', '-comp', 'xz'], ignore_fail=False)
 970         logging.debug(msg)
 971         check_size = os.path.getsize(suffixed)
 972         if check_size < (1024 * 1024):
 973             logging.warning(
 974                 "%s appears to be too small! %s bytes",
 975                 suffixed, check_size)
 976         else:
 977             logging.debug("squashed size: %s", check_size)
 978             os.unlink(self.settings['image'])
 979             self.settings['image'] = suffixed
 980             logging.debug(
 981                 "%s usage: %s", self.settings['image'],
 982                 self.runcmd(['du', self.settings['image']]))
 983
 984     def cleanup_system(self):
 985         # Clean up after any errors.
 986
 987         self.message('Cleaning up')
 988
 989         # Umount in the reverse mount order
 990         if self.settings['image']:
 991             for i in range(len(self.mount_points) - 1, -1, -1):
 992                 mount_point = self.mount_points[i]
 993                 try:
 994                     self.runcmd(['umount', mount_point], ignore_fail=False)
 995                 except cliapp.AppException:
 996                     logging.debug("umount failed, sleeping and trying again")
 997                     time.sleep(5)
 998                     self.runcmd(['umount', mount_point], ignore_fail=False)
 999
1000             self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1001
1002         for dirname in self.remove_dirs:
1003             shutil.rmtree(dirname)
1004
1005     def customize(self, rootdir):
1006         script = self.settings['customize']
1007         if not script:
1008             return
1009         if not os.path.exists(script):
1010             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1011             if not os.path.exists(example):
1012                 self.message("Unable to find %s" % script)
1013                 return
1014             script = example
1015         self.message('Running customize script %s' % script)
1016         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1017         logging.debug(
1018             "%s usage: %s", self.settings['image'],
1019             self.runcmd(['du', self.settings['image']]))
1020         with open('/dev/tty', 'w') as tty:
1021             try:
1022                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1023             except IOError:
1024                 subprocess.call([script, rootdir, self.settings['image']])
1025         logging.debug(
1026             "%s usage: %s", self.settings['image'],
1027             self.runcmd(['du', self.settings['image']]))
1028
1029     def create_tarball(self, rootdir):
1030         # Create a tarball of the disk's contents
1031         # shell out to runcmd since it more easily handles rootdir
1032         self.message('Creating tarball of disk contents')
1033         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1034
1035     def chown(self):
1036         # Change image owner after completed build
1037         if self.settings['image']:
1038             filename = self.settings['image']
1039         elif self.settings['tarball']:
1040             filename = self.settings['tarball']
1041         else:
1042             return
1043         self.message("Changing owner to %s" % self.settings["owner"])
1044         subprocess.call(["chown", self.settings["owner"], filename])
1045
1046     def list_installed_pkgs(self, rootdir):
1047         # output the list of installed packages for sources identification
1048         self.message("Creating a list of installed binary package names")
1049         out = self.runcmd(['chroot', rootdir,
1050                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1051         with open('dpkg.list', 'w') as dpkg:
1052             dpkg.write(out)
1053
1054     def configure_apt(self, rootdir):
1055         # use the distribution and mirror to create an apt source
1056         self.message("Configuring apt to use distribution and mirror")
1057         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1058         logging.debug('configure apt %s', conf)
1059         mirror = self.settings['mirror']
1060         if self.settings['apt-mirror']:
1061             mirror = self.settings['apt-mirror']
1062             self.message("Setting apt mirror to %s" % mirror)
1063         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1064         f = open(conf, 'w')
1065         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1066         f.write(line)
1067         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1068         f.write(line)
1069         f.close()
1070         # ensure the apt sources have valid lists
1071         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1072
1073 if __name__ == '__main__':
1074     VmDebootstrap(version=__version__).run()