vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import glob
  24 import re
  25 import sys
  26 import shutil
  27 import datetime
  28 import subprocess
  29 import tempfile
  30 import time
  31 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  32
  33
  34 __version__ = '0.11'
  35
  36 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  37
  38
  39 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  40
  41     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  42         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  43         self.remove_dirs = []
  44         self.mount_points = []
  45         self.debian_info = DebianDistroInfo()
  46         self.ubuntu_info = UbuntuDistroInfo()
  47         self.bootdir = None
  48         self.efi_arch_table = {
  49             'amd64': {
  50                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  51                 'install': '/EFI/debian/grubx64.efi',  # package location
  52                 'package': 'grub-efi-amd64',  # bootstrap package
  53                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  54                 'extra': 'i386',  # architecture to add binary package
  55                 'exclusive': False,  # only EFI supported for this arch.
  56                 'target': 'x86_64-efi',  # grub target name
  57             },
  58             'i386': {
  59                 'removable': '/EFI/boot/bootia32.efi',
  60                 'install': '/EFI/debian/grubia32.efi',
  61                 'package': 'grub-efi-ia32',
  62                 'bin_package': 'grub-efi-ia32-bin',
  63                 'extra': None,
  64                 'exclusive': False,
  65                 'target': 'i386-efi',
  66             },
  67             'arm64': {
  68                 'removable': '/EFI/boot/bootaa64.efi',
  69                 'install': '/EFI/debian/grubaa64.efi',
  70                 'package': 'grub-efi-arm64',
  71                 'bin_package': 'grub-efi-arm64-bin',
  72                 'extra': None,
  73                 'exclusive': True,
  74                 'target': 'arm64-efi',
  75             }
  76         }
  77
  78     def add_settings(self):
  79         default_arch = subprocess.check_output(
  80             ["dpkg", "--print-architecture"]).strip()
  81
  82         self.settings.boolean(
  83             ['verbose'], 'report what is going on')
  84         self.settings.string(
  85             ['image'], 'put created disk image in FILE',
  86             metavar='FILE')
  87         self.settings.bytesize(
  88             ['size'],
  89             'create a disk image of size SIZE (%default)',
  90             metavar='SIZE',
  91             default='1G')
  92         self.settings.bytesize(
  93             ['bootsize'],
  94             'create boot partition of size SIZE (%default)',
  95             metavar='BOOTSIZE',
  96             default='0%')
  97         self.settings.string(
  98             ['boottype'],
  99             'specify file system type for /boot/',
 100             default='ext2')
 101         self.settings.bytesize(
 102             ['bootoffset'],
 103             'Space to leave at start of the image for bootloader',
 104             default='0')
 105         self.settings.boolean(
 106             ['use-uefi'],
 107             'Setup image for UEFI boot',
 108             default=False)
 109         self.settings.bytesize(
 110             ['esp-size'],
 111             'Size of EFI System Partition - requires use-uefi',
 112             default='5mib')
 113         self.settings.string(
 114             ['part-type'],
 115             'Partition type to use for this image',
 116             default='msdos')
 117         self.settings.string(
 118             ['roottype'],
 119             'specify file system type for /',
 120             default='ext4')
 121         self.settings.bytesize(
 122             ['swap'],
 123             'create swap space of size SIZE (min 256Mb)')
 124         self.settings.string(
 125             ['foreign'],
 126             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 127         self.settings.string(
 128             ['variant'],
 129             'select debootstrap variant if not using the default [deprecated]')
 130         self.settings.string_list(
 131             ['debootstrapopts'],
 132             'pass additional options to debootstrap'),
 133         self.settings.boolean(
 134             ['extlinux'],
 135             'install extlinux?',
 136             default=True)
 137         self.settings.string(
 138             ['tarball'],
 139             "tar up the disk's contents in FILE",
 140             metavar='FILE')
 141         self.settings.string(
 142             ['apt-mirror'],
 143             'configure apt to use MIRROR',
 144             metavar='URL')
 145         self.settings.string(
 146             ['mirror'],
 147             'use MIRROR as package source (%default)',
 148             metavar='URL',
 149             default='http://http.debian.net/debian/')
 150         self.settings.string(
 151             ['arch'],
 152             'architecture to use (%default)',
 153             metavar='ARCH',
 154             default=default_arch)
 155         self.settings.string(
 156             ['distribution'],
 157             'release to use (%default)',
 158             metavar='NAME',
 159             default='stable')
 160         self.settings.string_list(
 161             ['package'],
 162             'install PACKAGE onto system')
 163         self.settings.string_list(
 164             ['custom-package'],
 165             'install package in DEB file onto system (not from mirror)',
 166             metavar='DEB')
 167         self.settings.boolean(
 168             ['no-kernel'],
 169             'do not install a linux package')
 170         self.settings.string(
 171             ['kernel-package'],
 172             'install PACKAGE instead of the default kernel package',
 173             metavar='PACKAGE')
 174         self.settings.boolean(
 175             ['enable-dhcp'],
 176             'enable DHCP on eth0')
 177         self.settings.string(
 178             ['root-password'],
 179             'set root password',
 180             metavar='PASSWORD')
 181         self.settings.boolean(
 182             ['lock-root-password'],
 183             'lock root account so they cannot login?')
 184         self.settings.string(
 185             ['customize'],
 186             'run SCRIPT after setting up system',
 187             metavar='SCRIPT')
 188         self.settings.string(
 189             ['hostname'],
 190             'set name to HOSTNAME (%default)',
 191             metavar='HOSTNAME',
 192             default='debian')
 193         self.settings.string_list(
 194             ['user'],
 195             'create USER with PASSWORD',
 196             metavar='USER/PASSWORD')
 197         self.settings.boolean(
 198             ['serial-console'],
 199             'configure image to use a serial console')
 200         self.settings.string(
 201             ['serial-console-command'],
 202             'command to manage the serial console, appended to /etc/inittab (%default)',
 203             metavar='COMMAND',
 204             default='/sbin/getty -L ttyS0 115200 vt100')
 205         self.settings.boolean(
 206             ['sudo'],
 207             'install sudo, and if user is created, add them to sudo group')
 208         self.settings.string(
 209             ['owner'],
 210             'the user who will own the image when the build is complete.')
 211         self.settings.boolean(
 212             ['squash'],
 213             'use squashfs on the final image.')
 214         self.settings.boolean(
 215             ['configure-apt'],
 216             'Create an apt source based on the distribution and mirror selected.')
 217         self.settings.boolean(
 218             ['mbr'],
 219             'Run install-mbr (default if extlinux used)')
 220         self.settings.boolean(
 221             ['grub'],
 222             'Install and configure grub2 - disables extlinux.')
 223         self.settings.boolean(
 224             ['sparse'],
 225             'Do not fill the image with zeros to keep a sparse disk image',
 226             default=False)
 227         self.settings.boolean(
 228             ['pkglist'],
 229             'Create a list of package names included in the image.')
 230         self.settings.boolean(
 231             ['no-acpid'],
 232             'do not install the acpid package',
 233             default=False)
 234
 235     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 236         if not self.settings['image'] and not self.settings['tarball']:
 237             raise cliapp.AppException(
 238                 'You must give disk image filename, or tarball filename')
 239         if self.settings['image'] and not self.settings['size']:
 240             raise cliapp.AppException(
 241                 'If disk image is specified, you must give image size.')
 242         if not self.debian_info.valid(self.settings['distribution']):
 243             if not self.ubuntu_info.valid(self.settings['distribution']):
 244                 raise cliapp.AppException(
 245                     '%s is not a valid Debian or Ubuntu suite or codename.'
 246                     % self.settings['distribution'])
 247         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 248             raise cliapp.AppException(
 249                 'You must specify use-uefi for esp-size to have effect')
 250         if self.settings['arch'] in self.efi_arch_table and\
 251                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 252                 not self.settings['use-uefi']:
 253             raise cliapp.AppException(
 254                 'Only UEFI is supported on %s' % self.settings['arch'])
 255         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 256             raise cliapp.AppException(
 257                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 258         if self.settings['use-uefi'] and (
 259                 self.settings['bootsize'] or
 260                 self.settings['bootoffset']):
 261             raise cliapp.AppException(
 262                 'A separate boot partition is not supported with UEFI')
 263
 264         if self.settings['use-uefi'] and not self.settings['grub']:
 265             raise cliapp.AppException(
 266                 'UEFI without Grub is not supported.')
 267
 268         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 269         if self.was_oldstable(datetime.date(2015, 4, 26)):
 270             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 271                 raise cliapp.AppException(
 272                     'Only amd64 supports UEFI in Wheezy')
 273
 274         if os.geteuid() != 0:
 275             sys.exit("You need to have root privileges to run this script.")
 276         rootdir = None
 277         try:
 278             rootdev = None
 279             roottype = self.settings['roottype']
 280             bootdev = None
 281             boottype = None
 282             if self.settings['image']:
 283                 self.create_empty_image()
 284                 self.partition_image()
 285                 if self.settings['mbr'] or self.settings['extlinux']:
 286                     self.install_mbr()
 287                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 288                 if self.settings['swap'] > 0:
 289                     self.message("Creating swap space")
 290                     self.runcmd(['mkswap', swapdev])
 291                 self.mkfs(rootdev, fstype=roottype)
 292                 rootdir = self.mount(rootdev)
 293                 if self.settings['use-uefi']:
 294                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 295                     logging.debug("bootdir:%s", self.bootdir)
 296                     self.mkfs(bootdev, fstype='vfat')
 297                     os.makedirs(self.bootdir)
 298                     self.mount(bootdev, self.bootdir)
 299                 elif bootdev:
 300                     if self.settings['boottype']:
 301                         boottype = self.settings['boottype']
 302                     else:
 303                         boottype = 'ext2'
 304                     self.mkfs(bootdev, fstype=boottype)
 305                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 306                     os.mkdir(self.bootdir)
 307                     self.mount(bootdev, self.bootdir)
 308             else:
 309                 rootdir = self.mkdtemp()
 310             self.debootstrap(rootdir)
 311             self.set_hostname(rootdir)
 312             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 313             self.install_debs(rootdir)
 314             self.set_root_password(rootdir)
 315             self.create_users(rootdir)
 316             self.remove_udev_persistent_rules(rootdir)
 317             self.setup_networking(rootdir)
 318             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 319                 self.configure_apt(rootdir)
 320             self.customize(rootdir)
 321             self.cleanup_apt_cache(rootdir)
 322             self.update_initramfs(rootdir)
 323
 324             if self.settings['image']:
 325                 if self.settings['use-uefi']:
 326                     self.install_grub_uefi(rootdir)
 327                 elif self.settings['grub']:
 328                     self.install_grub2(rootdev, rootdir)
 329                 elif self.settings['extlinux']:
 330                     self.install_extlinux(rootdev, rootdir)
 331                 self.append_serial_console(rootdir)
 332                 self.optimize_image(rootdir)
 333                 if self.settings['squash']:
 334                     self.squash()
 335             if self.settings['pkglist']:
 336                 self.list_installed_pkgs(rootdir)
 337
 338             if self.settings['foreign']:
 339                 os.unlink('%s/usr/bin/%s' %
 340                           (rootdir, os.path.basename(self.settings['foreign'])))
 341
 342             if self.settings['tarball']:
 343                 self.create_tarball(rootdir)
 344
 345             if self.settings['owner']:
 346                 self.chown()
 347         except BaseException as e:
 348             self.message('EEEK! Something bad happened...')
 349             if rootdir:
 350                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 351                 if os.path.exists(db_log):
 352                     shutil.copy(db_log, os.getcwd())
 353             self.message(e)
 354             self.cleanup_system()
 355             raise
 356         else:
 357             self.cleanup_system()
 358
 359     def message(self, msg):
 360         logging.info(msg)
 361         if self.settings['verbose']:
 362             print msg
 363
 364     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 365         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 366         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 367                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 368                              env=env, **kwargs)
 369         out, err = p.communicate(stdin)
 370         if p.returncode != 0:
 371             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 372             logging.error(msg)
 373             if not ignore_fail:
 374                 raise cliapp.AppException(msg)
 375         return out
 376
 377     def mkdtemp(self):
 378         dirname = tempfile.mkdtemp()
 379         self.remove_dirs.append(dirname)
 380         logging.debug('mkdir %s', dirname)
 381         return dirname
 382
 383     def mount(self, device, path=None):
 384         if not path:
 385             mount_point = self.mkdtemp()
 386         else:
 387             mount_point = path
 388         self.message('Mounting %s on %s' % (device, mount_point))
 389         self.runcmd(['mount', device, mount_point])
 390         self.mount_points.append(mount_point)
 391         logging.debug('mounted %s on %s', device, mount_point)
 392         return mount_point
 393
 394     def create_empty_image(self):
 395         self.message('Creating disk image')
 396         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 397                      self.settings['image'],
 398                      str(self.settings['size'])])
 399
 400     def partition_image(self):
 401         """
 402         Uses fat16 (msdos) partitioning by default, use part-type to change.
 403         If bootoffset is specified, the first actual partition
 404         starts at that offset to allow customisation scripts to
 405         put bootloader images into the space, e.g. u-boot.
 406         """
 407         self.message('Creating partitions')
 408         self.runcmd(['parted', '-s', self.settings['image'],
 409                      'mklabel', self.settings['part-type']])
 410         partoffset = 0
 411         extent = '100%'
 412         swap = 256 * 1024 * 1024
 413         if self.settings['swap'] > 0:
 414             if self.settings['swap'] > swap:
 415                 swap = self.settings['swap']
 416             else:
 417                 # minimum 256Mb as default qemu ram is 128Mb
 418                 logging.debug("Setting minimum 256Mb swap space")
 419             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 420
 421         if self.settings['use-uefi']:
 422             espsize = self.settings['esp-size'] / (1024 * 1024)
 423             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 424             self.runcmd(['parted', '-s', self.settings['image'],
 425                          'mkpart', 'primary', 'fat32',
 426                          '1', str(espsize)])
 427             self.runcmd(['parted', '-s', self.settings['image'],
 428                          'set', '1', 'boot', 'on'])
 429             self.runcmd(['parted', '-s', self.settings['image'],
 430                          'set', '1', 'esp', 'on'])
 431
 432         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 433             # turn v.small offsets into something at least possible to create.
 434             if self.settings['bootoffset'] < 1048576:
 435                 partoffset = 1
 436                 logging.info(
 437                     "Setting bootoffset %smib to allow for %s bytes",
 438                     partoffset, self.settings['bootoffset'])
 439             else:
 440                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 441                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 442         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 443             if self.settings['grub'] and not partoffset:
 444                 partoffset = 1
 445             bootsize = self.settings['bootsize'] / (1024 * 1024)
 446             bootsize += partoffset
 447             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 448             logging.debug("Starting boot partition at %sMb", bootsize)
 449             self.runcmd(['parted', '-s', self.settings['image'],
 450                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 451             logging.debug("Starting root partition at %sMb", partoffset)
 452             self.runcmd(['parted', '-s', self.settings['image'],
 453                          'mkpart', 'primary', str(bootsize), extent])
 454         elif self.settings['use-uefi']:
 455             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 456             self.runcmd(['parted', '-s', self.settings['image'],
 457                          'mkpart', 'primary', str(bootsize), extent])
 458         else:
 459             self.runcmd(['parted', '-s', self.settings['image'],
 460                          'mkpart', 'primary', '0%', extent])
 461         self.runcmd(['parted', '-s', self.settings['image'],
 462                      'set', '1', 'boot', 'on'])
 463         if self.settings['swap'] > 0:
 464             logging.debug("Creating swap partition")
 465             self.runcmd(['parted', '-s', self.settings['image'],
 466                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 467
 468     def update_initramfs(self, rootdir):
 469         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 470         if os.path.exists(os.path.join(rootdir, cmd)):
 471             self.message("Updating the initramfs")
 472             self.runcmd(['chroot', rootdir, cmd, '-u'])
 473
 474     def install_mbr(self):
 475         if os.path.exists("/sbin/install-mbr"):
 476             self.message('Installing MBR')
 477             self.runcmd(['install-mbr', self.settings['image']])
 478         else:
 479             msg = "mbr enabled but /sbin/install-mbr not found" \
 480                   " - please install the mbr package."
 481             raise cliapp.AppException(msg)
 482
 483     def setup_kpartx(self):
 484         bootindex = None
 485         swapindex = None
 486         if 'freebsd' in os.sys.platform:
 487             out = self.runcmd(['mdconfig', '-a', '-t', 'vnode', '-f',
 488                                self.settings['image']])
 489         else:
 490             out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 491         if self.settings['bootsize'] and self.settings['swap'] > 0:
 492             bootindex = 0
 493             rootindex = 1
 494             swapindex = 2
 495             parts = 3
 496         elif self.settings['use-uefi']:
 497             bootindex = 0
 498             rootindex = 1
 499             parts = 2
 500         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 501             bootindex = 0
 502             rootindex = 1
 503             swapindex = 2
 504             parts = 3
 505         elif self.settings['bootsize']:
 506             bootindex = 0
 507             rootindex = 1
 508             parts = 2
 509         elif self.settings['swap'] > 0:
 510             rootindex = 0
 511             swapindex = 1
 512             parts = 2
 513         else:
 514             rootindex = 0
 515             parts = 1
 516         boot = None
 517         swap = None
 518         if 'freebsd' in os.sys.platform:
 519             devices = glob.glob("/dev/%ss*" % out.strip())
 520         else:
 521             devices = ['/dev/mapper/%s' % line.split()[2]
 522                        for line in out.splitlines()
 523                        if line.startswith('add map ')]
 524         if len(devices) != parts:
 525             msg = 'Surprising number of partitions - check output of losetup -a'
 526             logging.debug("%s", self.runcmd(['losetup', '-a']))
 527             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 528             raise cliapp.AppException(msg)
 529         root = devices[rootindex]
 530         if self.settings['bootsize'] or self.settings['use-uefi']:
 531             boot = devices[bootindex]
 532         if self.settings['swap'] > 0:
 533             swap = devices[swapindex]
 534         return root, boot, swap
 535
 536     def _efi_packages(self):
 537         packages = []
 538         pkg = self.efi_arch_table[self.settings['arch']]['package']
 539         self.message("Adding %s" % pkg)
 540         packages.append(pkg)
 541         extra = self.efi_arch_table[self.settings['arch']]['extra']
 542         if extra and isinstance(extra, str):
 543             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 544             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 545             packages.append(bin_pkg)
 546         return packages
 547
 548     def _copy_efi_binary(self, efi_removable, efi_install):
 549         logging.debug("using bootdir=%s", self.bootdir)
 550         logging.debug("moving %s to %s", efi_removable, efi_install)
 551         if efi_removable.startswith('/'):
 552             efi_removable = efi_removable[1:]
 553         if efi_install.startswith('/'):
 554             efi_install = efi_install[1:]
 555         efi_output = os.path.join(self.bootdir, efi_removable)
 556         efi_input = os.path.join(self.bootdir, efi_install)
 557         if not os.path.exists(efi_input):
 558             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 559             raise cliapp.AppException("Missing %s" % efi_install)
 560         if not os.path.exists(os.path.dirname(efi_output)):
 561             os.makedirs(os.path.dirname(efi_output))
 562         logging.debug(
 563             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 564         if os.path.exists(efi_output):
 565             os.unlink(efi_output)
 566         os.rename(efi_input, efi_output)
 567
 568     def configure_efi(self):
 569         """
 570         Copy the bootloader file from the package into the location
 571         so needs to be after grub and kernel already installed.
 572         """
 573         self.message('Configuring EFI')
 574         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 575         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 576         self.message('Installing UEFI support binary')
 577         self._copy_efi_binary(efi_removable, efi_install)
 578
 579     def configure_extra_efi(self):
 580         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 581         if extra:
 582             efi_removable = str(self.efi_arch_table[extra]['removable'])
 583             efi_install = str(self.efi_arch_table[extra]['install'])
 584             self.message('Copying UEFI support binary for %s' % extra)
 585             self._copy_efi_binary(efi_removable, efi_install)
 586
 587     def mkfs(self, device, fstype):
 588         self.message('Creating filesystem %s' % fstype)
 589         self.runcmd(['mkfs', '-t', fstype, device])
 590
 591     def suite_to_codename(self, distro):
 592         suite = self.debian_info.codename(distro, datetime.date.today())
 593         if not suite:
 594             return distro
 595         return suite
 596
 597     def was_oldstable(self, limit):
 598         suite = self.suite_to_codename(self.settings['distribution'])
 599         # this check is only for debian
 600         if not self.debian_info.valid(suite):
 601             return False
 602         return suite == self.debian_info.old(limit)
 603
 604     def was_stable(self, limit):
 605         suite = self.suite_to_codename(self.settings['distribution'])
 606         # this check is only for debian
 607         if not self.debian_info.valid(suite):
 608             return False
 609         return suite == self.debian_info.stable(limit)
 610
 611     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 612         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 613         self.message(
 614             'Debootstrapping %s [%s] %s' % (
 615                 self.settings['distribution'], self.settings['arch'], msg))
 616
 617         include = self.settings['package']
 618
 619         if not self.settings['foreign'] and not self.settings['no-acpid']:
 620             include.append('acpid')
 621
 622         if self.settings['grub']:
 623             if self.settings['use-uefi']:
 624                 include.extend(self._efi_packages())
 625             else:
 626                 include.append('grub-pc')
 627
 628         if not self.settings['no-kernel']:
 629             if self.settings['kernel-package']:
 630                 kernel_image = self.settings['kernel-package']
 631             else:
 632                 if self.settings['arch'] == 'i386':
 633                     # wheezy (which became oldstable on 04/25/2015) used '486'
 634                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 635                         kernel_arch = '486'
 636                     else:
 637                         kernel_arch = '586'
 638                 elif self.settings['arch'] == 'armhf':
 639                     kernel_arch = 'armmp'
 640                 else:
 641                     kernel_arch = self.settings['arch']
 642                 kernel_image = 'linux-image-%s' % kernel_arch
 643             include.append(kernel_image)
 644
 645         if self.settings['sudo'] and 'sudo' not in include:
 646             include.append('sudo')
 647
 648         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 649
 650         if self.settings['package']:
 651             args.append(
 652                 '--include=%s' % ','.join(include))
 653         if self.settings['foreign']:
 654             args.append('--foreign')
 655         if self.settings['debootstrapopts']:
 656             for opt in self.settings['debootstrapopts']:
 657                 for part in opt.split(' '):
 658                     args.append('--%s' % part)
 659         elif self.settings['variant']:
 660             args.append('--variant')
 661             args.append(self.settings['variant'])
 662         args += [self.settings['distribution'],
 663                  rootdir, self.settings['mirror']]
 664         logging.debug(" ".join(args))
 665         self.runcmd(args)
 666         if self.settings['foreign']:
 667             # set a noninteractive debconf environment for secondstage
 668             env = {
 669                 "DEBIAN_FRONTEND": "noninteractive",
 670                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 671                 "LC_ALL": "C"
 672             }
 673             # add the mapping to the complete environment.
 674             env.update(os.environ)
 675             # First copy the binfmt handler over
 676             self.message('Setting up binfmt handler')
 677             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 678             # Next, run the package install scripts etc.
 679             self.message('Running debootstrap second stage')
 680             self.runcmd(['chroot', rootdir,
 681                          '/debootstrap/debootstrap', '--second-stage'],
 682                         env=env)
 683
 684     def set_hostname(self, rootdir):
 685         hostname = self.settings['hostname']
 686         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 687             f.write('%s\n' % hostname)
 688
 689         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 690         try:
 691             with open(etc_hosts, 'r') as f:
 692                 data = f.read()
 693             with open(etc_hosts, 'w') as f:
 694                 for line in data.splitlines():
 695                     if line.startswith('127.0.0.1'):
 696                         line += ' %s' % hostname
 697                     f.write('%s\n' % line)
 698         except IOError:
 699             pass
 700
 701     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 702         def fsuuid(device):
 703             if 'freebsd' in os.sys.platform:
 704                 out = self.runcmd(['grub-probe', '-d', device, '-t', 'fs_uuid'])
 705                 return "/dev/ufsid/%s" % out.strip()
 706             else:
 707                 out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 708                                 '-s', 'UUID', device])
 709                 return "UUID=%s" % out.splitlines()[0].strip()
 710
 711         if rootdev:
 712             rootdevstr = fsuuid(rootdev)
 713         else:
 714             rootdevstr = '/dev/sda1'
 715
 716         if bootdev and not self.settings['use-uefi']:
 717             bootdevstr = fsuuid(bootdev)
 718         else:
 719             bootdevstr = None
 720
 721         fstab = os.path.join(rootdir, 'etc', 'fstab')
 722         with open(fstab, 'w') as f:
 723             f.write('proc /proc proc defaults 0 0\n')
 724             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 725             if bootdevstr:
 726                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 727                 if self.settings['swap'] > 0:
 728                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 729             elif self.settings['swap'] > 0:
 730                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 731
 732     def install_debs(self, rootdir):
 733         if not self.settings['custom-package']:
 734             return
 735         self.message('Installing custom packages')
 736         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 737         os.mkdir(tmp)
 738         for deb in self.settings['custom-package']:
 739             shutil.copy(deb, tmp)
 740         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 741                      for deb in self.settings['custom-package']]
 742         out, err, _ = \
 743             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 744         logging.debug('stdout:\n%s', out)
 745         logging.debug('stderr:\n%s', err)
 746         out = self.runcmd(['chroot', rootdir,
 747                            'apt-get', '-f', '--no-remove', 'install'])
 748         logging.debug('stdout:\n%s', out)
 749         shutil.rmtree(tmp)
 750
 751     def cleanup_apt_cache(self, rootdir):
 752         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 753         logging.debug('stdout:\n%s', out)
 754
 755     def set_root_password(self, rootdir):
 756         if self.settings['root-password']:
 757             self.message('Setting root password')
 758             self.set_password(rootdir, 'root', self.settings['root-password'])
 759         elif self.settings['lock-root-password']:
 760             self.message('Locking root password')
 761             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 762         else:
 763             self.message('Give root an empty password')
 764             self.delete_password(rootdir, 'root')
 765
 766     def create_users(self, rootdir):
 767         def create_user(vmuser):
 768             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 769                          '--disabled-password', vmuser])
 770             if self.settings['sudo']:
 771                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 772
 773         for userpass in self.settings['user']:
 774             if '/' in userpass:
 775                 user, password = userpass.split('/', 1)
 776                 create_user(user)
 777                 self.set_password(rootdir, user, password)
 778             else:
 779                 create_user(userpass)
 780                 self.delete_password(rootdir, userpass)
 781
 782     def set_password(self, rootdir, user, password):
 783         encrypted = crypt.crypt(password, '..')
 784         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 785
 786     def delete_password(self, rootdir, user):
 787         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 788
 789     def remove_udev_persistent_rules(self, rootdir):
 790         self.message('Removing udev persistent cd and net rules')
 791         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 792             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 793             if os.path.exists(pathname):
 794                 logging.debug('rm %s', pathname)
 795                 os.remove(pathname)
 796             else:
 797                 logging.debug('not removing non-existent %s', pathname)
 798
 799     def mask_udev_predictable_rules(self, rootdir):
 800         """
 801         This can be reset later but to get networking working immediately
 802         on boot, the interface we're going to use must be known without
 803         reference to the eventual machine.
 804         http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
 805         """
 806         self.message('Disabling systemd predictable interface names')
 807         udev_path = os.path.join(
 808             'etc', 'udev', 'rules.d', '80-net-setup-link.rules')
 809         self.runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path])
 810
 811     def setup_networking(self, rootdir):
 812         self.message('Setting up networking')
 813         ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 814         ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d')
 815
 816         # unconditionally write for wheezy (which became oldstable 2015.04.25)
 817         if self.was_oldstable(datetime.date(2015, 4, 26)):
 818             with open(ifc_file, 'w') as netfile:
 819                 netfile.write('source /etc/network/interfaces.d/*\n')
 820         elif not os.path.exists(ifc_file):
 821             with open(ifc_file, 'a') as netfile:
 822                 netfile.write('source-directory /etc/network/interfaces.d\n')
 823
 824         if not os.path.exists(ifc_d):
 825             os.mkdir(ifc_d)
 826         ethpath = os.path.join(ifc_d, 'setup')
 827         with open(ethpath, 'w') as eth:
 828             eth.write('auto lo\n')
 829             eth.write('iface lo inet loopback\n')
 830
 831             if self.settings['enable-dhcp']:
 832                 eth.write('\n')
 833                 eth.write('auto eth0\n')
 834                 eth.write('iface eth0 inet dhcp\n')
 835                 # force predictable interface names
 836                 self.mask_udev_predictable_rules(rootdir)
 837
 838     def append_serial_console(self, rootdir):
 839         if self.settings['serial-console']:
 840             serial_command = self.settings['serial-console-command']
 841             logging.debug('adding getty to serial console')
 842             inittab = os.path.join(rootdir, 'etc/inittab')
 843             # to autologin, serial_command can contain '-a root'
 844             with open(inittab, 'a') as f:
 845                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 846
 847     # pylint: disable=no-self-use
 848     def _grub_serial_console(self, rootdir):
 849         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 850         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 851         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 852         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 853         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 854         with open(grub_cfg, 'a+') as cfg:
 855             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 856             cfg.write("%s\n" % cmdline)
 857             cfg.write("%s\n" % terminal)
 858             cfg.write("%s\n" % command)
 859
 860     def _mount_wrapper(self, rootdir):
 861         self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 862                      '%s' % os.path.join(rootdir, 'dev')])
 863         self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 864                      '%s' % os.path.join(rootdir, 'proc')])
 865         self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 866                      '%s' % os.path.join(rootdir, 'sys')])
 867
 868     def _umount_wrapper(self, rootdir):
 869         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 870         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 871         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 872
 873     def install_grub_uefi(self, rootdir):
 874         self.message("Configuring grub-uefi")
 875         target = self.efi_arch_table[self.settings['arch']]['target']
 876         grub_opts = "--target=%s" % target
 877         logging.debug("Running grub-install with options: %s", grub_opts)
 878         self._mount_wrapper(rootdir)
 879         try:
 880             self.runcmd(['chroot', rootdir, 'update-grub'])
 881             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 882         except cliapp.AppException as exc:
 883             logging.warning(exc)
 884             self.message(
 885                 "Failed to configure grub-uefi for %s" %
 886                 self.settings['arch'])
 887             self._umount_wrapper(rootdir)
 888         self.configure_efi()
 889         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 890         if extra:
 891             target = self.efi_arch_table[extra]['target']
 892             grub_opts = "--target=%s" % target
 893             try:
 894                 self.runcmd(['chroot', rootdir, 'update-grub'])
 895                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 896             except cliapp.AppException as exc:
 897                 logging.warning(exc)
 898                 self.message(
 899                     "Failed to configure grub-uefi for %s" % extra)
 900             self.configure_extra_efi()
 901         self._umount_wrapper(rootdir)
 902
 903     def install_grub2(self, rootdev, rootdir):
 904         self.message("Configuring grub2")
 905         # rely on kpartx using consistent naming to map loop0p1 to loop0
 906         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 907         if self.settings['serial-console']:
 908             self._grub_serial_console(rootdir)
 909         logging.debug("Running grub-install with options: %s", grub_opts)
 910         self._mount_wrapper(rootdir)
 911         try:
 912             self.runcmd(['chroot', rootdir, 'update-grub'])
 913             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 914         except cliapp.AppException as exc:
 915             logging.warning(exc)
 916             self.message("Failed. Is grub2-common installed? Using extlinux.")
 917             self.install_extlinux(rootdev, rootdir)
 918         self._umount_wrapper(rootdir)
 919
 920     def install_extlinux(self, rootdev, rootdir):
 921         if not os.path.exists("/usr/bin/extlinux"):
 922             self.message("extlinux not installed, skipping.")
 923             return
 924         self.message('Installing extlinux')
 925
 926         def find(pattern):
 927             dirname = os.path.join(rootdir, 'boot')
 928             basenames = os.listdir(dirname)
 929             logging.debug('find: %s', basenames)
 930             for basename in basenames:
 931                 if re.search(pattern, basename):
 932                     return os.path.join('boot', basename)
 933             raise cliapp.AppException('Cannot find match: %s' % pattern)
 934
 935         try:
 936             kernel_image = find('vmlinuz-.*')
 937             initrd_image = find('initrd.img-.*')
 938         except cliapp.AppException as e:
 939             self.message("Unable to find kernel. Not installing extlinux.")
 940             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 941             return
 942
 943         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 944                            '-s', 'UUID', rootdev])
 945         uuid = out.splitlines()[0].strip()
 946
 947         conf = os.path.join(rootdir, 'extlinux.conf')
 948         logging.debug('configure extlinux %s', conf)
 949         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 950         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 951         msg = '''
 952 default linux
 953 timeout 1
 954
 955 label linux
 956 kernel %(kernel)s
 957 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 958 %(extserial)s
 959 ''' % {
 960             'kernel': kernel_image,  # pylint: disable=bad-continuation
 961             'initrd': initrd_image,  # pylint: disable=bad-continuation
 962             'uuid': uuid,  # pylint: disable=bad-continuation
 963             'kserial': kserial,  # pylint: disable=bad-continuation
 964             'extserial': extserial,  # pylint: disable=bad-continuation
 965         }  # pylint: disable=bad-continuation
 966         logging.debug("extlinux config:\n%s", msg)
 967
 968         # python multiline string substitution is just ugly.
 969         # use an external file or live with the mangling, no point in
 970         # mangling the string to remove spaces just to keep it pretty in source.
 971         f = open(conf, 'w')
 972         f.write(msg)
 973
 974         self.runcmd(['extlinux', '--install', rootdir])
 975         self.runcmd(['sync'])
 976         time.sleep(2)
 977
 978     def optimize_image(self, rootdir):
 979         """
 980         Filing up the image with zeros will increase its compression rate
 981         """
 982         if not self.settings['sparse']:
 983             zeros = os.path.join(rootdir, 'ZEROS')
 984             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 985             self.runcmd(['rm', '-f', zeros])
 986
 987     def squash(self):
 988         """
 989         Run squashfs on the image.
 990         """
 991         if not os.path.exists('/usr/bin/mksquashfs'):
 992             logging.warning("Squash selected but mksquashfs not found!")
 993             return
 994         logging.debug(
 995             "%s usage: %s", self.settings['image'],
 996             self.runcmd(['du', self.settings['image']]))
 997         self.message("Running mksquashfs")
 998         suffixed = "%s.squashfs" % self.settings['image']
 999         if os.path.exists(suffixed):
1000             os.unlink(suffixed)
1001         msg = self.runcmd(
1002             ['mksquashfs', self.settings['image'],
1003              suffixed,
1004              '-no-progress', '-comp', 'xz'], ignore_fail=False)
1005         logging.debug(msg)
1006         check_size = os.path.getsize(suffixed)
1007         if check_size < (1024 * 1024):
1008             logging.warning(
1009                 "%s appears to be too small! %s bytes",
1010                 suffixed, check_size)
1011         else:
1012             logging.debug("squashed size: %s", check_size)
1013             os.unlink(self.settings['image'])
1014             self.settings['image'] = suffixed
1015             logging.debug(
1016                 "%s usage: %s", self.settings['image'],
1017                 self.runcmd(['du', self.settings['image']]))
1018
1019     def cleanup_system(self):
1020         # Clean up after any errors.
1021
1022         self.message('Cleaning up')
1023
1024         # Umount in the reverse mount order
1025         if self.settings['image']:
1026             for i in range(len(self.mount_points) - 1, -1, -1):
1027                 mount_point = self.mount_points[i]
1028                 try:
1029                     self.runcmd(['umount', mount_point], ignore_fail=False)
1030                 except cliapp.AppException:
1031                     logging.debug("umount failed, sleeping and trying again")
1032                     time.sleep(5)
1033                     self.runcmd(['umount', mount_point], ignore_fail=False)
1034
1035             if 'freebsd' in os.sys.platform:
1036                 out = self.runcmd(['mdconfig', '-l', '-f', self.settings['image']])
1037                 for devid in out.split():
1038                     self.runcmd(['mdconfig', '-d', '-u', devid],
1039                                 ignore_fail=True)
1040             else:
1041                 self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1042
1043         for dirname in self.remove_dirs:
1044             shutil.rmtree(dirname)
1045
1046     def customize(self, rootdir):
1047         script = self.settings['customize']
1048         if not script:
1049             return
1050         if not os.path.exists(script):
1051             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1052             if not os.path.exists(example):
1053                 self.message("Unable to find %s" % script)
1054                 return
1055             script = example
1056         self.message('Running customize script %s' % script)
1057         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1058         logging.debug(
1059             "%s usage: %s", self.settings['image'],
1060             self.runcmd(['du', self.settings['image']]))
1061         try:
1062             with open('/dev/tty', 'w') as tty:
1063                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1064         except IOError:
1065             logging.debug('tty unavailable, trying in headless mode.')
1066             subprocess.call([script, rootdir, self.settings['image']])
1067
1068     def create_tarball(self, rootdir):
1069         # Create a tarball of the disk's contents
1070         # shell out to runcmd since it more easily handles rootdir
1071         self.message('Creating tarball of disk contents')
1072         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1073
1074     def chown(self):
1075         # Change image owner after completed build
1076         if self.settings['image']:
1077             filename = self.settings['image']
1078         elif self.settings['tarball']:
1079             filename = self.settings['tarball']
1080         else:
1081             return
1082         self.message("Changing owner to %s" % self.settings["owner"])
1083         subprocess.call(["chown", self.settings["owner"], filename])
1084
1085     def list_installed_pkgs(self, rootdir):
1086         # output the list of installed packages for sources identification
1087         self.message("Creating a list of installed binary package names")
1088         out = self.runcmd(['chroot', rootdir,
1089                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1090         with open('dpkg.list', 'w') as dpkg:
1091             dpkg.write(out)
1092
1093     def configure_apt(self, rootdir):
1094         # use the distribution and mirror to create an apt source
1095         self.message("Configuring apt to use distribution and mirror")
1096         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1097         logging.debug('configure apt %s', conf)
1098         mirror = self.settings['mirror']
1099         if self.settings['apt-mirror']:
1100             mirror = self.settings['apt-mirror']
1101             self.message("Setting apt mirror to %s" % mirror)
1102         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1103         f = open(conf, 'w')
1104         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1105         f.write(line)
1106         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1107         f.write(line)
1108         f.close()
1109         # ensure the apt sources have valid lists
1110         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1111
1112 if __name__ == '__main__':
1113     VmDebootstrap(version=__version__).run()