vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import glob
  24 import re
  25 import sys
  26 import shutil
  27 import datetime
  28 import subprocess
  29 import tempfile
  30 import time
  31 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  32
  33
  34 __version__ = '0.11'
  35
  36 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  37
  38
  39 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  40
  41     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  42         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  43         self.remove_dirs = []
  44         self.mount_points = []
  45         self.debian_info = DebianDistroInfo()
  46         self.ubuntu_info = UbuntuDistroInfo()
  47         self.bootdir = None
  48         self.efi_arch_table = {
  49             'amd64': {
  50                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  51                 'install': '/EFI/debian/grubx64.efi',  # package location
  52                 'package': 'grub-efi-amd64',  # bootstrap package
  53                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  54                 'extra': 'i386',  # architecture to add binary package
  55                 'exclusive': False,  # only EFI supported for this arch.
  56                 'target': 'x86_64-efi',  # grub target name
  57             },
  58             'i386': {
  59                 'removable': '/EFI/boot/bootia32.efi',
  60                 'install': '/EFI/debian/grubia32.efi',
  61                 'package': 'grub-efi-ia32',
  62                 'bin_package': 'grub-efi-ia32-bin',
  63                 'extra': None,
  64                 'exclusive': False,
  65                 'target': 'i386-efi',
  66             },
  67             'arm64': {
  68                 'removable': '/EFI/boot/bootaa64.efi',
  69                 'install': '/EFI/debian/grubaa64.efi',
  70                 'package': 'grub-efi-arm64',
  71                 'bin_package': 'grub-efi-arm64-bin',
  72                 'extra': None,
  73                 'exclusive': True,
  74                 'target': 'arm64-efi',
  75             }
  76         }
  77
  78     def add_settings(self):
  79         default_arch = subprocess.check_output(
  80             ["dpkg", "--print-architecture"]).strip()
  81
  82         self.settings.boolean(
  83             ['verbose'], 'report what is going on')
  84         self.settings.string(
  85             ['image'], 'put created disk image in FILE',
  86             metavar='FILE')
  87         self.settings.bytesize(
  88             ['size'],
  89             'create a disk image of size SIZE (%default)',
  90             metavar='SIZE',
  91             default='1G')
  92         self.settings.bytesize(
  93             ['bootsize'],
  94             'create boot partition of size SIZE (%default)',
  95             metavar='BOOTSIZE',
  96             default='0%')
  97         self.settings.string(
  98             ['boottype'],
  99             'specify file system type for /boot/',
 100             default='ext2')
 101         self.settings.bytesize(
 102             ['bootoffset'],
 103             'Space to leave at start of the image for bootloader',
 104             default='0')
 105         self.settings.boolean(
 106             ['use-uefi'],
 107             'Setup image for UEFI boot',
 108             default=False)
 109         self.settings.bytesize(
 110             ['esp-size'],
 111             'Size of EFI System Partition - requires use-uefi',
 112             default='5mib')
 113         self.settings.string(
 114             ['part-type'],
 115             'Partition type to use for this image',
 116             default='msdos')
 117         self.settings.string(
 118             ['roottype'],
 119             'specify file system type for /',
 120             default='ext4')
 121         self.settings.bytesize(
 122             ['swap'],
 123             'create swap space of size SIZE (min 256Mb)')
 124         self.settings.string(
 125             ['foreign'],
 126             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 127         self.settings.string(
 128             ['variant'],
 129             'select debootstrap variant if not using the default [deprecated]')
 130         self.settings.string_list(
 131             ['debootstrapopts'],
 132             'pass additional options to debootstrap'),
 133         self.settings.boolean(
 134             ['extlinux'],
 135             'install extlinux?',
 136             default=True)
 137         self.settings.string(
 138             ['tarball'],
 139             "tar up the disk's contents in FILE",
 140             metavar='FILE')
 141         self.settings.string(
 142             ['apt-mirror'],
 143             'configure apt to use MIRROR',
 144             metavar='URL')
 145         self.settings.string(
 146             ['mirror'],
 147             'use MIRROR as package source (%default)',
 148             metavar='URL',
 149             default='http://http.debian.net/debian/')
 150         self.settings.string(
 151             ['arch'],
 152             'architecture to use (%default)',
 153             metavar='ARCH',
 154             default=default_arch)
 155         self.settings.string(
 156             ['distribution'],
 157             'release to use (%default)',
 158             metavar='NAME',
 159             default='stable')
 160         self.settings.string_list(
 161             ['package'],
 162             'install PACKAGE onto system')
 163         self.settings.string_list(
 164             ['custom-package'],
 165             'install package in DEB file onto system (not from mirror)',
 166             metavar='DEB')
 167         self.settings.boolean(
 168             ['no-kernel'],
 169             'do not install a linux package')
 170         self.settings.string(
 171             ['kernel-package'],
 172             'install PACKAGE instead of the default kernel package',
 173             metavar='PACKAGE')
 174         self.settings.boolean(
 175             ['enable-dhcp'],
 176             'enable DHCP on eth0')
 177         self.settings.string(
 178             ['root-password'],
 179             'set root password',
 180             metavar='PASSWORD')
 181         self.settings.boolean(
 182             ['lock-root-password'],
 183             'lock root account so they cannot login?')
 184         self.settings.string(
 185             ['customize'],
 186             'run SCRIPT after setting up system',
 187             metavar='SCRIPT')
 188         self.settings.string(
 189             ['hostname'],
 190             'set name to HOSTNAME (%default)',
 191             metavar='HOSTNAME',
 192             default='debian')
 193         self.settings.string_list(
 194             ['user'],
 195             'create USER with PASSWORD',
 196             metavar='USER/PASSWORD')
 197         self.settings.boolean(
 198             ['serial-console'],
 199             'configure image to use a serial console')
 200         self.settings.string(
 201             ['serial-console-command'],
 202             'command to manage the serial console, appended to /etc/inittab (%default)',
 203             metavar='COMMAND',
 204             default='/sbin/getty -L ttyS0 115200 vt100')
 205         self.settings.boolean(
 206             ['sudo'],
 207             'install sudo, and if user is created, add them to sudo group')
 208         self.settings.string(
 209             ['owner'],
 210             'the user who will own the image when the build is complete.')
 211         self.settings.boolean(
 212             ['squash'],
 213             'use squashfs on the final image.')
 214         self.settings.boolean(
 215             ['configure-apt'],
 216             'Create an apt source based on the distribution and mirror selected.')
 217         self.settings.boolean(
 218             ['mbr'],
 219             'Run install-mbr (default if extlinux used)')
 220         self.settings.boolean(
 221             ['grub'],
 222             'Install and configure grub2 - disables extlinux.')
 223         self.settings.boolean(
 224             ['sparse'],
 225             'Do not fill the image with zeros to keep a sparse disk image',
 226             default=False)
 227         self.settings.boolean(
 228             ['pkglist'],
 229             'Create a list of package names included in the image.')
 230         self.settings.boolean(
 231             ['no-acpid'],
 232             'do not install the acpid package',
 233             default=False)
 234
 235     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 236         if not self.settings['image'] and not self.settings['tarball']:
 237             raise cliapp.AppException(
 238                 'You must give disk image filename, or tarball filename')
 239         if self.settings['image'] and not self.settings['size']:
 240             raise cliapp.AppException(
 241                 'If disk image is specified, you must give image size.')
 242         if not self.debian_info.valid(self.settings['distribution']):
 243             if not self.ubuntu_info.valid(self.settings['distribution']):
 244                 raise cliapp.AppException(
 245                     '%s is not a valid Debian or Ubuntu suite or codename.'
 246                     % self.settings['distribution'])
 247         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 248             raise cliapp.AppException(
 249                 'You must specify use-uefi for esp-size to have effect')
 250         if self.settings['arch'] in self.efi_arch_table and\
 251                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 252                 not self.settings['use-uefi']:
 253             raise cliapp.AppException(
 254                 'Only UEFI is supported on %s' % self.settings['arch'])
 255         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 256             raise cliapp.AppException(
 257                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 258         if self.settings['use-uefi'] and (
 259                 self.settings['bootsize'] or
 260                 self.settings['bootoffset']):
 261             raise cliapp.AppException(
 262                 'A separate boot partition is not supported with UEFI')
 263
 264         if self.settings['use-uefi'] and not self.settings['grub']:
 265             raise cliapp.AppException(
 266                 'UEFI without Grub is not supported.')
 267
 268         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 269         if self.was_oldstable(datetime.date(2015, 4, 26)):
 270             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 271                 raise cliapp.AppException(
 272                     'Only amd64 supports UEFI in Wheezy')
 273
 274         if os.geteuid() != 0:
 275             sys.exit("You need to have root privileges to run this script.")
 276         rootdir = None
 277         try:
 278             rootdev = None
 279             roottype = self.settings['roottype']
 280             bootdev = None
 281             boottype = None
 282             if self.settings['image']:
 283                 self.create_empty_image()
 284                 self.partition_image()
 285                 if self.settings['mbr'] or self.settings['extlinux']:
 286                     self.install_mbr()
 287                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 288                 if self.settings['swap'] > 0:
 289                     self.message("Creating swap space")
 290                     self.runcmd(['mkswap', swapdev])
 291                 self.mkfs(rootdev, fstype=roottype)
 292                 rootdir = self.mount(rootdev)
 293                 if self.settings['use-uefi']:
 294                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 295                     logging.debug("bootdir:%s", self.bootdir)
 296                     self.mkfs(bootdev, fstype='vfat')
 297                     os.makedirs(self.bootdir)
 298                     self.mount(bootdev, self.bootdir)
 299                 elif bootdev:
 300                     if self.settings['boottype']:
 301                         boottype = self.settings['boottype']
 302                     else:
 303                         boottype = 'ext2'
 304                     self.mkfs(bootdev, fstype=boottype)
 305                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 306                     os.mkdir(self.bootdir)
 307                     self.mount(bootdev, self.bootdir)
 308             else:
 309                 rootdir = self.mkdtemp()
 310             self.debootstrap(rootdir)
 311             self.set_hostname(rootdir)
 312             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 313             self.install_debs(rootdir)
 314             self.set_root_password(rootdir)
 315             self.create_users(rootdir)
 316             self.remove_udev_persistent_rules(rootdir)
 317             self.setup_networking(rootdir)
 318             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 319                 self.configure_apt(rootdir)
 320             self.customize(rootdir)
 321             self.cleanup_apt_cache(rootdir)
 322             self.update_initramfs(rootdir)
 323
 324             if self.settings['image']:
 325                 if self.settings['use-uefi']:
 326                     self.install_grub_uefi(rootdir)
 327                 elif self.settings['grub']:
 328                     self.install_grub2(rootdev, rootdir)
 329                 elif self.settings['extlinux']:
 330                     self.install_extlinux(rootdev, rootdir)
 331                 self.append_serial_console(rootdir)
 332                 self.optimize_image(rootdir)
 333                 if self.settings['squash']:
 334                     self.squash()
 335             if self.settings['pkglist']:
 336                 self.list_installed_pkgs(rootdir)
 337
 338             if self.settings['foreign']:
 339                 os.unlink('%s/usr/bin/%s' %
 340                           (rootdir, os.path.basename(self.settings['foreign'])))
 341
 342             if self.settings['tarball']:
 343                 self.create_tarball(rootdir)
 344
 345             if self.settings['owner']:
 346                 self.chown()
 347         except BaseException as e:
 348             self.message('EEEK! Something bad happened...')
 349             if rootdir:
 350                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 351                 if os.path.exists(db_log):
 352                     shutil.copy(db_log, os.getcwd())
 353             self.message(e)
 354             self.cleanup_system()
 355             raise
 356         else:
 357             self.cleanup_system()
 358
 359     def message(self, msg):
 360         logging.info(msg)
 361         if self.settings['verbose']:
 362             print msg
 363
 364     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 365         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 366         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 367                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 368                              env=env, **kwargs)
 369         out, err = p.communicate(stdin)
 370         if p.returncode != 0:
 371             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 372             logging.error(msg)
 373             if not ignore_fail:
 374                 raise cliapp.AppException(msg)
 375         return out
 376
 377     def mkdtemp(self):
 378         dirname = tempfile.mkdtemp()
 379         self.remove_dirs.append(dirname)
 380         logging.debug('mkdir %s', dirname)
 381         return dirname
 382
 383     def mount(self, device, path=None):
 384         if not path:
 385             mount_point = self.mkdtemp()
 386         else:
 387             mount_point = path
 388         self.message('Mounting %s on %s' % (device, mount_point))
 389         self.runcmd(['mount', device, mount_point])
 390         self.mount_points.append(mount_point)
 391         logging.debug('mounted %s on %s', device, mount_point)
 392         return mount_point
 393
 394     def create_empty_image(self):
 395         self.message('Creating disk image')
 396         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 397                      self.settings['image'],
 398                      str(self.settings['size'])])
 399
 400     def partition_image(self):
 401         """
 402         Uses fat16 (msdos) partitioning by default, use part-type to change.
 403         If bootoffset is specified, the first actual partition
 404         starts at that offset to allow customisation scripts to
 405         put bootloader images into the space, e.g. u-boot.
 406         """
 407         self.message('Creating partitions')
 408         self.runcmd(['parted', '-s', self.settings['image'],
 409                      'mklabel', self.settings['part-type']])
 410         partoffset = 0
 411         extent = '100%'
 412         swap = 256 * 1024 * 1024
 413         if self.settings['swap'] > 0:
 414             if self.settings['swap'] > swap:
 415                 swap = self.settings['swap']
 416             else:
 417                 # minimum 256Mb as default qemu ram is 128Mb
 418                 logging.debug("Setting minimum 256Mb swap space")
 419             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 420
 421         if self.settings['use-uefi']:
 422             espsize = self.settings['esp-size'] / (1024 * 1024)
 423             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 424             self.runcmd(['parted', '-s', self.settings['image'],
 425                          'mkpart', 'primary', 'fat32',
 426                          '1', str(espsize)])
 427             self.runcmd(['parted', '-s', self.settings['image'],
 428                          'set', '1', 'boot', 'on'])
 429             self.runcmd(['parted', '-s', self.settings['image'],
 430                          'set', '1', 'esp', 'on'])
 431
 432         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 433             # turn v.small offsets into something at least possible to create.
 434             if self.settings['bootoffset'] < 1048576:
 435                 partoffset = 1
 436                 logging.info(
 437                     "Setting bootoffset %smib to allow for %s bytes",
 438                     partoffset, self.settings['bootoffset'])
 439             else:
 440                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 441                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 442         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 443             if self.settings['grub'] and not partoffset:
 444                 partoffset = 1
 445             bootsize = self.settings['bootsize'] / (1024 * 1024)
 446             bootsize += partoffset
 447             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 448             logging.debug("Starting boot partition at %sMb", bootsize)
 449             self.runcmd(['parted', '-s', self.settings['image'],
 450                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 451             logging.debug("Starting root partition at %sMb", partoffset)
 452             self.runcmd(['parted', '-s', self.settings['image'],
 453                          'mkpart', 'primary', str(bootsize), extent])
 454         elif self.settings['use-uefi']:
 455             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 456             self.runcmd(['parted', '-s', self.settings['image'],
 457                          'mkpart', 'primary', str(bootsize), extent])
 458         else:
 459             self.runcmd(['parted', '-s', self.settings['image'],
 460                          'mkpart', 'primary', '0%', extent])
 461         self.runcmd(['parted', '-s', self.settings['image'],
 462                      'set', '1', 'boot', 'on'])
 463         if self.settings['swap'] > 0:
 464             logging.debug("Creating swap partition")
 465             self.runcmd(['parted', '-s', self.settings['image'],
 466                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 467
 468     def update_initramfs(self, rootdir):
 469         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 470         if os.path.exists(os.path.join(rootdir, cmd)):
 471             self.message("Updating the initramfs")
 472             self.runcmd(['chroot', rootdir, cmd, '-u'])
 473
 474     def install_mbr(self):
 475         if os.path.exists("/sbin/install-mbr"):
 476             self.message('Installing MBR')
 477             self.runcmd(['install-mbr', self.settings['image']])
 478         else:
 479             msg = "mbr enabled but /sbin/install-mbr not found" \
 480                   " - please install the mbr package."
 481             raise cliapp.AppException(msg)
 482
 483     def setup_kpartx(self):
 484         bootindex = None
 485         swapindex = None
 486         if 'freebsd' in os.sys.platform:
 487             out = self.runcmd(['mdconfig', '-a', '-t', 'vnode', '-f',
 488                                self.settings['image']])
 489         else:
 490             out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 491         if self.settings['bootsize'] and self.settings['swap'] > 0:
 492             bootindex = 0
 493             rootindex = 1
 494             swapindex = 2
 495             parts = 3
 496         elif self.settings['use-uefi']:
 497             bootindex = 0
 498             rootindex = 1
 499             parts = 2
 500         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 501             bootindex = 0
 502             rootindex = 1
 503             swapindex = 2
 504             parts = 3
 505         elif self.settings['bootsize']:
 506             bootindex = 0
 507             rootindex = 1
 508             parts = 2
 509         elif self.settings['swap'] > 0:
 510             rootindex = 0
 511             swapindex = 1
 512             parts = 2
 513         else:
 514             rootindex = 0
 515             parts = 1
 516         boot = None
 517         swap = None
 518         if 'freebsd' in os.sys.platform:
 519             devices = glob.glob("/dev/%ss*" % out.strip())
 520         else:
 521             devices = ['/dev/mapper/%s' % line.split()[2]
 522                        for line in out.splitlines()
 523                        if line.startswith('add map ')]
 524         if len(devices) != parts:
 525             msg = 'Surprising number of partitions - check output of losetup -a'
 526             logging.debug("%s", self.runcmd(['losetup', '-a']))
 527             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 528             raise cliapp.AppException(msg)
 529         root = devices[rootindex]
 530         if self.settings['bootsize'] or self.settings['use-uefi']:
 531             boot = devices[bootindex]
 532         if self.settings['swap'] > 0:
 533             swap = devices[swapindex]
 534         return root, boot, swap
 535
 536     def _efi_packages(self):
 537         packages = []
 538         pkg = self.efi_arch_table[self.settings['arch']]['package']
 539         self.message("Adding %s" % pkg)
 540         packages.append(pkg)
 541         extra = self.efi_arch_table[self.settings['arch']]['extra']
 542         if extra and isinstance(extra, str):
 543             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 544             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 545             packages.append(bin_pkg)
 546         return packages
 547
 548     def _copy_efi_binary(self, efi_removable, efi_install):
 549         logging.debug("using bootdir=%s", self.bootdir)
 550         logging.debug("moving %s to %s", efi_removable, efi_install)
 551         if efi_removable.startswith('/'):
 552             efi_removable = efi_removable[1:]
 553         if efi_install.startswith('/'):
 554             efi_install = efi_install[1:]
 555         efi_output = os.path.join(self.bootdir, efi_removable)
 556         efi_input = os.path.join(self.bootdir, efi_install)
 557         if not os.path.exists(efi_input):
 558             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 559             raise cliapp.AppException("Missing %s" % efi_install)
 560         if not os.path.exists(os.path.dirname(efi_output)):
 561             os.makedirs(os.path.dirname(efi_output))
 562         logging.debug(
 563             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 564         if os.path.exists(efi_output):
 565             os.unlink(efi_output)
 566         os.rename(efi_input, efi_output)
 567
 568     def configure_efi(self):
 569         """
 570         Copy the bootloader file from the package into the location
 571         so needs to be after grub and kernel already installed.
 572         """
 573         self.message('Configuring EFI')
 574         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 575         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 576         self.message('Installing UEFI support binary')
 577         self._copy_efi_binary(efi_removable, efi_install)
 578
 579     def configure_extra_efi(self):
 580         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 581         if extra:
 582             efi_removable = str(self.efi_arch_table[extra]['removable'])
 583             efi_install = str(self.efi_arch_table[extra]['install'])
 584             self.message('Copying UEFI support binary for %s' % extra)
 585             self._copy_efi_binary(efi_removable, efi_install)
 586
 587     def mkfs(self, device, fstype):
 588         self.message('Creating filesystem %s' % fstype)
 589         self.runcmd(['mkfs', '-t', fstype, device])
 590
 591     def suite_to_codename(self, distro):
 592         suite = self.debian_info.codename(distro, datetime.date.today())
 593         if not suite:
 594             return distro
 595         return suite
 596
 597     def was_oldstable(self, limit):
 598         suite = self.suite_to_codename(self.settings['distribution'])
 599         # this check is only for debian
 600         if not self.debian_info.valid(suite):
 601             return False
 602         return suite == self.debian_info.old(limit)
 603
 604     def was_stable(self, limit):
 605         suite = self.suite_to_codename(self.settings['distribution'])
 606         # this check is only for debian
 607         if not self.debian_info.valid(suite):
 608             return False
 609         return suite == self.debian_info.stable(limit)
 610
 611     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 612         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 613         self.message(
 614             'Debootstrapping %s [%s] %s' % (
 615                 self.settings['distribution'], self.settings['arch'], msg))
 616
 617         include = self.settings['package']
 618
 619         if not self.settings['foreign'] and not self.settings['no-acpid']:
 620             include.append('acpid')
 621
 622         if self.settings['grub']:
 623             if self.settings['use-uefi']:
 624                 include.extend(self._efi_packages())
 625             else:
 626                 include.append('grub-pc')
 627
 628         if not self.settings['no-kernel']:
 629             if self.settings['kernel-package']:
 630                 kernel_image = self.settings['kernel-package']
 631             else:
 632                 if self.settings['arch'] == 'i386':
 633                     # wheezy (which became oldstable on 04/25/2015) used '486'
 634                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 635                         kernel_arch = '486'
 636                     else:
 637                         kernel_arch = '586'
 638                 elif self.settings['arch'] == 'armhf':
 639                     kernel_arch = 'armmp'
 640                 else:
 641                     kernel_arch = self.settings['arch']
 642                 kernel_image = 'linux-image-%s' % kernel_arch
 643             include.append(kernel_image)
 644
 645         if self.settings['sudo'] and 'sudo' not in include:
 646             include.append('sudo')
 647
 648         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 649
 650         if self.settings['package']:
 651             args.append(
 652                 '--include=%s' % ','.join(include))
 653         if self.settings['foreign']:
 654             args.append('--foreign')
 655         if self.settings['debootstrapopts']:
 656             for opt in self.settings['debootstrapopts']:
 657                 for part in opt.split(' '):
 658                     args.append('--%s' % part)
 659         elif self.settings['variant']:
 660             args.append('--variant')
 661             args.append(self.settings['variant'])
 662         args += [self.settings['distribution'],
 663                  rootdir, self.settings['mirror']]
 664         logging.debug(" ".join(args))
 665         self.runcmd(args)
 666         if self.settings['foreign']:
 667             # set a noninteractive debconf environment for secondstage
 668             env = {
 669                 "DEBIAN_FRONTEND": "noninteractive",
 670                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 671                 "LC_ALL": "C"
 672             }
 673             # add the mapping to the complete environment.
 674             env.update(os.environ)
 675             # First copy the binfmt handler over
 676             self.message('Setting up binfmt handler')
 677             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 678             # Next, run the package install scripts etc.
 679             self.message('Running debootstrap second stage')
 680             self.runcmd(['chroot', rootdir,
 681                          '/debootstrap/debootstrap', '--second-stage'],
 682                         env=env)
 683
 684     def set_hostname(self, rootdir):
 685         hostname = self.settings['hostname']
 686         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 687             f.write('%s\n' % hostname)
 688
 689         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 690         try:
 691             with open(etc_hosts, 'r') as f:
 692                 data = f.read()
 693             with open(etc_hosts, 'w') as f:
 694                 for line in data.splitlines():
 695                     if line.startswith('127.0.0.1'):
 696                         line += ' %s' % hostname
 697                     f.write('%s\n' % line)
 698         except IOError:
 699             pass
 700
 701     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 702         def fsuuid(device):
 703             out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 704                                '-s', 'UUID', device])
 705             return out.splitlines()[0].strip()
 706
 707         if rootdev:
 708             rootdevstr = 'UUID=%s' % fsuuid(rootdev)
 709         else:
 710             rootdevstr = '/dev/sda1'
 711
 712         if bootdev and not self.settings['use-uefi']:
 713             bootdevstr = 'UUID=%s' % fsuuid(bootdev)
 714         else:
 715             bootdevstr = None
 716
 717         fstab = os.path.join(rootdir, 'etc', 'fstab')
 718         with open(fstab, 'w') as f:
 719             f.write('proc /proc proc defaults 0 0\n')
 720             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 721             if bootdevstr:
 722                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 723                 if self.settings['swap'] > 0:
 724                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 725             elif self.settings['swap'] > 0:
 726                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 727
 728     def install_debs(self, rootdir):
 729         if not self.settings['custom-package']:
 730             return
 731         self.message('Installing custom packages')
 732         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 733         os.mkdir(tmp)
 734         for deb in self.settings['custom-package']:
 735             shutil.copy(deb, tmp)
 736         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 737                      for deb in self.settings['custom-package']]
 738         out, err, _ = \
 739             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 740         logging.debug('stdout:\n%s', out)
 741         logging.debug('stderr:\n%s', err)
 742         out = self.runcmd(['chroot', rootdir,
 743                            'apt-get', '-f', '--no-remove', 'install'])
 744         logging.debug('stdout:\n%s', out)
 745         shutil.rmtree(tmp)
 746
 747     def cleanup_apt_cache(self, rootdir):
 748         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 749         logging.debug('stdout:\n%s', out)
 750
 751     def set_root_password(self, rootdir):
 752         if self.settings['root-password']:
 753             self.message('Setting root password')
 754             self.set_password(rootdir, 'root', self.settings['root-password'])
 755         elif self.settings['lock-root-password']:
 756             self.message('Locking root password')
 757             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 758         else:
 759             self.message('Give root an empty password')
 760             self.delete_password(rootdir, 'root')
 761
 762     def create_users(self, rootdir):
 763         def create_user(vmuser):
 764             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 765                          '--disabled-password', vmuser])
 766             if self.settings['sudo']:
 767                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 768
 769         for userpass in self.settings['user']:
 770             if '/' in userpass:
 771                 user, password = userpass.split('/', 1)
 772                 create_user(user)
 773                 self.set_password(rootdir, user, password)
 774             else:
 775                 create_user(userpass)
 776                 self.delete_password(rootdir, userpass)
 777
 778     def set_password(self, rootdir, user, password):
 779         encrypted = crypt.crypt(password, '..')
 780         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 781
 782     def delete_password(self, rootdir, user):
 783         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 784
 785     def remove_udev_persistent_rules(self, rootdir):
 786         self.message('Removing udev persistent cd and net rules')
 787         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 788             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 789             if os.path.exists(pathname):
 790                 logging.debug('rm %s', pathname)
 791                 os.remove(pathname)
 792             else:
 793                 logging.debug('not removing non-existent %s', pathname)
 794
 795     def mask_udev_predictable_rules(self, rootdir):
 796         """
 797         This can be reset later but to get networking working immediately
 798         on boot, the interface we're going to use must be known without
 799         reference to the eventual machine.
 800         http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
 801         """
 802         self.message('Disabling systemd predictable interface names')
 803         udev_path = os.path.join(
 804             'etc', 'udev', 'rules.d', '80-net-setup-link.rules')
 805         self.runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path])
 806
 807     def setup_networking(self, rootdir):
 808         self.message('Setting up networking')
 809         ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 810         ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d')
 811
 812         # unconditionally write for wheezy (which became oldstable 2015.04.25)
 813         if self.was_oldstable(datetime.date(2015, 4, 26)):
 814             with open(ifc_file, 'w') as netfile:
 815                 netfile.write('source /etc/network/interfaces.d/*\n')
 816         elif not os.path.exists(ifc_file):
 817             with open(ifc_file, 'a') as netfile:
 818                 netfile.write('source-directory /etc/network/interfaces.d\n')
 819
 820         if not os.path.exists(ifc_d):
 821             os.mkdir(ifc_d)
 822         ethpath = os.path.join(ifc_d, 'setup')
 823         with open(ethpath, 'w') as eth:
 824             eth.write('auto lo\n')
 825             eth.write('iface lo inet loopback\n')
 826
 827             if self.settings['enable-dhcp']:
 828                 eth.write('\n')
 829                 eth.write('auto eth0\n')
 830                 eth.write('iface eth0 inet dhcp\n')
 831                 # force predictable interface names
 832                 self.mask_udev_predictable_rules(rootdir)
 833
 834     def append_serial_console(self, rootdir):
 835         if self.settings['serial-console']:
 836             serial_command = self.settings['serial-console-command']
 837             logging.debug('adding getty to serial console')
 838             inittab = os.path.join(rootdir, 'etc/inittab')
 839             # to autologin, serial_command can contain '-a root'
 840             with open(inittab, 'a') as f:
 841                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 842
 843     # pylint: disable=no-self-use
 844     def _grub_serial_console(self, rootdir):
 845         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 846         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 847         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 848         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 849         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 850         with open(grub_cfg, 'a+') as cfg:
 851             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 852             cfg.write("%s\n" % cmdline)
 853             cfg.write("%s\n" % terminal)
 854             cfg.write("%s\n" % command)
 855
 856     def _mount_wrapper(self, rootdir):
 857         self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 858                      '%s' % os.path.join(rootdir, 'dev')])
 859         self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 860                      '%s' % os.path.join(rootdir, 'proc')])
 861         self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 862                      '%s' % os.path.join(rootdir, 'sys')])
 863
 864     def _umount_wrapper(self, rootdir):
 865         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 866         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 867         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 868
 869     def install_grub_uefi(self, rootdir):
 870         self.message("Configuring grub-uefi")
 871         target = self.efi_arch_table[self.settings['arch']]['target']
 872         grub_opts = "--target=%s" % target
 873         logging.debug("Running grub-install with options: %s", grub_opts)
 874         self._mount_wrapper(rootdir)
 875         try:
 876             self.runcmd(['chroot', rootdir, 'update-grub'])
 877             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 878         except cliapp.AppException as exc:
 879             logging.warning(exc)
 880             self.message(
 881                 "Failed to configure grub-uefi for %s" %
 882                 self.settings['arch'])
 883             self._umount_wrapper(rootdir)
 884         self.configure_efi()
 885         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 886         if extra:
 887             target = self.efi_arch_table[extra]['target']
 888             grub_opts = "--target=%s" % target
 889             try:
 890                 self.runcmd(['chroot', rootdir, 'update-grub'])
 891                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 892             except cliapp.AppException as exc:
 893                 logging.warning(exc)
 894                 self.message(
 895                     "Failed to configure grub-uefi for %s" % extra)
 896             self.configure_extra_efi()
 897         self._umount_wrapper(rootdir)
 898
 899     def install_grub2(self, rootdev, rootdir):
 900         self.message("Configuring grub2")
 901         # rely on kpartx using consistent naming to map loop0p1 to loop0
 902         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 903         if self.settings['serial-console']:
 904             self._grub_serial_console(rootdir)
 905         logging.debug("Running grub-install with options: %s", grub_opts)
 906         self._mount_wrapper(rootdir)
 907         try:
 908             self.runcmd(['chroot', rootdir, 'update-grub'])
 909             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 910         except cliapp.AppException as exc:
 911             logging.warning(exc)
 912             self.message("Failed. Is grub2-common installed? Using extlinux.")
 913             self.install_extlinux(rootdev, rootdir)
 914         self._umount_wrapper(rootdir)
 915
 916     def install_extlinux(self, rootdev, rootdir):
 917         if not os.path.exists("/usr/bin/extlinux"):
 918             self.message("extlinux not installed, skipping.")
 919             return
 920         self.message('Installing extlinux')
 921
 922         def find(pattern):
 923             dirname = os.path.join(rootdir, 'boot')
 924             basenames = os.listdir(dirname)
 925             logging.debug('find: %s', basenames)
 926             for basename in basenames:
 927                 if re.search(pattern, basename):
 928                     return os.path.join('boot', basename)
 929             raise cliapp.AppException('Cannot find match: %s' % pattern)
 930
 931         try:
 932             kernel_image = find('vmlinuz-.*')
 933             initrd_image = find('initrd.img-.*')
 934         except cliapp.AppException as e:
 935             self.message("Unable to find kernel. Not installing extlinux.")
 936             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 937             return
 938
 939         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 940                            '-s', 'UUID', rootdev])
 941         uuid = out.splitlines()[0].strip()
 942
 943         conf = os.path.join(rootdir, 'extlinux.conf')
 944         logging.debug('configure extlinux %s', conf)
 945         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 946         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 947         msg = '''
 948 default linux
 949 timeout 1
 950
 951 label linux
 952 kernel %(kernel)s
 953 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 954 %(extserial)s
 955 ''' % {
 956             'kernel': kernel_image,  # pylint: disable=bad-continuation
 957             'initrd': initrd_image,  # pylint: disable=bad-continuation
 958             'uuid': uuid,  # pylint: disable=bad-continuation
 959             'kserial': kserial,  # pylint: disable=bad-continuation
 960             'extserial': extserial,  # pylint: disable=bad-continuation
 961         }  # pylint: disable=bad-continuation
 962         logging.debug("extlinux config:\n%s", msg)
 963
 964         # python multiline string substitution is just ugly.
 965         # use an external file or live with the mangling, no point in
 966         # mangling the string to remove spaces just to keep it pretty in source.
 967         f = open(conf, 'w')
 968         f.write(msg)
 969
 970         self.runcmd(['extlinux', '--install', rootdir])
 971         self.runcmd(['sync'])
 972         time.sleep(2)
 973
 974     def optimize_image(self, rootdir):
 975         """
 976         Filing up the image with zeros will increase its compression rate
 977         """
 978         if not self.settings['sparse']:
 979             zeros = os.path.join(rootdir, 'ZEROS')
 980             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 981             self.runcmd(['rm', '-f', zeros])
 982
 983     def squash(self):
 984         """
 985         Run squashfs on the image.
 986         """
 987         if not os.path.exists('/usr/bin/mksquashfs'):
 988             logging.warning("Squash selected but mksquashfs not found!")
 989             return
 990         logging.debug(
 991             "%s usage: %s", self.settings['image'],
 992             self.runcmd(['du', self.settings['image']]))
 993         self.message("Running mksquashfs")
 994         suffixed = "%s.squashfs" % self.settings['image']
 995         if os.path.exists(suffixed):
 996             os.unlink(suffixed)
 997         msg = self.runcmd(
 998             ['mksquashfs', self.settings['image'],
 999              suffixed,
1000              '-no-progress', '-comp', 'xz'], ignore_fail=False)
1001         logging.debug(msg)
1002         check_size = os.path.getsize(suffixed)
1003         if check_size < (1024 * 1024):
1004             logging.warning(
1005                 "%s appears to be too small! %s bytes",
1006                 suffixed, check_size)
1007         else:
1008             logging.debug("squashed size: %s", check_size)
1009             os.unlink(self.settings['image'])
1010             self.settings['image'] = suffixed
1011             logging.debug(
1012                 "%s usage: %s", self.settings['image'],
1013                 self.runcmd(['du', self.settings['image']]))
1014
1015     def cleanup_system(self):
1016         # Clean up after any errors.
1017
1018         self.message('Cleaning up')
1019
1020         # Umount in the reverse mount order
1021         if self.settings['image']:
1022             for i in range(len(self.mount_points) - 1, -1, -1):
1023                 mount_point = self.mount_points[i]
1024                 try:
1025                     self.runcmd(['umount', mount_point], ignore_fail=False)
1026                 except cliapp.AppException:
1027                     logging.debug("umount failed, sleeping and trying again")
1028                     time.sleep(5)
1029                     self.runcmd(['umount', mount_point], ignore_fail=False)
1030
1031             if 'freebsd' in os.sys.platform:
1032                 out = self.runcmd(['mdconfig', '-l', '-f', self.settings['image']])
1033                 for devid in out.split():
1034                     self.runcmd(['mdconfig', '-d', '-u', devid],
1035                                 ignore_fail=True)
1036             else:
1037                 self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1038
1039         for dirname in self.remove_dirs:
1040             shutil.rmtree(dirname)
1041
1042     def customize(self, rootdir):
1043         script = self.settings['customize']
1044         if not script:
1045             return
1046         if not os.path.exists(script):
1047             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1048             if not os.path.exists(example):
1049                 self.message("Unable to find %s" % script)
1050                 return
1051             script = example
1052         self.message('Running customize script %s' % script)
1053         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1054         logging.debug(
1055             "%s usage: %s", self.settings['image'],
1056             self.runcmd(['du', self.settings['image']]))
1057         try:
1058             with open('/dev/tty', 'w') as tty:
1059                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1060         except IOError:
1061             logging.debug('tty unavailable, trying in headless mode.')
1062             subprocess.call([script, rootdir, self.settings['image']])
1063
1064     def create_tarball(self, rootdir):
1065         # Create a tarball of the disk's contents
1066         # shell out to runcmd since it more easily handles rootdir
1067         self.message('Creating tarball of disk contents')
1068         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1069
1070     def chown(self):
1071         # Change image owner after completed build
1072         if self.settings['image']:
1073             filename = self.settings['image']
1074         elif self.settings['tarball']:
1075             filename = self.settings['tarball']
1076         else:
1077             return
1078         self.message("Changing owner to %s" % self.settings["owner"])
1079         subprocess.call(["chown", self.settings["owner"], filename])
1080
1081     def list_installed_pkgs(self, rootdir):
1082         # output the list of installed packages for sources identification
1083         self.message("Creating a list of installed binary package names")
1084         out = self.runcmd(['chroot', rootdir,
1085                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1086         with open('dpkg.list', 'w') as dpkg:
1087             dpkg.write(out)
1088
1089     def configure_apt(self, rootdir):
1090         # use the distribution and mirror to create an apt source
1091         self.message("Configuring apt to use distribution and mirror")
1092         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1093         logging.debug('configure apt %s', conf)
1094         mirror = self.settings['mirror']
1095         if self.settings['apt-mirror']:
1096             mirror = self.settings['apt-mirror']
1097             self.message("Setting apt mirror to %s" % mirror)
1098         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1099         f = open(conf, 'w')
1100         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1101         f.write(line)
1102         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1103         f.write(line)
1104         f.close()
1105         # ensure the apt sources have valid lists
1106         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1107
1108 if __name__ == '__main__':
1109     VmDebootstrap(version=__version__).run()