vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import glob
  24 import re
  25 import sys
  26 import shutil
  27 import datetime
  28 import subprocess
  29 import tempfile
  30 import time
  31 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  32
  33
  34 __version__ = '0.11'
  35
  36 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  37
  38
  39 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  40
  41     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  42         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  43         self.remove_dirs = []
  44         self.mount_points = []
  45         self.debian_info = DebianDistroInfo()
  46         self.ubuntu_info = UbuntuDistroInfo()
  47         self.bootdir = None
  48         self.efi_arch_table = {
  49             'amd64': {
  50                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  51                 'install': '/EFI/debian/grubx64.efi',  # package location
  52                 'package': 'grub-efi-amd64',  # bootstrap package
  53                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  54                 'extra': 'i386',  # architecture to add binary package
  55                 'exclusive': False,  # only EFI supported for this arch.
  56                 'target': 'x86_64-efi',  # grub target name
  57             },
  58             'i386': {
  59                 'removable': '/EFI/boot/bootia32.efi',
  60                 'install': '/EFI/debian/grubia32.efi',
  61                 'package': 'grub-efi-ia32',
  62                 'bin_package': 'grub-efi-ia32-bin',
  63                 'extra': None,
  64                 'exclusive': False,
  65                 'target': 'i386-efi',
  66             },
  67             'arm64': {
  68                 'removable': '/EFI/boot/bootaa64.efi',
  69                 'install': '/EFI/debian/grubaa64.efi',
  70                 'package': 'grub-efi-arm64',
  71                 'bin_package': 'grub-efi-arm64-bin',
  72                 'extra': None,
  73                 'exclusive': True,
  74                 'target': 'arm64-efi',
  75             }
  76         }
  77
  78     def add_settings(self):
  79         default_arch = subprocess.check_output(
  80             ["dpkg", "--print-architecture"]).strip()
  81
  82         self.settings.boolean(
  83             ['verbose'], 'report what is going on')
  84         self.settings.string(
  85             ['image'], 'put created disk image in FILE',
  86             metavar='FILE')
  87         self.settings.bytesize(
  88             ['size'],
  89             'create a disk image of size SIZE (%default)',
  90             metavar='SIZE',
  91             default='1G')
  92         self.settings.bytesize(
  93             ['bootsize'],
  94             'create boot partition of size SIZE (%default)',
  95             metavar='BOOTSIZE',
  96             default='0%')
  97         self.settings.string(
  98             ['boottype'],
  99             'specify file system type for /boot/',
 100             default='ext2')
 101         self.settings.bytesize(
 102             ['bootoffset'],
 103             'Space to leave at start of the image for bootloader',
 104             default='0')
 105         self.settings.boolean(
 106             ['use-uefi'],
 107             'Setup image for UEFI boot',
 108             default=False)
 109         self.settings.bytesize(
 110             ['esp-size'],
 111             'Size of EFI System Partition - requires use-uefi',
 112             default='5mib')
 113         self.settings.string(
 114             ['part-type'],
 115             'Partition type to use for this image',
 116             default='msdos')
 117         self.settings.string(
 118             ['roottype'],
 119             'specify file system type for /',
 120             default='ext4')
 121         self.settings.bytesize(
 122             ['swap'],
 123             'create swap space of size SIZE (min 256Mb)')
 124         self.settings.string(
 125             ['foreign'],
 126             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 127         self.settings.string(
 128             ['variant'],
 129             'select debootstrap variant if not using the default [deprecated]')
 130         self.settings.string_list(
 131             ['debootstrapopts'],
 132             'pass additional options to debootstrap'),
 133         self.settings.boolean(
 134             ['extlinux'],
 135             'install extlinux?',
 136             default=True)
 137         self.settings.string(
 138             ['tarball'],
 139             "tar up the disk's contents in FILE",
 140             metavar='FILE')
 141         self.settings.string(
 142             ['apt-mirror'],
 143             'configure apt to use MIRROR',
 144             metavar='URL')
 145         self.settings.string(
 146             ['mirror'],
 147             'use MIRROR as package source (%default)',
 148             metavar='URL',
 149             default='http://http.debian.net/debian/')
 150         self.settings.string(
 151             ['arch'],
 152             'architecture to use (%default)',
 153             metavar='ARCH',
 154             default=default_arch)
 155         self.settings.string(
 156             ['distribution'],
 157             'release to use (%default)',
 158             metavar='NAME',
 159             default='stable')
 160         self.settings.string_list(
 161             ['package'],
 162             'install PACKAGE onto system')
 163         self.settings.string_list(
 164             ['custom-package'],
 165             'install package in DEB file onto system (not from mirror)',
 166             metavar='DEB')
 167         self.settings.boolean(
 168             ['no-kernel'],
 169             'do not install a linux package')
 170         self.settings.string(
 171             ['kernel-package'],
 172             'install PACKAGE instead of the default kernel package',
 173             metavar='PACKAGE')
 174         self.settings.boolean(
 175             ['enable-dhcp'],
 176             'enable DHCP on eth0')
 177         self.settings.string(
 178             ['root-password'],
 179             'set root password',
 180             metavar='PASSWORD')
 181         self.settings.boolean(
 182             ['lock-root-password'],
 183             'lock root account so they cannot login?')
 184         self.settings.string(
 185             ['customize'],
 186             'run SCRIPT after setting up system',
 187             metavar='SCRIPT')
 188         self.settings.string(
 189             ['hostname'],
 190             'set name to HOSTNAME (%default)',
 191             metavar='HOSTNAME',
 192             default='debian')
 193         self.settings.string_list(
 194             ['user'],
 195             'create USER with PASSWORD',
 196             metavar='USER/PASSWORD')
 197         self.settings.boolean(
 198             ['serial-console'],
 199             'configure image to use a serial console')
 200         self.settings.string(
 201             ['serial-console-command'],
 202             'command to manage the serial console, appended to /etc/inittab (%default)',
 203             metavar='COMMAND',
 204             default='/sbin/getty -L ttyS0 115200 vt100')
 205         self.settings.boolean(
 206             ['sudo'],
 207             'install sudo, and if user is created, add them to sudo group')
 208         self.settings.string(
 209             ['owner'],
 210             'the user who will own the image when the build is complete.')
 211         self.settings.boolean(
 212             ['squash'],
 213             'use squashfs on the final image.')
 214         self.settings.boolean(
 215             ['configure-apt'],
 216             'Create an apt source based on the distribution and mirror selected.')
 217         self.settings.boolean(
 218             ['mbr'],
 219             'Run install-mbr (default if extlinux used)')
 220         self.settings.boolean(
 221             ['grub'],
 222             'Install and configure grub2 - disables extlinux.')
 223         self.settings.boolean(
 224             ['sparse'],
 225             'Do not fill the image with zeros to keep a sparse disk image',
 226             default=False)
 227         self.settings.boolean(
 228             ['pkglist'],
 229             'Create a list of package names included in the image.')
 230         self.settings.boolean(
 231             ['no-acpid'],
 232             'do not install the acpid package',
 233             default=False)
 234
 235     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 236         if not self.settings['image'] and not self.settings['tarball']:
 237             raise cliapp.AppException(
 238                 'You must give disk image filename, or tarball filename')
 239         if self.settings['image'] and not self.settings['size']:
 240             raise cliapp.AppException(
 241                 'If disk image is specified, you must give image size.')
 242         if not self.debian_info.valid(self.settings['distribution']):
 243             if not self.ubuntu_info.valid(self.settings['distribution']):
 244                 raise cliapp.AppException(
 245                     '%s is not a valid Debian or Ubuntu suite or codename.'
 246                     % self.settings['distribution'])
 247         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 248             raise cliapp.AppException(
 249                 'You must specify use-uefi for esp-size to have effect')
 250         if self.settings['arch'] in self.efi_arch_table and\
 251                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 252                 not self.settings['use-uefi']:
 253             raise cliapp.AppException(
 254                 'Only UEFI is supported on %s' % self.settings['arch'])
 255         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 256             raise cliapp.AppException(
 257                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 258         if self.settings['use-uefi'] and (
 259                 self.settings['bootsize'] or
 260                 self.settings['bootoffset']):
 261             raise cliapp.AppException(
 262                 'A separate boot partition is not supported with UEFI')
 263
 264         if self.settings['use-uefi'] and not self.settings['grub']:
 265             raise cliapp.AppException(
 266                 'UEFI without Grub is not supported.')
 267
 268         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 269         if self.was_oldstable(datetime.date(2015, 4, 26)):
 270             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 271                 raise cliapp.AppException(
 272                     'Only amd64 supports UEFI in Wheezy')
 273
 274         if os.geteuid() != 0:
 275             sys.exit("You need to have root privileges to run this script.")
 276         rootdir = None
 277         try:
 278             rootdev = None
 279             roottype = self.settings['roottype']
 280             bootdev = None
 281             boottype = None
 282             if self.settings['image']:
 283                 self.create_empty_image()
 284                 self.partition_image()
 285                 if self.settings['mbr'] or self.settings['extlinux']:
 286                     self.install_mbr()
 287                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 288                 if self.settings['swap'] > 0:
 289                     self.message("Creating swap space")
 290                     self.runcmd(['mkswap', swapdev])
 291                 self.mkfs(rootdev, fstype=roottype)
 292                 rootdir = self.mount(rootdev)
 293                 if self.settings['use-uefi']:
 294                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 295                     logging.debug("bootdir:%s", self.bootdir)
 296                     self.mkfs(bootdev, fstype='vfat')
 297                     os.makedirs(self.bootdir)
 298                     self.mount(bootdev, self.bootdir)
 299                 elif bootdev:
 300                     if self.settings['boottype']:
 301                         boottype = self.settings['boottype']
 302                     else:
 303                         boottype = 'ext2'
 304                     self.mkfs(bootdev, fstype=boottype)
 305                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 306                     os.mkdir(self.bootdir)
 307                     self.mount(bootdev, self.bootdir)
 308             else:
 309                 rootdir = self.mkdtemp()
 310             self.debootstrap(rootdir)
 311             self.set_hostname(rootdir)
 312             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 313             self.install_debs(rootdir)
 314             self.set_root_password(rootdir)
 315             self.create_users(rootdir)
 316             self.remove_udev_persistent_rules(rootdir)
 317             self.setup_networking(rootdir)
 318             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 319                 self.configure_apt(rootdir)
 320             self.customize(rootdir)
 321             self.cleanup_apt_cache(rootdir)
 322             self.update_initramfs(rootdir)
 323
 324             if self.settings['image']:
 325                 if self.settings['use-uefi']:
 326                     self.install_grub_uefi(rootdir)
 327                 elif self.settings['grub']:
 328                     self.install_grub2(rootdev, rootdir)
 329                 elif self.settings['extlinux']:
 330                     self.install_extlinux(rootdev, rootdir)
 331                 self.append_serial_console(rootdir)
 332                 self.optimize_image(rootdir)
 333                 if self.settings['squash']:
 334                     self.squash()
 335             if self.settings['pkglist']:
 336                 self.list_installed_pkgs(rootdir)
 337
 338             if self.settings['foreign']:
 339                 os.unlink('%s/usr/bin/%s' %
 340                           (rootdir, os.path.basename(self.settings['foreign'])))
 341
 342             if self.settings['tarball']:
 343                 self.create_tarball(rootdir)
 344
 345             if self.settings['owner']:
 346                 self.chown()
 347         except BaseException as e:
 348             self.message('EEEK! Something bad happened...')
 349             if rootdir:
 350                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 351                 if os.path.exists(db_log):
 352                     shutil.copy(db_log, os.getcwd())
 353             self.message(e)
 354             self.cleanup_system()
 355             raise
 356         else:
 357             self.cleanup_system()
 358
 359     def message(self, msg):
 360         logging.info(msg)
 361         if self.settings['verbose']:
 362             print msg
 363
 364     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 365         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 366         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 367                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 368                              env=env, **kwargs)
 369         out, err = p.communicate(stdin)
 370         if p.returncode != 0:
 371             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 372             logging.error(msg)
 373             if not ignore_fail:
 374                 raise cliapp.AppException(msg)
 375         return out
 376
 377     def mkdtemp(self):
 378         dirname = tempfile.mkdtemp()
 379         self.remove_dirs.append(dirname)
 380         logging.debug('mkdir %s', dirname)
 381         return dirname
 382
 383     def mount(self, device, path=None):
 384         if not path:
 385             mount_point = self.mkdtemp()
 386         else:
 387             mount_point = path
 388         self.message('Mounting %s on %s' % (device, mount_point))
 389         self.runcmd(['mount', device, mount_point])
 390         self.mount_points.append(mount_point)
 391         logging.debug('mounted %s on %s', device, mount_point)
 392         return mount_point
 393
 394     def create_empty_image(self):
 395         self.message('Creating disk image')
 396         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 397                      self.settings['image'],
 398                      str(self.settings['size'])])
 399
 400     def partition_image(self):
 401         """
 402         Uses fat16 (msdos) partitioning by default, use part-type to change.
 403         If bootoffset is specified, the first actual partition
 404         starts at that offset to allow customisation scripts to
 405         put bootloader images into the space, e.g. u-boot.
 406         """
 407         self.message('Creating partitions')
 408         self.runcmd(['parted', '-s', self.settings['image'],
 409                      'mklabel', self.settings['part-type']])
 410         partoffset = 0
 411         extent = '100%'
 412         swap = 256 * 1024 * 1024
 413         if self.settings['swap'] > 0:
 414             if self.settings['swap'] > swap:
 415                 swap = self.settings['swap']
 416             else:
 417                 # minimum 256Mb as default qemu ram is 128Mb
 418                 logging.debug("Setting minimum 256Mb swap space")
 419             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 420
 421         if self.settings['use-uefi']:
 422             espsize = self.settings['esp-size'] / (1024 * 1024)
 423             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 424             self.runcmd(['parted', '-s', self.settings['image'],
 425                          'mkpart', 'primary', 'fat32',
 426                          '1', str(espsize)])
 427             self.runcmd(['parted', '-s', self.settings['image'],
 428                          'set', '1', 'boot', 'on'])
 429             self.runcmd(['parted', '-s', self.settings['image'],
 430                          'set', '1', 'esp', 'on'])
 431
 432         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 433             # turn v.small offsets into something at least possible to create.
 434             if self.settings['bootoffset'] < 1048576:
 435                 partoffset = 1
 436                 logging.info(
 437                     "Setting bootoffset %smib to allow for %s bytes",
 438                     partoffset, self.settings['bootoffset'])
 439             else:
 440                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 441                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 442         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 443             if self.settings['grub'] and not partoffset:
 444                 partoffset = 1
 445             bootsize = self.settings['bootsize'] / (1024 * 1024)
 446             bootsize += partoffset
 447             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 448             logging.debug("Starting boot partition at %sMb", bootsize)
 449             self.runcmd(['parted', '-s', self.settings['image'],
 450                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 451             logging.debug("Starting root partition at %sMb", partoffset)
 452             self.runcmd(['parted', '-s', self.settings['image'],
 453                          'mkpart', 'primary', str(bootsize), extent])
 454         elif self.settings['use-uefi']:
 455             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 456             self.runcmd(['parted', '-s', self.settings['image'],
 457                          'mkpart', 'primary', str(bootsize), extent])
 458         else:
 459             self.runcmd(['parted', '-s', self.settings['image'],
 460                          'mkpart', 'primary', '0%', extent])
 461         self.runcmd(['parted', '-s', self.settings['image'],
 462                      'set', '1', 'boot', 'on'])
 463         if self.settings['swap'] > 0:
 464             logging.debug("Creating swap partition")
 465             self.runcmd(['parted', '-s', self.settings['image'],
 466                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 467
 468     def update_initramfs(self, rootdir):
 469         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 470         if os.path.exists(os.path.join(rootdir, cmd)):
 471             self.message("Updating the initramfs")
 472             self.runcmd(['chroot', rootdir, cmd, '-u'])
 473
 474     def install_mbr(self):
 475         if os.path.exists("/sbin/install-mbr"):
 476             self.message('Installing MBR')
 477             self.runcmd(['install-mbr', self.settings['image']])
 478         else:
 479             msg = "mbr enabled but /sbin/install-mbr not found" \
 480                   " - please install the mbr package."
 481             raise cliapp.AppException(msg)
 482
 483     def setup_kpartx(self):
 484         bootindex = None
 485         swapindex = None
 486         if 'freebsd' in os.sys.platform:
 487             out = self.runcmd(['mdconfig', '-a', '-t', 'vnode', '-f',
 488                                self.settings['image']])
 489         else:
 490             out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 491         if self.settings['bootsize'] and self.settings['swap'] > 0:
 492             bootindex = 0
 493             rootindex = 1
 494             swapindex = 2
 495             parts = 3
 496         elif self.settings['use-uefi']:
 497             bootindex = 0
 498             rootindex = 1
 499             parts = 2
 500         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 501             bootindex = 0
 502             rootindex = 1
 503             swapindex = 2
 504             parts = 3
 505         elif self.settings['bootsize']:
 506             bootindex = 0
 507             rootindex = 1
 508             parts = 2
 509         elif self.settings['swap'] > 0:
 510             rootindex = 0
 511             swapindex = 1
 512             parts = 2
 513         else:
 514             rootindex = 0
 515             parts = 1
 516         boot = None
 517         swap = None
 518         if 'freebsd' in os.sys.platform:
 519             devices = glob.glob("/dev/%ss*" % out.strip())
 520         else:
 521             devices = ['/dev/mapper/%s' % line.split()[2]
 522                        for line in out.splitlines()
 523                        if line.startswith('add map ')]
 524         if len(devices) != parts:
 525             msg = 'Surprising number of partitions - check output of losetup -a'
 526             logging.debug("%s", self.runcmd(['losetup', '-a']))
 527             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 528             raise cliapp.AppException(msg)
 529         root = devices[rootindex]
 530         if self.settings['bootsize'] or self.settings['use-uefi']:
 531             boot = devices[bootindex]
 532         if self.settings['swap'] > 0:
 533             swap = devices[swapindex]
 534         return root, boot, swap
 535
 536     def _efi_packages(self):
 537         packages = []
 538         pkg = self.efi_arch_table[self.settings['arch']]['package']
 539         self.message("Adding %s" % pkg)
 540         packages.append(pkg)
 541         extra = self.efi_arch_table[self.settings['arch']]['extra']
 542         if extra and isinstance(extra, str):
 543             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 544             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 545             packages.append(bin_pkg)
 546         return packages
 547
 548     def _copy_efi_binary(self, efi_removable, efi_install):
 549         logging.debug("using bootdir=%s", self.bootdir)
 550         logging.debug("moving %s to %s", efi_removable, efi_install)
 551         if efi_removable.startswith('/'):
 552             efi_removable = efi_removable[1:]
 553         if efi_install.startswith('/'):
 554             efi_install = efi_install[1:]
 555         efi_output = os.path.join(self.bootdir, efi_removable)
 556         efi_input = os.path.join(self.bootdir, efi_install)
 557         if not os.path.exists(efi_input):
 558             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 559             raise cliapp.AppException("Missing %s" % efi_install)
 560         if not os.path.exists(os.path.dirname(efi_output)):
 561             os.makedirs(os.path.dirname(efi_output))
 562         logging.debug(
 563             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 564         if os.path.exists(efi_output):
 565             os.unlink(efi_output)
 566         os.rename(efi_input, efi_output)
 567
 568     def configure_efi(self):
 569         """
 570         Copy the bootloader file from the package into the location
 571         so needs to be after grub and kernel already installed.
 572         """
 573         self.message('Configuring EFI')
 574         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 575         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 576         self.message('Installing UEFI support binary')
 577         self._copy_efi_binary(efi_removable, efi_install)
 578
 579     def configure_extra_efi(self):
 580         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 581         if extra:
 582             efi_removable = str(self.efi_arch_table[extra]['removable'])
 583             efi_install = str(self.efi_arch_table[extra]['install'])
 584             self.message('Copying UEFI support binary for %s' % extra)
 585             self._copy_efi_binary(efi_removable, efi_install)
 586
 587     def mkfs(self, device, fstype):
 588         self.message('Creating filesystem %s' % fstype)
 589         self.runcmd(['mkfs', '-t', fstype, device])
 590
 591     def suite_to_codename(self, distro):
 592         suite = self.debian_info.codename(distro, datetime.date.today())
 593         if not suite:
 594             return distro
 595         return suite
 596
 597     def was_oldstable(self, limit):
 598         suite = self.suite_to_codename(self.settings['distribution'])
 599         # this check is only for debian
 600         if not self.debian_info.valid(suite):
 601             return False
 602         return suite == self.debian_info.old(limit)
 603
 604     def was_stable(self, limit):
 605         suite = self.suite_to_codename(self.settings['distribution'])
 606         # this check is only for debian
 607         if not self.debian_info.valid(suite):
 608             return False
 609         return suite == self.debian_info.stable(limit)
 610
 611     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 612         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 613         self.message(
 614             'Debootstrapping %s [%s] %s' % (
 615                 self.settings['distribution'], self.settings['arch'], msg))
 616
 617         include = self.settings['package']
 618
 619         if not self.settings['foreign'] and not self.settings['no-acpid']:
 620             include.append('acpid')
 621
 622         if self.settings['grub']:
 623             if self.settings['use-uefi']:
 624                 include.extend(self._efi_packages())
 625             else:
 626                 include.append('grub-pc')
 627
 628         if not self.settings['no-kernel']:
 629             if self.settings['kernel-package']:
 630                 kernel_image = self.settings['kernel-package']
 631             else:
 632                 if self.settings['arch'] == 'i386':
 633                     # wheezy (which became oldstable on 04/25/2015) used '486'
 634                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 635                         kernel_arch = '486'
 636                     else:
 637                         kernel_arch = '586'
 638                 elif self.settings['arch'] == 'armhf':
 639                     kernel_arch = 'armmp'
 640                 else:
 641                     kernel_arch = self.settings['arch']
 642                 kernel_image = 'linux-image-%s' % kernel_arch
 643             include.append(kernel_image)
 644
 645         if self.settings['sudo'] and 'sudo' not in include:
 646             include.append('sudo')
 647
 648         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 649
 650         if self.settings['package']:
 651             args.append(
 652                 '--include=%s' % ','.join(include))
 653         if self.settings['foreign']:
 654             args.append('--foreign')
 655         if self.settings['debootstrapopts']:
 656             for opt in self.settings['debootstrapopts']:
 657                 for part in opt.split(' '):
 658                     args.append('--%s' % part)
 659         elif self.settings['variant']:
 660             args.append('--variant')
 661             args.append(self.settings['variant'])
 662         args += [self.settings['distribution'],
 663                  rootdir, self.settings['mirror']]
 664         logging.debug(" ".join(args))
 665         self.runcmd(args)
 666         if self.settings['foreign']:
 667             # set a noninteractive debconf environment for secondstage
 668             env = {
 669                 "DEBIAN_FRONTEND": "noninteractive",
 670                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 671                 "LC_ALL": "C"
 672             }
 673             # add the mapping to the complete environment.
 674             env.update(os.environ)
 675             # First copy the binfmt handler over
 676             self.message('Setting up binfmt handler')
 677             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 678             # Next, run the package install scripts etc.
 679             self.message('Running debootstrap second stage')
 680             self.runcmd(['chroot', rootdir,
 681                          '/debootstrap/debootstrap', '--second-stage'],
 682                         env=env)
 683
 684     def set_hostname(self, rootdir):
 685         hostname = self.settings['hostname']
 686         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 687             f.write('%s\n' % hostname)
 688
 689         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 690         try:
 691             with open(etc_hosts, 'r') as f:
 692                 data = f.read()
 693             with open(etc_hosts, 'w') as f:
 694                 for line in data.splitlines():
 695                     if line.startswith('127.0.0.1'):
 696                         line += ' %s' % hostname
 697                     f.write('%s\n' % line)
 698         except IOError:
 699             pass
 700
 701     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 702         def fsuuid(device):
 703             if 'freebsd' in os.sys.platform:
 704                 out = self.runcmd(['grub-probe', '-d', device, '-t', 'fs_uuid'])
 705                 return "/dev/ufsid/%s" % out.strip()
 706             else:
 707                 out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 708                                 '-s', 'UUID', device])
 709                 return "UUID=%s" % out.splitlines()[0].strip()
 710
 711         if rootdev:
 712             rootdevstr = fsuuid(rootdev)
 713         else:
 714             rootdevstr = '/dev/sda1'
 715
 716         if bootdev and not self.settings['use-uefi']:
 717             bootdevstr = fsuuid(bootdev)
 718         else:
 719             bootdevstr = None
 720
 721         fstab = os.path.join(rootdir, 'etc', 'fstab')
 722         with open(fstab, 'w') as f:
 723             f.write('proc /proc proc defaults 0 0\n')
 724             f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
 725             if bootdevstr:
 726                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 727                 if self.settings['swap'] > 0:
 728                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 729             elif self.settings['swap'] > 0:
 730                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 731
 732     def install_debs(self, rootdir):
 733         if not self.settings['custom-package']:
 734             return
 735         self.message('Installing custom packages')
 736         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 737         os.mkdir(tmp)
 738         for deb in self.settings['custom-package']:
 739             shutil.copy(deb, tmp)
 740         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 741                      for deb in self.settings['custom-package']]
 742         out, err, _ = \
 743             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 744         logging.debug('stdout:\n%s', out)
 745         logging.debug('stderr:\n%s', err)
 746         out = self.runcmd(['chroot', rootdir,
 747                            'apt-get', '-f', '--no-remove', 'install'])
 748         logging.debug('stdout:\n%s', out)
 749         shutil.rmtree(tmp)
 750
 751     def cleanup_apt_cache(self, rootdir):
 752         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 753         logging.debug('stdout:\n%s', out)
 754
 755     def set_root_password(self, rootdir):
 756         if self.settings['root-password']:
 757             self.message('Setting root password')
 758             self.set_password(rootdir, 'root', self.settings['root-password'])
 759         elif self.settings['lock-root-password']:
 760             self.message('Locking root password')
 761             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 762         else:
 763             self.message('Give root an empty password')
 764             self.delete_password(rootdir, 'root')
 765
 766     def create_users(self, rootdir):
 767         def create_user(vmuser):
 768             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 769                          '--disabled-password', vmuser])
 770             if self.settings['sudo']:
 771                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 772
 773         for userpass in self.settings['user']:
 774             if '/' in userpass:
 775                 user, password = userpass.split('/', 1)
 776                 create_user(user)
 777                 self.set_password(rootdir, user, password)
 778             else:
 779                 create_user(userpass)
 780                 self.delete_password(rootdir, userpass)
 781
 782     def set_password(self, rootdir, user, password):
 783         encrypted = crypt.crypt(password, '..')
 784         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 785
 786     def delete_password(self, rootdir, user):
 787         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 788
 789     def remove_udev_persistent_rules(self, rootdir):
 790         self.message('Removing udev persistent cd and net rules')
 791         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 792             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 793             if os.path.exists(pathname):
 794                 logging.debug('rm %s', pathname)
 795                 os.remove(pathname)
 796             else:
 797                 logging.debug('not removing non-existent %s', pathname)
 798
 799     def mask_udev_predictable_rules(self, rootdir):
 800         """
 801         This can be reset later but to get networking working immediately
 802         on boot, the interface we're going to use must be known without
 803         reference to the eventual machine.
 804         http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
 805         """
 806         self.message('Disabling systemd predictable interface names')
 807         udev_path = os.path.join(
 808             'etc', 'udev', 'rules.d', '80-net-setup-link.rules')
 809         self.runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path])
 810
 811     def setup_networking(self, rootdir):
 812         self.message('Setting up networking')
 813         ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 814         ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d')
 815
 816         # unconditionally write for wheezy (which became oldstable 2015.04.25)
 817         if self.was_oldstable(datetime.date(2015, 4, 26)):
 818             with open(ifc_file, 'w') as netfile:
 819                 netfile.write('source /etc/network/interfaces.d/*\n')
 820         elif not os.path.exists(ifc_file):
 821             with open(ifc_file, 'a') as netfile:
 822                 netfile.write('source-directory /etc/network/interfaces.d\n')
 823
 824         if not os.path.exists(ifc_d):
 825             os.mkdir(ifc_d)
 826         ethpath = os.path.join(ifc_d, 'setup')
 827         with open(ethpath, 'w') as eth:
 828             eth.write('auto lo\n')
 829             eth.write('iface lo inet loopback\n')
 830
 831             if self.settings['enable-dhcp']:
 832                 eth.write('\n')
 833                 eth.write('auto eth0\n')
 834                 eth.write('iface eth0 inet dhcp\n')
 835                 # force predictable interface names
 836                 self.mask_udev_predictable_rules(rootdir)
 837
 838     def append_serial_console(self, rootdir):
 839         if self.settings['serial-console']:
 840             serial_command = self.settings['serial-console-command']
 841             logging.debug('adding getty to serial console')
 842             inittab = os.path.join(rootdir, 'etc/inittab')
 843             # to autologin, serial_command can contain '-a root'
 844             with open(inittab, 'a') as f:
 845                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 846
 847     # pylint: disable=no-self-use
 848     def _grub_serial_console(self, rootdir):
 849         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 850         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 851         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 852         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 853         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 854         with open(grub_cfg, 'a+') as cfg:
 855             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 856             cfg.write("%s\n" % cmdline)
 857             cfg.write("%s\n" % terminal)
 858             cfg.write("%s\n" % command)
 859
 860     def _mount_wrapper(self, rootdir):
 861         if 'freebsd' in os.sys.platform:
 862             self.runcmd(['mount', 'dev', '-t', 'devfs',
 863                         '%s' % os.path.join(rootdir, 'dev')])
 864             self.runcmd(['mount', 'proc', '-t', 'linprocfs',
 865                         '%s' % os.path.join(rootdir, 'proc')])
 866             self.runcmd(['mount', 'sys', '-t', 'linsysfs',
 867                         '%s' % os.path.join(rootdir, 'sys')])
 868         else:
 869             self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 870                         '%s' % os.path.join(rootdir, 'dev')])
 871             self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 872                         '%s' % os.path.join(rootdir, 'proc')])
 873             self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 874                         '%s' % os.path.join(rootdir, 'sys')])
 875
 876     def _umount_wrapper(self, rootdir):
 877         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 878         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 879         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 880
 881     def install_grub_uefi(self, rootdir):
 882         self.message("Configuring grub-uefi")
 883         target = self.efi_arch_table[self.settings['arch']]['target']
 884         grub_opts = "--target=%s" % target
 885         logging.debug("Running grub-install with options: %s", grub_opts)
 886         self._mount_wrapper(rootdir)
 887         try:
 888             self.runcmd(['chroot', rootdir, 'update-grub'])
 889             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 890         except cliapp.AppException as exc:
 891             logging.warning(exc)
 892             self.message(
 893                 "Failed to configure grub-uefi for %s" %
 894                 self.settings['arch'])
 895             self._umount_wrapper(rootdir)
 896         self.configure_efi()
 897         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 898         if extra:
 899             target = self.efi_arch_table[extra]['target']
 900             grub_opts = "--target=%s" % target
 901             try:
 902                 self.runcmd(['chroot', rootdir, 'update-grub'])
 903                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 904             except cliapp.AppException as exc:
 905                 logging.warning(exc)
 906                 self.message(
 907                     "Failed to configure grub-uefi for %s" % extra)
 908             self.configure_extra_efi()
 909         self._umount_wrapper(rootdir)
 910
 911     def install_grub2(self, rootdev, rootdir):
 912         self.message("Configuring grub2")
 913         # rely on kpartx using consistent naming to map loop0p1 to loop0
 914         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 915         if self.settings['serial-console']:
 916             self._grub_serial_console(rootdir)
 917         logging.debug("Running grub-install with options: %s", grub_opts)
 918         self._mount_wrapper(rootdir)
 919         try:
 920             self.runcmd(['chroot', rootdir, 'update-grub'])
 921             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 922         except cliapp.AppException as exc:
 923             logging.warning(exc)
 924             self.message("Failed. Is grub2-common installed? Using extlinux.")
 925             self.install_extlinux(rootdev, rootdir)
 926         self._umount_wrapper(rootdir)
 927
 928     def install_extlinux(self, rootdev, rootdir):
 929         if not os.path.exists("/usr/bin/extlinux"):
 930             self.message("extlinux not installed, skipping.")
 931             return
 932         self.message('Installing extlinux')
 933
 934         def find(pattern):
 935             dirname = os.path.join(rootdir, 'boot')
 936             basenames = os.listdir(dirname)
 937             logging.debug('find: %s', basenames)
 938             for basename in basenames:
 939                 if re.search(pattern, basename):
 940                     return os.path.join('boot', basename)
 941             raise cliapp.AppException('Cannot find match: %s' % pattern)
 942
 943         try:
 944             kernel_image = find('vmlinuz-.*')
 945             initrd_image = find('initrd.img-.*')
 946         except cliapp.AppException as e:
 947             self.message("Unable to find kernel. Not installing extlinux.")
 948             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 949             return
 950
 951         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 952                            '-s', 'UUID', rootdev])
 953         uuid = out.splitlines()[0].strip()
 954
 955         conf = os.path.join(rootdir, 'extlinux.conf')
 956         logging.debug('configure extlinux %s', conf)
 957         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 958         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 959         msg = '''
 960 default linux
 961 timeout 1
 962
 963 label linux
 964 kernel %(kernel)s
 965 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 966 %(extserial)s
 967 ''' % {
 968             'kernel': kernel_image,  # pylint: disable=bad-continuation
 969             'initrd': initrd_image,  # pylint: disable=bad-continuation
 970             'uuid': uuid,  # pylint: disable=bad-continuation
 971             'kserial': kserial,  # pylint: disable=bad-continuation
 972             'extserial': extserial,  # pylint: disable=bad-continuation
 973         }  # pylint: disable=bad-continuation
 974         logging.debug("extlinux config:\n%s", msg)
 975
 976         # python multiline string substitution is just ugly.
 977         # use an external file or live with the mangling, no point in
 978         # mangling the string to remove spaces just to keep it pretty in source.
 979         f = open(conf, 'w')
 980         f.write(msg)
 981
 982         self.runcmd(['extlinux', '--install', rootdir])
 983         self.runcmd(['sync'])
 984         time.sleep(2)
 985
 986     def optimize_image(self, rootdir):
 987         """
 988         Filing up the image with zeros will increase its compression rate
 989         """
 990         if not self.settings['sparse']:
 991             zeros = os.path.join(rootdir, 'ZEROS')
 992             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 993             self.runcmd(['rm', '-f', zeros])
 994
 995     def squash(self):
 996         """
 997         Run squashfs on the image.
 998         """
 999         if not os.path.exists('/usr/bin/mksquashfs'):
1000             logging.warning("Squash selected but mksquashfs not found!")
1001             return
1002         logging.debug(
1003             "%s usage: %s", self.settings['image'],
1004             self.runcmd(['du', self.settings['image']]))
1005         self.message("Running mksquashfs")
1006         suffixed = "%s.squashfs" % self.settings['image']
1007         if os.path.exists(suffixed):
1008             os.unlink(suffixed)
1009         msg = self.runcmd(
1010             ['mksquashfs', self.settings['image'],
1011              suffixed,
1012              '-no-progress', '-comp', 'xz'], ignore_fail=False)
1013         logging.debug(msg)
1014         check_size = os.path.getsize(suffixed)
1015         if check_size < (1024 * 1024):
1016             logging.warning(
1017                 "%s appears to be too small! %s bytes",
1018                 suffixed, check_size)
1019         else:
1020             logging.debug("squashed size: %s", check_size)
1021             os.unlink(self.settings['image'])
1022             self.settings['image'] = suffixed
1023             logging.debug(
1024                 "%s usage: %s", self.settings['image'],
1025                 self.runcmd(['du', self.settings['image']]))
1026
1027     def cleanup_system(self):
1028         # Clean up after any errors.
1029
1030         self.message('Cleaning up')
1031
1032         # Umount in the reverse mount order
1033         if self.settings['image']:
1034             for i in range(len(self.mount_points) - 1, -1, -1):
1035                 mount_point = self.mount_points[i]
1036                 try:
1037                     self.runcmd(['umount', mount_point], ignore_fail=False)
1038                 except cliapp.AppException:
1039                     logging.debug("umount failed, sleeping and trying again")
1040                     time.sleep(5)
1041                     self.runcmd(['umount', mount_point], ignore_fail=False)
1042
1043             if 'freebsd' in os.sys.platform:
1044                 out = self.runcmd(['mdconfig', '-l', '-f', self.settings['image']])
1045                 for devid in out.split():
1046                     self.runcmd(['mdconfig', '-d', '-u', devid],
1047                                 ignore_fail=True)
1048             else:
1049                 self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1050
1051         for dirname in self.remove_dirs:
1052             shutil.rmtree(dirname)
1053
1054     def customize(self, rootdir):
1055         script = self.settings['customize']
1056         if not script:
1057             return
1058         if not os.path.exists(script):
1059             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1060             if not os.path.exists(example):
1061                 self.message("Unable to find %s" % script)
1062                 return
1063             script = example
1064         self.message('Running customize script %s' % script)
1065         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1066         logging.debug(
1067             "%s usage: %s", self.settings['image'],
1068             self.runcmd(['du', self.settings['image']]))
1069         try:
1070             with open('/dev/tty', 'w') as tty:
1071                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1072         except IOError:
1073             logging.debug('tty unavailable, trying in headless mode.')
1074             subprocess.call([script, rootdir, self.settings['image']])
1075
1076     def create_tarball(self, rootdir):
1077         # Create a tarball of the disk's contents
1078         # shell out to runcmd since it more easily handles rootdir
1079         self.message('Creating tarball of disk contents')
1080         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1081
1082     def chown(self):
1083         # Change image owner after completed build
1084         if self.settings['image']:
1085             filename = self.settings['image']
1086         elif self.settings['tarball']:
1087             filename = self.settings['tarball']
1088         else:
1089             return
1090         self.message("Changing owner to %s" % self.settings["owner"])
1091         subprocess.call(["chown", self.settings["owner"], filename])
1092
1093     def list_installed_pkgs(self, rootdir):
1094         # output the list of installed packages for sources identification
1095         self.message("Creating a list of installed binary package names")
1096         out = self.runcmd(['chroot', rootdir,
1097                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1098         with open('dpkg.list', 'w') as dpkg:
1099             dpkg.write(out)
1100
1101     def configure_apt(self, rootdir):
1102         # use the distribution and mirror to create an apt source
1103         self.message("Configuring apt to use distribution and mirror")
1104         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1105         logging.debug('configure apt %s', conf)
1106         mirror = self.settings['mirror']
1107         if self.settings['apt-mirror']:
1108             mirror = self.settings['apt-mirror']
1109             self.message("Setting apt mirror to %s" % mirror)
1110         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1111         f = open(conf, 'w')
1112         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1113         f.write(line)
1114         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1115         f.write(line)
1116         f.close()
1117         # ensure the apt sources have valid lists
1118         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1119
1120 if __name__ == '__main__':
1121     VmDebootstrap(version=__version__).run()