vmdebootstrap

   1 #! /usr/bin/python
   2 # Copyright 2011-2013  Lars Wirzenius
   3 # Copyright 2012  Codethink Limited
   4 # Copyright 2014-2015 Neil Williams <codehelp@debian.org>
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import cliapp
  20 import crypt
  21 import logging
  22 import os
  23 import glob
  24 import re
  25 import sys
  26 import shutil
  27 import datetime
  28 import subprocess
  29 import tempfile
  30 import time
  31 from distro_info import DebianDistroInfo, UbuntuDistroInfo
  32
  33
  34 __version__ = '0.11'
  35
  36 # pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches
  37
  38
  39 class VmDebootstrap(cliapp.Application):  # pylint: disable=too-many-public-methods
  40
  41     def __init__(self, progname=None, version=__version__, description=None, epilog=None):
  42         super(VmDebootstrap, self).__init__(progname, version, description, epilog)
  43         self.remove_dirs = []
  44         self.mount_points = []
  45         self.debian_info = DebianDistroInfo()
  46         self.ubuntu_info = UbuntuDistroInfo()
  47         self.bootdir = None
  48         self.efi_arch_table = {
  49             'amd64': {
  50                 'removable': '/EFI/boot/bootx64.efi',  # destination location
  51                 'install': '/EFI/debian/grubx64.efi',  # package location
  52                 'package': 'grub-efi-amd64',  # bootstrap package
  53                 'bin_package': 'grub-efi-amd64-bin',  # binary only
  54                 'extra': 'i386',  # architecture to add binary package
  55                 'exclusive': False,  # only EFI supported for this arch.
  56                 'target': 'x86_64-efi',  # grub target name
  57             },
  58             'i386': {
  59                 'removable': '/EFI/boot/bootia32.efi',
  60                 'install': '/EFI/debian/grubia32.efi',
  61                 'package': 'grub-efi-ia32',
  62                 'bin_package': 'grub-efi-ia32-bin',
  63                 'extra': None,
  64                 'exclusive': False,
  65                 'target': 'i386-efi',
  66             },
  67             'arm64': {
  68                 'removable': '/EFI/boot/bootaa64.efi',
  69                 'install': '/EFI/debian/grubaa64.efi',
  70                 'package': 'grub-efi-arm64',
  71                 'bin_package': 'grub-efi-arm64-bin',
  72                 'extra': None,
  73                 'exclusive': True,
  74                 'target': 'arm64-efi',
  75             }
  76         }
  77
  78     def add_settings(self):
  79         default_arch = subprocess.check_output(
  80             ["dpkg", "--print-architecture"]).strip()
  81
  82         self.settings.boolean(
  83             ['verbose'], 'report what is going on')
  84         self.settings.string(
  85             ['image'], 'put created disk image in FILE',
  86             metavar='FILE')
  87         self.settings.bytesize(
  88             ['size'],
  89             'create a disk image of size SIZE (%default)',
  90             metavar='SIZE',
  91             default='1G')
  92         self.settings.bytesize(
  93             ['bootsize'],
  94             'create boot partition of size SIZE (%default)',
  95             metavar='BOOTSIZE',
  96             default='0%')
  97         self.settings.string(
  98             ['boottype'],
  99             'specify file system type for /boot/',
 100             default='ext2')
 101         self.settings.bytesize(
 102             ['bootoffset'],
 103             'Space to leave at start of the image for bootloader',
 104             default='0')
 105         self.settings.boolean(
 106             ['use-uefi'],
 107             'Setup image for UEFI boot',
 108             default=False)
 109         self.settings.bytesize(
 110             ['esp-size'],
 111             'Size of EFI System Partition - requires use-uefi',
 112             default='5mib')
 113         self.settings.string(
 114             ['part-type'],
 115             'Partition type to use for this image',
 116             default='msdos')
 117         self.settings.string(
 118             ['roottype'],
 119             'specify file system type for /',
 120             default='ext4')
 121         self.settings.bytesize(
 122             ['swap'],
 123             'create swap space of size SIZE (min 256Mb)')
 124         self.settings.string(
 125             ['foreign'],
 126             'set up foreign debootstrap environment using provided program (ie binfmt handler)')
 127         self.settings.string(
 128             ['variant'],
 129             'select debootstrap variant if not using the default [deprecated]')
 130         self.settings.string_list(
 131             ['debootstrapopts'],
 132             'pass additional options to debootstrap'),
 133         self.settings.boolean(
 134             ['extlinux'],
 135             'install extlinux?',
 136             default=True)
 137         self.settings.string(
 138             ['tarball'],
 139             "tar up the disk's contents in FILE",
 140             metavar='FILE')
 141         self.settings.string(
 142             ['apt-mirror'],
 143             'configure apt to use MIRROR',
 144             metavar='URL')
 145         self.settings.string(
 146             ['mirror'],
 147             'use MIRROR as package source (%default)',
 148             metavar='URL',
 149             default='http://http.debian.net/debian/')
 150         self.settings.string(
 151             ['arch'],
 152             'architecture to use (%default)',
 153             metavar='ARCH',
 154             default=default_arch)
 155         self.settings.string(
 156             ['distribution'],
 157             'release to use (%default)',
 158             metavar='NAME',
 159             default='stable')
 160         self.settings.string_list(
 161             ['package'],
 162             'install PACKAGE onto system')
 163         self.settings.string_list(
 164             ['custom-package'],
 165             'install package in DEB file onto system (not from mirror)',
 166             metavar='DEB')
 167         self.settings.boolean(
 168             ['no-kernel'],
 169             'do not install a linux package')
 170         self.settings.string(
 171             ['kernel-package'],
 172             'install PACKAGE instead of the default kernel package',
 173             metavar='PACKAGE')
 174         self.settings.boolean(
 175             ['enable-dhcp'],
 176             'enable DHCP on eth0')
 177         self.settings.string(
 178             ['root-password'],
 179             'set root password',
 180             metavar='PASSWORD')
 181         self.settings.boolean(
 182             ['lock-root-password'],
 183             'lock root account so they cannot login?')
 184         self.settings.string(
 185             ['customize'],
 186             'run SCRIPT after setting up system',
 187             metavar='SCRIPT')
 188         self.settings.string(
 189             ['hostname'],
 190             'set name to HOSTNAME (%default)',
 191             metavar='HOSTNAME',
 192             default='debian')
 193         self.settings.string_list(
 194             ['user'],
 195             'create USER with PASSWORD',
 196             metavar='USER/PASSWORD')
 197         self.settings.boolean(
 198             ['serial-console'],
 199             'configure image to use a serial console')
 200         self.settings.string(
 201             ['serial-console-command'],
 202             'command to manage the serial console, appended to /etc/inittab (%default)',
 203             metavar='COMMAND',
 204             default='/sbin/getty -L ttyS0 115200 vt100')
 205         self.settings.boolean(
 206             ['sudo'],
 207             'install sudo, and if user is created, add them to sudo group')
 208         self.settings.string(
 209             ['owner'],
 210             'the user who will own the image when the build is complete.')
 211         self.settings.boolean(
 212             ['squash'],
 213             'use squashfs on the final image.')
 214         self.settings.boolean(
 215             ['configure-apt'],
 216             'Create an apt source based on the distribution and mirror selected.')
 217         self.settings.boolean(
 218             ['mbr'],
 219             'Run install-mbr (default if extlinux used)')
 220         self.settings.boolean(
 221             ['grub'],
 222             'Install and configure grub2 - disables extlinux.')
 223         self.settings.boolean(
 224             ['sparse'],
 225             'Do not fill the image with zeros to keep a sparse disk image',
 226             default=False)
 227         self.settings.boolean(
 228             ['pkglist'],
 229             'Create a list of package names included in the image.')
 230         self.settings.boolean(
 231             ['no-acpid'],
 232             'do not install the acpid package',
 233             default=False)
 234
 235     def process_args(self, args):  # pylint: disable=too-many-branches,too-many-statements
 236         if not self.settings['image'] and not self.settings['tarball']:
 237             raise cliapp.AppException(
 238                 'You must give disk image filename, or tarball filename')
 239         if self.settings['image'] and not self.settings['size']:
 240             raise cliapp.AppException(
 241                 'If disk image is specified, you must give image size.')
 242         if not self.debian_info.valid(self.settings['distribution']):
 243             if not self.ubuntu_info.valid(self.settings['distribution']):
 244                 raise cliapp.AppException(
 245                     '%s is not a valid Debian or Ubuntu suite or codename.'
 246                     % self.settings['distribution'])
 247         if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880:
 248             raise cliapp.AppException(
 249                 'You must specify use-uefi for esp-size to have effect')
 250         if self.settings['arch'] in self.efi_arch_table and\
 251                 self.efi_arch_table[self.settings['arch']]['exclusive'] and\
 252                 not self.settings['use-uefi']:
 253             raise cliapp.AppException(
 254                 'Only UEFI is supported on %s' % self.settings['arch'])
 255         elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table:
 256             raise cliapp.AppException(
 257                 '%s is not a supported UEFI architecture' % self.settings['arch'])
 258         if self.settings['use-uefi'] and (
 259                 self.settings['bootsize'] or
 260                 self.settings['bootoffset']):
 261             raise cliapp.AppException(
 262                 'A separate boot partition is not supported with UEFI')
 263
 264         if self.settings['use-uefi'] and not self.settings['grub']:
 265             raise cliapp.AppException(
 266                 'UEFI without Grub is not supported.')
 267
 268         # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi
 269         if self.was_oldstable(datetime.date(2015, 4, 26)):
 270             if self.settings['use-uefi'] and self.settings['arch'] != 'amd64':
 271                 raise cliapp.AppException(
 272                     'Only amd64 supports UEFI in Wheezy')
 273
 274         if os.geteuid() != 0:
 275             sys.exit("You need to have root privileges to run this script.")
 276         rootdir = None
 277         try:
 278             rootdev = None
 279             roottype = self.settings['roottype']
 280             bootdev = None
 281             boottype = None
 282             if self.settings['image']:
 283                 self.create_empty_image()
 284                 self.partition_image()
 285                 if self.settings['mbr'] or self.settings['extlinux']:
 286                     self.install_mbr()
 287                 (rootdev, bootdev, swapdev) = self.setup_kpartx()
 288                 if self.settings['swap'] > 0:
 289                     self.message("Creating swap space")
 290                     self.runcmd(['mkswap', swapdev])
 291                 self.mkfs(rootdev, fstype=roottype)
 292                 rootdir = self.mount(rootdev)
 293                 if self.settings['use-uefi']:
 294                     self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi')
 295                     logging.debug("bootdir:%s", self.bootdir)
 296                     self.mkfs(bootdev, fstype='vfat')
 297                     os.makedirs(self.bootdir)
 298                     self.mount(bootdev, self.bootdir)
 299                 elif bootdev:
 300                     if self.settings['boottype']:
 301                         boottype = self.settings['boottype']
 302                     else:
 303                         boottype = 'ext2'
 304                     self.mkfs(bootdev, fstype=boottype)
 305                     self.bootdir = '%s/%s' % (rootdir, 'boot/')
 306                     os.mkdir(self.bootdir)
 307                     self.mount(bootdev, self.bootdir)
 308             else:
 309                 rootdir = self.mkdtemp()
 310             self.debootstrap(rootdir)
 311             self.set_hostname(rootdir)
 312             self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
 313             self.install_debs(rootdir)
 314             self.set_root_password(rootdir)
 315             self.create_users(rootdir)
 316             self.remove_udev_persistent_rules(rootdir)
 317             self.setup_networking(rootdir)
 318             if self.settings['configure-apt'] or self.settings['apt-mirror']:
 319                 self.configure_apt(rootdir)
 320             self.customize(rootdir)
 321             self.cleanup_apt_cache(rootdir)
 322             self.update_initramfs(rootdir)
 323
 324             if self.settings['image']:
 325                 if self.settings['use-uefi']:
 326                     self.install_grub_uefi(rootdir)
 327                 elif self.settings['grub']:
 328                     self.install_grub2(rootdev, rootdir)
 329                 elif self.settings['extlinux']:
 330                     self.install_extlinux(rootdev, rootdir)
 331                 self.append_serial_console(rootdir)
 332                 self.optimize_image(rootdir)
 333                 if self.settings['squash']:
 334                     self.squash()
 335             if self.settings['pkglist']:
 336                 self.list_installed_pkgs(rootdir)
 337
 338             if self.settings['foreign']:
 339                 os.unlink('%s/usr/bin/%s' %
 340                           (rootdir, os.path.basename(self.settings['foreign'])))
 341
 342             if self.settings['tarball']:
 343                 self.create_tarball(rootdir)
 344
 345             if self.settings['owner']:
 346                 self.chown()
 347         except BaseException as e:
 348             self.message('EEEK! Something bad happened...')
 349             if rootdir:
 350                 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
 351                 if os.path.exists(db_log):
 352                     shutil.copy(db_log, os.getcwd())
 353             self.message(e)
 354             self.cleanup_system()
 355             raise
 356         else:
 357             self.cleanup_system()
 358
 359     def message(self, msg):
 360         logging.info(msg)
 361         if self.settings['verbose']:
 362             print msg
 363
 364     def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
 365         logging.debug('runcmd: %s %s %s', argv, env, kwargs)
 366         p = subprocess.Popen(argv, stdin=subprocess.PIPE,
 367                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 368                              env=env, **kwargs)
 369         out, err = p.communicate(stdin)
 370         if p.returncode != 0:
 371             msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
 372             logging.error(msg)
 373             if not ignore_fail:
 374                 raise cliapp.AppException(msg)
 375         return out
 376
 377     def mkdtemp(self):
 378         dirname = tempfile.mkdtemp()
 379         self.remove_dirs.append(dirname)
 380         logging.debug('mkdir %s', dirname)
 381         return dirname
 382
 383     def mount(self, device, path=None):
 384         if not path:
 385             mount_point = self.mkdtemp()
 386         else:
 387             mount_point = path
 388         self.message('Mounting %s on %s' % (device, mount_point))
 389         self.runcmd(['mount', device, mount_point])
 390         self.mount_points.append(mount_point)
 391         logging.debug('mounted %s on %s', device, mount_point)
 392         return mount_point
 393
 394     def create_empty_image(self):
 395         self.message('Creating disk image')
 396         self.runcmd(['qemu-img', 'create', '-f', 'raw',
 397                      self.settings['image'],
 398                      str(self.settings['size'])])
 399
 400     def partition_image(self):
 401         """
 402         Uses fat16 (msdos) partitioning by default, use part-type to change.
 403         If bootoffset is specified, the first actual partition
 404         starts at that offset to allow customisation scripts to
 405         put bootloader images into the space, e.g. u-boot.
 406         """
 407         self.message('Creating partitions')
 408         self.runcmd(['parted', '-s', self.settings['image'],
 409                      'mklabel', self.settings['part-type']])
 410         partoffset = 0
 411         extent = '100%'
 412         swap = 256 * 1024 * 1024
 413         if self.settings['swap'] > 0:
 414             if self.settings['swap'] > swap:
 415                 swap = self.settings['swap']
 416             else:
 417                 # minimum 256Mb as default qemu ram is 128Mb
 418                 logging.debug("Setting minimum 256Mb swap space")
 419             extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size'])
 420
 421         if self.settings['use-uefi']:
 422             espsize = self.settings['esp-size'] / (1024 * 1024)
 423             self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size']))
 424             self.runcmd(['parted', '-s', self.settings['image'],
 425                          'mkpart', 'primary', 'fat32',
 426                          '1', str(espsize)])
 427             self.runcmd(['parted', '-s', self.settings['image'],
 428                          'set', '1', 'boot', 'on'])
 429             self.runcmd(['parted', '-s', self.settings['image'],
 430                          'set', '1', 'esp', 'on'])
 431
 432         if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0':
 433             # turn v.small offsets into something at least possible to create.
 434             if self.settings['bootoffset'] < 1048576:
 435                 partoffset = 1
 436                 logging.info(
 437                     "Setting bootoffset %smib to allow for %s bytes",
 438                     partoffset, self.settings['bootoffset'])
 439             else:
 440                 partoffset = self.settings['bootoffset'] / (1024 * 1024)
 441                 self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset']))
 442         if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
 443             if self.settings['grub'] and not partoffset:
 444                 partoffset = 1
 445             bootsize = self.settings['bootsize'] / (1024 * 1024)
 446             bootsize += partoffset
 447             self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize']))
 448             logging.debug("Starting boot partition at %sMb", bootsize)
 449             self.runcmd(['parted', '-s', self.settings['image'],
 450                          'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)])
 451             logging.debug("Starting root partition at %sMb", partoffset)
 452             self.runcmd(['parted', '-s', self.settings['image'],
 453                          'mkpart', 'primary', str(bootsize), extent])
 454         elif self.settings['use-uefi']:
 455             bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
 456             self.runcmd(['parted', '-s', self.settings['image'],
 457                          'mkpart', 'primary', str(bootsize), extent])
 458         else:
 459             self.runcmd(['parted', '-s', self.settings['image'],
 460                          'mkpart', 'primary', '0%', extent])
 461         self.runcmd(['parted', '-s', self.settings['image'],
 462                      'set', '1', 'boot', 'on'])
 463         if self.settings['swap'] > 0:
 464             logging.debug("Creating swap partition")
 465             self.runcmd(['parted', '-s', self.settings['image'],
 466                          'mkpart', 'primary', 'linux-swap', extent, '100%'])
 467
 468     def update_initramfs(self, rootdir):
 469         cmd = os.path.join('usr', 'sbin', 'update-initramfs')
 470         if os.path.exists(os.path.join(rootdir, cmd)):
 471             self.message("Updating the initramfs")
 472             self.runcmd(['chroot', rootdir, cmd, '-u'])
 473
 474     def install_mbr(self):
 475         if os.path.exists("/sbin/install-mbr"):
 476             self.message('Installing MBR')
 477             self.runcmd(['install-mbr', self.settings['image']])
 478         else:
 479             msg = "mbr enabled but /sbin/install-mbr not found" \
 480                   " - please install the mbr package."
 481             raise cliapp.AppException(msg)
 482
 483     def setup_kpartx(self):
 484         bootindex = None
 485         swapindex = None
 486         if 'freebsd' in os.sys.platform:
 487             out = self.runcmd(['mdconfig', '-a', '-t', 'vnode', '-f',
 488                                self.settings['image']])
 489         else:
 490             out = self.runcmd(['kpartx', '-avs', self.settings['image']])
 491         if self.settings['bootsize'] and self.settings['swap'] > 0:
 492             bootindex = 0
 493             rootindex = 1
 494             swapindex = 2
 495             parts = 3
 496         elif self.settings['use-uefi']:
 497             bootindex = 0
 498             rootindex = 1
 499             parts = 2
 500         elif self.settings['use-uefi'] and self.settings['swap'] > 0:
 501             bootindex = 0
 502             rootindex = 1
 503             swapindex = 2
 504             parts = 3
 505         elif self.settings['bootsize']:
 506             bootindex = 0
 507             rootindex = 1
 508             parts = 2
 509         elif self.settings['swap'] > 0:
 510             rootindex = 0
 511             swapindex = 1
 512             parts = 2
 513         else:
 514             rootindex = 0
 515             parts = 1
 516         boot = None
 517         swap = None
 518         if 'freebsd' in os.sys.platform:
 519             devices = glob.glob("/dev/%ss*" % out.strip())
 520         else:
 521             devices = ['/dev/mapper/%s' % line.split()[2]
 522                        for line in out.splitlines()
 523                        if line.startswith('add map ')]
 524         if len(devices) != parts:
 525             msg = 'Surprising number of partitions - check output of losetup -a'
 526             logging.debug("%s", self.runcmd(['losetup', '-a']))
 527             logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
 528             raise cliapp.AppException(msg)
 529         root = devices[rootindex]
 530         if self.settings['bootsize'] or self.settings['use-uefi']:
 531             boot = devices[bootindex]
 532         if self.settings['swap'] > 0:
 533             swap = devices[swapindex]
 534         return root, boot, swap
 535
 536     def _efi_packages(self):
 537         packages = []
 538         pkg = self.efi_arch_table[self.settings['arch']]['package']
 539         self.message("Adding %s" % pkg)
 540         packages.append(pkg)
 541         extra = self.efi_arch_table[self.settings['arch']]['extra']
 542         if extra and isinstance(extra, str):
 543             bin_pkg = self.efi_arch_table[str(extra)]['bin_package']
 544             self.message("Adding support for %s using %s" % (extra, bin_pkg))
 545             packages.append(bin_pkg)
 546         return packages
 547
 548     def _copy_efi_binary(self, efi_removable, efi_install):
 549         logging.debug("using bootdir=%s", self.bootdir)
 550         logging.debug("moving %s to %s", efi_removable, efi_install)
 551         if efi_removable.startswith('/'):
 552             efi_removable = efi_removable[1:]
 553         if efi_install.startswith('/'):
 554             efi_install = efi_install[1:]
 555         efi_output = os.path.join(self.bootdir, efi_removable)
 556         efi_input = os.path.join(self.bootdir, efi_install)
 557         if not os.path.exists(efi_input):
 558             logging.warning("%s does not exist (%s)", efi_input, efi_install)
 559             raise cliapp.AppException("Missing %s" % efi_install)
 560         if not os.path.exists(os.path.dirname(efi_output)):
 561             os.makedirs(os.path.dirname(efi_output))
 562         logging.debug(
 563             'Moving UEFI support: %s -> %s', efi_input, efi_output)
 564         if os.path.exists(efi_output):
 565             os.unlink(efi_output)
 566         os.rename(efi_input, efi_output)
 567
 568     def configure_efi(self):
 569         """
 570         Copy the bootloader file from the package into the location
 571         so needs to be after grub and kernel already installed.
 572         """
 573         self.message('Configuring EFI')
 574         efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable'])
 575         efi_install = str(self.efi_arch_table[self.settings['arch']]['install'])
 576         self.message('Installing UEFI support binary')
 577         self._copy_efi_binary(efi_removable, efi_install)
 578
 579     def configure_extra_efi(self):
 580         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 581         if extra:
 582             efi_removable = str(self.efi_arch_table[extra]['removable'])
 583             efi_install = str(self.efi_arch_table[extra]['install'])
 584             self.message('Copying UEFI support binary for %s' % extra)
 585             self._copy_efi_binary(efi_removable, efi_install)
 586
 587     def mkfs(self, device, fstype):
 588         self.message('Creating filesystem %s' % fstype)
 589         self.runcmd(['mkfs', '-t', fstype, device])
 590
 591     def suite_to_codename(self, distro):
 592         suite = self.debian_info.codename(distro, datetime.date.today())
 593         if not suite:
 594             return distro
 595         return suite
 596
 597     def was_oldstable(self, limit):
 598         suite = self.suite_to_codename(self.settings['distribution'])
 599         # this check is only for debian
 600         if not self.debian_info.valid(suite):
 601             return False
 602         return suite == self.debian_info.old(limit)
 603
 604     def was_stable(self, limit):
 605         suite = self.suite_to_codename(self.settings['distribution'])
 606         # this check is only for debian
 607         if not self.debian_info.valid(suite):
 608             return False
 609         return suite == self.debian_info.stable(limit)
 610
 611     def debootstrap(self, rootdir):  # pylint: disable=too-many-statements
 612         msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else ''
 613         self.message(
 614             'Debootstrapping %s [%s] %s' % (
 615                 self.settings['distribution'], self.settings['arch'], msg))
 616
 617         include = self.settings['package']
 618
 619         if not self.settings['foreign'] and not self.settings['no-acpid']:
 620             include.append('acpid')
 621
 622         if self.settings['grub']:
 623             if self.settings['use-uefi']:
 624                 include.extend(self._efi_packages())
 625             else:
 626                 include.append('grub-pc')
 627
 628         if not self.settings['no-kernel']:
 629             if self.settings['kernel-package']:
 630                 kernel_image = self.settings['kernel-package']
 631             else:
 632                 if self.settings['arch'] == 'i386':
 633                     # wheezy (which became oldstable on 04/25/2015) used '486'
 634                     if self.was_oldstable(datetime.date(2015, 4, 26)):
 635                         kernel_arch = '486'
 636                     else:
 637                         kernel_arch = '586'
 638                 elif self.settings['arch'] == 'armhf':
 639                     kernel_arch = 'armmp'
 640                 else:
 641                     kernel_arch = self.settings['arch']
 642                 kernel_image = 'linux-image-%s' % kernel_arch
 643             include.append(kernel_image)
 644
 645         if self.settings['sudo'] and 'sudo' not in include:
 646             include.append('sudo')
 647
 648         args = ['debootstrap', '--arch=%s' % self.settings['arch']]
 649
 650         if self.settings['package']:
 651             args.append(
 652                 '--include=%s' % ','.join(include))
 653         if self.settings['foreign']:
 654             args.append('--foreign')
 655         if self.settings['debootstrapopts']:
 656             for opt in self.settings['debootstrapopts']:
 657                 for part in opt.split(' '):
 658                     args.append('--%s' % part)
 659         elif self.settings['variant']:
 660             args.append('--variant')
 661             args.append(self.settings['variant'])
 662         args += [self.settings['distribution'],
 663                  rootdir, self.settings['mirror']]
 664         logging.debug(" ".join(args))
 665         self.runcmd(args)
 666         if self.settings['foreign']:
 667             # set a noninteractive debconf environment for secondstage
 668             env = {
 669                 "DEBIAN_FRONTEND": "noninteractive",
 670                 "DEBCONF_NONINTERACTIVE_SEEN": "true",
 671                 "LC_ALL": "C"
 672             }
 673             # add the mapping to the complete environment.
 674             env.update(os.environ)
 675             # First copy the binfmt handler over
 676             self.message('Setting up binfmt handler')
 677             shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
 678             # Next, run the package install scripts etc.
 679             self.message('Running debootstrap second stage')
 680             self.runcmd(['chroot', rootdir,
 681                          '/debootstrap/debootstrap', '--second-stage'],
 682                         env=env)
 683
 684     def set_hostname(self, rootdir):
 685         hostname = self.settings['hostname']
 686         with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
 687             f.write('%s\n' % hostname)
 688
 689         etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
 690         try:
 691             with open(etc_hosts, 'r') as f:
 692                 data = f.read()
 693             with open(etc_hosts, 'w') as f:
 694                 for line in data.splitlines():
 695                     if line.startswith('127.0.0.1'):
 696                         line += ' %s' % hostname
 697                     f.write('%s\n' % line)
 698         except IOError:
 699             pass
 700
 701     def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):  # pylint: disable=too-many-arguments
 702         def fsuuid(device):
 703             if 'freebsd' in os.sys.platform:
 704                 out = self.runcmd(['grub-probe', '-d', device, '-t', 'fs_uuid'])
 705                 return "/dev/ufsid/%s" % out.strip()
 706             else:
 707                 out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 708                                 '-s', 'UUID', device])
 709                 return "UUID=%s" % out.splitlines()[0].strip()
 710
 711         if rootdev:
 712             rootdevstr = fsuuid(rootdev)
 713         else:
 714             rootdevstr = '/dev/sda1'
 715
 716         if bootdev and not self.settings['use-uefi']:
 717             bootdevstr = fsuuid(bootdev)
 718         else:
 719             bootdevstr = None
 720
 721         fstab = os.path.join(rootdir, 'etc', 'fstab')
 722         with open(fstab, 'w') as f:
 723             if 'freebsd' in os.sys.platform:
 724                 f.write('proc  /proc   linprocfs rw 0 0\n')
 725                 f.write('sys   /sys    linsysfs  rw 0 0\n')
 726                 f.write('fdesc /dev/fd fdescfs   rw 0 0\n')
 727             else:
 728                 f.write('proc /proc proc defaults 0 0\n')
 729             f.write('%s / %s rw 0 1\n' % (rootdevstr, roottype))
 730             if bootdevstr:
 731                 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
 732                 if self.settings['swap'] > 0:
 733                     f.write("/dev/sda3 swap swap defaults 0 0\n")
 734             elif self.settings['swap'] > 0:
 735                 f.write("/dev/sda2 swap swap defaults 0 0\n")
 736
 737     def install_debs(self, rootdir):
 738         if not self.settings['custom-package']:
 739             return
 740         self.message('Installing custom packages')
 741         tmp = os.path.join(rootdir, 'tmp', 'install_debs')
 742         os.mkdir(tmp)
 743         for deb in self.settings['custom-package']:
 744             shutil.copy(deb, tmp)
 745         filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
 746                      for deb in self.settings['custom-package']]
 747         out, err, _ = \
 748             self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
 749         logging.debug('stdout:\n%s', out)
 750         logging.debug('stderr:\n%s', err)
 751         out = self.runcmd(['chroot', rootdir,
 752                            'apt-get', '-f', '--no-remove', 'install'])
 753         logging.debug('stdout:\n%s', out)
 754         shutil.rmtree(tmp)
 755
 756     def cleanup_apt_cache(self, rootdir):
 757         out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
 758         logging.debug('stdout:\n%s', out)
 759
 760     def set_root_password(self, rootdir):
 761         if self.settings['root-password']:
 762             self.message('Setting root password')
 763             self.set_password(rootdir, 'root', self.settings['root-password'])
 764         elif self.settings['lock-root-password']:
 765             self.message('Locking root password')
 766             self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
 767         else:
 768             self.message('Give root an empty password')
 769             self.delete_password(rootdir, 'root')
 770
 771     def create_users(self, rootdir):
 772         def create_user(vmuser):
 773             self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser,
 774                          '--disabled-password', vmuser])
 775             if self.settings['sudo']:
 776                 self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo'])
 777
 778         for userpass in self.settings['user']:
 779             if '/' in userpass:
 780                 user, password = userpass.split('/', 1)
 781                 create_user(user)
 782                 self.set_password(rootdir, user, password)
 783             else:
 784                 create_user(userpass)
 785                 self.delete_password(rootdir, userpass)
 786
 787     def set_password(self, rootdir, user, password):
 788         encrypted = crypt.crypt(password, '..')
 789         self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
 790
 791     def delete_password(self, rootdir, user):
 792         self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
 793
 794     def remove_udev_persistent_rules(self, rootdir):
 795         self.message('Removing udev persistent cd and net rules')
 796         for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
 797             pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
 798             if os.path.exists(pathname):
 799                 logging.debug('rm %s', pathname)
 800                 os.remove(pathname)
 801             else:
 802                 logging.debug('not removing non-existent %s', pathname)
 803
 804     def mask_udev_predictable_rules(self, rootdir):
 805         """
 806         This can be reset later but to get networking working immediately
 807         on boot, the interface we're going to use must be known without
 808         reference to the eventual machine.
 809         http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
 810         """
 811         self.message('Disabling systemd predictable interface names')
 812         udev_path = os.path.join(
 813             'etc', 'udev', 'rules.d', '80-net-setup-link.rules')
 814         self.runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path])
 815
 816     def setup_networking(self, rootdir):
 817         self.message('Setting up networking')
 818         ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces')
 819         ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d')
 820
 821         # unconditionally write for wheezy (which became oldstable 2015.04.25)
 822         if self.was_oldstable(datetime.date(2015, 4, 26)):
 823             with open(ifc_file, 'w') as netfile:
 824                 netfile.write('source /etc/network/interfaces.d/*\n')
 825         elif not os.path.exists(ifc_file):
 826             with open(ifc_file, 'a') as netfile:
 827                 netfile.write('source-directory /etc/network/interfaces.d\n')
 828
 829         if not os.path.exists(ifc_d):
 830             os.mkdir(ifc_d)
 831         ethpath = os.path.join(ifc_d, 'setup')
 832         with open(ethpath, 'w') as eth:
 833             eth.write('auto lo\n')
 834             eth.write('iface lo inet loopback\n')
 835
 836             if self.settings['enable-dhcp']:
 837                 eth.write('\n')
 838                 eth.write('auto eth0\n')
 839                 eth.write('iface eth0 inet dhcp\n')
 840                 # force predictable interface names
 841                 self.mask_udev_predictable_rules(rootdir)
 842
 843     def append_serial_console(self, rootdir):
 844         if self.settings['serial-console']:
 845             serial_command = self.settings['serial-console-command']
 846             logging.debug('adding getty to serial console')
 847             inittab = os.path.join(rootdir, 'etc/inittab')
 848             # to autologin, serial_command can contain '-a root'
 849             with open(inittab, 'a') as f:
 850                 f.write('\nS0:23:respawn:%s\n' % serial_command)
 851
 852     # pylint: disable=no-self-use
 853     def _grub_serial_console(self, rootdir):
 854         cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"'
 855         terminal = 'GRUB_TERMINAL="serial gfxterm"'
 856         command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"'
 857         grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub')
 858         logging.debug("Allowing serial output in grub config %s", grub_cfg)
 859         with open(grub_cfg, 'a+') as cfg:
 860             cfg.write("# %s serial support\n" % os.path.basename(__file__))
 861             cfg.write("%s\n" % cmdline)
 862             cfg.write("%s\n" % terminal)
 863             cfg.write("%s\n" % command)
 864
 865     def _mount_wrapper(self, rootdir):
 866         if 'freebsd' in os.sys.platform:
 867             self.runcmd(['mount', 'dev', '-t', 'devfs',
 868                         '%s' % os.path.join(rootdir, 'dev')])
 869             self.runcmd(['mount', 'proc', '-t', 'linprocfs',
 870                         '%s' % os.path.join(rootdir, 'proc')])
 871             self.runcmd(['mount', 'sys', '-t', 'linsysfs',
 872                         '%s' % os.path.join(rootdir, 'sys')])
 873         else:
 874             self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
 875                         '%s' % os.path.join(rootdir, 'dev')])
 876             self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
 877                         '%s' % os.path.join(rootdir, 'proc')])
 878             self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
 879                         '%s' % os.path.join(rootdir, 'sys')])
 880
 881     def _umount_wrapper(self, rootdir):
 882         self.runcmd(['umount', os.path.join(rootdir, 'sys')])
 883         self.runcmd(['umount', os.path.join(rootdir, 'proc')])
 884         self.runcmd(['umount', os.path.join(rootdir, 'dev')])
 885
 886     def install_grub_uefi(self, rootdir):
 887         self.message("Configuring grub-uefi")
 888         target = self.efi_arch_table[self.settings['arch']]['target']
 889         grub_opts = "--target=%s" % target
 890         logging.debug("Running grub-install with options: %s", grub_opts)
 891         self._mount_wrapper(rootdir)
 892         try:
 893             self.runcmd(['chroot', rootdir, 'update-grub'])
 894             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 895         except cliapp.AppException as exc:
 896             logging.warning(exc)
 897             self.message(
 898                 "Failed to configure grub-uefi for %s" %
 899                 self.settings['arch'])
 900             self._umount_wrapper(rootdir)
 901         self.configure_efi()
 902         extra = str(self.efi_arch_table[self.settings['arch']]['extra'])
 903         if extra:
 904             target = self.efi_arch_table[extra]['target']
 905             grub_opts = "--target=%s" % target
 906             try:
 907                 self.runcmd(['chroot', rootdir, 'update-grub'])
 908                 self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 909             except cliapp.AppException as exc:
 910                 logging.warning(exc)
 911                 self.message(
 912                     "Failed to configure grub-uefi for %s" % extra)
 913             self.configure_extra_efi()
 914         self._umount_wrapper(rootdir)
 915
 916     def install_grub2(self, rootdev, rootdir):
 917         self.message("Configuring grub2")
 918         # rely on kpartx using consistent naming to map loop0p1 to loop0
 919         grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
 920         if self.settings['serial-console']:
 921             self._grub_serial_console(rootdir)
 922         logging.debug("Running grub-install with options: %s", grub_opts)
 923         self._mount_wrapper(rootdir)
 924         try:
 925             self.runcmd(['chroot', rootdir, 'update-grub'])
 926             self.runcmd(['chroot', rootdir, 'grub-install', grub_opts])
 927         except cliapp.AppException as exc:
 928             logging.warning(exc)
 929             self.message("Failed. Is grub2-common installed? Using extlinux.")
 930             self.install_extlinux(rootdev, rootdir)
 931         self._umount_wrapper(rootdir)
 932
 933     def install_extlinux(self, rootdev, rootdir):
 934         if not os.path.exists("/usr/bin/extlinux"):
 935             self.message("extlinux not installed, skipping.")
 936             return
 937         self.message('Installing extlinux')
 938
 939         def find(pattern):
 940             dirname = os.path.join(rootdir, 'boot')
 941             basenames = os.listdir(dirname)
 942             logging.debug('find: %s', basenames)
 943             for basename in basenames:
 944                 if re.search(pattern, basename):
 945                     return os.path.join('boot', basename)
 946             raise cliapp.AppException('Cannot find match: %s' % pattern)
 947
 948         try:
 949             kernel_image = find('vmlinuz-.*')
 950             initrd_image = find('initrd.img-.*')
 951         except cliapp.AppException as e:
 952             self.message("Unable to find kernel. Not installing extlinux.")
 953             logging.debug("No kernel found. %s. Skipping install of extlinux.", e)
 954             return
 955
 956         out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
 957                            '-s', 'UUID', rootdev])
 958         uuid = out.splitlines()[0].strip()
 959
 960         conf = os.path.join(rootdir, 'extlinux.conf')
 961         logging.debug('configure extlinux %s', conf)
 962         kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
 963         extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
 964         msg = '''
 965 default linux
 966 timeout 1
 967
 968 label linux
 969 kernel %(kernel)s
 970 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
 971 %(extserial)s
 972 ''' % {
 973             'kernel': kernel_image,  # pylint: disable=bad-continuation
 974             'initrd': initrd_image,  # pylint: disable=bad-continuation
 975             'uuid': uuid,  # pylint: disable=bad-continuation
 976             'kserial': kserial,  # pylint: disable=bad-continuation
 977             'extserial': extserial,  # pylint: disable=bad-continuation
 978         }  # pylint: disable=bad-continuation
 979         logging.debug("extlinux config:\n%s", msg)
 980
 981         # python multiline string substitution is just ugly.
 982         # use an external file or live with the mangling, no point in
 983         # mangling the string to remove spaces just to keep it pretty in source.
 984         f = open(conf, 'w')
 985         f.write(msg)
 986
 987         self.runcmd(['extlinux', '--install', rootdir])
 988         self.runcmd(['sync'])
 989         time.sleep(2)
 990
 991     def optimize_image(self, rootdir):
 992         """
 993         Filing up the image with zeros will increase its compression rate
 994         """
 995         if not self.settings['sparse']:
 996             zeros = os.path.join(rootdir, 'ZEROS')
 997             self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
 998             self.runcmd(['rm', '-f', zeros])
 999
1000     def squash(self):
1001         """
1002         Run squashfs on the image.
1003         """
1004         if not os.path.exists('/usr/bin/mksquashfs'):
1005             logging.warning("Squash selected but mksquashfs not found!")
1006             return
1007         logging.debug(
1008             "%s usage: %s", self.settings['image'],
1009             self.runcmd(['du', self.settings['image']]))
1010         self.message("Running mksquashfs")
1011         suffixed = "%s.squashfs" % self.settings['image']
1012         if os.path.exists(suffixed):
1013             os.unlink(suffixed)
1014         msg = self.runcmd(
1015             ['mksquashfs', self.settings['image'],
1016              suffixed,
1017              '-no-progress', '-comp', 'xz'], ignore_fail=False)
1018         logging.debug(msg)
1019         check_size = os.path.getsize(suffixed)
1020         if check_size < (1024 * 1024):
1021             logging.warning(
1022                 "%s appears to be too small! %s bytes",
1023                 suffixed, check_size)
1024         else:
1025             logging.debug("squashed size: %s", check_size)
1026             os.unlink(self.settings['image'])
1027             self.settings['image'] = suffixed
1028             logging.debug(
1029                 "%s usage: %s", self.settings['image'],
1030                 self.runcmd(['du', self.settings['image']]))
1031
1032     def cleanup_system(self):
1033         # Clean up after any errors.
1034
1035         self.message('Cleaning up')
1036
1037         # Umount in the reverse mount order
1038         if self.settings['image']:
1039             for i in range(len(self.mount_points) - 1, -1, -1):
1040                 mount_point = self.mount_points[i]
1041                 try:
1042                     self.runcmd(['umount', mount_point], ignore_fail=False)
1043                 except cliapp.AppException:
1044                     logging.debug("umount failed, sleeping and trying again")
1045                     time.sleep(5)
1046                     self.runcmd(['umount', mount_point], ignore_fail=False)
1047
1048             if 'freebsd' in os.sys.platform:
1049                 out = self.runcmd(['mdconfig', '-l', '-f', self.settings['image']])
1050                 for devid in out.split():
1051                     self.runcmd(['mdconfig', '-d', '-u', devid],
1052                                 ignore_fail=True)
1053             else:
1054                 self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
1055
1056         for dirname in self.remove_dirs:
1057             shutil.rmtree(dirname)
1058
1059     def customize(self, rootdir):
1060         script = self.settings['customize']
1061         if not script:
1062             return
1063         if not os.path.exists(script):
1064             example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
1065             if not os.path.exists(example):
1066                 self.message("Unable to find %s" % script)
1067                 return
1068             script = example
1069         self.message('Running customize script %s' % script)
1070         logging.info("rootdir=%s image=%s", rootdir, self.settings['image'])
1071         logging.debug(
1072             "%s usage: %s", self.settings['image'],
1073             self.runcmd(['du', self.settings['image']]))
1074         try:
1075             with open('/dev/tty', 'w') as tty:
1076                 cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty)
1077         except IOError:
1078             logging.debug('tty unavailable, trying in headless mode.')
1079             subprocess.call([script, rootdir, self.settings['image']])
1080
1081     def create_tarball(self, rootdir):
1082         # Create a tarball of the disk's contents
1083         # shell out to runcmd since it more easily handles rootdir
1084         self.message('Creating tarball of disk contents')
1085         self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
1086
1087     def chown(self):
1088         # Change image owner after completed build
1089         if self.settings['image']:
1090             filename = self.settings['image']
1091         elif self.settings['tarball']:
1092             filename = self.settings['tarball']
1093         else:
1094             return
1095         self.message("Changing owner to %s" % self.settings["owner"])
1096         subprocess.call(["chown", self.settings["owner"], filename])
1097
1098     def list_installed_pkgs(self, rootdir):
1099         # output the list of installed packages for sources identification
1100         self.message("Creating a list of installed binary package names")
1101         out = self.runcmd(['chroot', rootdir,
1102                            'dpkg-query', '-W', "-f='${Package}.deb\n'"])
1103         with open('dpkg.list', 'w') as dpkg:
1104             dpkg.write(out)
1105
1106     def configure_apt(self, rootdir):
1107         # use the distribution and mirror to create an apt source
1108         self.message("Configuring apt to use distribution and mirror")
1109         conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
1110         logging.debug('configure apt %s', conf)
1111         mirror = self.settings['mirror']
1112         if self.settings['apt-mirror']:
1113             mirror = self.settings['apt-mirror']
1114             self.message("Setting apt mirror to %s" % mirror)
1115         os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list'))
1116         f = open(conf, 'w')
1117         line = 'deb %s %s main\n' % (mirror, self.settings['distribution'])
1118         f.write(line)
1119         line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution'])
1120         f.write(line)
1121         f.close()
1122         # ensure the apt sources have valid lists
1123         self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update'])
1124
1125 if __name__ == '__main__':
1126     VmDebootstrap(version=__version__).run()