X-Git-Url: https://git.siccegge.de//index.cgi?p=forks%2Fvmdebootstrap.git;a=blobdiff_plain;f=vmdebootstrap;h=c9764ae67c44fffe66cdff7b42d23fe4acbca7a6;hp=dd729251d4aadc628ceb32539ee90cf8759bbbde;hb=HEAD;hpb=e3571547c2a9ce6b37739aae626c03de2aea96cb diff --git a/vmdebootstrap b/vmdebootstrap index dd72925..c9764ae 100755 --- a/vmdebootstrap +++ b/vmdebootstrap @@ -1,16 +1,18 @@ -#!/usr/bin/python -# Copyright 2011 Lars Wirzenius -# +#! /usr/bin/python +# Copyright 2011-2013 Lars Wirzenius +# Copyright 2012 Codethink Limited +# Copyright 2014-2015 Neil Williams +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program. If not, see . @@ -18,96 +20,352 @@ import cliapp import crypt import logging import os +import glob import re +import sys import shutil +import datetime import subprocess import tempfile +import time +from distro_info import DebianDistroInfo, UbuntuDistroInfo -class VmDebootstrap(cliapp.Application): +__version__ = '0.11' + +# pylint: disable=invalid-name,line-too-long,missing-docstring,too-many-branches - def add_settings(self): - default_arch = 'amd64' - - self.settings.boolean(['verbose'], 'report what is going on') - self.settings.string(['image'], 'put created disk image in FILE', - metavar='FILE') - self.settings.bytesize(['size'], - 'create a disk image of size SIZE (%default)', - metavar='SIZE', - default='1G') - self.settings.string(['mirror'], - 'use MIRROR as package source (%default)', - metavar='URL', - default='http://cdn.debian.net/debian/') - self.settings.string(['arch'], 'architecture to use (%default)', - metavar='ARCH', - default=default_arch) - self.settings.string(['distribution'], - 'release to use (%default)', - metavar='NAME', - default='stable') - self.settings.string_list(['package'], 'install PACKAGE onto system') - self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0') - self.settings.string(['root-password'], 'set root password', - metavar='PASSWORD') - self.settings.string(['customize'], - 'run SCRIPT after setting up system', - metavar='SCRIPT') - self.settings.string(['hostname'], - 'set name to HOSTNAME (%default)', - metavar='HOSTNAME', - default='debian') - self.settings.string_list(['user'], - 'create USER with PASSWORD', - metavar='USER/PASSWORD') - self.settings.boolean(['serial-console'], - 'configure image to use a serial console') - self.settings.boolean(['sudo'], - 'install sudo, and if user is created, add them ' - 'to sudo group') - - def process_args(self, args): - if not self.settings['image']: - raise cliapp.AppException('You must give image filename.') - if not self.settings['size']: - raise cliapp.AppException('You must give image size.') +class VmDebootstrap(cliapp.Application): # pylint: disable=too-many-public-methods + + def __init__(self, progname=None, version=__version__, description=None, epilog=None): + super(VmDebootstrap, self).__init__(progname, version, description, epilog) self.remove_dirs = [] self.mount_points = [] + self.debian_info = DebianDistroInfo() + self.ubuntu_info = UbuntuDistroInfo() + self.bootdir = None + self.efi_arch_table = { + 'amd64': { + 'removable': '/EFI/boot/bootx64.efi', # destination location + 'install': '/EFI/debian/grubx64.efi', # package location + 'package': 'grub-efi-amd64', # bootstrap package + 'bin_package': 'grub-efi-amd64-bin', # binary only + 'extra': 'i386', # architecture to add binary package + 'exclusive': False, # only EFI supported for this arch. + 'target': 'x86_64-efi', # grub target name + }, + 'i386': { + 'removable': '/EFI/boot/bootia32.efi', + 'install': '/EFI/debian/grubia32.efi', + 'package': 'grub-efi-ia32', + 'bin_package': 'grub-efi-ia32-bin', + 'extra': None, + 'exclusive': False, + 'target': 'i386-efi', + }, + 'arm64': { + 'removable': '/EFI/boot/bootaa64.efi', + 'install': '/EFI/debian/grubaa64.efi', + 'package': 'grub-efi-arm64', + 'bin_package': 'grub-efi-arm64-bin', + 'extra': None, + 'exclusive': True, + 'target': 'arm64-efi', + } + } + + def add_settings(self): + default_arch = subprocess.check_output( + ["dpkg", "--print-architecture"]).strip() + + self.settings.boolean( + ['verbose'], 'report what is going on') + self.settings.string( + ['image'], 'put created disk image in FILE', + metavar='FILE') + self.settings.bytesize( + ['size'], + 'create a disk image of size SIZE (%default)', + metavar='SIZE', + default='1G') + self.settings.bytesize( + ['bootsize'], + 'create boot partition of size SIZE (%default)', + metavar='BOOTSIZE', + default='0%') + self.settings.string( + ['boottype'], + 'specify file system type for /boot/', + default='ext2') + self.settings.bytesize( + ['bootoffset'], + 'Space to leave at start of the image for bootloader', + default='0') + self.settings.boolean( + ['use-uefi'], + 'Setup image for UEFI boot', + default=False) + self.settings.bytesize( + ['esp-size'], + 'Size of EFI System Partition - requires use-uefi', + default='5mib') + self.settings.string( + ['part-type'], + 'Partition type to use for this image', + default='msdos') + self.settings.string( + ['roottype'], + 'specify file system type for /', + default='ext4') + self.settings.bytesize( + ['swap'], + 'create swap space of size SIZE (min 256Mb)') + self.settings.string( + ['foreign'], + 'set up foreign debootstrap environment using provided program (ie binfmt handler)') + self.settings.string( + ['variant'], + 'select debootstrap variant if not using the default [deprecated]') + self.settings.string_list( + ['debootstrapopts'], + 'pass additional options to debootstrap'), + self.settings.boolean( + ['extlinux'], + 'install extlinux?', + default=True) + self.settings.string( + ['tarball'], + "tar up the disk's contents in FILE", + metavar='FILE') + self.settings.string( + ['apt-mirror'], + 'configure apt to use MIRROR', + metavar='URL') + self.settings.string( + ['mirror'], + 'use MIRROR as package source (%default)', + metavar='URL', + default='http://http.debian.net/debian/') + self.settings.string( + ['arch'], + 'architecture to use (%default)', + metavar='ARCH', + default=default_arch) + self.settings.string( + ['distribution'], + 'release to use (%default)', + metavar='NAME', + default='stable') + self.settings.string_list( + ['package'], + 'install PACKAGE onto system') + self.settings.string_list( + ['custom-package'], + 'install package in DEB file onto system (not from mirror)', + metavar='DEB') + self.settings.boolean( + ['no-kernel'], + 'do not install a linux package') + self.settings.string( + ['kernel-package'], + 'install PACKAGE instead of the default kernel package', + metavar='PACKAGE') + self.settings.boolean( + ['enable-dhcp'], + 'enable DHCP on eth0') + self.settings.string( + ['root-password'], + 'set root password', + metavar='PASSWORD') + self.settings.boolean( + ['lock-root-password'], + 'lock root account so they cannot login?') + self.settings.string( + ['customize'], + 'run SCRIPT after setting up system', + metavar='SCRIPT') + self.settings.string( + ['hostname'], + 'set name to HOSTNAME (%default)', + metavar='HOSTNAME', + default='debian') + self.settings.string_list( + ['user'], + 'create USER with PASSWORD', + metavar='USER/PASSWORD') + self.settings.boolean( + ['serial-console'], + 'configure image to use a serial console') + self.settings.string( + ['serial-console-command'], + 'command to manage the serial console, appended to /etc/inittab (%default)', + metavar='COMMAND', + default='/sbin/getty -L ttyS0 115200 vt100') + self.settings.boolean( + ['sudo'], + 'install sudo, and if user is created, add them to sudo group') + self.settings.string( + ['owner'], + 'the user who will own the image when the build is complete.') + self.settings.boolean( + ['squash'], + 'use squashfs on the final image.') + self.settings.boolean( + ['configure-apt'], + 'Create an apt source based on the distribution and mirror selected.') + self.settings.boolean( + ['mbr'], + 'Run install-mbr (default if extlinux used)') + self.settings.boolean( + ['grub'], + 'Install and configure grub2 - disables extlinux.') + self.settings.boolean( + ['sparse'], + 'Do not fill the image with zeros to keep a sparse disk image', + default=False) + self.settings.boolean( + ['pkglist'], + 'Create a list of package names included in the image.') + self.settings.boolean( + ['no-acpid'], + 'do not install the acpid package', + default=False) + + def process_args(self, args): # pylint: disable=too-many-branches,too-many-statements + if not self.settings['image'] and not self.settings['tarball']: + raise cliapp.AppException( + 'You must give disk image filename, or tarball filename') + if self.settings['image'] and not self.settings['size']: + raise cliapp.AppException( + 'If disk image is specified, you must give image size.') + if not self.debian_info.valid(self.settings['distribution']): + if not self.ubuntu_info.valid(self.settings['distribution']): + raise cliapp.AppException( + '%s is not a valid Debian or Ubuntu suite or codename.' + % self.settings['distribution']) + if not self.settings['use-uefi'] and self.settings['esp-size'] != 5242880: + raise cliapp.AppException( + 'You must specify use-uefi for esp-size to have effect') + if self.settings['arch'] in self.efi_arch_table and\ + self.efi_arch_table[self.settings['arch']]['exclusive'] and\ + not self.settings['use-uefi']: + raise cliapp.AppException( + 'Only UEFI is supported on %s' % self.settings['arch']) + elif self.settings['use-uefi'] and self.settings['arch'] not in self.efi_arch_table: + raise cliapp.AppException( + '%s is not a supported UEFI architecture' % self.settings['arch']) + if self.settings['use-uefi'] and ( + self.settings['bootsize'] or + self.settings['bootoffset']): + raise cliapp.AppException( + 'A separate boot partition is not supported with UEFI') + if self.settings['use-uefi'] and not self.settings['grub']: + raise cliapp.AppException( + 'UEFI without Grub is not supported.') + + # wheezy (which became oldstable on 04/25/2015) only had amd64 uefi + if self.was_oldstable(datetime.date(2015, 4, 26)): + if self.settings['use-uefi'] and self.settings['arch'] != 'amd64': + raise cliapp.AppException( + 'Only amd64 supports UEFI in Wheezy') + + if os.geteuid() != 0: + sys.exit("You need to have root privileges to run this script.") + rootdir = None try: - self.create_empty_image() - self.partition_image() - self.install_mbr() - rootdev = self.setup_kpartx() - self.mkfs(rootdev) - rootdir = self.mount(rootdev) + rootdev = None + roottype = self.settings['roottype'] + bootdev = None + boottype = None + if self.settings['image']: + self.create_empty_image() + self.partition_image() + if self.settings['mbr'] or self.settings['extlinux']: + self.install_mbr() + (rootdev, bootdev, swapdev) = self.setup_kpartx() + if self.settings['swap'] > 0: + self.message("Creating swap space") + self.runcmd(['mkswap', swapdev]) + self.mkfs(rootdev, fstype=roottype) + rootdir = self.mount(rootdev) + if self.settings['use-uefi']: + self.bootdir = '%s/%s/%s' % (rootdir, 'boot', 'efi') + logging.debug("bootdir:%s", self.bootdir) + self.mkfs(bootdev, fstype='vfat') + os.makedirs(self.bootdir) + self.mount(bootdev, self.bootdir) + elif bootdev: + if self.settings['boottype']: + boottype = self.settings['boottype'] + else: + boottype = 'ext2' + self.mkfs(bootdev, fstype=boottype) + self.bootdir = '%s/%s' % (rootdir, 'boot/') + os.mkdir(self.bootdir) + self.mount(bootdev, self.bootdir) + else: + rootdir = self.mkdtemp() self.debootstrap(rootdir) self.set_hostname(rootdir) - self.create_fstab(rootdir) + self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype) + self.install_debs(rootdir) self.set_root_password(rootdir) self.create_users(rootdir) self.remove_udev_persistent_rules(rootdir) self.setup_networking(rootdir) - self.install_extlinux(rootdev, rootdir) + if self.settings['configure-apt'] or self.settings['apt-mirror']: + self.configure_apt(rootdir) self.customize(rootdir) - except BaseException, e: + self.cleanup_apt_cache(rootdir) + self.update_initramfs(rootdir) + + if self.settings['image']: + if self.settings['use-uefi']: + self.install_grub_uefi(rootdir) + elif self.settings['grub']: + self.install_grub2(rootdev, rootdir) + elif self.settings['extlinux']: + self.install_extlinux(rootdev, rootdir) + self.append_serial_console(rootdir) + self.optimize_image(rootdir) + if self.settings['squash']: + self.squash() + if self.settings['pkglist']: + self.list_installed_pkgs(rootdir) + + if self.settings['foreign']: + os.unlink('%s/usr/bin/%s' % + (rootdir, os.path.basename(self.settings['foreign']))) + + if self.settings['tarball']: + self.create_tarball(rootdir) + + if self.settings['owner']: + self.chown() + except BaseException as e: self.message('EEEK! Something bad happened...') - self.cleanup() + if rootdir: + db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log') + if os.path.exists(db_log): + shutil.copy(db_log, os.getcwd()) + self.message(e) + self.cleanup_system() raise else: - self.cleanup() + self.cleanup_system() def message(self, msg): + logging.info(msg) if self.settings['verbose']: print msg - def runcmd(self, argv, stdin='', ignore_fail=False, **kwargs): - logging.debug('runcmd: %s %s' % (argv, kwargs)) - p = subprocess.Popen(argv, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs) + def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs): + logging.debug('runcmd: %s %s %s', argv, env, kwargs) + p = subprocess.Popen(argv, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env, **kwargs) out, err = p.communicate(stdin) if p.returncode != 0: msg = 'command failed: %s\n%s\n%s' % (argv, out, err) @@ -119,103 +377,403 @@ class VmDebootstrap(cliapp.Application): def mkdtemp(self): dirname = tempfile.mkdtemp() self.remove_dirs.append(dirname) - logging.debug('mkdir %s' % dirname) + logging.debug('mkdir %s', dirname) return dirname - - def mount(self, device): - self.message('Mounting %s' % device) - mount_point = self.mkdtemp() + + def mount(self, device, path=None): + if not path: + mount_point = self.mkdtemp() + else: + mount_point = path + self.message('Mounting %s on %s' % (device, mount_point)) self.runcmd(['mount', device, mount_point]) self.mount_points.append(mount_point) - logging.debug('mounted %s on %s' % (device, mount_point)) + logging.debug('mounted %s on %s', device, mount_point) return mount_point def create_empty_image(self): self.message('Creating disk image') - self.runcmd(['qemu-img', 'create', '-f', 'raw', + self.runcmd(['qemu-img', 'create', '-f', 'raw', self.settings['image'], str(self.settings['size'])]) def partition_image(self): + """ + Uses fat16 (msdos) partitioning by default, use part-type to change. + If bootoffset is specified, the first actual partition + starts at that offset to allow customisation scripts to + put bootloader images into the space, e.g. u-boot. + """ self.message('Creating partitions') self.runcmd(['parted', '-s', self.settings['image'], - 'mklabel', 'msdos']) - self.runcmd(['parted', '-s', self.settings['image'], - 'mkpart', 'primary', '0%', '100%']) + 'mklabel', self.settings['part-type']]) + partoffset = 0 + extent = '100%' + swap = 256 * 1024 * 1024 + if self.settings['swap'] > 0: + if self.settings['swap'] > swap: + swap = self.settings['swap'] + else: + # minimum 256Mb as default qemu ram is 128Mb + logging.debug("Setting minimum 256Mb swap space") + extent = "%s%%" % int(100 * (self.settings['size'] - swap) / self.settings['size']) + + if self.settings['use-uefi']: + espsize = self.settings['esp-size'] / (1024 * 1024) + self.message("Using ESP size: %smib %s bytes" % (espsize, self.settings['esp-size'])) + self.runcmd(['parted', '-s', self.settings['image'], + 'mkpart', 'primary', 'fat32', + '1', str(espsize)]) + self.runcmd(['parted', '-s', self.settings['image'], + 'set', '1', 'boot', 'on']) + self.runcmd(['parted', '-s', self.settings['image'], + 'set', '1', 'esp', 'on']) + + if self.settings['bootoffset'] and self.settings['bootoffset'] is not '0': + # turn v.small offsets into something at least possible to create. + if self.settings['bootoffset'] < 1048576: + partoffset = 1 + logging.info( + "Setting bootoffset %smib to allow for %s bytes", + partoffset, self.settings['bootoffset']) + else: + partoffset = self.settings['bootoffset'] / (1024 * 1024) + self.message("Using bootoffset: %smib %s bytes" % (partoffset, self.settings['bootoffset'])) + if self.settings['bootsize'] and self.settings['bootsize'] is not '0%': + if self.settings['grub'] and not partoffset: + partoffset = 1 + bootsize = self.settings['bootsize'] / (1024 * 1024) + bootsize += partoffset + self.message("Using bootsize %smib: %s bytes" % (bootsize, self.settings['bootsize'])) + logging.debug("Starting boot partition at %sMb", bootsize) + self.runcmd(['parted', '-s', self.settings['image'], + 'mkpart', 'primary', 'fat16', str(partoffset), str(bootsize)]) + logging.debug("Starting root partition at %sMb", partoffset) + self.runcmd(['parted', '-s', self.settings['image'], + 'mkpart', 'primary', str(bootsize), extent]) + elif self.settings['use-uefi']: + bootsize = self.settings['esp-size'] / (1024 * 1024) + 1 + self.runcmd(['parted', '-s', self.settings['image'], + 'mkpart', 'primary', str(bootsize), extent]) + else: + self.runcmd(['parted', '-s', self.settings['image'], + 'mkpart', 'primary', '0%', extent]) self.runcmd(['parted', '-s', self.settings['image'], 'set', '1', 'boot', 'on']) + if self.settings['swap'] > 0: + logging.debug("Creating swap partition") + self.runcmd(['parted', '-s', self.settings['image'], + 'mkpart', 'primary', 'linux-swap', extent, '100%']) + + def update_initramfs(self, rootdir): + cmd = os.path.join('usr', 'sbin', 'update-initramfs') + if os.path.exists(os.path.join(rootdir, cmd)): + self.message("Updating the initramfs") + self.runcmd(['chroot', rootdir, cmd, '-u']) def install_mbr(self): - self.message('Installing MBR') - self.runcmd(['install-mbr', self.settings['image']]) + if os.path.exists("/sbin/install-mbr"): + self.message('Installing MBR') + self.runcmd(['install-mbr', self.settings['image']]) + else: + msg = "mbr enabled but /sbin/install-mbr not found" \ + " - please install the mbr package." + raise cliapp.AppException(msg) def setup_kpartx(self): - out = self.runcmd(['kpartx', '-av', self.settings['image']]) - devices = [line.split()[2] - for line in out.splitlines() - if line.startswith('add map ')] - if len(devices) != 1: - raise cliapp.AppException('Surprising number of partitions') - return '/dev/mapper/%s' % devices[0] - - def mkfs(self, device): - self.message('Creating filesystem') - self.runcmd(['mkfs', '-t', 'ext2', device]) - - def debootstrap(self, rootdir): - self.message('Debootstrapping') - - if self.settings['arch'] == 'i386': - kernel_arch = '686' + bootindex = None + swapindex = None + if 'freebsd' in os.sys.platform: + out = self.runcmd(['mdconfig', '-a', '-t', 'vnode', '-f', + self.settings['image']]) + else: + out = self.runcmd(['kpartx', '-avs', self.settings['image']]) + if self.settings['bootsize'] and self.settings['swap'] > 0: + bootindex = 0 + rootindex = 1 + swapindex = 2 + parts = 3 + elif self.settings['use-uefi']: + bootindex = 0 + rootindex = 1 + parts = 2 + elif self.settings['use-uefi'] and self.settings['swap'] > 0: + bootindex = 0 + rootindex = 1 + swapindex = 2 + parts = 3 + elif self.settings['bootsize']: + bootindex = 0 + rootindex = 1 + parts = 2 + elif self.settings['swap'] > 0: + rootindex = 0 + swapindex = 1 + parts = 2 else: - kernel_arch = self.settings['arch'] - kernel_image = 'linux-image-2.6-%s' % kernel_arch + rootindex = 0 + parts = 1 + boot = None + swap = None + if 'freebsd' in os.sys.platform: + devices = glob.glob("/dev/%ss*" % out.strip()) + else: + devices = ['/dev/mapper/%s' % line.split()[2] + for line in out.splitlines() + if line.startswith('add map ')] + if len(devices) != parts: + msg = 'Surprising number of partitions - check output of losetup -a' + logging.debug("%s", self.runcmd(['losetup', '-a'])) + logging.debug("%s: devices=%s parts=%s", msg, devices, parts) + raise cliapp.AppException(msg) + root = devices[rootindex] + if self.settings['bootsize'] or self.settings['use-uefi']: + boot = devices[bootindex] + if self.settings['swap'] > 0: + swap = devices[swapindex] + return root, boot, swap + + def _efi_packages(self): + packages = [] + pkg = self.efi_arch_table[self.settings['arch']]['package'] + self.message("Adding %s" % pkg) + packages.append(pkg) + extra = self.efi_arch_table[self.settings['arch']]['extra'] + if extra and isinstance(extra, str): + bin_pkg = self.efi_arch_table[str(extra)]['bin_package'] + self.message("Adding support for %s using %s" % (extra, bin_pkg)) + packages.append(bin_pkg) + return packages + + def _copy_efi_binary(self, efi_removable, efi_install): + logging.debug("using bootdir=%s", self.bootdir) + logging.debug("moving %s to %s", efi_removable, efi_install) + if efi_removable.startswith('/'): + efi_removable = efi_removable[1:] + if efi_install.startswith('/'): + efi_install = efi_install[1:] + efi_output = os.path.join(self.bootdir, efi_removable) + efi_input = os.path.join(self.bootdir, efi_install) + if not os.path.exists(efi_input): + logging.warning("%s does not exist (%s)", efi_input, efi_install) + raise cliapp.AppException("Missing %s" % efi_install) + if not os.path.exists(os.path.dirname(efi_output)): + os.makedirs(os.path.dirname(efi_output)) + logging.debug( + 'Moving UEFI support: %s -> %s', efi_input, efi_output) + if os.path.exists(efi_output): + os.unlink(efi_output) + os.rename(efi_input, efi_output) + + def configure_efi(self): + """ + Copy the bootloader file from the package into the location + so needs to be after grub and kernel already installed. + """ + self.message('Configuring EFI') + efi_removable = str(self.efi_arch_table[self.settings['arch']]['removable']) + efi_install = str(self.efi_arch_table[self.settings['arch']]['install']) + self.message('Installing UEFI support binary') + self._copy_efi_binary(efi_removable, efi_install) + + def configure_extra_efi(self): + extra = str(self.efi_arch_table[self.settings['arch']]['extra']) + if extra: + efi_removable = str(self.efi_arch_table[extra]['removable']) + efi_install = str(self.efi_arch_table[extra]['install']) + self.message('Copying UEFI support binary for %s' % extra) + self._copy_efi_binary(efi_removable, efi_install) + + def mkfs(self, device, fstype): + self.message('Creating filesystem %s' % fstype) + self.runcmd(['mkfs', '-t', fstype, device]) + + def suite_to_codename(self, distro): + suite = self.debian_info.codename(distro, datetime.date.today()) + if not suite: + return distro + return suite + + def was_oldstable(self, limit): + suite = self.suite_to_codename(self.settings['distribution']) + # this check is only for debian + if not self.debian_info.valid(suite): + return False + return suite == self.debian_info.old(limit) + + def was_stable(self, limit): + suite = self.suite_to_codename(self.settings['distribution']) + # this check is only for debian + if not self.debian_info.valid(suite): + return False + return suite == self.debian_info.stable(limit) + + def debootstrap(self, rootdir): # pylint: disable=too-many-statements + msg = "(%s)" % self.settings['variant'] if self.settings['variant'] else '' + self.message( + 'Debootstrapping %s [%s] %s' % ( + self.settings['distribution'], self.settings['arch'], msg)) + + include = self.settings['package'] + + if not self.settings['foreign'] and not self.settings['no-acpid']: + include.append('acpid') + + if self.settings['grub']: + if self.settings['use-uefi']: + include.extend(self._efi_packages()) + else: + include.append('grub-pc') + + if not self.settings['no-kernel']: + if self.settings['kernel-package']: + kernel_image = self.settings['kernel-package'] + else: + if self.settings['arch'] == 'i386': + # wheezy (which became oldstable on 04/25/2015) used '486' + if self.was_oldstable(datetime.date(2015, 4, 26)): + kernel_arch = '486' + else: + kernel_arch = '586' + elif self.settings['arch'] == 'armhf': + kernel_arch = 'armmp' + else: + kernel_arch = self.settings['arch'] + kernel_image = 'linux-image-%s' % kernel_arch + include.append(kernel_image) - include = [kernel_image] + self.settings['package'] if self.settings['sudo'] and 'sudo' not in include: include.append('sudo') - self.runcmd(['debootstrap', - '--arch=%s' % self.settings['arch'], - '--include=%s' % ','.join(include), - self.settings['distribution'], - rootdir, - self.settings['mirror']]) + args = ['debootstrap', '--arch=%s' % self.settings['arch']] + + if self.settings['package']: + args.append( + '--include=%s' % ','.join(include)) + if self.settings['foreign']: + args.append('--foreign') + if self.settings['debootstrapopts']: + for opt in self.settings['debootstrapopts']: + for part in opt.split(' '): + args.append('--%s' % part) + elif self.settings['variant']: + args.append('--variant') + args.append(self.settings['variant']) + args += [self.settings['distribution'], + rootdir, self.settings['mirror']] + logging.debug(" ".join(args)) + self.runcmd(args) + if self.settings['foreign']: + # set a noninteractive debconf environment for secondstage + env = { + "DEBIAN_FRONTEND": "noninteractive", + "DEBCONF_NONINTERACTIVE_SEEN": "true", + "LC_ALL": "C" + } + # add the mapping to the complete environment. + env.update(os.environ) + # First copy the binfmt handler over + self.message('Setting up binfmt handler') + shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir) + # Next, run the package install scripts etc. + self.message('Running debootstrap second stage') + self.runcmd(['chroot', rootdir, + '/debootstrap/debootstrap', '--second-stage'], + env=env) def set_hostname(self, rootdir): hostname = self.settings['hostname'] with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f: f.write('%s\n' % hostname) - + etc_hosts = os.path.join(rootdir, 'etc', 'hosts') - with open(etc_hosts, 'r') as f: - data = f.read() - with open(etc_hosts, 'w') as f: - for line in data.splitlines(): - if line.startswith('127.0.0.1'): - line += ' %s' % hostname - f.write('%s\n' % line) - - def create_fstab(self, rootdir): + try: + with open(etc_hosts, 'r') as f: + data = f.read() + with open(etc_hosts, 'w') as f: + for line in data.splitlines(): + if line.startswith('127.0.0.1'): + line += ' %s' % hostname + f.write('%s\n' % line) + except IOError: + pass + + def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype): # pylint: disable=too-many-arguments + def fsuuid(device): + if 'freebsd' in os.sys.platform: + out = self.runcmd(['grub-probe', '-d', device, '-t', 'fs_uuid']) + return "/dev/ufsid/%s" % out.strip() + else: + out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value', + '-s', 'UUID', device]) + return "UUID=%s" % out.splitlines()[0].strip() + + if rootdev: + rootdevstr = fsuuid(rootdev) + else: + rootdevstr = '/dev/sda1' + + if bootdev and not self.settings['use-uefi']: + bootdevstr = fsuuid(bootdev) + else: + bootdevstr = None + fstab = os.path.join(rootdir, 'etc', 'fstab') with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('/dev/sda1 / ext4 errors=remount-ro 0 1\n') + if 'freebsd' in os.sys.platform: + f.write('proc /proc linprocfs rw 0 0\n') + f.write('sys /sys linsysfs rw 0 0\n') + f.write('fdesc /dev/fd fdescfs rw 0 0\n') + else: + f.write('proc /proc proc defaults 0 0\n') + f.write('%s / %s rw 0 1\n' % (rootdevstr, roottype)) + if bootdevstr: + f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype)) + if self.settings['swap'] > 0: + f.write("/dev/sda3 swap swap defaults 0 0\n") + elif self.settings['swap'] > 0: + f.write("/dev/sda2 swap swap defaults 0 0\n") + + def install_debs(self, rootdir): + if not self.settings['custom-package']: + return + self.message('Installing custom packages') + tmp = os.path.join(rootdir, 'tmp', 'install_debs') + os.mkdir(tmp) + for deb in self.settings['custom-package']: + shutil.copy(deb, tmp) + filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb)) + for deb in self.settings['custom-package']] + out, err, _ = \ + self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames) + logging.debug('stdout:\n%s', out) + logging.debug('stderr:\n%s', err) + out = self.runcmd(['chroot', rootdir, + 'apt-get', '-f', '--no-remove', 'install']) + logging.debug('stdout:\n%s', out) + shutil.rmtree(tmp) + + def cleanup_apt_cache(self, rootdir): + out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean']) + logging.debug('stdout:\n%s', out) def set_root_password(self, rootdir): if self.settings['root-password']: self.message('Setting root password') self.set_password(rootdir, 'root', self.settings['root-password']) - else: + elif self.settings['lock-root-password']: self.message('Locking root password') self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root']) + else: + self.message('Give root an empty password') + self.delete_password(rootdir, 'root') def create_users(self, rootdir): - def create_user(user): - self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user, - '--disabled-password', user]) + def create_user(vmuser): + self.runcmd(['chroot', rootdir, 'adduser', '--gecos', vmuser, + '--disabled-password', vmuser]) if self.settings['sudo']: - self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo']) + self.runcmd(['chroot', rootdir, 'adduser', vmuser, 'sudo']) for userpass in self.settings['user']: if '/' in userpass: @@ -224,105 +782,345 @@ class VmDebootstrap(cliapp.Application): self.set_password(rootdir, user, password) else: create_user(userpass) + self.delete_password(rootdir, userpass) def set_password(self, rootdir, user, password): encrypted = crypt.crypt(password, '..') self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user]) + def delete_password(self, rootdir, user): + self.runcmd(['chroot', rootdir, 'passwd', '-d', user]) + def remove_udev_persistent_rules(self, rootdir): self.message('Removing udev persistent cd and net rules') for x in ['70-persistent-cd.rules', '70-persistent-net.rules']: pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x) if os.path.exists(pathname): - logging.debug('rm %s' % pathname) + logging.debug('rm %s', pathname) os.remove(pathname) else: - logging.debug('not removing non-existent %s' % pathname) + logging.debug('not removing non-existent %s', pathname) + + def mask_udev_predictable_rules(self, rootdir): + """ + This can be reset later but to get networking working immediately + on boot, the interface we're going to use must be known without + reference to the eventual machine. + http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/ + """ + self.message('Disabling systemd predictable interface names') + udev_path = os.path.join( + 'etc', 'udev', 'rules.d', '80-net-setup-link.rules') + self.runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path]) def setup_networking(self, rootdir): self.message('Setting up networking') - - f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w') - f.write('auto lo\n') - f.write('iface lo inet loopback\n') - - if self.settings['enable-dhcp']: - f.write('\n') - f.write('allow-hotplug eth0\n') - f.write('iface eth0 inet dhcp\n') - - f.close() + ifc_file = os.path.join(rootdir, 'etc', 'network', 'interfaces') + ifc_d = os.path.join(rootdir, 'etc', 'network', 'interfaces.d') + + # unconditionally write for wheezy (which became oldstable 2015.04.25) + if self.was_oldstable(datetime.date(2015, 4, 26)): + with open(ifc_file, 'w') as netfile: + netfile.write('source /etc/network/interfaces.d/*\n') + elif not os.path.exists(ifc_file): + with open(ifc_file, 'a') as netfile: + netfile.write('source-directory /etc/network/interfaces.d\n') + + if not os.path.exists(ifc_d): + os.mkdir(ifc_d) + ethpath = os.path.join(ifc_d, 'setup') + with open(ethpath, 'w') as eth: + eth.write('auto lo\n') + eth.write('iface lo inet loopback\n') + + if self.settings['enable-dhcp']: + eth.write('\n') + eth.write('auto eth0\n') + eth.write('iface eth0 inet dhcp\n') + # force predictable interface names + self.mask_udev_predictable_rules(rootdir) + + def append_serial_console(self, rootdir): + if self.settings['serial-console']: + serial_command = self.settings['serial-console-command'] + logging.debug('adding getty to serial console') + inittab = os.path.join(rootdir, 'etc/inittab') + # to autologin, serial_command can contain '-a root' + with open(inittab, 'a') as f: + f.write('\nS0:23:respawn:%s\n' % serial_command) + + # pylint: disable=no-self-use + def _grub_serial_console(self, rootdir): + cmdline = 'GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=tty1 console=ttyS0,38400n8"' + terminal = 'GRUB_TERMINAL="serial gfxterm"' + command = 'GRUB_SERIAL_COMMAND="serial --speed=38400 --unit=0 --parity=no --stop=1"' + grub_cfg = os.path.join(rootdir, 'etc', 'default', 'grub') + logging.debug("Allowing serial output in grub config %s", grub_cfg) + with open(grub_cfg, 'a+') as cfg: + cfg.write("# %s serial support\n" % os.path.basename(__file__)) + cfg.write("%s\n" % cmdline) + cfg.write("%s\n" % terminal) + cfg.write("%s\n" % command) + + def _mount_wrapper(self, rootdir): + if 'freebsd' in os.sys.platform: + self.runcmd(['mount', 'dev', '-t', 'devfs', + '%s' % os.path.join(rootdir, 'dev')]) + self.runcmd(['mount', 'proc', '-t', 'linprocfs', + '%s' % os.path.join(rootdir, 'proc')]) + self.runcmd(['mount', 'sys', '-t', 'linsysfs', + '%s' % os.path.join(rootdir, 'sys')]) + else: + self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind', + '%s' % os.path.join(rootdir, 'dev')]) + self.runcmd(['mount', '/proc', '-t', 'proc', '-obind', + '%s' % os.path.join(rootdir, 'proc')]) + self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind', + '%s' % os.path.join(rootdir, 'sys')]) + + def _umount_wrapper(self, rootdir): + self.runcmd(['umount', os.path.join(rootdir, 'sys')]) + self.runcmd(['umount', os.path.join(rootdir, 'proc')]) + self.runcmd(['umount', os.path.join(rootdir, 'dev')]) + + def install_grub_uefi(self, rootdir): + self.message("Configuring grub-uefi") + target = self.efi_arch_table[self.settings['arch']]['target'] + grub_opts = "--target=%s" % target + logging.debug("Running grub-install with options: %s", grub_opts) + self._mount_wrapper(rootdir) + try: + self.runcmd(['chroot', rootdir, 'update-grub']) + self.runcmd(['chroot', rootdir, 'grub-install', grub_opts]) + except cliapp.AppException as exc: + logging.warning(exc) + self.message( + "Failed to configure grub-uefi for %s" % + self.settings['arch']) + self._umount_wrapper(rootdir) + self.configure_efi() + extra = str(self.efi_arch_table[self.settings['arch']]['extra']) + if extra: + target = self.efi_arch_table[extra]['target'] + grub_opts = "--target=%s" % target + try: + self.runcmd(['chroot', rootdir, 'update-grub']) + self.runcmd(['chroot', rootdir, 'grub-install', grub_opts]) + except cliapp.AppException as exc: + logging.warning(exc) + self.message( + "Failed to configure grub-uefi for %s" % extra) + self.configure_extra_efi() + self._umount_wrapper(rootdir) + + def install_grub2(self, rootdev, rootdir): + self.message("Configuring grub2") + # rely on kpartx using consistent naming to map loop0p1 to loop0 + grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2]) + if self.settings['serial-console']: + self._grub_serial_console(rootdir) + logging.debug("Running grub-install with options: %s", grub_opts) + self._mount_wrapper(rootdir) + try: + self.runcmd(['chroot', rootdir, 'update-grub']) + self.runcmd(['chroot', rootdir, 'grub-install', grub_opts]) + except cliapp.AppException as exc: + logging.warning(exc) + self.message("Failed. Is grub2-common installed? Using extlinux.") + self.install_extlinux(rootdev, rootdir) + self._umount_wrapper(rootdir) def install_extlinux(self, rootdev, rootdir): + if not os.path.exists("/usr/bin/extlinux"): + self.message("extlinux not installed, skipping.") + return self.message('Installing extlinux') def find(pattern): dirname = os.path.join(rootdir, 'boot') basenames = os.listdir(dirname) - logging.debug('find: %s' % basenames) + logging.debug('find: %s', basenames) for basename in basenames: if re.search(pattern, basename): return os.path.join('boot', basename) raise cliapp.AppException('Cannot find match: %s' % pattern) - kernel_image = find('vmlinuz-.*') - initrd_image = find('initrd.img-.*') + try: + kernel_image = find('vmlinuz-.*') + initrd_image = find('initrd.img-.*') + except cliapp.AppException as e: + self.message("Unable to find kernel. Not installing extlinux.") + logging.debug("No kernel found. %s. Skipping install of extlinux.", e) + return out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', rootdev]) uuid = out.splitlines()[0].strip() conf = os.path.join(rootdir, 'extlinux.conf') - logging.debug('configure extlinux %s' % conf) - f = open(conf, 'w') - f.write(''' + logging.debug('configure extlinux %s', conf) + kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else '' + extserial = 'serial 0 115200' if self.settings['serial-console'] else '' + msg = ''' default linux timeout 1 label linux kernel %(kernel)s -append initrd=%(initrd)s root=UUID=%(uuid)s ro quiet %(kserial)s +append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s %(extserial)s ''' % { - 'kernel': kernel_image, - 'initrd': initrd_image, - 'uuid': uuid, - 'kserial': - 'console=ttyS0,115200' if self.settings['serial-console'] else '', - 'extserial': 'serial 0 115200' if self.settings['serial-console'] else '', -}) - f.close() - - if self.settings['serial-console']: - logging.debug('adding getty to serial console') - inittab = os.path.join(rootdir, 'etc/inittab') - with open(inittab, 'a') as f: - f.write('\nS0:23:respawn:/sbin/getty -L ttyS0 115200 vt100\n') + 'kernel': kernel_image, # pylint: disable=bad-continuation + 'initrd': initrd_image, # pylint: disable=bad-continuation + 'uuid': uuid, # pylint: disable=bad-continuation + 'kserial': kserial, # pylint: disable=bad-continuation + 'extserial': extserial, # pylint: disable=bad-continuation + } # pylint: disable=bad-continuation + logging.debug("extlinux config:\n%s", msg) + + # python multiline string substitution is just ugly. + # use an external file or live with the mangling, no point in + # mangling the string to remove spaces just to keep it pretty in source. + f = open(conf, 'w') + f.write(msg) self.runcmd(['extlinux', '--install', rootdir]) self.runcmd(['sync']) - import time; time.sleep(2) - - def cleanup(self): + time.sleep(2) + + def optimize_image(self, rootdir): + """ + Filing up the image with zeros will increase its compression rate + """ + if not self.settings['sparse']: + zeros = os.path.join(rootdir, 'ZEROS') + self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M']) + self.runcmd(['rm', '-f', zeros]) + + def squash(self): + """ + Run squashfs on the image. + """ + if not os.path.exists('/usr/bin/mksquashfs'): + logging.warning("Squash selected but mksquashfs not found!") + return + logging.debug( + "%s usage: %s", self.settings['image'], + self.runcmd(['du', self.settings['image']])) + self.message("Running mksquashfs") + suffixed = "%s.squashfs" % self.settings['image'] + if os.path.exists(suffixed): + os.unlink(suffixed) + msg = self.runcmd( + ['mksquashfs', self.settings['image'], + suffixed, + '-no-progress', '-comp', 'xz'], ignore_fail=False) + logging.debug(msg) + check_size = os.path.getsize(suffixed) + if check_size < (1024 * 1024): + logging.warning( + "%s appears to be too small! %s bytes", + suffixed, check_size) + else: + logging.debug("squashed size: %s", check_size) + os.unlink(self.settings['image']) + self.settings['image'] = suffixed + logging.debug( + "%s usage: %s", self.settings['image'], + self.runcmd(['du', self.settings['image']])) + + def cleanup_system(self): # Clean up after any errors. self.message('Cleaning up') - for mount_point in self.mount_points: - self.runcmd(['umount', mount_point], ignore_fail=True) + # Umount in the reverse mount order + if self.settings['image']: + for i in range(len(self.mount_points) - 1, -1, -1): + mount_point = self.mount_points[i] + try: + self.runcmd(['umount', mount_point], ignore_fail=False) + except cliapp.AppException: + logging.debug("umount failed, sleeping and trying again") + time.sleep(5) + self.runcmd(['umount', mount_point], ignore_fail=False) + + if 'freebsd' in os.sys.platform: + out = self.runcmd(['mdconfig', '-l', '-f', self.settings['image']]) + for devid in out.split(): + self.runcmd(['mdconfig', '-d', '-u', devid], + ignore_fail=True) + else: + self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True) - self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True) - for dirname in self.remove_dirs: shutil.rmtree(dirname) def customize(self, rootdir): script = self.settings['customize'] - if script: - self.message('Running customize script %s' % script) - self.runcmd([script, rootdir]) + if not script: + return + if not os.path.exists(script): + example = os.path.join("/usr/share/vmdebootstrap/examples/", script) + if not os.path.exists(example): + self.message("Unable to find %s" % script) + return + script = example + self.message('Running customize script %s' % script) + logging.info("rootdir=%s image=%s", rootdir, self.settings['image']) + logging.debug( + "%s usage: %s", self.settings['image'], + self.runcmd(['du', self.settings['image']])) + try: + with open('/dev/tty', 'w') as tty: + cliapp.runcmd([script, rootdir, self.settings['image']], stdout=tty, stderr=tty) + except IOError: + logging.debug('tty unavailable, trying in headless mode.') + subprocess.call([script, rootdir, self.settings['image']]) + def create_tarball(self, rootdir): + # Create a tarball of the disk's contents + # shell out to runcmd since it more easily handles rootdir + self.message('Creating tarball of disk contents') + self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.']) -if __name__ == '__main__': - VmDebootstrap().run() + def chown(self): + # Change image owner after completed build + if self.settings['image']: + filename = self.settings['image'] + elif self.settings['tarball']: + filename = self.settings['tarball'] + else: + return + self.message("Changing owner to %s" % self.settings["owner"]) + subprocess.call(["chown", self.settings["owner"], filename]) + + def list_installed_pkgs(self, rootdir): + # output the list of installed packages for sources identification + self.message("Creating a list of installed binary package names") + out = self.runcmd(['chroot', rootdir, + 'dpkg-query', '-W', "-f='${Package}.deb\n'"]) + with open('dpkg.list', 'w') as dpkg: + dpkg.write(out) + + def configure_apt(self, rootdir): + # use the distribution and mirror to create an apt source + self.message("Configuring apt to use distribution and mirror") + conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list') + logging.debug('configure apt %s', conf) + mirror = self.settings['mirror'] + if self.settings['apt-mirror']: + mirror = self.settings['apt-mirror'] + self.message("Setting apt mirror to %s" % mirror) + os.unlink(os.path.join(rootdir, 'etc', 'apt', 'sources.list')) + f = open(conf, 'w') + line = 'deb %s %s main\n' % (mirror, self.settings['distribution']) + f.write(line) + line = '#deb-src %s %s main\n' % (mirror, self.settings['distribution']) + f.write(line) + f.close() + # ensure the apt sources have valid lists + self.runcmd(['chroot', rootdir, 'apt-get', '-qq', 'update']) +if __name__ == '__main__': + VmDebootstrap(version=__version__).run()