]> git.siccegge.de Git - forks/vmdebootstrap.git/blob - vmdebootstrap
Add sparse option to not fill image with zeros
[forks/vmdebootstrap.git] / vmdebootstrap
1 #!/usr/bin/python
2 # Copyright 2011-2013 Lars Wirzenius
3 # Copyright 2012 Codethink Limited
4 # Copyright 2014 Neil Williams <codehelp@debian.org>
5 #
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19 import cliapp
20 import crypt
21 import logging
22 import os
23 import re
24 import shutil
25 import subprocess
26 import tempfile
27 import time
28
29
30 __version__ = '0.4'
31
32
33 class VmDebootstrap(cliapp.Application):
34
35 def add_settings(self):
36 default_arch = subprocess.check_output(
37 ["dpkg", "--print-architecture"]).strip()
38
39 self.settings.boolean(['verbose'], 'report what is going on')
40 self.settings.string(['image'], 'put created disk image in FILE',
41 metavar='FILE')
42 self.settings.bytesize(['size'],
43 'create a disk image of size SIZE (%default)',
44 metavar='SIZE',
45 default='1G')
46 self.settings.bytesize(['bootsize'],
47 'create boot partition of size SIZE (%default)',
48 metavar='BOOTSIZE',
49 default='0%')
50 self.settings.string(['boottype'],
51 'specify file system type for /boot/',
52 default='ext2')
53 self.settings.string(['foreign'],
54 'set up foreign debootstrap environment using provided program (ie binfmt handler)')
55 self.settings.string(['variant'],
56 'select debootstrap variant it not using the default')
57 self.settings.boolean(['extlinux'], 'install extlinux?', default=True)
58 self.settings.string(['tarball'], "tar up the disk's contents in FILE",
59 metavar='FILE')
60 self.settings.string(['mirror'],
61 'use MIRROR as package source (%default)',
62 metavar='URL',
63 default='http://http.debian.net/debian/')
64 self.settings.string(['arch'], 'architecture to use (%default)',
65 metavar='ARCH',
66 default=default_arch)
67 self.settings.string(['distribution'],
68 'release to use (%default)',
69 metavar='NAME',
70 default='stable')
71 self.settings.string_list(['package'], 'install PACKAGE onto system')
72 self.settings.string_list(['custom-package'],
73 'install package in DEB file onto system '
74 '(not from mirror)',
75 metavar='DEB')
76 self.settings.boolean(['no-kernel'], 'do not install a linux package')
77 self.settings.boolean(['enable-dhcp'], 'enable DHCP on eth0')
78 self.settings.string(['root-password'], 'set root password',
79 metavar='PASSWORD')
80 self.settings.boolean(['lock-root-password'],
81 'lock root account so they cannot login?')
82 self.settings.string(['customize'],
83 'run SCRIPT after setting up system',
84 metavar='SCRIPT')
85 self.settings.string(['hostname'],
86 'set name to HOSTNAME (%default)',
87 metavar='HOSTNAME',
88 default='debian')
89 self.settings.string_list(['user'],
90 'create USER with PASSWORD',
91 metavar='USER/PASSWORD')
92 self.settings.boolean(['serial-console'],
93 'configure image to use a serial console')
94 self.settings.string(['serial-console-command'],
95 'command to manage the serial console, appended '
96 'to /etc/inittab (%default)',
97 metavar='COMMAND',
98 default='/sbin/getty -L ttyS0 115200 vt100')
99 self.settings.boolean(['sudo'],
100 'install sudo, and if user is created, add them '
101 'to sudo group')
102 self.settings.string(['owner'],
103 'the user who will own the image when the build '
104 'is complete.')
105 self.settings.boolean(['squash'],
106 'use squashfs on the final image.')
107 self.settings.boolean(['configure-apt'],
108 'Create an apt source based on the distribution '
109 'and mirror selected.')
110 self.settings.boolean(['mbr'],
111 'Run install-mbr (no longer done by default)')
112 self.settings.boolean(['grub'],
113 'Install and configure grub2 - disables '
114 'extlinux.')
115 self.settings.boolean(['sparse'],
116 'Dont fill the image with zeros to keep a sparse disk image',
117 default=False)
118
119 def process_args(self, args):
120 if not self.settings['image'] and not self.settings['tarball']:
121 raise cliapp.AppException('You must give disk image filename, '
122 'or tarball filename')
123 if self.settings['image'] and not self.settings['size']:
124 raise cliapp.AppException('If disk image is specified, '
125 'You must give image size.')
126
127 self.remove_dirs = []
128 self.mount_points = []
129
130 try:
131 rootdev = None
132 roottype = 'ext4'
133 bootdev = None
134 boottype = None
135 rootdir = None
136 if self.settings['image']:
137 self.create_empty_image()
138 self.partition_image()
139 if self.settings['mbr']:
140 self.install_mbr()
141 (rootdev, bootdev) = self.setup_kpartx()
142 self.mkfs(rootdev, type=roottype)
143 rootdir = self.mount(rootdev)
144 if bootdev:
145 if self.settings['boottype']:
146 boottype = self.settings['boottype']
147 else:
148 boottype = 'ext2'
149 self.mkfs(bootdev, type=boottype)
150 bootdir = '%s/%s' % (rootdir, 'boot/')
151 os.mkdir(bootdir)
152 bootdir = self.mount(bootdev, bootdir)
153 else:
154 rootdir = self.mkdtemp()
155 self.debootstrap(rootdir)
156 self.set_hostname(rootdir)
157 self.create_fstab(rootdir, rootdev, roottype, bootdev, boottype)
158 self.install_debs(rootdir)
159 self.cleanup_apt_cache(rootdir)
160 self.set_root_password(rootdir)
161 self.create_users(rootdir)
162 self.remove_udev_persistent_rules(rootdir)
163 self.setup_networking(rootdir)
164 if self.settings['configure-apt']:
165 self.configure_apt(rootdir)
166 self.customize(rootdir)
167 self.update_initramfs(rootdir)
168
169 if self.settings['image']:
170 if self.settings['grub']:
171 self.install_grub2(rootdev, rootdir)
172 elif self.settings['extlinux']:
173 self.install_extlinux(rootdev, rootdir)
174 self.append_serial_console(rootdir)
175 self.optimize_image(rootdir)
176 if self.settings['squash']:
177 self.squash()
178
179 if self.settings['foreign']:
180 os.unlink('%s/usr/bin/%s' %
181 (rootdir, os.path.basename(self.settings['foreign'])))
182
183 if self.settings['tarball']:
184 self.create_tarball(rootdir)
185
186 if self.settings['owner']:
187 self.chown(rootdir)
188 except BaseException, e:
189 self.message('EEEK! Something bad happened...')
190 if rootdir:
191 db_log = os.path.join(rootdir, 'debootstrap', 'debootstrap.log')
192 if os.path.exists(db_log):
193 shutil.copy(db_log, os.getcwd())
194 self.message(e)
195 self.cleanup_system()
196 raise
197 else:
198 self.cleanup_system()
199
200 def message(self, msg):
201 logging.info(msg)
202 if self.settings['verbose']:
203 print msg
204
205 def runcmd(self, argv, stdin='', ignore_fail=False, env=None, **kwargs):
206 logging.debug('runcmd: %s %s %s' % (argv, env, kwargs))
207 p = subprocess.Popen(argv, stdin=subprocess.PIPE,
208 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
209 env=env, **kwargs)
210 out, err = p.communicate(stdin)
211 if p.returncode != 0:
212 msg = 'command failed: %s\n%s\n%s' % (argv, out, err)
213 logging.error(msg)
214 if not ignore_fail:
215 raise cliapp.AppException(msg)
216 return out
217
218 def mkdtemp(self):
219 dirname = tempfile.mkdtemp()
220 self.remove_dirs.append(dirname)
221 logging.debug('mkdir %s' % dirname)
222 return dirname
223
224 def mount(self, device, path=None):
225 if not path:
226 mount_point = self.mkdtemp()
227 else:
228 mount_point = path
229 self.message('Mounting %s on %s' % (device, mount_point))
230 self.runcmd(['mount', device, mount_point])
231 self.mount_points.append(mount_point)
232 logging.debug('mounted %s on %s' % (device, mount_point))
233 return mount_point
234
235 def create_empty_image(self):
236 self.message('Creating disk image')
237 self.runcmd(['qemu-img', 'create', '-f', 'raw',
238 self.settings['image'],
239 str(self.settings['size'])])
240
241 def partition_image(self):
242 self.message('Creating partitions')
243 self.runcmd(['parted', '-s', self.settings['image'],
244 'mklabel', 'msdos'])
245 if self.settings['bootsize'] and self.settings['bootsize'] is not '0%':
246 bootsize = str(self.settings['bootsize'] / (1024 * 1024))
247 self.runcmd(['parted', '-s', self.settings['image'],
248 'mkpart', 'primary', 'fat16', '0', bootsize])
249 else:
250 bootsize = '0%'
251 self.runcmd(['parted', '-s', self.settings['image'],
252 'mkpart', 'primary', bootsize, '100%'])
253 self.runcmd(['parted', '-s', self.settings['image'],
254 'set', '1', 'boot', 'on'])
255
256 def update_initramfs(self, rootdir):
257 cmd = os.path.join('usr', 'sbin', 'update-initramfs')
258 if os.path.exists(os.path.join(rootdir, cmd)):
259 self.message("Updating the initramfs")
260 self.runcmd(['chroot', rootdir, cmd, '-u'])
261
262 def install_mbr(self):
263 if os.path.exists("/sbin/install-mbr"):
264 self.message('Installing MBR')
265 self.runcmd(['install-mbr', self.settings['image']])
266
267 def setup_kpartx(self):
268 out = self.runcmd(['kpartx', '-avs', self.settings['image']])
269 if self.settings['bootsize']:
270 bootindex = 0
271 rootindex = 1
272 parts = 2
273 else:
274 rootindex = 0
275 parts = 1
276 boot = None
277 devices = [line.split()[2]
278 for line in out.splitlines()
279 if line.startswith('add map ')]
280 if len(devices) != parts:
281 raise cliapp.AppException('Surprising number of partitions')
282 root = '/dev/mapper/%s' % devices[rootindex]
283 if self.settings['bootsize']:
284 boot = '/dev/mapper/%s' % devices[bootindex]
285 return (root, boot)
286
287 def mkfs(self, device, type):
288 self.message('Creating filesystem %s' % type)
289 self.runcmd(['mkfs', '-t', type, device])
290
291 def debootstrap(self, rootdir):
292 self.message('Debootstrapping')
293
294 if self.settings['foreign']:
295 necessary_packages = []
296 else:
297 necessary_packages = ['acpid']
298
299 if self.settings['grub']:
300 necessary_packages.append('grub2')
301
302 include = self.settings['package']
303
304 if not self.settings['no-kernel']:
305 if self.settings['arch'] == 'i386':
306 kernel_arch = '486'
307 else:
308 kernel_arch = self.settings['arch']
309 kernel_image = 'linux-image-%s' % kernel_arch
310 include.append(kernel_image)
311
312 if self.settings['sudo'] and 'sudo' not in include:
313 include.append('sudo')
314
315 args = ['debootstrap', '--arch=%s' % self.settings['arch']]
316 if self.settings['package'] and len(necessary_packages) > 0:
317 args.append(
318 '--include=%s' % ','.join(necessary_packages + include))
319 if self.settings['foreign']:
320 args.append('--foreign')
321 if self.settings['variant']:
322 args.append('--variant')
323 args.append(self.settings['variant'])
324 args += [self.settings['distribution'],
325 rootdir, self.settings['mirror']]
326 logging.debug(" ".join(args))
327 self.runcmd(args)
328 if self.settings['foreign']:
329 # set a noninteractive debconf environment for secondstage
330 env = {
331 "DEBIAN_FRONTEND": "noninteractive",
332 "DEBCONF_NONINTERACTIVE_SEEN": "true",
333 "LC_ALL": "C"
334 }
335 # add the mapping to the complete environment.
336 env.update(os.environ)
337 # First copy the binfmt handler over
338 self.message('Setting up binfmt handler')
339 shutil.copy(self.settings['foreign'], '%s/usr/bin/' % rootdir)
340 # Next, run the package install scripts etc.
341 self.message('Running debootstrap second stage')
342 self.runcmd(['chroot', rootdir,
343 '/debootstrap/debootstrap', '--second-stage'],
344 env=env)
345
346 def set_hostname(self, rootdir):
347 hostname = self.settings['hostname']
348 with open(os.path.join(rootdir, 'etc', 'hostname'), 'w') as f:
349 f.write('%s\n' % hostname)
350
351 etc_hosts = os.path.join(rootdir, 'etc', 'hosts')
352 try:
353 with open(etc_hosts, 'r') as f:
354 data = f.read()
355 with open(etc_hosts, 'w') as f:
356 for line in data.splitlines():
357 if line.startswith('127.0.0.1'):
358 line += ' %s' % hostname
359 f.write('%s\n' % line)
360 except IOError, e:
361 pass
362
363 def create_fstab(self, rootdir, rootdev, roottype, bootdev, boottype):
364 def fsuuid(device):
365 out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
366 '-s', 'UUID', device])
367 return out.splitlines()[0].strip()
368
369 if rootdev:
370 rootdevstr = 'UUID=%s' % fsuuid(rootdev)
371 else:
372 rootdevstr = '/dev/sda1'
373
374 if bootdev:
375 bootdevstr = 'UUID=%s' % fsuuid(bootdev)
376 else:
377 bootdevstr = None
378
379 fstab = os.path.join(rootdir, 'etc', 'fstab')
380 with open(fstab, 'w') as f:
381 f.write('proc /proc proc defaults 0 0\n')
382 f.write('%s / %s errors=remount-ro 0 1\n' % (rootdevstr, roottype))
383 if bootdevstr:
384 f.write('%s /boot %s errors=remount-ro 0 2\n' % (bootdevstr, boottype))
385
386 def install_debs(self, rootdir):
387 if not self.settings['custom-package']:
388 return
389 self.message('Installing custom packages')
390 tmp = os.path.join(rootdir, 'tmp', 'install_debs')
391 os.mkdir(tmp)
392 for deb in self.settings['custom-package']:
393 shutil.copy(deb, tmp)
394 filenames = [os.path.join('/tmp/install_debs', os.path.basename(deb))
395 for deb in self.settings['custom-package']]
396 out, err, exit = \
397 self.runcmd_unchecked(['chroot', rootdir, 'dpkg', '-i'] + filenames)
398 logging.debug('stdout:\n%s' % out)
399 logging.debug('stderr:\n%s' % err)
400 out = self.runcmd(['chroot', rootdir,
401 'apt-get', '-f', '--no-remove', 'install'])
402 logging.debug('stdout:\n%s' % out)
403 shutil.rmtree(tmp)
404
405 def cleanup_apt_cache(self, rootdir):
406 out = self.runcmd(['chroot', rootdir, 'apt-get', 'clean'])
407 logging.debug('stdout:\n%s' % out)
408
409 def set_root_password(self, rootdir):
410 if self.settings['root-password']:
411 self.message('Setting root password')
412 self.set_password(rootdir, 'root', self.settings['root-password'])
413 elif self.settings['lock-root-password']:
414 self.message('Locking root password')
415 self.runcmd(['chroot', rootdir, 'passwd', '-l', 'root'])
416 else:
417 self.message('Give root an empty password')
418 self.delete_password(rootdir, 'root')
419
420 def create_users(self, rootdir):
421 def create_user(user):
422 self.runcmd(['chroot', rootdir, 'adduser', '--gecos', user,
423 '--disabled-password', user])
424 if self.settings['sudo']:
425 self.runcmd(['chroot', rootdir, 'adduser', user, 'sudo'])
426
427 for userpass in self.settings['user']:
428 if '/' in userpass:
429 user, password = userpass.split('/', 1)
430 create_user(user)
431 self.set_password(rootdir, user, password)
432 else:
433 create_user(userpass)
434 self.delete_password(rootdir, userpass)
435
436 def set_password(self, rootdir, user, password):
437 encrypted = crypt.crypt(password, '..')
438 self.runcmd(['chroot', rootdir, 'usermod', '-p', encrypted, user])
439
440 def delete_password(self, rootdir, user):
441 self.runcmd(['chroot', rootdir, 'passwd', '-d', user])
442
443 def remove_udev_persistent_rules(self, rootdir):
444 self.message('Removing udev persistent cd and net rules')
445 for x in ['70-persistent-cd.rules', '70-persistent-net.rules']:
446 pathname = os.path.join(rootdir, 'etc', 'udev', 'rules.d', x)
447 if os.path.exists(pathname):
448 logging.debug('rm %s' % pathname)
449 os.remove(pathname)
450 else:
451 logging.debug('not removing non-existent %s' % pathname)
452
453 def setup_networking(self, rootdir):
454 self.message('Setting up networking')
455
456 f = open(os.path.join(rootdir, 'etc', 'network', 'interfaces'), 'w')
457 f.write('auto lo\n')
458 f.write('iface lo inet loopback\n')
459
460 if self.settings['enable-dhcp']:
461 f.write('\n')
462 f.write('auto eth0\n')
463 f.write('iface eth0 inet dhcp\n')
464
465 f.close()
466
467 def append_serial_console(self, rootdir):
468 if self.settings['serial-console']:
469 serial_command = self.settings['serial-console-command']
470 logging.debug('adding getty to serial console')
471 inittab = os.path.join(rootdir, 'etc/inittab')
472 with open(inittab, 'a') as f:
473 f.write('\nS0:23:respawn:%s\n' % serial_command)
474
475 def install_grub2(self, rootdev, rootdir):
476 self.message("Configuring grub2")
477 # rely on kpartx using consistent naming to map loop0p1 to loop0
478 install_dev = os.path.join('/dev', os.path.basename(rootdev)[:-2])
479 self.runcmd(['mount', '/dev', '-t', 'devfs', '-obind',
480 '%s' % os.path.join(rootdir, 'dev')])
481 self.runcmd(['mount', '/proc', '-t', 'proc', '-obind',
482 '%s' % os.path.join(rootdir, 'proc')])
483 self.runcmd(['mount', '/sys', '-t', 'sysfs', '-obind',
484 '%s' % os.path.join(rootdir, 'sys')])
485 try:
486 self.runcmd(['chroot', rootdir, 'update-grub'])
487 self.runcmd(['chroot', rootdir, 'grub-install', install_dev])
488 except cliapp.AppException as e:
489 self.message("Failed. Is grub2-common installed? Using extlinux.")
490 self.runcmd(['umount', os.path.join(rootdir, 'sys')])
491 self.runcmd(['umount', os.path.join(rootdir, 'proc')])
492 self.runcmd(['umount', os.path.join(rootdir, 'dev')])
493 self.install_extlinux(rootdev, rootdir)
494
495 def install_extlinux(self, rootdev, rootdir):
496 if not os.path.exists("/usr/bin/extlinux"):
497 self.message("extlinux not installed, skipping.")
498 return
499 self.message('Installing extlinux')
500
501 def find(pattern):
502 dirname = os.path.join(rootdir, 'boot')
503 basenames = os.listdir(dirname)
504 logging.debug('find: %s' % basenames)
505 for basename in basenames:
506 if re.search(pattern, basename):
507 return os.path.join('boot', basename)
508 raise cliapp.AppException('Cannot find match: %s' % pattern)
509
510 try:
511 kernel_image = find('vmlinuz-.*')
512 initrd_image = find('initrd.img-.*')
513 except cliapp.AppException as e:
514 self.message("Unable to find kernel. Not installing extlinux.")
515 logging.debug("No kernel found. %s. Skipping install of extlinux." % e)
516 return
517
518 out = self.runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
519 '-s', 'UUID', rootdev])
520 uuid = out.splitlines()[0].strip()
521
522 conf = os.path.join(rootdir, 'extlinux.conf')
523 logging.debug('configure extlinux %s' % conf)
524 f = open(conf, 'w')
525 f.write('''
526 default linux
527 timeout 1
528
529 label linux
530 kernel %(kernel)s
531 append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
532 %(extserial)s
533 ''' % {
534 'kernel': kernel_image,
535 'initrd': initrd_image,
536 'uuid': uuid,
537 'kserial':
538 'console=ttyS0,115200' if self.settings['serial-console'] else '',
539 'extserial': 'serial 0 115200' if self.settings['serial-console'] else '',
540 })
541 f.close()
542
543 self.runcmd(['extlinux', '--install', rootdir])
544 self.runcmd(['sync'])
545 time.sleep(2)
546
547 def optimize_image(self, rootdir):
548 """
549 Filing up the image with zeros will increase its compression rate
550 """
551 if not self.settings['sparse']:
552 zeros = os.path.join(rootdir, 'ZEROS')
553 self.runcmd_unchecked(['dd', 'if=/dev/zero', 'of=' + zeros, 'bs=1M'])
554 self.runcmd(['rm', '-f', zeros])
555
556 def squash(self):
557 """
558 Run squashfs on the image.
559 """
560 if not os.path.exists('/usr/bin/mksquashfs'):
561 logging.warning("Squash selected but mksquashfs not found!")
562 return
563 self.message("Running mksquashfs")
564 suffixed = "%s.squashfs" % self.settings['image']
565 self.runcmd(['mksquashfs', self.settings['image'],
566 suffixed,
567 '-no-progress', '-comp', 'xz'], ignore_fail=False)
568 os.unlink(self.settings['image'])
569 self.settings['image'] = suffixed
570
571 def cleanup_system(self):
572 # Clean up after any errors.
573
574 self.message('Cleaning up')
575
576 # Umount in the reverse mount order
577 if self.settings['image']:
578 for i in xrange(len(self.mount_points) - 1, -1, -1):
579 mount_point = self.mount_points[i]
580 try:
581 self.runcmd(['umount', mount_point], ignore_fail=False)
582 except cliapp.AppException:
583 logging.debug("umount failed, sleeping and trying again")
584 time.sleep(5)
585 self.runcmd(['umount', mount_point], ignore_fail=False)
586
587 self.runcmd(['kpartx', '-d', self.settings['image']], ignore_fail=True)
588
589 for dirname in self.remove_dirs:
590 shutil.rmtree(dirname)
591
592 def customize(self, rootdir):
593 script = self.settings['customize']
594 if not script:
595 return
596 if not os.path.exists(script):
597 example = os.path.join("/usr/share/vmdebootstrap/examples/", script)
598 if not os.path.exists(example):
599 self.message("Unable to find %s" % script)
600 return
601 script = example
602 self.message('Running customize script %s' % script)
603 with open('/dev/tty', 'w') as tty:
604 cliapp.runcmd([script, rootdir], stdout=tty, stderr=tty)
605
606 def create_tarball(self, rootdir):
607 # Create a tarball of the disk's contents
608 # shell out to runcmd since it more easily handles rootdir
609 self.message('Creating tarball of disk contents')
610 self.runcmd(['tar', '-cf', self.settings['tarball'], '-C', rootdir, '.'])
611
612 def chown(self, rootdir):
613 # Change image owner after completed build
614 self.message("Changing owner to %s" % self.settings["owner"])
615 subprocess.call(["chown",
616 self.settings["owner"],
617 self.settings["image"]])
618
619 def configure_apt(self, rootdir):
620 # use the distribution and mirror to create an apt source
621 self.message("Configuring apt to use distribution and mirror")
622 conf = os.path.join(rootdir, 'etc', 'apt', 'sources.list.d', 'base.list')
623 logging.debug('configure apt %s' % conf)
624 f = open(conf, 'w')
625 f.write('''
626 deb %(mirror)s %(distribution)s main
627 #deb-src %(mirror)s %(distribution)s main
628 ''' % {
629 'mirror': self.settings['mirror'],
630 'distribution': self.settings['distribution']
631 })
632 f.close()
633
634 if __name__ == '__main__':
635 VmDebootstrap(version=__version__).run()