diff options
author | intrigeri <intrigeri@boum.org> | 2018-11-27 16:30:08 +0000 |
---|---|---|
committer | intrigeri <intrigeri@boum.org> | 2018-11-27 16:30:08 +0000 |
commit | f504fe0f3cc7d4f90cde8a3d2cf62798d86000f6 (patch) | |
tree | 5f8063a133594c3e7b491e84d341642ff066623b | |
parent | f4b602f6442abfecdd286c1f036af018d913652d (diff) | |
parent | 7f567ade99ed5e15cec3e9db5a488014de4116b1 (diff) |
Merge branch 'stable' into feature/14596-automated-tests-for-ASP-gui-on-stable
-rw-r--r-- | Rakefile | 2 | ||||
-rwxr-xr-x | auto/build | 69 | ||||
-rwxr-xr-x | auto/scripts/create-usb-image-from-iso | 388 | ||||
-rwxr-xr-x | vagrant/definitions/tails-builder/generate-tails-builder-box.sh | 1 | ||||
-rwxr-xr-x | vagrant/definitions/tails-builder/postinstall.sh | 9 | ||||
-rwxr-xr-x | vagrant/provision/assets/build-tails | 2 | ||||
-rw-r--r-- | wiki/src/contribute/release_process/test/setup.mdwn | 13 | ||||
-rw-r--r-- | wiki/src/contribute/working_together/roles/debian_maintainer.mdwn | 5 |
8 files changed, 437 insertions, 52 deletions
@@ -314,7 +314,7 @@ end def list_artifacts user = vagrant_ssh_config('User') stdout = capture_vagrant_ssh("find '/home/#{user}/amnesia/' -maxdepth 1 " + - "-name 'tails-*.iso*'").first + "-name 'tails-amd64-*'").first stdout.split("\n") rescue VagrantCommandError return Array.new @@ -122,8 +122,6 @@ export MKSQUASHFS_OPTIONS case "$LB_BINARY_IMAGES" in iso) - BUILD_FILENAME_EXT=iso - BUILD_FILENAME=binary which isohybrid >/dev/null || fatal 'Cannot find isohybrid in $PATH' installed_syslinux_utils_upstream_version="$(syslinux_utils_upstream_version)" if dpkg --compare-versions \ @@ -135,27 +133,16 @@ case "$LB_BINARY_IMAGES" in "while we need at least '${REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION}'." fi ;; - iso-hybrid) - BUILD_FILENAME_EXT=iso - BUILD_FILENAME=binary-hybrid - ;; - tar) - BUILD_FILENAME_EXT=tar.gz - BUILD_FILENAME=binary-tar - ;; - usb-hdd) - BUILD_FILENAME_EXT=img - BUILD_FILENAME=binary - ;; *) fatal "Image type ${LB_BINARY_IMAGES} is not supported." ;; esac -BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}" -BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest" -BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources" -BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages" -BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog" +BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso" +BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest" +BUILD_APT_SOURCES="${BUILD_BASENAME}.apt-sources" +BUILD_PACKAGES="${BUILD_BASENAME}.packages" +BUILD_LOG="${BUILD_BASENAME}.buildlog" +BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img" # Clone all output, from this point on, to the log file exec > >(tee -a "$BUILD_LOG") @@ -172,27 +159,25 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM cat config/chroot_sources/*.chroot ) > "$BUILD_APT_SOURCES" -echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..." -set -o pipefail +echo "Building ISO image ${BUILD_ISO_FILENAME}..." time lb build noauto ${@} -RET=$? -if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then - echo "Image was successfully created" - [ "$RET" -eq 0 ] || \ - echo "Warning: lb build exited with code $RET" - if [ "$LB_BINARY_IMAGES" = iso ]; then - ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" - print_iso_size "$ISO_FILE" - echo "Hybriding it..." - isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE" || fatal "isohybrid failed" - print_iso_size "$ISO_FILE" - truncate -s %2048 "$ISO_FILE" - print_iso_size "$ISO_FILE" - fi - echo "Renaming generated files..." - mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}" - mv -i binary.packages "${BUILD_PACKAGES}" - generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}" -else - fatal "lb build failed ($?)." -fi +[ -e binary.iso ] || fatal "lb build failed ($?)." + +echo "ISO image was successfully created" +print_iso_size binary.iso + +echo "Hybriding it..." +isohybrid $AMNESIA_ISOHYBRID_OPTS binary.iso || fatal "isohybrid failed" +print_iso_size binary.iso +truncate -s %2048 binary.iso +print_iso_size binary.iso + +echo "Renaming generated files..." +mv -i binary.iso "${BUILD_ISO_FILENAME}" +mv -i binary.packages "${BUILD_PACKAGES}" + +echo "Generating build manifest..." +generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}" + +echo "Creating USB image ${BUILD_USB_IMAGE_FILENAME}..." +create-usb-image-from-iso "${BUILD_ISO_FILENAME}" diff --git a/auto/scripts/create-usb-image-from-iso b/auto/scripts/create-usb-image-from-iso new file mode 100755 index 0000000..8600ef1 --- /dev/null +++ b/auto/scripts/create-usb-image-from-iso @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 + +import argparse +import os +import logging +from contextlib import contextmanager +import re +import time +import subprocess + +import gi +gi.require_version('UDisks', '2.0') +from gi.repository import UDisks, GLib, Gio + + +logger = logging.getLogger(__name__) + +SYSTEM_PARTITION_FLAGS = ( + 1 << 0 | # system partition + 1 << 2 | # legacy BIOS bootable + 1 << 60 | # read-only + 1 << 62 | # hidden + 1 << 63 # do not automount +) + +# EFI System Partition +ESP_GUID = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B' + +PARTITION_LABEL = 'Tails' +FILESYSTEM_LABEL = 'Tails' + +GET_UDISKS_OBJECT_TIMEOUT = 2 +WAIT_FOR_PARTITION_TIMEOUT = 2 + +# The size of the system partition (in MiB) will be: +# +# SYSTEM_PARTITION_ADDITIONAL_SIZE + size of the ISO +# +# SYSTEM_PARTITION_ADDITIONAL_SIZE must be large enough to fit +# the partition table, reserved sectors, and filesystem metadata. +SYSTEM_PARTITION_ADDITIONAL_SIZE = 10 + +SYSLINUX_COM32MODULES_DIR = '/usr/lib/syslinux/modules/bios' + + +class ImageCreationError(Exception): + pass + + +class ImageCreator(object): + + def __init__(self, iso: str, image: str, free_space: int): + self.iso = iso + self.image = image + self.free_space = free_space + self._loop_device = None # type: str + self._partition = None # type: str + self._system_partition_size = None # type: int + self.mountpoint = None # type: str + + @property + def loop_device(self) -> UDisks.ObjectProxy: + if not self._loop_device: + raise ImageCreationError("Loop device not set up") + return self.try_getting_udisks_object(self._loop_device) + + @property + def partition(self) -> UDisks.ObjectProxy: + if not self._partition: + raise ImageCreationError("Partition not created") + + return self.try_getting_udisks_object(self._partition) + + @property + def system_partition_size(self) -> int: + if self._system_partition_size is None: + self._system_partition_size = get_file_size(self.iso) + SYSTEM_PARTITION_ADDITIONAL_SIZE + + return self._system_partition_size + + def try_getting_udisks_object(self, object_path: str) -> UDisks.Object: + start_time = time.perf_counter() + while time.perf_counter() - start_time < GET_UDISKS_OBJECT_TIMEOUT: + with self.get_udisks_client() as udisks_client: + udisks_object = udisks_client.get_object(object_path) + if udisks_object: + return udisks_object + time.sleep(0.1) + raise ImageCreationError("Couldn't get UDisksObject for path '%s' (timeout: %s)" % + (object_path, GET_UDISKS_OBJECT_TIMEOUT)) + + @contextmanager + def get_udisks_client(self): + client = UDisks.Client().new_sync() + yield client + client.settle() + + def create_image(self): + self.create_empty_image() + + with self.setup_loop_device(): + self.create_gpt() + self.create_partition() + self.set_partition_flags() + # XXX: Rescan? + self.format_partition() + with self.mount_partition(): + self.extract_iso() + self.set_permissions() + self.update_configs() + self.install_mbr() + self.copy_syslinux_modules() + + # We have to install syslinux after the partition was unmounted. + # This sleep is a workaround for a race condition which causes the + # syslinux installation to return without errors, even though the + # bootloader isn't actually installed + # XXX: Investigate and report this race condition + # Might it be https://bugs.chromium.org/p/chromium/issues/detail?id=508713 ? + time.sleep(1) + self.install_syslinux() + self.set_guids() + self.set_fsuuid() + + with self.mount_partition(): + self.reset_timestamps() + + def extract_iso(self): + logger.info("Extracting ISO contents to the partition") + execute(['7z', 'x', self.iso, '-x![BOOT]', '-y', '-o%s' % self.mountpoint]) + + def create_empty_image(self): + logger.info("Creating empty image %r", self.image) + image_size = self.system_partition_size + self.free_space + execute(["dd", "if=/dev/zero", "of=%s" % self.image, "bs=1M", "count=%s" % image_size]) + + @contextmanager + def setup_loop_device(self): + logger.info("Setting up loop device") + with self.get_udisks_client() as udisks_client: + manager = udisks_client.get_manager() + + image_fd = os.open(self.image, os.O_RDWR) + resulting_device, fd_list = manager.call_loop_setup_sync( + arg_fd=GLib.Variant('h', 0), + arg_options=GLib.Variant('a{sv}', None), + fd_list=Gio.UnixFDList.new_from_array([image_fd]), + cancellable=None, + ) + + if not resulting_device: + raise ImageCreationError("Failed to set up loop device") + + logger.info("Loop device: %r", resulting_device) + self._loop_device = resulting_device + + try: + yield + finally: + logger.info("Tearing down loop device") + self.loop_device.props.loop.call_delete_sync( + arg_options=GLib.Variant('a{sv}', None), + cancellable=None, + ) + + def create_gpt(self): + logger.info("Creating GPT") + self.loop_device.props.block.call_format_sync( + arg_type='gpt', + arg_options=GLib.Variant('a{sv}', None), + cancellable=None + ) + + def create_partition(self): + logger.info("Creating partition") + partition = self.loop_device.props.partition_table.call_create_partition_sync( + arg_offset=0, + arg_size=self.system_partition_size * 2**20, + arg_type=ESP_GUID, + arg_name=PARTITION_LABEL, + arg_options=GLib.Variant('a{sv}', None), + cancellable=None + ) + # XXX: Tails Installer ignores GLib errors here + + logger.info("Partition: %r", partition) + self._partition = partition + + def set_partition_flags(self): + logger.info("Setting partition flags") + + start_time = time.perf_counter() + while time.perf_counter() - start_time < WAIT_FOR_PARTITION_TIMEOUT: + try: + self.partition.props.partition.call_set_flags_sync( + arg_flags=SYSTEM_PARTITION_FLAGS, + arg_options=GLib.Variant('a{sv}', None), + cancellable=None + ) + except GLib.Error as e: + if "GDBus.Error:org.freedesktop.DBus.Error.UnknownMethod: No such interface" in e.message: + time.sleep(0.1) + continue + raise + return + + def format_partition(self): + logger.info("Formatting partition") + options = GLib.Variant('a{sv}', { + 'label': GLib.Variant('s', FILESYSTEM_LABEL), + 'update-partition-type': GLib.Variant('b', False) + }) + + self.partition.props.block.call_format_sync( + arg_type='vfat', + arg_options=options, + cancellable=None + ) + + @contextmanager + def mount_partition(self): + logger.info("Mounting partition") + try: + self.mountpoint = self.partition.props.filesystem.call_mount_sync( + arg_options=GLib.Variant('a{sv}', None), + cancellable=None + ) + except GLib.Error as e: + if "org.freedesktop.UDisks2.Error.AlreadyMounted" in e.message and \ + self.partition.props.filesystem.props.mount_points: + self.mountpoint = self.partition.props.filesystem.props.mount_points[0] + logger.info("Partition is already mounted at {}".format(self.mountpoint)) + else: + raise + + try: + yield + finally: + logger.info("Unmounting partition") + self.partition.props.filesystem.call_unmount_sync( + arg_options=GLib.Variant('a{sv}', {'force': GLib.Variant('b', True)}), + cancellable=None, + ) + + def set_permissions(self): + logger.info("Setting file access permissions") + for root, dirs, files in os.walk(self.mountpoint): + for d in dirs: + os.chmod(os.path.join(root, d), 0o755) + for f in files: + os.chmod(os.path.join(root, f), 0o644) + + def update_configs(self): + logger.info("Updating config files") + grubconf = os.path.join(self.mountpoint, "EFI", "BOOT", "grub.conf") + bootconf = os.path.join(self.mountpoint, "EFI", "BOOT", "boot.conf") + isolinux_dir = os.path.join(self.mountpoint, "isolinux") + syslinux_dir = os.path.join(self.mountpoint, "syslinux") + isolinux_cfg = os.path.join(syslinux_dir, "isolinux.cfg") + + files_to_update = [ + (os.path.join(self.mountpoint, "isolinux", "isolinux.cfg"), + os.path.join(self.mountpoint, "isolinux", "syslinux.cfg")), + (os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg"), + os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg")), + (os.path.join(self.mountpoint, "isolinux", "exithelp.cfg"), + os.path.join(self.mountpoint, "isolinux", "exithelp.cfg")), + (os.path.join(self.mountpoint, "EFI", "BOOT", "isolinux.cfg"), + os.path.join(self.mountpoint, "EFI", "BOOT", "syslinux.cfg")), + (grubconf, bootconf) + ] + + for (infile, outfile) in files_to_update: + if os.path.exists(infile): + self.update_config(infile, outfile) + + if os.path.exists(isolinux_dir): + execute(["mv", isolinux_dir, syslinux_dir]) + + if os.path.exists(isolinux_cfg): + os.remove(isolinux_cfg) + + def update_config(self, infile, outfile): + with open(infile) as f_in: + lines = [re.sub('/isolinux/', '/syslinux/', line) for line in f_in] + with open(outfile, "w") as f_out: + f_out.writelines(lines) + + def install_mbr(self): + logger.info("Installing MBR") + mbr_path = os.path.join(self.mountpoint, "utils/mbr/mbr.bin") + execute(["dd", "bs=440", "count=1", "conv=notrunc", "if=%s" % mbr_path, "of=%s" % self.image]) + + # Only required if using the running system's syslinux instead of the one on the ISO + def copy_syslinux_modules(self): + logger.info("Copying syslinux modules to device") + + syslinux_dir = os.path.join(self.mountpoint, 'syslinux') + com32modules = [f for f in os.listdir(syslinux_dir) if f.endswith('.c32')] + + for module in com32modules: + src_path = os.path.join(SYSLINUX_COM32MODULES_DIR, module) + if not os.path.isfile(src_path): + raise ImageCreationError("Could not find the '%s' COM32 module" % module) + + logger.debug('Copying %s to the device' % src_path) + execute(["cp", "-a", src_path, os.path.join(syslinux_dir, module)]) + + def install_syslinux(self): + logger.info("Installing bootloader") + # We install syslinux directly on the image. Installing it on the loop + # device would cause this issue: + # https://bugs.chromium.org/p/chromium/issues/detail?id=508713#c8 + execute([ + 'syslinux', + '--offset', str(self.partition.props.partition.props.offset), + '--directory', '/syslinux/', + '--install', self.image + ], + as_root=True # XXX: Why does this only work as root? + ) + + def reset_timestamps(self): + logger.info("Resetting timestamps") + for root, dirs, files in os.walk(self.mountpoint): + os.utime(root, (0, 0), follow_symlinks=False) + for file in files: + os.utime(os.path.join(root, file), (0, 0), follow_symlinks=False) + + def set_guids(self): + logger.info("Setting disk and partition GUID") + execute(["/sbin/sgdisk", "--disk-guid", "17B81DA0-8B1E-4269-9C39-FE5C7B9B58A3", + "--partition-guid", "1:34BF027A-8001-4B93-8243-1F9D3DCE7DE7", self.image]) + + def set_fsuuid(self): + """Set a fixed filesystem UUID aka. FAT Volume ID / serial number""" + logger.info("Setting filesystem UUID") + with set_env("MTOOLS_SKIP_CHECK", "1"): + execute(["mlabel", "-i", self.partition.props.block.props.device, "-N", "a69020d2"]) + + +def execute(cmd: list, as_root=False): + if as_root and os.geteuid() != 0: + cmd = ['pkexec'] + cmd + logger.info("Executing '%s'" % ' '.join(cmd)) + subprocess.check_call(cmd) + + +@contextmanager +def set_env(name: str, value:str): + old_value = os.getenv(name) + os.putenv(name, value) + try: + yield + finally: + if old_value is not None: + os.putenv(name, value) + else: + os.unsetenv(name) + + +def get_file_size(path: str) -> int: + """Returns the size of a file in MiB""" + size_in_bytes = os.path.getsize(path) + return round(size_in_bytes // 1024 ** 2) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("ISO", help="Path to the ISO") + parser.add_argument("-d", "--directory", default=".", help="Output directory for the resulting image (the current directory by default)") + parser.add_argument("--free-space", type=int, default=0, help="Additional free space (for a persistent volume) in MiB") + args = parser.parse_args() + if not args.ISO.endswith(".iso"): + parser.error("Input file is not an ISO (no .iso extension)") + + logging.basicConfig(level=logging.INFO) + logging.getLogger('sh').setLevel(logging.WARNING) + + iso = args.ISO + image = os.path.realpath(os.path.join(args.directory, os.path.basename(iso).replace(".iso", ".img"))) + + image_creator = ImageCreator(iso, image, args.free_space) + image_creator.create_image() + + +if __name__ == "__main__": + main() diff --git a/vagrant/definitions/tails-builder/generate-tails-builder-box.sh b/vagrant/definitions/tails-builder/generate-tails-builder-box.sh index 33eff1a..40f8f6f 100755 --- a/vagrant/definitions/tails-builder/generate-tails-builder-box.sh +++ b/vagrant/definitions/tails-builder/generate-tails-builder-box.sh @@ -1,6 +1,7 @@ #!/bin/sh set -e set -u +set -x # Based on ypcs' scripts found at: # https://github.com/ypcs/vmdebootstrap-vagrant/ diff --git a/vagrant/definitions/tails-builder/postinstall.sh b/vagrant/definitions/tails-builder/postinstall.sh index a0ab2e5..76c02f9 100755 --- a/vagrant/definitions/tails-builder/postinstall.sh +++ b/vagrant/definitions/tails-builder/postinstall.sh @@ -72,10 +72,13 @@ sed -i 's,^GRUB_TIMEOUT=5,GRUB_TIMEOUT=1,g' /etc/default/grub echo "I: Installing Tails build dependencies." apt-get -y install \ debootstrap \ + dosfstools \ dpkg-dev \ eatmydata \ faketime \ + gdisk \ gettext \ + gir1.2-udisks-2.0 \ git \ ikiwiki \ intltool \ @@ -93,12 +96,18 @@ apt-get -y install \ libyaml-syck-perl \ live-build \ lsof \ + mtools \ + p7zip-full \ perlmagick \ psmisc \ + python3-gi \ rsync \ ruby \ + syslinux \ + syslinux-common \ syslinux-utils \ time \ + udisks2 \ whois # Ensure we can use timedatectl diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails index 87fd478..050416e 100755 --- a/vagrant/provision/assets/build-tails +++ b/vagrant/provision/assets/build-tails @@ -34,7 +34,7 @@ remove_build_dirs() { tries=0 sudo lsof | grep --fixed-strings "${mountpoint}" || true while ! sudo umount -f --verbose "${mountpoint}" && [ $tries -lt 12 ]; do - sudo fuser --ismountpoint --mount "${mountpoint}" --kill + sudo fuser --ismountpoint --mount "${mountpoint}" --kill || true sleep 5 tries=$(expr $tries + 1) done diff --git a/wiki/src/contribute/release_process/test/setup.mdwn b/wiki/src/contribute/release_process/test/setup.mdwn index a20d0cc..2d9a1e6 100644 --- a/wiki/src/contribute/release_process/test/setup.mdwn +++ b/wiki/src/contribute/release_process/test/setup.mdwn @@ -88,11 +88,14 @@ the content of `features/misc_files/` in the Git checkout. AppArmor tweaks --------------- -If libvirt has the `apparmor` security driver enabled: - -* you may need to add the `/tmp/TailsToaster/TailsToasterStorage/* - rw,` line to `/etc/apparmor.d/libvirt/TEMPLATE.qemu`, in the - `profile LIBVIRT_TEMPLATE` section. +If you have AppArmor enabled: + +* You need to add the `/tmp/TailsToaster/** rwk,` line + to `/etc/apparmor.d/libvirt/TEMPLATE.qemu`, in the + `profile LIBVIRT_TEMPLATE` section; then delete + `/etc/apparmor.d/libvirt/libvirt-*` and retry. + If you use a custom `TMPDIR` to run the test suite, + replace `/tmp/TailsToaster` with the value of that `$TMPDIR`. Special use cases ================= diff --git a/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn b/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn index 7bb10b7..a1d8d38 100644 --- a/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn +++ b/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn @@ -47,6 +47,5 @@ Calendar ======== * [Debian release schedule](https://www.debian.org/releases/) -* [Ubuntu release schedule](https://wiki.ubuntu.com/ReleaseSchedule) - * Upcoming: BionicBeaver, 18.04, April 26th 2018 - +* [Ubuntu release schedule](https://wiki.ubuntu.com/Release), generally + in April and October |