Merge branch 'stable' into feature/14596-automated-tests-for-ASP-gui-on-stable

8 files changed, 437 insertions, 52 deletions

diff --git a/Rakefile b/Rakefile
index 921fe44..eae44c8 100644
--- a/Rakefile
+++ b/Rakefile
@@ -314,7 +314,7 @@ end
 def list_artifacts
   user = vagrant_ssh_config('User')
   stdout = capture_vagrant_ssh("find '/home/#{user}/amnesia/' -maxdepth 1 " +
-                                        "-name 'tails-*.iso*'").first
+                                        "-name 'tails-amd64-*'").first
   stdout.split("\n")
 rescue VagrantCommandError
   return Array.new
diff --git a/auto/build b/auto/build
index b1da886..72321ec 100755
--- a/auto/build
+++ b/auto/build
@@ -122,8 +122,6 @@ export MKSQUASHFS_OPTIONS
 
 case "$LB_BINARY_IMAGES" in
    iso)
-      BUILD_FILENAME_EXT=iso
-      BUILD_FILENAME=binary
       which isohybrid >/dev/null || fatal 'Cannot find isohybrid in $PATH'
       installed_syslinux_utils_upstream_version="$(syslinux_utils_upstream_version)"
       if dpkg --compare-versions \
@@ -135,27 +133,16 @@ case "$LB_BINARY_IMAGES" in
 	      "while we need at least '${REQUIRED_SYSLINUX_UTILS_UPSTREAM_VERSION}'."
       fi
       ;;
-   iso-hybrid)
-      BUILD_FILENAME_EXT=iso
-      BUILD_FILENAME=binary-hybrid
-      ;;
-   tar)
-      BUILD_FILENAME_EXT=tar.gz
-      BUILD_FILENAME=binary-tar
-      ;;
-   usb-hdd)
-      BUILD_FILENAME_EXT=img
-      BUILD_FILENAME=binary
-      ;;
    *)
       fatal "Image type ${LB_BINARY_IMAGES} is not supported."
       ;;
 esac
-BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}"
-BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest"
-BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources"
-BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages"
-BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog"
+BUILD_ISO_FILENAME="${BUILD_BASENAME}.iso"
+BUILD_MANIFEST="${BUILD_BASENAME}.build-manifest"
+BUILD_APT_SOURCES="${BUILD_BASENAME}.apt-sources"
+BUILD_PACKAGES="${BUILD_BASENAME}.packages"
+BUILD_LOG="${BUILD_BASENAME}.buildlog"
+BUILD_USB_IMAGE_FILENAME="${BUILD_BASENAME}.img"
 
 # Clone all output, from this point on, to the log file
 exec >  >(tee -a "$BUILD_LOG")
@@ -172,27 +159,25 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
    cat config/chroot_sources/*.chroot
 ) > "$BUILD_APT_SOURCES"
 
-echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..."
-set -o pipefail
+echo "Building ISO image ${BUILD_ISO_FILENAME}..."
 time lb build noauto ${@}
-RET=$?
-if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
-   echo "Image was successfully created"
-   [ "$RET" -eq 0 ] || \
-       echo "Warning: lb build exited with code $RET"
-   if [ "$LB_BINARY_IMAGES" = iso ]; then
-       ISO_FILE="${BUILD_FILENAME}.${BUILD_FILENAME_EXT}"
-       print_iso_size "$ISO_FILE"
-       echo "Hybriding it..."
-       isohybrid $AMNESIA_ISOHYBRID_OPTS "$ISO_FILE" || fatal "isohybrid failed"
-       print_iso_size "$ISO_FILE"
-       truncate -s %2048 "$ISO_FILE"
-       print_iso_size "$ISO_FILE"
-   fi
-   echo "Renaming generated files..."
-   mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
-   mv -i binary.packages "${BUILD_PACKAGES}"
-   generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
-else
-   fatal "lb build failed ($?)."
-fi
+[ -e binary.iso ] || fatal "lb build failed ($?)."
+
+echo "ISO image was successfully created"
+print_iso_size binary.iso
+
+echo "Hybriding it..."
+isohybrid $AMNESIA_ISOHYBRID_OPTS binary.iso || fatal "isohybrid failed"
+print_iso_size binary.iso
+truncate -s %2048 binary.iso
+print_iso_size binary.iso
+
+echo "Renaming generated files..."
+mv -i binary.iso "${BUILD_ISO_FILENAME}"
+mv -i binary.packages "${BUILD_PACKAGES}"
+
+echo "Generating build manifest..."
+generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
+
+echo "Creating USB image ${BUILD_USB_IMAGE_FILENAME}..."
+create-usb-image-from-iso "${BUILD_ISO_FILENAME}"
diff --git a/auto/scripts/create-usb-image-from-iso b/auto/scripts/create-usb-image-from-iso
new file mode 100755
index 0000000..8600ef1
--- /dev/null
+++ b/auto/scripts/create-usb-image-from-iso
@@ -0,0 +1,388 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import logging
+from contextlib import contextmanager
+import re
+import time
+import subprocess
+
+import gi
+gi.require_version('UDisks', '2.0')
+from gi.repository import UDisks, GLib, Gio
+
+
+logger = logging.getLogger(__name__)
+
+SYSTEM_PARTITION_FLAGS = (
+    1 << 0 |   # system partition
+    1 << 2 |   # legacy BIOS bootable
+    1 << 60 |  # read-only
+    1 << 62 |  # hidden
+    1 << 63    # do not automount
+)
+
+# EFI System Partition
+ESP_GUID = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B'
+
+PARTITION_LABEL = 'Tails'
+FILESYSTEM_LABEL = 'Tails'
+
+GET_UDISKS_OBJECT_TIMEOUT = 2
+WAIT_FOR_PARTITION_TIMEOUT = 2
+
+# The size of the system partition (in MiB) will be:
+#
+#   SYSTEM_PARTITION_ADDITIONAL_SIZE + size of the ISO
+#
+# SYSTEM_PARTITION_ADDITIONAL_SIZE must be large enough to fit
+# the partition table, reserved sectors, and filesystem metadata.
+SYSTEM_PARTITION_ADDITIONAL_SIZE = 10
+
+SYSLINUX_COM32MODULES_DIR = '/usr/lib/syslinux/modules/bios'
+
+
+class ImageCreationError(Exception):
+    pass
+
+
+class ImageCreator(object):
+
+    def __init__(self, iso: str, image: str, free_space: int):
+        self.iso = iso
+        self.image = image
+        self.free_space = free_space
+        self._loop_device = None  # type: str
+        self._partition = None  # type: str
+        self._system_partition_size = None  # type: int
+        self.mountpoint = None  # type: str
+
+    @property
+    def loop_device(self) -> UDisks.ObjectProxy:
+        if not self._loop_device:
+            raise ImageCreationError("Loop device not set up")
+        return self.try_getting_udisks_object(self._loop_device)
+
+    @property
+    def partition(self) -> UDisks.ObjectProxy:
+        if not self._partition:
+            raise ImageCreationError("Partition not created")
+
+        return self.try_getting_udisks_object(self._partition)
+
+    @property
+    def system_partition_size(self) -> int:
+        if self._system_partition_size is None:
+            self._system_partition_size = get_file_size(self.iso) + SYSTEM_PARTITION_ADDITIONAL_SIZE
+
+        return self._system_partition_size
+
+    def try_getting_udisks_object(self, object_path: str) -> UDisks.Object:
+        start_time = time.perf_counter()
+        while time.perf_counter() - start_time < GET_UDISKS_OBJECT_TIMEOUT:
+            with self.get_udisks_client() as udisks_client:
+                udisks_object = udisks_client.get_object(object_path)
+            if udisks_object:
+                return udisks_object
+            time.sleep(0.1)
+        raise ImageCreationError("Couldn't get UDisksObject for path '%s' (timeout: %s)" %
+                                 (object_path, GET_UDISKS_OBJECT_TIMEOUT))
+
+    @contextmanager
+    def get_udisks_client(self):
+        client = UDisks.Client().new_sync()
+        yield client
+        client.settle()
+
+    def create_image(self):
+        self.create_empty_image()
+
+        with self.setup_loop_device():
+            self.create_gpt()
+            self.create_partition()
+            self.set_partition_flags()
+            # XXX: Rescan?
+            self.format_partition()
+            with self.mount_partition():
+                self.extract_iso()
+                self.set_permissions()
+                self.update_configs()
+                self.install_mbr()
+                self.copy_syslinux_modules()
+
+            # We have to install syslinux after the partition was unmounted.
+            # This sleep is a workaround for a race condition which causes the
+            # syslinux installation to return without errors, even though the
+            # bootloader isn't actually installed
+            # XXX: Investigate and report this race condition
+            # Might it be https://bugs.chromium.org/p/chromium/issues/detail?id=508713 ?
+            time.sleep(1)
+            self.install_syslinux()
+            self.set_guids()
+            self.set_fsuuid()
+
+            with self.mount_partition():
+                self.reset_timestamps()
+
+    def extract_iso(self):
+        logger.info("Extracting ISO contents to the partition")
+        execute(['7z', 'x', self.iso, '-x![BOOT]', '-y', '-o%s' % self.mountpoint])
+
+    def create_empty_image(self):
+        logger.info("Creating empty image %r", self.image)
+        image_size = self.system_partition_size + self.free_space
+        execute(["dd", "if=/dev/zero", "of=%s" % self.image, "bs=1M", "count=%s" % image_size])
+
+    @contextmanager
+    def setup_loop_device(self):
+        logger.info("Setting up loop device")
+        with self.get_udisks_client() as udisks_client:
+            manager = udisks_client.get_manager()
+
+            image_fd = os.open(self.image, os.O_RDWR)
+            resulting_device, fd_list = manager.call_loop_setup_sync(
+                arg_fd=GLib.Variant('h', 0),
+                arg_options=GLib.Variant('a{sv}', None),
+                fd_list=Gio.UnixFDList.new_from_array([image_fd]),
+                cancellable=None,
+            )
+
+            if not resulting_device:
+                raise ImageCreationError("Failed to set up loop device")
+
+        logger.info("Loop device: %r", resulting_device)
+        self._loop_device = resulting_device
+
+        try:
+            yield
+        finally:
+            logger.info("Tearing down loop device")
+            self.loop_device.props.loop.call_delete_sync(
+                arg_options=GLib.Variant('a{sv}', None),
+                cancellable=None,
+            )
+
+    def create_gpt(self):
+        logger.info("Creating GPT")
+        self.loop_device.props.block.call_format_sync(
+            arg_type='gpt',
+            arg_options=GLib.Variant('a{sv}', None),
+            cancellable=None
+        )
+
+    def create_partition(self):
+        logger.info("Creating partition")
+        partition = self.loop_device.props.partition_table.call_create_partition_sync(
+            arg_offset=0,
+            arg_size=self.system_partition_size * 2**20,
+            arg_type=ESP_GUID,
+            arg_name=PARTITION_LABEL,
+            arg_options=GLib.Variant('a{sv}', None),
+            cancellable=None
+        )
+        # XXX: Tails Installer ignores GLib errors here
+
+        logger.info("Partition: %r", partition)
+        self._partition = partition
+
+    def set_partition_flags(self):
+        logger.info("Setting partition flags")
+
+        start_time = time.perf_counter()
+        while time.perf_counter() - start_time < WAIT_FOR_PARTITION_TIMEOUT:
+            try:
+                self.partition.props.partition.call_set_flags_sync(
+                    arg_flags=SYSTEM_PARTITION_FLAGS,
+                    arg_options=GLib.Variant('a{sv}', None),
+                    cancellable=None
+                )
+            except GLib.Error as e:
+                if "GDBus.Error:org.freedesktop.DBus.Error.UnknownMethod: No such interface" in e.message:
+                    time.sleep(0.1)
+                    continue
+                raise
+            return
+
+    def format_partition(self):
+        logger.info("Formatting partition")
+        options = GLib.Variant('a{sv}', {
+                'label': GLib.Variant('s', FILESYSTEM_LABEL),
+                'update-partition-type': GLib.Variant('b', False)
+        })
+
+        self.partition.props.block.call_format_sync(
+            arg_type='vfat',
+            arg_options=options,
+            cancellable=None
+        )
+
+    @contextmanager
+    def mount_partition(self):
+        logger.info("Mounting partition")
+        try:
+            self.mountpoint = self.partition.props.filesystem.call_mount_sync(
+                arg_options=GLib.Variant('a{sv}', None),
+                cancellable=None
+            )
+        except GLib.Error as e:
+            if "org.freedesktop.UDisks2.Error.AlreadyMounted" in e.message and \
+                    self.partition.props.filesystem.props.mount_points:
+                self.mountpoint = self.partition.props.filesystem.props.mount_points[0]
+                logger.info("Partition is already mounted at {}".format(self.mountpoint))
+            else:
+                raise
+
+        try:
+            yield
+        finally:
+            logger.info("Unmounting partition")
+            self.partition.props.filesystem.call_unmount_sync(
+                arg_options=GLib.Variant('a{sv}', {'force': GLib.Variant('b', True)}),
+                cancellable=None,
+            )
+
+    def set_permissions(self):
+        logger.info("Setting file access permissions")
+        for root, dirs, files in os.walk(self.mountpoint):
+            for d in dirs:
+                os.chmod(os.path.join(root, d), 0o755)
+            for f in files:
+                os.chmod(os.path.join(root, f), 0o644)
+
+    def update_configs(self):
+        logger.info("Updating config files")
+        grubconf = os.path.join(self.mountpoint, "EFI", "BOOT", "grub.conf")
+        bootconf = os.path.join(self.mountpoint, "EFI", "BOOT", "boot.conf")
+        isolinux_dir = os.path.join(self.mountpoint, "isolinux")
+        syslinux_dir = os.path.join(self.mountpoint, "syslinux")
+        isolinux_cfg = os.path.join(syslinux_dir, "isolinux.cfg")
+
+        files_to_update = [
+            (os.path.join(self.mountpoint, "isolinux", "isolinux.cfg"),
+             os.path.join(self.mountpoint, "isolinux", "syslinux.cfg")),
+            (os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg"),
+             os.path.join(self.mountpoint, "isolinux", "stdmenu.cfg")),
+            (os.path.join(self.mountpoint, "isolinux", "exithelp.cfg"),
+             os.path.join(self.mountpoint, "isolinux", "exithelp.cfg")),
+            (os.path.join(self.mountpoint, "EFI", "BOOT", "isolinux.cfg"),
+             os.path.join(self.mountpoint, "EFI", "BOOT", "syslinux.cfg")),
+            (grubconf, bootconf)
+        ]
+
+        for (infile, outfile) in files_to_update:
+            if os.path.exists(infile):
+                self.update_config(infile, outfile)
+
+        if os.path.exists(isolinux_dir):
+            execute(["mv", isolinux_dir, syslinux_dir])
+
+        if os.path.exists(isolinux_cfg):
+            os.remove(isolinux_cfg)
+
+    def update_config(self, infile, outfile):
+        with open(infile) as f_in:
+            lines = [re.sub('/isolinux/', '/syslinux/', line) for line in f_in]
+        with open(outfile, "w") as f_out:
+            f_out.writelines(lines)
+
+    def install_mbr(self):
+        logger.info("Installing MBR")
+        mbr_path = os.path.join(self.mountpoint, "utils/mbr/mbr.bin")
+        execute(["dd", "bs=440", "count=1", "conv=notrunc", "if=%s" % mbr_path, "of=%s" % self.image])
+
+    # Only required if using the running system's syslinux instead of the one on the ISO
+    def copy_syslinux_modules(self):
+        logger.info("Copying syslinux modules to device")
+
+        syslinux_dir = os.path.join(self.mountpoint, 'syslinux')
+        com32modules = [f for f in os.listdir(syslinux_dir) if f.endswith('.c32')]
+
+        for module in com32modules:
+            src_path = os.path.join(SYSLINUX_COM32MODULES_DIR, module)
+            if not os.path.isfile(src_path):
+                raise ImageCreationError("Could not find the '%s' COM32 module" % module)
+
+            logger.debug('Copying %s to the device' % src_path)
+            execute(["cp", "-a", src_path, os.path.join(syslinux_dir, module)])
+
+    def install_syslinux(self):
+        logger.info("Installing bootloader")
+        # We install syslinux directly on the image. Installing it on the loop
+        # device would cause this issue:
+        # https://bugs.chromium.org/p/chromium/issues/detail?id=508713#c8
+        execute([
+            'syslinux',
+            '--offset', str(self.partition.props.partition.props.offset),
+            '--directory', '/syslinux/',
+            '--install', self.image
+        ],
+            as_root=True  # XXX: Why does this only work as root?
+        )
+
+    def reset_timestamps(self):
+        logger.info("Resetting timestamps")
+        for root, dirs, files in os.walk(self.mountpoint):
+            os.utime(root, (0, 0), follow_symlinks=False)
+            for file in files:
+                os.utime(os.path.join(root, file), (0, 0), follow_symlinks=False)
+
+    def set_guids(self):
+        logger.info("Setting disk and partition GUID")
+        execute(["/sbin/sgdisk", "--disk-guid", "17B81DA0-8B1E-4269-9C39-FE5C7B9B58A3",
+                 "--partition-guid", "1:34BF027A-8001-4B93-8243-1F9D3DCE7DE7", self.image])
+
+    def set_fsuuid(self):
+        """Set a fixed filesystem UUID aka. FAT Volume ID / serial number"""
+        logger.info("Setting filesystem UUID")
+        with set_env("MTOOLS_SKIP_CHECK", "1"):
+            execute(["mlabel", "-i", self.partition.props.block.props.device, "-N", "a69020d2"])
+
+
+def execute(cmd: list, as_root=False):
+    if as_root and os.geteuid() != 0:
+        cmd = ['pkexec'] + cmd
+    logger.info("Executing '%s'" % ' '.join(cmd))
+    subprocess.check_call(cmd)
+
+
+@contextmanager
+def set_env(name: str, value:str):
+    old_value = os.getenv(name)
+    os.putenv(name, value)
+    try:
+        yield
+    finally:
+        if old_value is not None:
+            os.putenv(name, value)
+        else:
+            os.unsetenv(name)
+
+
+def get_file_size(path: str) -> int:
+    """Returns the size of a file in MiB"""
+    size_in_bytes = os.path.getsize(path)
+    return round(size_in_bytes // 1024 ** 2)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ISO", help="Path to the ISO")
+    parser.add_argument("-d", "--directory", default=".", help="Output directory for the resulting image (the current directory by default)")
+    parser.add_argument("--free-space", type=int, default=0, help="Additional free space (for a persistent volume) in MiB")
+    args = parser.parse_args()
+    if not args.ISO.endswith(".iso"):
+        parser.error("Input file is not an ISO (no .iso extension)")
+
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger('sh').setLevel(logging.WARNING)
+
+    iso = args.ISO
+    image = os.path.realpath(os.path.join(args.directory, os.path.basename(iso).replace(".iso", ".img")))
+
+    image_creator = ImageCreator(iso, image, args.free_space)
+    image_creator.create_image()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/vagrant/definitions/tails-builder/generate-tails-builder-box.sh b/vagrant/definitions/tails-builder/generate-tails-builder-box.sh
index 33eff1a..40f8f6f 100755
--- a/vagrant/definitions/tails-builder/generate-tails-builder-box.sh
+++ b/vagrant/definitions/tails-builder/generate-tails-builder-box.sh
@@ -1,6 +1,7 @@
 #!/bin/sh
 set -e
 set -u
+set -x
 
 # Based on ypcs' scripts found at:
 #     https://github.com/ypcs/vmdebootstrap-vagrant/
diff --git a/vagrant/definitions/tails-builder/postinstall.sh b/vagrant/definitions/tails-builder/postinstall.sh
index a0ab2e5..76c02f9 100755
--- a/vagrant/definitions/tails-builder/postinstall.sh
+++ b/vagrant/definitions/tails-builder/postinstall.sh
@@ -72,10 +72,13 @@ sed -i 's,^GRUB_TIMEOUT=5,GRUB_TIMEOUT=1,g' /etc/default/grub
 echo "I: Installing Tails build dependencies."
 apt-get -y install \
         debootstrap \
+        dosfstools \
         dpkg-dev \
         eatmydata \
         faketime \
+        gdisk \
         gettext \
+        gir1.2-udisks-2.0 \
         git \
         ikiwiki \
         intltool \
@@ -93,12 +96,18 @@ apt-get -y install \
         libyaml-syck-perl \
         live-build \
         lsof \
+        mtools \
+        p7zip-full \
         perlmagick \
         psmisc \
+        python3-gi \
         rsync \
         ruby \
+        syslinux \
+        syslinux-common \
         syslinux-utils \
         time \
+        udisks2 \
         whois
 
 # Ensure we can use timedatectl
diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails
index 87fd478..050416e 100755
--- a/vagrant/provision/assets/build-tails
+++ b/vagrant/provision/assets/build-tails
@@ -34,7 +34,7 @@ remove_build_dirs() {
 		tries=0
 		sudo lsof | grep --fixed-strings "${mountpoint}" || true
 		while ! sudo umount -f --verbose "${mountpoint}" && [ $tries -lt 12 ]; do
-			sudo fuser --ismountpoint --mount "${mountpoint}" --kill
+			sudo fuser --ismountpoint --mount "${mountpoint}" --kill || true
 			sleep 5
 			tries=$(expr $tries + 1)
 		done
diff --git a/wiki/src/contribute/release_process/test/setup.mdwn b/wiki/src/contribute/release_process/test/setup.mdwn
index a20d0cc..2d9a1e6 100644
--- a/wiki/src/contribute/release_process/test/setup.mdwn
+++ b/wiki/src/contribute/release_process/test/setup.mdwn
@@ -88,11 +88,14 @@ the content of `features/misc_files/` in the Git checkout.
 AppArmor tweaks
 ---------------
 
-If libvirt has the `apparmor` security driver enabled:
-
-* you may need to add the `/tmp/TailsToaster/TailsToasterStorage/*
-  rw,` line to `/etc/apparmor.d/libvirt/TEMPLATE.qemu`, in the
-  `profile LIBVIRT_TEMPLATE` section.
+If you have AppArmor enabled:
+
+* You need to add the `/tmp/TailsToaster/** rwk,` line
+  to `/etc/apparmor.d/libvirt/TEMPLATE.qemu`, in the
+  `profile LIBVIRT_TEMPLATE` section; then delete
+  `/etc/apparmor.d/libvirt/libvirt-*` and retry.
+  If you use a custom `TMPDIR` to run the test suite,
+  replace `/tmp/TailsToaster` with the value of that `$TMPDIR`.
 
 Special use cases
 =================
diff --git a/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn b/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn
index 7bb10b7..a1d8d38 100644
--- a/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn
+++ b/wiki/src/contribute/working_together/roles/debian_maintainer.mdwn
@@ -47,6 +47,5 @@ Calendar
 ========
 
 * [Debian release schedule](https://www.debian.org/releases/)
-* [Ubuntu release schedule](https://wiki.ubuntu.com/ReleaseSchedule)
-  * Upcoming: BionicBeaver, 18.04, April 26th 2018
-
+* [Ubuntu release schedule](https://wiki.ubuntu.com/Release), generally
+  in April and October