summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorintrigeri <intrigeri@boum.org>2020-01-14 20:39:02 +0000
committerintrigeri <intrigeri@boum.org>2020-01-14 20:39:02 +0000
commitc4eebf89756c56feb97f5fa55ac0c421fb1ca0ee (patch)
tree6f1bf9d1386e246986527284440241a34c3e62b9
parent2f4f61067de643f1bda86885fb487bdf6dd12531 (diff)
parentf5c75d4e7175123c68b92ecd1d1f91e8b303d6c4 (diff)
Merge branch 'stable' into develdevel
-rw-r--r--Rakefile135
-rwxr-xr-xauto/build13
-rwxr-xr-xauto/scripts/website-cache206
-rw-r--r--vagrant/Vagrantfile6
-rwxr-xr-xvagrant/definitions/tails-builder/postinstall.sh15
-rwxr-xr-xvagrant/provision/assets/build-tails88
-rwxr-xr-xvagrant/provision/setup-tails-builder110
-rw-r--r--wiki/src/contribute/build.mdwn16
-rw-r--r--wiki/src/contribute/build/manually.mdwn113
9 files changed, 446 insertions, 256 deletions
diff --git a/Rakefile b/Rakefile
index 416b03d..529197c 100644
--- a/Rakefile
+++ b/Rakefile
@@ -43,6 +43,7 @@ EXPORTED_VARIABLES = [
'TAILS_PROXY',
'TAILS_PROXY_TYPE',
'TAILS_RAM_BUILD',
+ 'TAILS_WEBSITE_CACHE',
'GIT_COMMIT',
'GIT_REF',
'BASE_BRANCH_GIT_COMMIT',
@@ -118,7 +119,7 @@ rescue CommandError => e
end
# Runs the vagrant command, not letting stdout/stderr through, and
-# returns [stdout, stderr, Preocess:Status].
+# returns [stdout, stderr, Process::Status].
def capture_vagrant(*args)
capture_command('vagrant', *args, :chdir => './vagrant')
rescue CommandError => e
@@ -246,11 +247,20 @@ task :parse_build_options do
ENV['TAILS_PROXY_TYPE'] = 'noproxy'
when 'offline'
ENV['TAILS_OFFLINE_MODE'] = '1'
+ when 'cachewebsite'
+ if is_release?
+ $stderr.puts "Building a release ⇒ ignoring #{opt} build option"
+ ENV['TAILS_WEBSITE_CACHE'] = '0'
+ else
+ ENV['TAILS_WEBSITE_CACHE'] = '1'
+ end
# SquashFS compression settings
when 'fastcomp', 'gzipcomp'
- ENV['MKSQUASHFS_OPTIONS'] = '-comp xz'
if is_release?
- raise 'We must use the default compression when building releases!'
+ $stderr.puts "Building a release ⇒ ignoring #{opt} build option"
+ ENV['MKSQUASHFS_OPTIONS'] = nil
+ else
+ ENV['MKSQUASHFS_OPTIONS'] = '-comp xz -no-exports'
end
when 'defaultcomp'
ENV['MKSQUASHFS_OPTIONS'] = nil
@@ -389,8 +399,35 @@ task :maybe_clean_up_builder_vms do
clean_up_builder_vms if $force_cleanup
end
+task :ensure_correct_permissions do
+ FileUtils.chmod('go+x', '.')
+ FileUtils.chmod_R('go+rX', ['.git', 'submodules', 'vagrant'])
+
+ # Changing permissions outside of the working copy, in particular on
+ # parent directories such as $HOME, feels too blunt and can have
+ # problematic security consequences, so we don't forcibly do that.
+ # Instead, when the permissions are not OK, display a nicer error
+ # message than "Virtio-9p Failed to initialize fs-driver […]"
+ begin
+ capture_command('sudo', '-u', 'libvirt-qemu', 'stat', '.git')
+ rescue CommandError
+ abort <<-END_OF_MESSAGE.gsub(/^ /, '')
+
+ Incorrect permissions: the libvirt-qemu user needs to be allowed
+ to traverse the filesystem up to #{ENV['PWD']}.
+
+ To fix this, you can for example run the following command
+ on every parent directory of #{ENV['PWD']} up to #{ENV['HOME']}
+ (inclusive):
+
+ chmod g+x DIR && setfacl -m user:libvirt-qemu:x DIR
+
+ END_OF_MESSAGE
+ end
+end
+
desc 'Build Tails'
-task :build => ['parse_build_options', 'ensure_clean_repository', 'maybe_clean_up_builder_vms', 'validate_git_state', 'setup_environment', 'validate_http_proxy', 'vm:up', 'ensure_clean_home_directory'] do
+task :build => ['parse_build_options', 'ensure_clean_repository', 'maybe_clean_up_builder_vms', 'validate_git_state', 'setup_environment', 'validate_http_proxy', 'ensure_correct_permissions', 'vm:up', 'ensure_clean_home_directory'] do
begin
if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
@@ -419,38 +456,62 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'maybe_clean_u
exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
- run_vagrant_ssh("#{exported_env} build-tails")
-
- artifacts = list_artifacts
- raise 'No build artifacts were found!' if artifacts.empty?
- user = vagrant_ssh_config('User')
- hostname = vagrant_ssh_config('HostName')
- key_file = vagrant_ssh_config('IdentityFile')
- $stderr.puts "Retrieving artifacts from Vagrant build box."
- run_vagrant_ssh(
- "sudo chown #{user} " + artifacts.map { |a| "'#{a}'" } .join(' ')
- )
- fetch_command = [
- 'scp',
- '-i', key_file,
- # We need this since the user will not necessarily have a
- # known_hosts entry. It is safe since an attacker must
- # compromise libvirt's network config or the user running the
- # command to modify the #{hostname} below.
- '-o', 'StrictHostKeyChecking=no',
- '-o', 'UserKnownHostsFile=/dev/null',
- # Speed up the copy
- '-o', 'Compression=no',
- ]
- fetch_command += artifacts.map { |a| "#{user}@#{hostname}:#{a}" }
- fetch_command << ENV['ARTIFACTS']
- run_command(*fetch_command)
- clean_up_builder_vms unless $keep_running
+
+ begin
+ retrieved_artifacts = false
+ run_vagrant_ssh("#{exported_env} build-tails")
+ rescue VagrantCommandError
+ retrieve_artifacts(:missing_ok => true)
+ retrieved_artifacts = true
+ ensure
+ retrieve_artifacts(:missing_ok => false) unless retrieved_artifacts
+ clean_up_builder_vms unless $keep_running
+ end
ensure
clean_up_builder_vms if $force_cleanup
end
end
+desc "Retrieve build artifacts from the Vagrant box"
+task :retrieve_artifacts do
+ retrieve_artifacts
+end
+
+def retrieve_artifacts(missing_ok: false)
+ artifacts = list_artifacts
+ if artifacts.empty?
+ msg = 'No build artifacts were found!'
+ if missing_ok
+ $stderr.puts msg
+ return
+ else
+ raise msg
+ end
+ end
+ user = vagrant_ssh_config('User')
+ hostname = vagrant_ssh_config('HostName')
+ key_file = vagrant_ssh_config('IdentityFile')
+ $stderr.puts "Retrieving artifacts from Vagrant build box."
+ run_vagrant_ssh(
+ "sudo chown #{user} " + artifacts.map { |a| "'#{a}'" } .join(' ')
+ )
+ fetch_command = [
+ 'scp',
+ '-i', key_file,
+ # We need this since the user will not necessarily have a
+ # known_hosts entry. It is safe since an attacker must
+ # compromise libvirt's network config or the user running the
+ # command to modify the #{hostname} below.
+ '-o', 'StrictHostKeyChecking=no',
+ '-o', 'UserKnownHostsFile=/dev/null',
+ # Speed up the copy
+ '-o', 'Compression=no',
+ ]
+ fetch_command += artifacts.map { |a| "#{user}@#{hostname}:#{a}" }
+ fetch_command << ENV['ARTIFACTS']
+ run_command(*fetch_command)
+end
+
def has_box?
not(capture_vagrant('box', 'list').grep(/^#{box_name}\s+\(libvirt,/).empty?)
end
@@ -488,6 +549,14 @@ def clean_up_builder_vms
run_vagrant_ssh("sudo umount /var/cache/apt-cacher-ng")
run_vagrant_ssh("sudo sync")
end
+ begin
+ run_vagrant_ssh("mountpoint -q /var/cache/tails-website")
+ rescue VagrantCommandError
+ # Nothing to unmount.
+ else
+ run_vagrant_ssh("sudo umount /var/cache/tails-website")
+ run_vagrant_ssh("sudo sync")
+ end
end
clean_up_domain.call(previous_domain)
@@ -571,7 +640,7 @@ namespace :vm do
clean_up_builder_vms
end
begin
- run_vagrant('up')
+ run_vagrant('up', '--provision')
rescue VagrantCommandError => e
clean_up_builder_vms if $force_cleanup
raise e
@@ -593,7 +662,7 @@ namespace :vm do
run_vagrant('provision')
end
- desc "Destroy build virtual machine (clean up all files except the vmproxy's apt-cacher-ng data)"
+ desc "Destroy build virtual machine (clean up all files except the vmproxy's apt-cacher-ng data and the website cache)"
task :destroy do
clean_up_builder_vms
end
diff --git a/auto/build b/auto/build
index 99b15c8..83064c6 100755
--- a/auto/build
+++ b/auto/build
@@ -112,7 +112,18 @@ MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -mem 512M -wildcards -ef chroot/usr/sh
export MKSQUASHFS_OPTIONS
# build the doc wiki
-./build-website
+if [ "${TAILS_WEBSITE_CACHE:-0}" = 1 ]; then
+ export WEBSITE_DEST_DIR=.
+ export WEBSITE_CACHE_BASEDIR
+ website-cache gc
+ WEBSITE_CACHE_KEY=$(website-cache key)
+ if ! website-cache get "$WEBSITE_CACHE_KEY"; then
+ ./build-website
+ website-cache put "$WEBSITE_CACHE_KEY"
+ fi
+else
+ ./build-website
+fi
# refresh translations of our programs
./refresh-translations || fatal "refresh-translations failed ($?)."
diff --git a/auto/scripts/website-cache b/auto/scripts/website-cache
new file mode 100755
index 0000000..39bbed6
--- /dev/null
+++ b/auto/scripts/website-cache
@@ -0,0 +1,206 @@
+#!/usr/bin/python3
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+
+from datetime import datetime, timedelta
+from pathlib import Path
+
+LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
+log = logging.getLogger()
+
+# Input parameters that primarily determine the output of the build,
+# and therefore are used to generate the cache key: any change
+# to any of these input parameters will yield a different cache key.
+KEY_FILES = [
+ # A new implementation of the caching mechanism should
+ # invalidate older cached data
+ 'auto/scripts/website-cache',
+ # ikiwiki configuration
+ 'ikiwiki.setup',
+ # ikiwiki input directory
+ 'wiki/src',
+]
+KEY_PACKAGES = [
+ 'gettext',
+ 'ikiwiki',
+ 'libmarkdown2',
+ 'libtext-markdown-discount-perl',
+ 'perl',
+ 'po4a',
+]
+
+# Files to cache, relative to the root of this very Git repository
+CACHED_FILES = [
+ 'config/chroot_local-includes/usr/share/doc/tails/website',
+]
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Query and manage website cache"
+ )
+ parser.add_argument(
+ "--dest-dir", type=str, action="store",
+ default=os.getenv('WEBSITE_DEST_DIR', None),
+ help="Directory where to restore data from the cache")
+ parser.add_argument(
+ "--cache-base-dir", type=str, action="store",
+ default=os.getenv('WEBSITE_CACHE_BASEDIR', None),
+ help="Directory where the cache lives")
+ parser.add_argument("--debug", action="store_true", help="debug output")
+ subparsers = parser.add_subparsers(help="sub-command help", dest="command")
+
+ parser_gc = subparsers.add_parser(
+ "gc",
+ help="Garbage collect expired data from the cache")
+ parser_gc.add_argument(
+ "--max-days", type=int, action="store", default=30,
+ help="Number of days after which cached data expires")
+ parser_gc.set_defaults(func=gc)
+
+ parser_get = subparsers.add_parser("get", help="Copy data from the cache")
+ parser_get.add_argument("cache_key", type=str, action="store")
+ parser_get.set_defaults(func=get)
+
+ parser_put = subparsers.add_parser("put", help="Copy data to the cache")
+ parser_put.add_argument("cache_key", type=str, action="store")
+ parser_put.set_defaults(func=put)
+
+ parser_key = subparsers.add_parser(
+ "key",
+ help="Computing cache key for the source tree and build environment")
+ parser_key.set_defaults(func=key)
+
+ parser_forget = subparsers.add_parser("forget", help="Delete cached data")
+ parser_forget.add_argument("cache_key", type=str, action="store")
+ parser_forget.set_defaults(func=forget)
+
+ args = parser.parse_args()
+
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
+ else:
+ logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+
+ if args.cache_base_dir is None:
+ log.error("Please pass --cache-base-dir or set $WEBSITE_CACHE_BASEDIR")
+ sys.exit(1)
+
+ if args.dest_dir is None:
+ log.error("Please pass --dest-dir or set $WEBSITE_DEST_DIR")
+ sys.exit(1)
+
+ for key_file in KEY_FILES + [args.cache_base_dir]:
+ if not Path(key_file).exists():
+ log.error("%s does not exist" % (key_file))
+ sys.exit(1)
+
+ if args.command is None:
+ parser.print_help()
+ else:
+ args.func(args)
+
+
+def gc(args):
+ log.info("Garbage collecting expired data from the cache…")
+ cache_dirs = [d for d in Path(args.cache_base_dir).iterdir() if d.is_dir()]
+ delete_before = datetime.utcnow() - timedelta(days=args.max_days)
+ log.debug("Will delete data created before %s" % (delete_before))
+ for cache_dir in cache_dirs:
+ mtime = datetime.utcfromtimestamp(cache_dir.stat().st_mtime)
+ if mtime < delete_before:
+ log.info(" - Deleting cache for %s with mtime %s"
+ % (cache_dir.name, mtime))
+ shutil.rmtree(cache_dir)
+ else:
+ log.debug(" - Cache for %s has mtime %s ⇒ keeping"
+ % (cache_dir.name, mtime))
+
+
+def get(args):
+ cache_dir = Path(args.cache_base_dir, args.cache_key)
+ if not cache_dir.exists():
+ raise LookupError("Found no cache dir for key %s" % (args.cache_key))
+
+ for file_to_get in [Path(f) for f in CACHED_FILES]:
+ cached_file = Path(cache_dir, file_to_get)
+ dest_file = Path(args.dest_dir, file_to_get)
+
+ if dest_file.exists():
+ raise FileExistsError("%s already exists locally, not overwriting"
+ % (dest_file))
+ if not cached_file.exists():
+ raise FileNotFoundError("Found no cached %s for key %s"
+ % (file_to_get, args.cache_key))
+
+ log.debug("Copying %s from the cache" % (file_to_get))
+ if cached_file.is_dir():
+ shutil.copytree(src=cached_file, dst=dest_file, symlinks=True)
+ else:
+ raise NotImplementedError(
+ "Retrieving non-directories is not supported yet")
+
+
+def put(args):
+ cache_dir = Path(args.cache_base_dir, args.cache_key)
+ cache_dir.mkdir()
+ for file_to_cache in [Path(f) for f in CACHED_FILES]:
+ cached_file = Path(cache_dir, file_to_cache)
+
+ if not file_to_cache.exists():
+ raise FileNotFoundError("Cannot store non-existing %s in the cache"
+ % file_to_cache)
+
+ log.debug("Caching %s with key %s" % (file_to_cache, args.cache_key))
+ cached_file.parent.mkdir(parents=True)
+ if file_to_cache.is_dir():
+ shutil.copytree(src=file_to_cache, dst=cached_file, symlinks=True)
+ else:
+ raise NotImplementedError(
+ "Caching non-directories is not supported yet")
+
+
+def forget(args):
+ cache_dir = Path(args.cache_base_dir, args.cache_key)
+ if cache_dir.exists():
+ log.info("Deleting cached data for key %s" % args.cache_key)
+ shutil.rmtree(cache_dir)
+ else:
+ log.info("No cached data to forget for key %s", args.cache_key)
+
+
+def package_version(package: str) -> str:
+ return subprocess.run(
+ ["dpkg-query", "--showformat", "${Version}", "--show", package],
+ stdout=subprocess.PIPE, universal_newlines=True,
+ check=True).stdout.rstrip()
+
+
+def compute_cache_key(key_files: [str], key_packages: [str]) -> str:
+ input_data = {
+ 'git_commit': subprocess.run(
+ ["git", "log", "-1", "--pretty=%H", "--", *key_files],
+ stdout=subprocess.PIPE, universal_newlines=True,
+ check=True).stdout.rstrip(),
+ 'packages': dict(
+ (package, package_version(package)) for package in sorted(key_packages)
+ ),
+ }
+ serialized = json.dumps(input_data, sort_keys=True)
+ log.debug("Serialized data: " + serialized)
+ return hashlib.sha1(bytes(serialized, encoding='utf-8')).hexdigest()
+
+
+def key(args):
+ print(compute_cache_key(KEY_FILES, KEY_PACKAGES))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 5ad9958..f58abb6 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -60,5 +60,11 @@ Vagrant.configure("2") do |config|
path: 'apt-cacher-ng-data.qcow2'
)
end
+ if ENV['TAILS_WEBSITE_CACHE' ] == '1'
+ domain.storage(
+ :file, size: '5G', allow_existing: true,
+ path: 'tails-website-cache.qcow2'
+ )
+ end
end
end
diff --git a/vagrant/definitions/tails-builder/postinstall.sh b/vagrant/definitions/tails-builder/postinstall.sh
index e77002b..7bc3d56 100755
--- a/vagrant/definitions/tails-builder/postinstall.sh
+++ b/vagrant/definitions/tails-builder/postinstall.sh
@@ -87,7 +87,6 @@ apt-get -y install \
debootstrap \
dosfstools \
dpkg-dev \
- eatmydata \
faketime \
gdisk \
gettext \
@@ -95,22 +94,15 @@ apt-get -y install \
git \
ikiwiki \
intltool \
- libfile-chdir-perl \
libfile-slurp-perl \
- libhtml-scrubber-perl \
- libhtml-template-perl \
liblist-moreutils-perl \
- libtext-multimarkdown-perl \
libtimedate-perl \
- liburi-perl libhtml-parser-perl \
- libxml-simple-perl \
- libyaml-libyaml-perl po4a \
- libyaml-perl \
+ po4a \
libyaml-syck-perl \
live-build \
lsof \
mtools \
- perlmagick \
+ libimage-magick-perl \
psmisc \
python3-gi \
rsync \
@@ -119,8 +111,7 @@ apt-get -y install \
syslinux-common \
syslinux-utils \
time \
- udisks2 \
- whois
+ udisks2
# Ensure we can use timedatectl
apt-get -y install dbus
diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails
index 10558af..1762a7f 100755
--- a/vagrant/provision/assets/build-tails
+++ b/vagrant/provision/assets/build-tails
@@ -1,4 +1,5 @@
#!/bin/sh
+# -*- mode: sh; sh-basic-offset: 8; tab-width: 8; indent-tabs-mode:t; -*-
set -e
set -x
@@ -7,6 +8,10 @@ if [ -n "${TAILS_PROXY:-}" ]; then
export http_proxy="${TAILS_PROXY}"
fi
+if [ "${TAILS_WEBSITE_CACHE}" = 1 ]; then
+ export WEBSITE_CACHE_BASEDIR=/var/cache/tails-website
+fi
+
as_root_do() {
sudo \
${RSYNC_PROXY:+RSYNC_PROXY="${RSYNC_PROXY}"} \
@@ -16,7 +21,9 @@ as_root_do() {
${no_proxy:+no_proxy="${no_proxy}"} \
${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS}"} \
${APT_SNAPSHOTS_SERIALS:+APT_SNAPSHOTS_SERIALS="${APT_SNAPSHOTS_SERIALS}"} \
- ${TAILS_MERGE_BASE_BRANCH:+TAILS_MERGE_BASE_BRANCH="${TAILS_MERGE_BASE_BRANCH}"} \
+ ${TAILS_WEBSITE_CACHE:+TAILS_WEBSITE_CACHE="${TAILS_WEBSITE_CACHE}"} \
+ ${WEBSITE_CACHE_BASEDIR:+WEBSITE_CACHE_BASEDIR="${WEBSITE_CACHE_BASEDIR}"} \
+ ${TAILS_MERGE_BASE_BRANCH:+TAILS_MERGE_BASE_BRANCH="${TAILS_MERGE_BASE_BRANCH}"} \
${GIT_COMMIT:+GIT_COMMIT="${GIT_COMMIT}"} \
${GIT_REF:+GIT_REF="${GIT_REF}"} \
${BASE_BRANCH_GIT_COMMIT:+BASE_BRANCH_GIT_COMMIT="${BASE_BRANCH_GIT_COMMIT}"} \
@@ -57,36 +64,50 @@ ntp_synchronized() {
timedatectl status | grep -qs -E '^\s*System\s+clock\s+synchronized:\s+yes$'
}
+shrink_acng_cache() {
+ local proxy_type="$1"
+ [ "${proxy_type}" = "vmproxy" ] || return 0
+ # The apt-cacher-ng cache disk is 15G, so let's ensure at most 10G
+ # of it is used so there is 5G before each build, which should be
+ # enough for any build, even if we have to download a complete set
+ # of new packages for a new Debian release.
+ as_root_do /usr/lib/apt-cacher-ng/acngtool shrink 10G -f || \
+ echo "The clean-up of apt-cacher-ng's cache failed: this is" \
+ "not fatal and most likely just means that some disk" \
+ "space could not be reclaimed -- in order to fix that" \
+ "situation you need to manually investigate " \
+ "/var/cache/apt-cacher-ng/apt-cacher-ng-log/main_*.html" >&2
+}
+
if [ "${TAILS_BUILD_FAILURE_RESCUE}" != 1 ]; then
trap cleanup EXIT
remove_build_dirs
fi
TAILS_GIT_DIR="/home/vagrant/amnesia"
-if [ ! -d "${TAILS_GIT_DIR}" ]; then
- # We use --shared as an time/space optimization, and it is safe
- # since our build process doesn't modify any objects (which would
- # fail since the host's .git directory is shared read-only).
- git clone --shared --local /amnesia.git/.git "${TAILS_GIT_DIR}"
- # When we locally Git clone the main repo over the filesystem
- # above, it will use the host's local repo as origin, but the
- # submodules will continue to use their remote repos. A problem
- # with this, beside unnecessary fetching of the network, is that
- # any unpublished commits in the host's submodule are
- # inaccessible, so if we want to build we first have to push those
- # commits to the submodules remote repo. To avoid this, and in
- # general try to make sure that the Git state in the builder is
- # the same as on the host, we just clone the submodules in the
- # same way we do the main repo.
- (
- cd "${TAILS_GIT_DIR}/submodules"
- for submodule in $(ls -1); do
- rm -rf "${submodule}"
- git clone --shared \
- "/amnesia.git/.git/modules/submodules/${submodule}"
- done
- )
-fi
+rm -rf "${TAILS_GIT_DIR}"
+# We use --shared as an time/space optimization, and it is safe
+# since our build process doesn't modify any objects (which would
+# fail since the host's .git directory is shared read-only).
+git clone --shared --local /amnesia.git/.git "${TAILS_GIT_DIR}"
+# When we locally Git clone the main repo over the filesystem
+# above, it will use the host's local repo as origin, but the
+# submodules will continue to use their remote repos. A problem
+# with this, beside unnecessary fetching of the network, is that
+# any unpublished commits in the host's submodule are
+# inaccessible, so if we want to build we first have to push those
+# commits to the submodules remote repo. To avoid this, and in
+# general try to make sure that the Git state in the builder is
+# the same as on the host, we just clone the submodules in the
+# same way we do the main repo.
+(
+ cd "${TAILS_GIT_DIR}/submodules"
+ for submodule in $(ls -1); do
+ rm -rf "${submodule}"
+ git clone --shared \
+ "/amnesia.git/.git/modules/submodules/${submodule}"
+ done
+)
cd "${TAILS_GIT_DIR}"
# Mirror the branches amnesia.git tracks on its "origin" remote as if
@@ -135,18 +156,7 @@ if [ -n "$TAILS_DATE_OFFSET" ]; then
as_root_do timedatectl set-time "$DESIRED_DATE"
fi
-if [ "${TAILS_PROXY_TYPE}" = "vmproxy" ]; then
- # The apt-cacher-ng cache disk is 15G, so let's ensure at most 10G
- # of it is used there is 5G before each build, which should be
- # enough for any build, even if we have to download a complete set
- # of new packages for a new Debian release.
- as_root_do /usr/lib/apt-cacher-ng/acngtool shrink 10G -f || \
- echo "The clean-up of apt-cacher-ng's cache failed: this is" \
- "not fatal and most likely just means that some disk" \
- "space could not be reclaimed -- in order to fix that" \
- "situation you need to manually investigate " \
- "/var/cache/apt-cacher-ng/apt-cacher-ng-log/main_*.html" >&2
-fi
+shrink_acng_cache "${TAILS_PROXY_TYPE}"
BUILD_DIR=$(mktemp -d /tmp/tails-build.XXXXXXXX)
if [ "${TAILS_RAM_BUILD}" ]; then
@@ -155,8 +165,10 @@ fi
as_root_do rsync -a "${TAILS_GIT_DIR}"/ "${BUILD_DIR}"/
cd "${BUILD_DIR}"
-as_root_do lb config --cache false
+as_root_do lb config --cache false
as_root_do lb build
+shrink_acng_cache "${TAILS_PROXY_TYPE}"
+
mv -f tails-* "${TAILS_GIT_DIR}/"
diff --git a/vagrant/provision/setup-tails-builder b/vagrant/provision/setup-tails-builder
index f0e35d5..56d1770 100755
--- a/vagrant/provision/setup-tails-builder
+++ b/vagrant/provision/setup-tails-builder
@@ -1,4 +1,5 @@
#!/bin/sh
+# -*- mode: sh; sh-basic-offset: 4; indent-tabs-mode:nil; -*-
set -e
@@ -6,21 +7,25 @@ export http_proxy="${TAILS_PROXY}"
export DEBIAN_FRONTEND=noninteractive
if [ ! -f /var/lib/vagrant_box_build_from ]; then
- echo "${GIT_REF}" > /var/lib/vagrant_box_build_from
+ echo "${GIT_REF}" > /var/lib/vagrant_box_build_from
else
- if [ "$(cat /var/lib/vagrant_box_build_from)" != "${GIT_REF}" ]; then
- echo "E: The current VM has been created to build $(cat /var/lib/vagrant_box_build_from)."
- echo "E: Please first use rake vm:destroy before trying to build again."
- exit 1
- fi
+ if [ "$(cat /var/lib/vagrant_box_build_from)" != "${GIT_REF}" ]; then
+ echo "E: The current VM has been created to build $(cat /var/lib/vagrant_box_build_from)."
+ echo "E: Please first use rake vm:destroy before trying to build again."
+ exit 1
+ fi
fi
latest_serial(){
- local archive="${1}"
- (
- cd /amnesia.git
- auto/scripts/apt-snapshots-serials get-latest --print-serials-only ${archive}
- )
+ local archive="${1}"
+ (
+ cd /amnesia.git
+ auto/scripts/apt-snapshots-serials get-latest --print-serials-only ${archive}
+ )
+}
+
+already_provisioned() {
+ [ -e /run/vagrant-already-provisioned ]
}
if [ "${TAILS_PROXY_TYPE}" = "vmproxy" ]; then
@@ -38,15 +43,38 @@ if [ "${TAILS_PROXY_TYPE}" = "vmproxy" ]; then
fi
# Install custom configuration for apt-cacher-ng and restart
- install -o root -g root -m 644 /vagrant/provision/assets/acng.conf /etc/apt-cacher-ng/acng.conf
- [ "${TAILS_OFFLINE_MODE}" = 1 ] || TAILS_OFFLINE_MODE=0
- echo "Offlinemode: ${TAILS_OFFLINE_MODE}" > /etc/apt-cacher-ng/network.conf
+ install -o root -g root -m 644 /vagrant/provision/assets/acng.conf \
+ /etc/apt-cacher-ng/acng.conf
+ rm -f /etc/apt-cacher-ng/network.conf
+ echo "Offlinemode: ${TAILS_OFFLINE_MODE:-0}" >> /etc/apt-cacher-ng/network.conf
if [ -n "${TAILS_ACNG_PROXY:-}" ]; then
echo "Proxy: ${TAILS_ACNG_PROXY}" >> /etc/apt-cacher-ng/network.conf
fi
systemctl restart apt-cacher-ng.service
fi
+if [ "$TAILS_WEBSITE_CACHE" = 1 ]; then
+ echo "I: Configuring website cache directory..."
+ if [ "${TAILS_PROXY_TYPE}" = "vmproxy" ]; then
+ WEBSITE_CACHE_DISK=/dev/vdc
+ else
+ WEBSITE_CACHE_DISK=/dev/vdb
+ fi
+ WEBSITE_CACHE_PARTITION="${WEBSITE_CACHE_DISK}1"
+ WEBSITE_CACHE_MOUNTPOINT=/var/cache/tails-website
+ mkdir -p "$WEBSITE_CACHE_MOUNTPOINT"
+ if [ ! -b "$WEBSITE_CACHE_PARTITION" ]; then
+ echo '1,,83' | sfdisk "$WEBSITE_CACHE_DISK"
+ mkfs.ext4 -m 0 "$WEBSITE_CACHE_PARTITION"
+ fi
+ if ! mountpoint -q "$WEBSITE_CACHE_MOUNTPOINT"; then
+ e2fsck -f -p "$WEBSITE_CACHE_PARTITION" || :
+ mount -o relatime,journal_checksum \
+ "$WEBSITE_CACHE_PARTITION" "$WEBSITE_CACHE_MOUNTPOINT"
+ chown -R vagrant:vagrant "$WEBSITE_CACHE_MOUNTPOINT"
+ fi
+fi
+
echo "I: Updating debian-security APT source..."
# Always set the latest serial for debian-security
stable_serial="$(grep -Po '\d{10}' /etc/apt/sources.list)"
@@ -56,43 +84,25 @@ sed -i -e "s/${stable_serial}/${security_serial}/g" /etc/apt/sources.list.d/bust
echo "I: Current APT sources are:"
cat /etc/apt/sources.list /etc/apt/sources.list.d/*
-echo "I: Upgrading system..."
-apt-key add /amnesia.git/config/chroot_sources/tails.binary.gpg
-apt-get update
-apt-get -y dist-upgrade
+if ! already_provisioned; then
+ echo "I: Upgrading system..."
+ apt-key add /amnesia.git/config/chroot_sources/tails.binary.gpg
+ apt-get update
+ apt-get -y dist-upgrade
-echo "I: Installing build script..."
-install -o root -g root -m 755 /vagrant/provision/assets/build-tails /usr/local/bin
+ echo "I: Cleaning up..."
+ apt-get -y autoremove
+ apt-get -y clean
-echo "I: Forcing live-build to use the mirrors configured in auto/config..."
+ echo "I: Configuring Git..."
+ # Necessary so that vagrant can merge the base branch
+ git config --global user.name vagrant
+ git config --global user.email vagrant@tailsbuilder
-disable_live_build_conf()
-{
- local var="$1"
+ echo "I: Installing build script..."
+ install -o root -g root -m 755 /vagrant/provision/assets/build-tails \
+ /usr/local/bin
- [ -e /etc/live/build.conf ] || return 0
- sed -e "/^[[:space:]]*$var=/d" -i /etc/live/build.conf
-}
-
-for prefix in MIRROR PARENT_MIRROR ; do
- for target in BOOTSTRAP BINARY CHROOT ; do
- for archive in '' BACKPORTS SECURITY UPDATES VOLATILE ; do
- if [ -z "$archive" ] ; then
- archive_suffix=''
- else
- archive_suffix="_${archive}"
- fi
- var="LB_${prefix}_${target}${archive_suffix}"
- disable_live_build_conf "$var"
- done
- done
-done
-
-echo "I: Cleaning up..."
-apt-get -y autoremove
-apt-get -y clean
-
-echo "I: Configuring Git..."
-# Necessary so that vagrant can merge the base branch
-git config --global user.name vagrant
-git config --global user.email vagrant@tailsbuilder
+ echo "I: Recording the fact this box was provisioned during this boot"
+ touch /run/vagrant-already-provisioned
+fi
diff --git a/wiki/src/contribute/build.mdwn b/wiki/src/contribute/build.mdwn
index 5ffbf71..6917b9a 100644
--- a/wiki/src/contribute/build.mdwn
+++ b/wiki/src/contribute/build.mdwn
@@ -72,13 +72,6 @@ image before building it.
# Known issues and workarounds
-* If Vagrant fails to start the Tails builder VM with an error similar
- to:
-
- Virtio-9p Failed to initialize fs-driver with id:fsdev-fs0
-
- … then see [[!tails_ticket 11411]].
-
* If Vagrant fails to start the Tails builder VM with:
Initialization parameters must be an attributes hash, got NilClass nil (ArgumentError)
@@ -120,7 +113,7 @@ You can customize the build system using two environment variables:
For example, you can speed up the build by setting:
- export TAILS_BUILD_OPTIONS="ram fastcomp"
+ export TAILS_BUILD_OPTIONS="ram fastcomp cachewebsite"
This will force the build to happen in RAM and SquashFS compression
will be done using faster, though less efficient size-wise, settings.
@@ -209,6 +202,10 @@ affect reproducibility of the ISO image:
* **rescue**: implies **keeprunning** and will also not clean up the
build directory, which is useful for investigating build failures.
+ * **cachewebsite**: enable caching of the built website.
+ The cache is keyed on the input parameters that primarily determine
+ the output of the website build.
+
## HTTP proxy settings
Building Tails requires downloading a little bit more than 2 GiB of
@@ -240,7 +237,8 @@ the following build options:
if a local HTTP proxy is set.
* **vmproxy+extproxy**: use the local proxy configured in the virtual
- machine but make it use the local HTTP proxy.
+ machine but make it use the external HTTP proxy configured through
+ the `http_proxy` environment variable.
* **noproxy**: do not use any HTTP proxy.
diff --git a/wiki/src/contribute/build/manually.mdwn b/wiki/src/contribute/build/manually.mdwn
deleted file mode 100644
index d34ab22..0000000
--- a/wiki/src/contribute/build/manually.mdwn
+++ /dev/null
@@ -1,113 +0,0 @@
-[[!meta title="Building a Tails image manually"]]
-
-[[!toc levels=3]]
-
-<div class="note">
-This page is totally outdated: all active developers use Vagrant,
-which is the only supported way to build Tails at the moment.
-</div>
-
-
-In order to build Tails manually, you need a running Debian Jessie
-system and some [backports](http://backports.debian.org/). Anything
-else will fail.
-
-# Dependencies
-
-The following Debian packages need to be installed:
-
-* our `live-build` 2.x package, adapted for Wheezy and later. Its version is
- something like *3.0.5+really+is+2.0.12-0.tails2*. One can install it
- from:
-
- deb http://deb.tails.boum.org/ builder-jessie main
-
- This APT repository's signing key can be found:
-
- - in our Git tree (that you have cloned already, right?):
- `config/chroot_sources/tails.chroot.gpg`
- - at <http://deb.tails.boum.org/key.asc>
- - on the keyservers.
-
- It is certified by the
- [[Tails signing key|doc/about/openpgp_keys#signing]], and its
- fingerprint is:
-
- 221F 9A3C 6FA3 E09E 182E 060B C798 8EA7 A358 D82E
-
- You should pin that repository, so that live-build isn't upgraded to
- the version of jessie.
-
- #/etc/apt/preferences.d/00-builder-jessie-pinning
- Package: *
- Pin: release o=Debian,a=stable
- Pin-Priority: 700
-
- Package: *
- Pin: origin deb.tails.boum.org
- Pin-Priority: 800
-
-
- Then install these dependencies from Jessie:
-
- apt install \
- dpkg-dev \
- gettext \
- intltool \
- libfile-slurp-perl \
- liblist-moreutils-perl \
- libyaml-libyaml-perl \
- libyaml-perl \
- libyaml-syck-perl \
- perlmagick \
- po4a \
- syslinux-utils \
- time \
- whois
-
- And install these dependencies from jessie-backports (please verify
- manually that the following command actually does install the
- expected versions):
-
- apt install \
- debootstrap/jessie-backports \
- ikiwiki/jessie-backports
-
-# Configure live-build
-
-Remove any line matching `/^\[[:space:]]*LB.*MIRROR.*=/` in
-`/etc/live/build.conf`.
-
-# Build process
-
-Every build command must be run as `root`, at the root of a clone of the
-[[`tails` repository|git]]. A local HTTP proxy is required.
-
-In short, a build shall be done using:
-
- lb clean --all && lb config --apt-http-proxy http://localhost:3142 && lb build
-
-Running `lb config` or `lb build` in an environment that wasn't full
-cleaned first is not supported.
-
-## Customize the build process if needed
-
-If you need to set custom build settings that are specific to your
-local environment, such as a custom Debian mirror or APT proxy, you
-probably want to configure live-build a bit.
-
-The most common customizations are documented on this wiki:
-
-* to avoid compressing the SquashFS using XZ (efficient, but very
- slow), `export MKSQUASHFS_OPTIONS='-comp gzip'` in your
- build environment;
-* to avoid downloading lots of Debian packages during every build, you
- can use [[!debpts apt-cacher-ng]]; however, the build system
- constantly switches APT sources for our
- [[APT repositories|contribute/APT_repository]], so some custom
- configuration is needed to make `apt-cacher-ng` useful: see the
- bits about `apt-cacher-ng` in
- [[!tails_gitweb vagrant/provision/assets/build-tails]].
-
-More documentation about this can be found in the [Debian Live
-Manual](http://live.debian.net/manual-2.x/html/live-manual.en.html).