summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorintrigeri <intrigeri@boum.org>2016-05-23 08:57:16 +0000
committerintrigeri <intrigeri@boum.org>2016-05-23 08:57:16 +0000
commitdc316ad2fe3e3c0fd478508e0423a4a8df300f4d (patch)
tree1c98fe07a444dcebeffc18fc9fcbe1c4a3eb03d9
parentd2276416e08f5a24226080c74ac90c6b81913a4d (diff)
parentf5cbf201407d88e864c63412d0b2766dcd6c6c69 (diff)
Merge branch 'devel' into test/11457-I-close-the-browser-is-fragiletest/11457-I-close-the-browser-is-fragile
-rw-r--r--.gitignore1
-rwxr-xr-xauto/build24
-rwxr-xr-xauto/config35
-rwxr-xr-xauto/scripts/apt-mirror75
-rwxr-xr-xauto/scripts/apt-snapshots-serials67
-rwxr-xr-xauto/scripts/generate-build-manifest13
-rwxr-xr-xauto/scripts/tails-custom-apt-sources33
-rwxr-xr-xauto/scripts/update-acng-config84
-rw-r--r--auto/scripts/utils.sh35
-rwxr-xr-xbin/freeze-apt-snapshots4
-rwxr-xr-xbin/tag-apt-snapshots54
-rw-r--r--config/APT_snapshots.d/.placeholder0
-rw-r--r--config/APT_snapshots.d/tails/serial1
-rwxr-xr-xconfig/binary_local-hooks/40-include_syslinux_in_ISO_filesystem4
-rw-r--r--config/chroot_apt/preferences126
-rwxr-xr-xconfig/chroot_local-hooks/19-install-tor-browser-AppArmor-profile6
-rwxr-xr-xconfig/chroot_local-hooks/51-module-assistant40
-rwxr-xr-xconfig/chroot_local-hooks/98-remove_unwanted_packages9
-rwxr-xr-xconfig/chroot_local-includes/lib/live/config/1500-reconfigure-APT68
-rw-r--r--config/chroot_local-includes/usr/share/applications/tails-reboot.desktop.in9
-rw-r--r--config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop.in9
-rw-r--r--config/chroot_local-packageslists/tails-common.list1
l---------config/chroot_sources/experimental.binary1
-rw-r--r--config/chroot_sources/experimental.chroot1
-rw-r--r--config/chroot_sources/sid.chroot2
-rw-r--r--config/chroot_sources/testing.chroot2
-rw-r--r--features/apt.feature5
-rw-r--r--features/build.feature210
-rw-r--r--features/i2p.feature2
-rw-r--r--features/step_definitions/build.rb41
-rw-r--r--features/support/env.rb36
-rw-r--r--features/unsafe_browser.feature12
-rw-r--r--po/POTFILES.in2
-rw-r--r--vagrant/provision/assets/acng.conf1
-rwxr-xr-xvagrant/provision/assets/build-tails5
-rw-r--r--wiki/src/blueprint/delete_obsolete_Git_branches.mdwn3
-rw-r--r--wiki/src/contribute.mdwn4
-rw-r--r--wiki/src/contribute/APT_repository.mdwn496
-rw-r--r--wiki/src/contribute/APT_repository/custom.mdwn366
-rw-r--r--wiki/src/contribute/APT_repository/tagged_snapshots.mdwn154
-rw-r--r--wiki/src/contribute/APT_repository/time-based_snapshots.mdwn431
-rw-r--r--wiki/src/contribute/build.mdwn12
-rw-r--r--wiki/src/contribute/build/custom_mirror.mdwn31
-rw-r--r--wiki/src/contribute/build/squid-deb-proxy.mdwn96
-rw-r--r--wiki/src/contribute/build/squid-deb-proxy/allowed-networks-src.acl11
-rw-r--r--wiki/src/contribute/build/squid-deb-proxy/mirror-dstdomain.acl18
-rwxr-xr-xwiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy174
-rw-r--r--wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy.conf124
-rw-r--r--wiki/src/contribute/design/I2P.mdwn2
-rw-r--r--wiki/src/contribute/release_process.mdwn145
-rw-r--r--wiki/src/contribute/release_schedule.mdwn3
-rw-r--r--wiki/src/contribute/working_together/roles/sysadmins.mdwn28
52 files changed, 2030 insertions, 1086 deletions
diff --git a/.gitignore b/.gitignore
index d290487..2aca529 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
*.po~
*.pot~
*.swp
+/*.apt-sources
/*.build-manifest
/*.buildlog
/*.img
diff --git a/auto/build b/auto/build
index d0bf0cc..43b0f56 100755
--- a/auto/build
+++ b/auto/build
@@ -77,7 +77,15 @@ chmod -R go+rX config/chroot_sources
# build the image
# we need /debootstrap/deburis to build a manifest of used packages:
-export DEBOOTSTRAP_OPTIONS='--keep-debootstrap-dir'
+DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keep-debootstrap-dir"
+
+# use our own APT repository's key:
+DEBOOTSTRAP_GNUPG_HOMEDIR=$(mktemp -d)
+gpg --homedir "$DEBOOTSTRAP_GNUPG_HOMEDIR" \
+ --import config/chroot_sources/tails.chroot.gpg
+DEBOOTSTRAP_OPTIONS="$DEBOOTSTRAP_OPTIONS --keyring=$DEBOOTSTRAP_GNUPG_HOMEDIR/pubring.gpg"
+
+export DEBOOTSTRAP_OPTIONS
: ${MKSQUASHFS_OPTIONS:='-comp xz -Xbcj x86 -b 1024K -Xdict-size 1024K'}
MKSQUASHFS_OPTIONS="${MKSQUASHFS_OPTIONS} -wildcards -ef chroot/usr/share/amnesia/build/mksquashfs-excludes"
@@ -165,7 +173,8 @@ case "$LB_BINARY_IMAGES" in
;;
esac
BUILD_DEST_FILENAME="${BUILD_BASENAME}.${BUILD_FILENAME_EXT}"
-BUILD_MANIFEST="${BUILD_DEST_FILENAME}.list"
+BUILD_MANIFEST="${BUILD_DEST_FILENAME}.build-manifest"
+BUILD_APT_SOURCES="${BUILD_DEST_FILENAME}.apt-sources"
BUILD_PACKAGES="${BUILD_DEST_FILENAME}.packages"
BUILD_LOG="${BUILD_DEST_FILENAME}.buildlog"
BUILD_START_FILENAME="${BUILD_DEST_FILENAME}.start.timestamp"
@@ -177,6 +186,15 @@ trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
exec 2> >(tee -a "$BUILD_LOG" >&2)
trap "kill -9 $! 2>/dev/null" EXIT HUP INT QUIT TERM
+(
+ echo "Mirrors:"
+ apt-mirror debian
+ apt-mirror debian-security
+ apt-mirror torproject
+ echo "Additional sources:"
+ cat config/chroot_sources/*.chroot
+) > "$BUILD_APT_SOURCES"
+
echo "Building $LB_BINARY_IMAGES image ${BUILD_BASENAME}..."
set -o pipefail
[ -z "$JENKINS_URL" ] || date --utc '+%s' > "$BUILD_START_FILENAME"
@@ -199,7 +217,7 @@ if [ -e "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" ]; then
echo "Renaming generated files..."
mv -i "${BUILD_FILENAME}.${BUILD_FILENAME_EXT}" "${BUILD_DEST_FILENAME}"
mv -i binary.packages "${BUILD_PACKAGES}"
- generate-build-manifest chroot/debootstrap "${BUILD_DEST_FILENAME}.build-manifest"
+ generate-build-manifest chroot/debootstrap "${BUILD_MANIFEST}"
else
fatal "lb build failed ($?)."
fi
diff --git a/auto/config b/auto/config
index 95c5517..ec9bf88 100755
--- a/auto/config
+++ b/auto/config
@@ -16,12 +16,39 @@ fi
export LB_BOOTSTRAP_INCLUDE='eatmydata'
+# sanity checks
+if grep -qs -E '^Pin:\s+release\s+.*a=' config/chroot_apt/preferences ; then
+ echo "Found unsupported a= syntax in config/chroot_apt/preferences,"
+ echo "use n= instead. Exiting."
+ exit 1
+fi
+if grep -qs -E '^Pin:\s+release\s+.*o=Debian Backports' \
+ config/chroot_apt/preferences ; then
+ echo "Found unsupported 'o=Debian Backports' syntax,"
+ echo "in config/chroot_apt/preferences. Use o=Debian instead. Exiting."
+ exit 1
+fi
+
# init variables
RUN_LB_CONFIG="lb config noauto"
# init config/ with defaults for the target distribution
$RUN_LB_CONFIG --distribution jessie ${@}
+# set up everything for time-based snapshots:
+apt-snapshots-serials prepare-build
+
+DEBIAN_MIRROR="$(apt-mirror debian)"
+DEBIAN_SECURITY_MIRROR="$(apt-mirror debian-security)"
+TORPROJECT_MIRROR="$(apt-mirror torproject)"
+
+perl -pi \
+ -E \
+ "s|^(deb(?:-src)?\s+)https?://ftp[.]us[.]debian[.]org/debian/?(\s+)|\$1$DEBIAN_MIRROR\$2| ; \
+ s|^(deb(?:-src)?\s+)https?://deb[.]torproject[.]org/torproject[.]org/?(\s+)|\$1$TORPROJECT_MIRROR\$2|" \
+ config/chroot_sources/*.chroot \
+ || exit 1
+
# set Amnesia's general options
$RUN_LB_CONFIG \
--verbose \
@@ -39,9 +66,11 @@ $RUN_LB_CONFIG \
--iso-publisher="https://tails.boum.org/" \
--iso-volume="TAILS ${AMNESIA_FULL_VERSION}" \
--memtest none \
- --mirror-binary "http://ftp.us.debian.org/debian/" \
- --mirror-bootstrap "http://ftp.us.debian.org/debian/" \
- --mirror-chroot "http://ftp.us.debian.org/debian/" \
+ --mirror-binary "$DEBIAN_MIRROR" \
+ --mirror-bootstrap "$DEBIAN_MIRROR" \
+ --mirror-chroot "$DEBIAN_MIRROR" \
+ --mirror-binary-security "$DEBIAN_SECURITY_MIRROR" \
+ --mirror-chroot-security "$DEBIAN_SECURITY_MIRROR" \
--packages-lists="standard" \
--tasks="standard" \
--linux-packages="linux-image-3.16.0-4" \
diff --git a/auto/scripts/apt-mirror b/auto/scripts/apt-mirror
new file mode 100755
index 0000000..4d4ccad
--- /dev/null
+++ b/auto/scripts/apt-mirror
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -e
+set -u
+
+. "$(dirname $0)/utils.sh"
+
+ARCHIVE="$1"
+
+output_tagged_snapshot() {
+ local archive="$1"
+ local tag="$2"
+ echo "http://tagged.snapshots.deb.tails.boum.org/$tag/$archive"
+}
+
+output_time_based_snapshot() {
+ local archive="$1"
+ local serial="$2"
+ echo "http://time-based.snapshots.deb.tails.boum.org/$archive/$serial"
+}
+
+### Sanity checks
+
+[ -n "$ARCHIVE" ] || exit 1
+
+### Main
+
+SERIAL=$(cat "config/APT_snapshots.d/$ARCHIVE/serial")
+RESOLVED_SERIAL=$(cat "tmp/APT_snapshots.d/$ARCHIVE/serial")
+
+if [ "$(base_branch)" = stable ]; then
+ if version_was_released "$(version_in_changelog)"; then
+ [ "$(current_branch)" = stable ] \
+ || fatal "Not building from stable, but last version in changelog" \
+ "was released"
+ output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
+ else
+ version_was_released "$(previous_version_in_changelog)" \
+ || fatal "None of the two last version in changelog were released"
+ case "$ARCHIVE" in
+ debian-security)
+ [ "$SERIAL" = latest ] \
+ || fatal "APT snapshots are frozen for debian-security, which" \
+ "should not happen on a branch based on stable"
+ output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
+ ;;
+ *)
+ if [ "$SERIAL" = latest ]; then
+ # In this case, "latest" means "do what I mean", that is stick
+ # to previous release's tagged snapshot
+ output_tagged_snapshot "$ARCHIVE" "$(previous_version_in_changelog)"
+ else
+ output_time_based_snapshot "$ARCHIVE" "$SERIAL"
+ fi
+ esac
+ fi
+elif [ "$(base_branch)" = testing ]; then
+ if version_was_released "$(version_in_changelog)"; then
+ [ "$(current_branch)" = testing ] \
+ || fatal "Not building from testing, but last version in changelog" \
+ "was released"
+ [ "$ARCHIVE" = debian-security ] || [ "$SERIAL" != latest ] \
+ || fatal "APT snapshots for $ARCHIVE are not frozen, which should" \
+ "not happen on a tagged testing branch"
+ output_tagged_snapshot "$ARCHIVE" "$(version_in_changelog)"
+ else
+ output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
+ fi
+else
+ if [ "$(base_branch)" = devel ] && [ "$SERIAL" != latest ]; then
+ fatal "APT snapshots are frozen, which should not happen on a branch" \
+ "based on the devel one"
+ fi
+ output_time_based_snapshot "$ARCHIVE" "$RESOLVED_SERIAL"
+fi
diff --git a/auto/scripts/apt-snapshots-serials b/auto/scripts/apt-snapshots-serials
new file mode 100755
index 0000000..faee29f
--- /dev/null
+++ b/auto/scripts/apt-snapshots-serials
@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e
+set -u
+
+set -o pipefail
+
+BASE_URL=http://time-based.snapshots.deb.tails.boum.org/
+CONFIG=config/APT_snapshots.d
+ORIGINS=$(cd $CONFIG; ls -d *)
+
+get_latest_serial() {
+ origin=$1
+ wget -q $BASE_URL/$origin/project/trace/$origin -O - \
+ | awk -F': ' '/^Archive serial: / {print $2}'
+}
+
+action="$1"
+case "$action" in
+ get-latest)
+ for origin in $ORIGINS; do
+ online=$(get_latest_serial $origin)
+ echo "$origin: $online"
+ done
+ ;;
+ freeze)
+ for origin in $ORIGINS; do
+ serial_file="$CONFIG/$origin/serial"
+ git=$(cat $serial_file)
+ case "$origin" in
+ debian-security)
+ new=latest
+ ;;
+ *)
+ new=$(get_latest_serial $origin)
+ esac
+ printf "Origin $origin:\n old: $git\n new: $new\n"
+ echo $new > $serial_file
+ done
+ printf "\nAll files ($CONFIG/*/serial) have been updated with new serials\n" >&2
+ ;;
+ thaw)
+ for origin in $ORIGINS; do
+ serial_file="$CONFIG/$origin/serial"
+ git=$(cat $serial_file)
+ printf "Origin $origin:\n old: $git\n new: latest\n"
+ echo 'latest' > $serial_file
+ done
+ ;;
+ prepare-build)
+ rm -rf tmp/APT_snapshots.d
+ mkdir -p tmp
+ cp -r config/APT_snapshots.d tmp/
+ $0 get-latest > tmp/cached_APT_snapshots_serials
+ for origin_dir in tmp/APT_snapshots.d/*; do
+ origin=$(basename $origin_dir)
+ if grep -qs '^latest$' $origin_dir/serial; then
+ awk -F': ' "/^$origin: / {print \$2}" \
+ tmp/cached_APT_snapshots_serials \
+ > $origin_dir/serial
+ fi
+ done
+ ;;
+ *)
+ printf "unknown action ($action), use either 'get-latest', 'prepare-build', 'freeze' or 'thaw'\n" >&2
+ exit 1
+ ;;
+esac
diff --git a/auto/scripts/generate-build-manifest b/auto/scripts/generate-build-manifest
index 20e2a5b..8798281 100755
--- a/auto/scripts/generate-build-manifest
+++ b/auto/scripts/generate-build-manifest
@@ -33,6 +33,12 @@ if (! -d $debootstrap) {
usage;
}
+if (-f "$debootstrap/unknown") {
+ print "E: actions unsupported by the apt-get wrapper were logged ",
+ "in $debootstrap/unknown. Aborting.";
+ exit 1;
+}
+
my $extra_packages_file = 'config/build-manifest-extra-packages.yml';
my $extra_packages;
@@ -82,11 +88,12 @@ foreach my $type (keys %package_type) {
}
}
-### Extract list of (origin, reference) from the build configuration:
+### Extract list of (origin, reference) from the build configuration
+### (the resolved serials, stored under tmp by "apt-snapshots-serials prepare-build"):
my %origin_reference;
-while (my $origin_dir = glob('config/APT_snapshots.d/*')) {
+while (my $origin_dir = glob('tmp/APT_snapshots.d/*')) {
my $origin_name = $origin_dir;
- $origin_name =~ s{\A config/APT_snapshots[.]d/}{}xms;
+ $origin_name =~ s{\A tmp/APT_snapshots[.]d/}{}xms;
$origin_reference{$origin_name} = read_file("$origin_dir/serial");
chomp $origin_reference{$origin_name};
$data->{origin_references}->{ $origin_name }->{reference} = $origin_reference{ $origin_name } || 'unknown';
diff --git a/auto/scripts/tails-custom-apt-sources b/auto/scripts/tails-custom-apt-sources
index e172926..76e5214 100755
--- a/auto/scripts/tails-custom-apt-sources
+++ b/auto/scripts/tails-custom-apt-sources
@@ -1,33 +1,14 @@
#!/bin/bash
set -e
+set -u
+
+. "$(dirname $0)/utils.sh"
APT_MIRROR_URL="http://deb.tails.boum.org/"
DEFAULT_COMPONENTS="main"
BASE_BRANCHES="stable testing devel feature/jessie"
-fatal() {
- echo "$*" >&2
- exit 1
-}
-
-git_tag_exists() {
- local tag="$1"
-
- test -n "$(git tag -l "$tag")"
-}
-
-version_was_released() {
- local version="$1"
-
- version="$(echo "$version" | tr '~' '-')"
- git_tag_exists "$version"
-}
-
-version_in_changelog() {
- dpkg-parsechangelog | awk '/^Version: / { print $2 }'
-}
-
output_apt_binary_source() {
local suite="$1"
local components="${2:-$DEFAULT_COMPONENTS}"
@@ -41,10 +22,6 @@ output_overlay_apt_binary_sources() {
done
}
-current_branch() {
- git branch | awk '/^\* / { print $2 }'
-}
-
on_base_branch() {
local current_branch=$(current_branch)
@@ -57,10 +34,6 @@ on_base_branch() {
return 1
}
-base_branch() {
- cat config/base_branch | head -n1
-}
-
branch_name_to_suite() {
local branch="$1"
diff --git a/auto/scripts/update-acng-config b/auto/scripts/update-acng-config
new file mode 100755
index 0000000..5e5695e
--- /dev/null
+++ b/auto/scripts/update-acng-config
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+list_origins () {
+ (
+ cd config/APT_snapshots.d/
+ ls --color=never -1 | grep -v --line-regexp '\.placeholder'
+ )
+}
+
+print_tagged_snapshots_pool_url () {
+ origin="$1"
+ version="$2"
+ printf \
+ 'http://tagged.snapshots.deb.tails.boum.org/%s/%s/pool/\n' \
+ "$version" "$origin"
+}
+
+conf=/etc/apt-cacher-ng/tails-snapshots.conf
+for origin in $(list_origins) ; do
+ [ "$origin" != .placeholder ] || continue
+ origin_without_dashes=$(echo "$origin" | sed -e 's,-,,g')
+ echo "Remap-tailssnapshots${origin_without_dashes}pool: file:tails-time-based-snapshots-$origin-pool.list file:tails-tagged-snapshots-$origin-pool.list"
+done > "$conf"
+chmod 644 "$conf"
+
+# Generate .list files for time-based snapshots
+for origin in $(list_origins) ; do
+ list="/etc/apt-cacher-ng/tails-time-based-snapshots-$origin-pool.list"
+ current_year=$(date '+%Y')
+ for year in $(seq $(($current_year - 1)) $(($current_year + 1))) ; do
+ for month in $(seq 1 12); do
+ # We need the config file to contain _at least_ everything
+ # that can possibly exists, and we don't care if it has some extra
+ # lines, so to simplify we do as if each month had 31 days.
+ for day in $(seq 1 31) ; do
+ for n in $(seq 1 4) ; do
+ printf 'http://time-based.snapshots.deb.tails.boum.org/%s/%04u%02u%02u%02u/pool/\n' \
+ "$origin" "$year" "$month" "$day" "$n"
+ done
+ done
+ done
+ done \
+ > "$list"
+ chmod 644 "$list"
+done
+
+# Generate .list files for tagged snapshots
+for origin in $(list_origins) ; do
+ list="/etc/apt-cacher-ng/tails-tagged-snapshots-$origin-pool.list"
+ # We need the config file to contain _at least_ everything
+ # that can possibly exists, and we don't care if it has some extra
+ # lines, so here we try to build the smallest possible superset of
+ # all realistic Tails version numbers; it could certainly be a tiny
+ # bit smaller, at the cost of more assumptions (=> more risk of not
+ # including some version number we'll end up using) or of more
+ # code complexity (=> higher maintenance cost).
+ #
+ # XXX: Stretch: bump the end of the range of major versions
+ for major in $(seq 2 3) ; do
+ for minor in $(seq 0 32); do
+ for suffix in "" alpha beta rc ; do
+ for suffix_n in "" $(seq 1 8); do
+ if [ -z "$suffix" ]; then
+ version="${major}.${minor}"
+ elif [ -z "$suffix_n" ]; then
+ version="${major}.${minor}-${suffix}"
+ else
+ version="${major}.${minor}-${suffix}${suffix_n}"
+ fi
+ print_tagged_snapshots_pool_url "$origin" "$version"
+ done
+ done
+ for emergency in $(seq 1 4) ; do
+ version="${major}.${minor}.${emergency}"
+ print_tagged_snapshots_pool_url "$origin" "$version"
+ done
+ done
+ done > "$list"
+ chmod 644 "$list"
+done
diff --git a/auto/scripts/utils.sh b/auto/scripts/utils.sh
new file mode 100644
index 0000000..4f95c22
--- /dev/null
+++ b/auto/scripts/utils.sh
@@ -0,0 +1,35 @@
+# This library is meant to be used in bash, with "set -e" and "set -u".
+
+current_branch() {
+ git branch | awk '/^\* / { print $2 }'
+}
+
+base_branch() {
+ cat config/base_branch | head -n1
+}
+
+fatal() {
+ echo "$*" >&2
+ exit 1
+}
+
+git_tag_exists() {
+ local tag="$1"
+
+ test -n "$(git tag -l "$tag")"
+}
+
+version_was_released() {
+ local version="$1"
+
+ version="$(echo "$version" | tr '~' '-')"
+ git_tag_exists "$version"
+}
+
+version_in_changelog() {
+ dpkg-parsechangelog | awk '/^Version: / { print $2 }'
+}
+
+previous_version_in_changelog() {
+ dpkg-parsechangelog --offset 1 --count 1 | awk '/^Version: / { print $2 }'
+}
diff --git a/bin/freeze-apt-snapshots b/bin/freeze-apt-snapshots
new file mode 100755
index 0000000..2970ea5
--- /dev/null
+++ b/bin/freeze-apt-snapshots
@@ -0,0 +1,4 @@
+# for each upstream APT repository:
+# tell time-based snapshots infra to keep last snapshot
+# -> returns us the corresponding serial
+# write serial of the last snapshot > config/APT_snapshots.d/$origin/serial
diff --git a/bin/tag-apt-snapshots b/bin/tag-apt-snapshots
new file mode 100755
index 0000000..c67a780
--- /dev/null
+++ b/bin/tag-apt-snapshots
@@ -0,0 +1,54 @@
+#!/bin/sh
+
+set -e
+set -u
+
+TIME_BASED_SNAPSHOTS_HOST='apt.lizard'
+TIME_BASED_SNAPSHOTS_USER='reprepro-time-based-snapshots'
+TIME_BASED_SNAPSHOTS_USER_AT_HOST="${TIME_BASED_SNAPSHOTS_USER}@${TIME_BASED_SNAPSHOTS_HOST}"
+
+fail_with_usage() {
+ echo "$(basename $0) BUILD_MANIFEST TAG" >&2
+}
+
+[ $# -eq 2 ] || fail_with_usage
+
+BUILD_MANIFEST="$1"
+TAG="$2"
+
+[ -r "$BUILD_MANIFEST" ] || fail_with_usage
+[ -n "$TAG" ] || fail_with_usage
+
+echo "I: Preparing a workspace on ${TIME_BASED_SNAPSHOTS_HOST}"
+ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" install -d '$HOME'/tmp
+REMOTE_BUILD_MANIFEST=$(ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
+ mktemp --tmpdir='$HOME'/tmp)
+REMOTE_DEST_DIR=$(ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
+ mktemp -d --tmpdir='$HOME'/tmp)
+
+echo "I: Sending build manifest to ${TIME_BASED_SNAPSHOTS_HOST}"
+scp "$BUILD_MANIFEST" \
+ "${TIME_BASED_SNAPSHOTS_USER_AT_HOST}:${REMOTE_BUILD_MANIFEST}"
+
+echo "I: Preparing reprepro configuration"
+ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
+ tails-prepare-tagged-apt-snapshot-import \
+ "$REMOTE_BUILD_MANIFEST" \
+ "$REMOTE_DEST_DIR"
+
+echo "I: Pulling packages from the time-based snapshots into the tagged ones"
+for archive in $(ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
+ ls "$REMOTE_DEST_DIR") ; do
+ echo "I: - $archive"
+ ssh "$TIME_BASED_SNAPSHOTS_USER_AT_HOST" \
+ reprepro --basedir "${REMOTE_DEST_DIR}/${archive}" update
+done
+
+echo "I: Publishing the tagged APT snapshot"
+ssh "${TIME_BASED_SNAPSHOTS_USER}@${TIME_BASED_SNAPSHOTS_HOST}" \
+ sudo -n /usr/local/sbin/tails-publish-tagged-apt-snapshot \
+ "$REMOTE_DEST_DIR" "$TAG"
+
+echo "I: Cleaning up"
+ssh "${TIME_BASED_SNAPSHOTS_USER}@${TIME_BASED_SNAPSHOTS_HOST}" \
+ rm "$REMOTE_BUILD_MANIFEST"
diff --git a/config/APT_snapshots.d/.placeholder b/config/APT_snapshots.d/.placeholder
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/config/APT_snapshots.d/.placeholder
diff --git a/config/APT_snapshots.d/tails/serial b/config/APT_snapshots.d/tails/serial
deleted file mode 100644
index a0f9a4b..0000000
--- a/config/APT_snapshots.d/tails/serial
+++ /dev/null
@@ -1 +0,0 @@
-latest
diff --git a/config/binary_local-hooks/40-include_syslinux_in_ISO_filesystem b/config/binary_local-hooks/40-include_syslinux_in_ISO_filesystem
index 37ec461..b33f22c 100755
--- a/config/binary_local-hooks/40-include_syslinux_in_ISO_filesystem
+++ b/config/binary_local-hooks/40-include_syslinux_in_ISO_filesystem
@@ -43,6 +43,10 @@ cp "$CHROOT_SYSLINUX_MBR" "$BINARY_MBR_DIR/mbr.bin"
cat chroot/etc/apt/sources.list chroot/etc/apt/sources.list.d/*.list \
| grep --extended-regexp --line-regexp --invert-match \
'deb\s+file:/root/local-packages\s+\./' \
+ | grep --extended-regexp --invert-match \
+ '^deb\s+http://tagged\.snapshots\.deb\.tails\.boum.org/[^/]+/torproject/' \
+ | grep --extended-regexp --invert-match \
+ '^deb\s+http://time-based\.snapshots\.deb\.tails\.boum.org/torproject/' \
| sed --regexp-extended -e 's,^deb(\s+),deb-src\1,' \
> "$CHROOT_TEMP_APT_SOURCES"
Chroot chroot apt-get --yes update
diff --git a/config/chroot_apt/preferences b/config/chroot_apt/preferences
index d460291..a6c0777 100644
--- a/config/chroot_apt/preferences
+++ b/config/chroot_apt/preferences
@@ -1,9 +1,9 @@
Package: apparmor-profiles-extra
-Pin: release o=Debian Backports,n=jessie-backports
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
Package: b43-fwcutter
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: electrum
@@ -11,127 +11,95 @@ Pin: release o=Debian,n=stretch
Pin-Priority: 999
Package: firmware-amd-graphics
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-atheros
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-b43-installer
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-b43legacy-installer
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-brcm80211
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-ipw2x00
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-iwlwifi
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-libertas
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-linux
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-linux-free
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-linux-nonfree
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-misc-nonfree
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-realtek
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
Package: firmware-zd1211
-Pin: release o=Debian,a=unstable
+Pin: release o=Debian,n=sid
Pin-Priority: 999
-Package: libnet-dbus-perl
-Pin: release o=Debian Backports,n=jessie-backports
-Pin-Priority: 999
-
-Package: linux-base
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-compiler-gcc-4.8-x86
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-headers-586
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-headers-686-pae
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-headers-amd64
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-headers-3.16.0-4-common
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-headers-3.16.0-4-586
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
-
-Package: linux-headers-3.16.0-4-686-pae
-Pin: release o=Debian,n=jessie
+Package: grub-common
+Pin: origin deb.tails.boum.org
Pin-Priority: 999
-Package: linux-headers-3.16.0-4-amd64
-Pin: release o=Debian,n=jessie
+Package: grub-efi-ia32
+Pin: origin deb.tails.boum.org
Pin-Priority: 999
-Package: linux-image-586
-Pin: release o=Debian,n=jessie
+Package: grub-efi-ia32-bin
+Pin: origin deb.tails.boum.org
Pin-Priority: 999
-Package: linux-image-686-pae
-Pin: release o=Debian,n=jessie
+Package: grub2-common
+Pin: origin deb.tails.boum.org
Pin-Priority: 999
-Package: linux-image-amd64
-Pin: release o=Debian,n=jessie
+Package: libdvd-pkg
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
-Package: linux-image-3.16.0-4-586
-Pin: release o=Debian,n=jessie
+Package: libnet-dbus-perl
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
-Package: linux-image-3.16.0-4-686-pae
-Pin: release o=Debian,n=jessie
+Package: monkeysphere
+Pin: release o=Debian,n=stretch
Pin-Priority: 999
-Package: linux-image-3.16.0-4-amd64
-Pin: release o=Debian,n=jessie
-Pin-Priority: 999
+Package: lib*-mesa*
+Pin: release o=Debian,n=jessie-backports
+Pin-Priority: 1006
-Package: linux-kbuild-3.16
-Pin: release o=Debian,n=jessie
+Package: libdrm*
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
Package: monkeysphere
@@ -142,28 +110,36 @@ Package: obfs4proxy
Pin: release o=TorProject,n=obfs4proxy
Pin-Priority: 990
+Package: onioncircuits
+Pin: origin deb.tails.boum.org
+Pin-Priority: 999
+
Package: pinentry-gtk2
-Pin: release o=Debian Backports,n=jessie-backports
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
Package: python-electrum
Pin: release o=Debian,n=stretch
Pin-Priority: 999
+Package: tails-installer
+Pin: origin deb.tails.boum.org
+Pin-Priority: 999
+
Package: ttdnsd
-Pin: release o=TorProject,a=unstable
+Pin: release o=TorProject,n=sid
Pin-Priority: 999
Package: torsocks
-Pin: release o=Debian Backports,n=jessie-backports
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
Package: xserver-xorg-video-intel
-Pin: release o=Debian Backports,n=jessie-backports
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
Package: xul-ext-torbirdy
-Pin: release o=Debian Backports,n=jessie-backports
+Pin: release o=Debian,n=jessie-backports
Pin-Priority: 999
Explanation: weirdness in chroot_apt install-binary
@@ -173,13 +149,17 @@ Pin-Priority: 1010
Package: *
Pin: origin deb.tails.boum.org
-Pin-Priority: 1005
+Pin-Priority: 990
Package: *
Pin: release o=Debian,n=jessie-updates
Pin-Priority: 990
Package: *
+Pin: release l=Debian-Security,n=jessie/updates
+Pin-Priority: 990
+
+Package: *
Pin: release o=Debian,n=jessie
Pin-Priority: 990
diff --git a/config/chroot_local-hooks/19-install-tor-browser-AppArmor-profile b/config/chroot_local-hooks/19-install-tor-browser-AppArmor-profile
index 4472f2f..ae39247 100755
--- a/config/chroot_local-hooks/19-install-tor-browser-AppArmor-profile
+++ b/config/chroot_local-hooks/19-install-tor-browser-AppArmor-profile
@@ -18,6 +18,10 @@ toggle_src_APT_sources() {
cat /etc/apt/sources.list /etc/apt/sources.list.d/*.list \
| grep --extended-regexp --line-regexp --invert-match \
'deb\s+file:/root/local-packages\s+\./' \
+ | grep --extended-regexp --invert-match \
+ '^deb\s+http://tagged\.snapshots\.deb\.tails\.boum.org/[^/]+/torproject/' \
+ | grep --extended-regexp --invert-match \
+ '^deb\s+http://time-based\.snapshots\.deb\.tails\.boum.org/torproject/' \
| sed --regexp-extended -e 's,^deb(\s+),deb-src\1,' \
> "$TEMP_APT_SOURCES"
;;
@@ -33,7 +37,7 @@ install_torbrowser_AppArmor_profile() {
tmpdir="$(mktemp -d)"
(
cd "$tmpdir"
- apt-get source torbrowser-launcher/testing
+ apt-get source torbrowser-launcher/stretch
install -m 0644 \
torbrowser-launcher-*/apparmor/torbrowser.Browser.firefox \
"$PROFILE"
diff --git a/config/chroot_local-hooks/51-module-assistant b/config/chroot_local-hooks/51-module-assistant
deleted file mode 100755
index 8967a0f..0000000
--- a/config/chroot_local-hooks/51-module-assistant
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# Install modules managed by module-assistant
-
-echo "Installing modules managed by module-assistant"
-MODULES=""
-MA="module-assistant --text-mode --non-inter"
-
-ma_install_tools ()
-{
- apt-get install --yes build-essential module-assistant debhelper
- $MA update
-}
-
-ma_install_modules ()
-{
- for MODULE in $@ ; do
- for KERNEL in /boot/vmlinuz-* ; do
- VERSION="$(basename ${KERNEL} | sed -e 's|vmlinuz-||')"
- echo "module-assistant: installing module ${MODULE} for kernel ${VERSION}"
- $MA --quiet auto-install ${MODULE} -l ${VERSION}
- done
- $MA clean ${MODULE}
- apt-get --yes purge ${MODULE}
- done
-}
-
-ma_cleanup ()
-{
- rm -f /usr/src/*.deb
-}
-
-if [ -n "${MODULES}" ] ; then
- ma_install_tools
- ma_install_modules ${MODULES}
- ma_cleanup
-fi
-
diff --git a/config/chroot_local-hooks/98-remove_unwanted_packages b/config/chroot_local-hooks/98-remove_unwanted_packages
index 93d1b40..db48f49 100755
--- a/config/chroot_local-hooks/98-remove_unwanted_packages
+++ b/config/chroot_local-hooks/98-remove_unwanted_packages
@@ -17,9 +17,8 @@ apt-get --yes purge \
build-essential debhelper dkms dpatch dpkg-dev \
gcc gcc-4.8 gcc-4.8-base gcc-4.9 \
intltool-debian \
- libc6-dev libgl1-mesa-dev libstdc++6-4.4-dev linux-libc-dev \
+ libc6-dev libgl1-mesa-dev linux-libc-dev \
make \
- module-assistant \
po-debconf \
rsyslog \
libdvdcss-dev \
@@ -28,21 +27,17 @@ apt-get --yes purge \
### Deinstall a few unwanted packages that were pulled by tasksel
### since they have Priority: standard.
apt-get --yes purge \
- apt-listchanges at bsd-mailx dc debian-faq doc-debian dselect \
+ apt-listchanges at bsd-mailx dc debian-faq doc-debian \
'^exim4*' ftp m4 mlocate mutt ncurses-term nfs-common portmap procmail \
python-reportbug reportbug telnet texinfo time w3m wamerican
### Deinstall some other unwanted packages.
apt-get --yes purge \
'^aptitude*' \
- db5.1-util \
'^geoclue*' \
krb5-locales \
live-build \
locales \
- '^openssh-blacklist*' \
- python2.6 \
- python2.6-minimal \
rpcbind \
tasksel \
tasksel-data \
diff --git a/config/chroot_local-includes/lib/live/config/1500-reconfigure-APT b/config/chroot_local-includes/lib/live/config/1500-reconfigure-APT
index 98c7492..1d92b8c 100755
--- a/config/chroot_local-includes/lib/live/config/1500-reconfigure-APT
+++ b/config/chroot_local-includes/lib/live/config/1500-reconfigure-APT
@@ -4,3 +4,71 @@ echo "- configuring APT sources"
sed -i 's,^\(\#\?\s*deb\(-src\)\?\s\+\)http://,\1tor+http://,' \
/etc/apt/sources.list /etc/apt/sources.list.d/*.list
+
+echo '
+
+### Time-based snapshots
+
+s{
+ ^
+ (deb(?:-src)?\s+)
+ tor[+]https?://time-based[.]snapshots[.]deb[.]tails[.]boum[.]org
+ /debian
+ /[0-9]{10} # serial
+ /?
+ (\s+)
+}{$1tor+http://ftp.us.debian.org/debian/$2}xms;
+
+s{
+ ^
+ (deb(?:-src)?\s+)
+ tor[+]https?://time-based[.]snapshots[.]deb[.]tails[.]boum[.]org
+ /debian-security
+ /[0-9]{10} # serial
+ /?
+ (\s+)
+}{$1tor+http://security.debian.org/$2}xms;
+
+s{
+ ^
+ (deb(?:-src)?\s+)
+ tor[+]https?://time-based[.]snapshots[.]deb[.]tails[.]boum[.]org
+ /torproject
+ /[0-9]{10} # serial
+ /?
+ (\s+)
+}{$1tor+http://deb.torproject.org/torproject.org/$2}xms;
+
+### Tagged snapshots
+
+s{
+ ^
+ (deb(?:-src)?\s+)
+ tor[+]https?://tagged[.]snapshots[.]deb[.]tails[.]boum[.]org
+ /[0-9a-z.-]+ # tag
+ /debian
+ /?
+ (\s+)
+}{$1tor+http://ftp.us.debian.org/debian/$2}xms;
+
+s{
+ ^
+ (deb(?:-src)?\s+)
+ tor[+]https?://tagged[.]snapshots[.]deb[.]tails[.]boum[.]org
+ /[0-9a-z.-]+ # tag
+ /debian-security
+ /?
+ (\s+)
+}{$1tor+http://security.debian.org/$2}xms;
+
+s{
+ ^
+ (deb(?:-src)?\s+)
+ tor[+]https?://tagged[.]snapshots[.]deb[.]tails[.]boum[.]org
+ /[0-9a-z.-]+ # tag
+ /torproject
+ /?
+ (\s+)
+}{$1tor+http://deb.torproject.org/torproject.org/$2}xms;
+
+' | perl -pi - /etc/apt/sources.list /etc/apt/sources.list.d/*.list
diff --git a/config/chroot_local-includes/usr/share/applications/tails-reboot.desktop.in b/config/chroot_local-includes/usr/share/applications/tails-reboot.desktop.in
deleted file mode 100644
index 2474270..0000000
--- a/config/chroot_local-includes/usr/share/applications/tails-reboot.desktop.in
+++ /dev/null
@@ -1,9 +0,0 @@
-[Desktop Entry]
-Version=1.0
-Type=Application
-Terminal=false
-Exec=sudo -n reboot
-_Name=Reboot
-_Comment=Immediately reboot computer
-Icon=gtk-refresh
-Categories=Application;System;Settings
diff --git a/config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop.in b/config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop.in
deleted file mode 100644
index e3b2189..0000000
--- a/config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop.in
+++ /dev/null
@@ -1,9 +0,0 @@
-[Desktop Entry]
-Version=1.0
-Type=Application
-Terminal=false
-Exec=sudo -n halt
-_Name=Power Off
-_Comment=Immediately shut down computer
-Icon=system-shutdown
-Categories=Application;System;Settings
diff --git a/config/chroot_local-packageslists/tails-common.list b/config/chroot_local-packageslists/tails-common.list
index 549f46a..187ecef 100644
--- a/config/chroot_local-packageslists/tails-common.list
+++ b/config/chroot_local-packageslists/tails-common.list
@@ -165,7 +165,6 @@ nautilus-wipe
nautilus-gtkhash
network-manager
network-manager-gnome
-nmh
ntfs-3g
obfs4proxy
libreoffice-calc
diff --git a/config/chroot_sources/experimental.binary b/config/chroot_sources/experimental.binary
deleted file mode 120000
index 321bcca..0000000
--- a/config/chroot_sources/experimental.binary
+++ /dev/null
@@ -1 +0,0 @@
-experimental.chroot \ No newline at end of file
diff --git a/config/chroot_sources/experimental.chroot b/config/chroot_sources/experimental.chroot
deleted file mode 100644
index 67f84e4..0000000
--- a/config/chroot_sources/experimental.chroot
+++ /dev/null
@@ -1 +0,0 @@
-deb http://ftp.us.debian.org/debian/ experimental main
diff --git a/config/chroot_sources/sid.chroot b/config/chroot_sources/sid.chroot
index c91091e..c8eea4f 100644
--- a/config/chroot_sources/sid.chroot
+++ b/config/chroot_sources/sid.chroot
@@ -1 +1 @@
-deb http://ftp.us.debian.org/debian/ unstable main contrib non-free
+deb http://ftp.us.debian.org/debian/ sid main contrib non-free
diff --git a/config/chroot_sources/testing.chroot b/config/chroot_sources/testing.chroot
index deb426b..715bb6e 100644
--- a/config/chroot_sources/testing.chroot
+++ b/config/chroot_sources/testing.chroot
@@ -1 +1 @@
-deb http://ftp.us.debian.org/debian/ testing main contrib non-free
+deb http://ftp.us.debian.org/debian/ stretch main contrib non-free
diff --git a/features/apt.feature b/features/apt.feature
index 01371c1..b0ece1f 100644
--- a/features/apt.feature
+++ b/features/apt.feature
@@ -1,5 +1,4 @@
-#10497: wait_until_tor_is_working
-@product @fragile
+@product
Feature: Installing packages through APT
As a Tails user
when I set an administration password in Tails Greeter
@@ -10,7 +9,7 @@ Feature: Installing packages through APT
Given I have started Tails from DVD and logged in with an administration password and the network is connected
Scenario: APT sources are configured correctly
- Then the only hosts in APT sources are "ftp.us.debian.org,security.debian.org,backports.debian.org,deb.tails.boum.org,deb.torproject.org,mozilla.debian.net"
+ Then the only hosts in APT sources are "ftp.us.debian.org,security.debian.org,deb.tails.boum.org,deb.torproject.org"
#10496: apt-get scenarios are fragile
@check_tor_leaks @fragile
diff --git a/features/build.feature b/features/build.feature
index 583a214..9042476 100644
--- a/features/build.feature
+++ b/features/build.feature
@@ -24,6 +24,34 @@ Feature: custom APT sources to build branches
And I should see the 'bugfix-bar' suite
But I should not see the '1.0' suite
+ Scenario: build from an untagged stable branch with no encoded time-based snapshot
+ Given I am working on the stable base branch
+ And Tails 0.10 has been released
+ And the last versions mentioned in debian/changelog are 0.10 and 1.0
+ And Tails 1.0 has not been released yet
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
+ Scenario: build from an untagged stable branch with encoded time-based snapshots
+ Given I am working on the stable base branch
+ And Tails 0.10 has been released
+ And the last versions mentioned in debian/changelog are 0.10 and 1.0
+ And Tails 1.0 has not been released yet
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
Scenario: build from a tagged stable branch where the config/APT_overlays.d directory is empty
Given Tails 0.10 has been released
And the last version mentioned in debian/changelog is 0.10
@@ -40,6 +68,32 @@ Feature: custom APT sources to build branches
When I run tails-custom-apt-sources
Then it should fail
+ Scenario: build from a tagged stable branch with no encoded time-based snapshot
+ Given I am working on the stable base branch
+ And Tails 0.10 has been released
+ And the last version mentioned in debian/changelog is 0.10
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see the 0.10 tagged snapshot
+
+ Scenario: build from a tagged stable branch with encoded time-based snapshots
+ Given I am working on the stable base branch
+ And Tails 0.10 has been released
+ And the last version mentioned in debian/changelog is 0.10
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see the 0.10 tagged snapshot
+
Scenario: build from a bugfix branch without overlays for a stable release
Given Tails 0.10 has been released
And the last version mentioned in debian/changelog is 0.10.1
@@ -62,6 +116,34 @@ Feature: custom APT sources to build branches
And I should see the 'bugfix-bar' suite
But I should not see the '0.10' suite
+ Scenario: build from a bugfix branch with no encoded time-based snapshot for a stable release
+ Given Tails 0.10 has been released
+ And the last version mentioned in debian/changelog is 0.10.1
+ And Tails 0.10.1 has not been released yet
+ And I am working on the bugfix/disable_gdomap branch based on stable
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see the 0.10 tagged snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
+ Scenario: build from a bugfix branch with encoded time-based snapshots for a stable release
+ Given Tails 0.10 has been released
+ And the last version mentioned in debian/changelog is 0.10.1
+ And Tails 0.10.1 has not been released yet
+ And I am working on the bugfix/disable_gdomap branch based on stable
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
Scenario: build from an untagged testing branch where the config/APT_overlays.d directory is empty
Given I am working on the testing base branch
And the last version mentioned in debian/changelog is 0.11
@@ -85,6 +167,34 @@ Feature: custom APT sources to build branches
And I should see the 'bugfix-bar' suite
But I should not see the '0.11' suite
+ Scenario: build from an untagged testing branch with no encoded time-based snapshot
+ Given I am working on the testing base branch
+ And Tails 0.10 has been released
+ And the last versions mentioned in debian/changelog are 0.10 and 1.0
+ And Tails 1.0 has not been released yet
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
+ Scenario: build from an untagged testing branch with encoded time-based snapshots
+ Given I am working on the testing base branch
+ And Tails 0.10 has been released
+ And the last versions mentioned in debian/changelog are 0.10 and 1.0
+ And Tails 1.0 has not been released yet
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
Scenario: build from a tagged testing branch where the config/APT_overlays.d directory is empty
Given I am working on the testing base branch
And the last version mentioned in debian/changelog is 0.11
@@ -101,6 +211,32 @@ Feature: custom APT sources to build branches
When I run tails-custom-apt-sources
Then it should fail
+ Scenario: build from a tagged testing branch with no encoded time-based snapshot
+ Given I am working on the testing base branch
+ And the last version mentioned in debian/changelog is 0.11
+ And Tails 0.11 has been released
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I run "apt-mirror debian"
+ Then it should fail
+ When I run "apt-mirror torproject"
+ Then it should fail
+ When I successfully run "apt-mirror debian-security"
+ Then I should see the 0.11 tagged snapshot
+
+ Scenario: build from a tagged testing branch with encoded time-based snapshots
+ Given I am working on the testing base branch
+ And the last version mentioned in debian/changelog is 0.11
+ And Tails 0.11 has been released
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see the 0.11 tagged snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see the 0.11 tagged snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see the 0.11 tagged snapshot
+
Scenario: build a release candidate from a tagged testing branch
Given I am working on the testing base branch
And Tails 0.11 has been released
@@ -119,6 +255,36 @@ Feature: custom APT sources to build branches
When I run tails-custom-apt-sources
Then it should fail
+ Scenario: build from a bugfix branch with no encoded time-based snapshot for a major release
+ Given I am working on the testing base branch
+ And Tails 0.10~rc1 has been released
+ And the last versions mentioned in debian/changelog are 0.10~rc1 and 0.10
+ And Tails 0.10 has not been released yet
+ And I am working on the bugfix/disable_gdomap branch based on testing
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
+ Scenario: build from a bugfix branch with encoded time-based snapshots for a major release
+ Given I am working on the testing base branch
+ And Tails 0.10~rc1 has been released
+ And the last versions mentioned in debian/changelog are 0.10~rc1 and 0.10
+ And Tails 0.10 has not been released yet
+ And I am working on the bugfix/disable_gdomap branch based on testing
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
Scenario: build from the devel branch without overlays
Given I am working on the devel base branch
And the config/APT_overlays.d directory is empty
@@ -134,6 +300,28 @@ Feature: custom APT sources to build branches
And I should see the 'feature-foo' suite
And I should see the 'bugfix-bar' suite
+ Scenario: build from the devel branch with no encoded time-based snapshot
+ Given I am working on the devel base branch
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
+ Scenario: build from the devel branch with encoded time-based snapshots
+ Given I am working on the devel base branch
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I run "apt-mirror debian"
+ Then it should fail
+ When I run "apt-mirror torproject"
+ Then it should fail
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
Scenario: build from the feature/jessie branch without overlays
Given I am working on the feature/jessie base branch
And the config/APT_overlays.d directory is empty
@@ -162,6 +350,28 @@ Feature: custom APT sources to build branches
When I successfully run tails-custom-apt-sources
Then I should see only the 'devel' suite
+ Scenario: build from a feature branch based on devel with no encoded time-based snapshot
+ Given I am working on the feature/icedove branch based on devel
+ And no frozen APT snapshot is encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I successfully run "apt-mirror debian"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror torproject"
+ Then I should see a time-based snapshot
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
+ Scenario: build from a feature branch based on devel with encoded time-based snapshots
+ Given I am working on the feature/icedove branch based on devel
+ And frozen APT snapshots are encoded in config/APT_snapshots.d
+ When I successfully run "apt-snapshots-serials prepare-build"
+ And I run "apt-mirror debian"
+ Then it should fail
+ When I run "apt-mirror torproject"
+ Then it should fail
+ When I successfully run "apt-mirror debian-security"
+ Then I should see a time-based snapshot
+
Scenario: build from a feature branch with overlays based on feature/jessie
Given I am working on the feature/7756-reintroduce-whisperback branch based on feature/jessie
And config/APT_overlays.d contains 'feature-7756-reintroduce-whisperback'
diff --git a/features/i2p.feature b/features/i2p.feature
index e115fa2b..cc64d54 100644
--- a/features/i2p.feature
+++ b/features/i2p.feature
@@ -31,7 +31,7 @@ Feature: I2P
Then the I2P router console is displayed in I2P Browser
And the I2P Browser uses all expected TBB shared libraries
- #10497, #11401, #11457
+ #10497, #11401, #11457, #11458
@fragile
Scenario: Closing the I2P Browser shows a stop notification and properly tears down the chroot.
Given I have started Tails from DVD with I2P enabled and logged in and the network is connected
diff --git a/features/step_definitions/build.rb b/features/step_definitions/build.rb
index fd001ff..a706430 100644
--- a/features/step_definitions/build.rb
+++ b/features/step_definitions/build.rb
@@ -1,4 +1,4 @@
-Given /^Tails ([[:alnum:].]+) has been released$/ do |version|
+Given /^Tails ([[:alnum:]~.]+) has been released$/ do |version|
create_git unless git_exists?
old_branch = current_branch
@@ -17,7 +17,7 @@ tails (#{version}) stable; urgency=low
END_OF_CHANGELOG
end
fatal_system "git commit --quiet debian/changelog -m 'Release #{version}'"
- fatal_system "git tag '#{version}'"
+ fatal_system "git tag '#{version.gsub('~', '-')}'"
if old_branch != 'stable'
fatal_system "git checkout --quiet '#{old_branch}'"
@@ -42,6 +42,31 @@ Given /^the last version mentioned in debian\/changelog is ([[:alnum:]~.]+)$/ do
end
end
+Given /^the last versions mentioned in debian\/changelog are ([[:alnum:]~.]+) and ([[:alnum:]~.]+)$/ do |version_a, version_b|
+ step "the last version mentioned in debian/changelog is #{version_a}"
+ step "the last version mentioned in debian/changelog is #{version_b}"
+end
+
+Given(/^no frozen APT snapshot is encoded in config\/APT_snapshots\.d$/) do
+ ['debian', 'debian-security', 'torproject'].map do |origin|
+ File.open("config/APT_snapshots.d/#{origin}/serial", 'w+') do |serial|
+ serial.write("latest\n")
+ end
+ end
+end
+
+Given(/^frozen APT snapshots are encoded in config\/APT_snapshots\.d$/) do
+ ['debian', 'torproject'].map do |origin|
+ File.open("config/APT_snapshots.d/#{origin}/serial", 'w+') do |serial|
+ serial.write("2016060602\n")
+ end
+ end
+ # We never freeze debian-security
+ File.open("config/APT_snapshots.d/debian-security/serial", 'w+') do |serial|
+ serial.write("latest\n")
+ end
+end
+
Given %r{I am working on the ([[:alnum:]./_-]+) base branch$} do |branch|
create_git unless git_exists?
@@ -66,12 +91,12 @@ Given %r{I am working on the ([[:alnum:]./_-]+) branch based on ([[:alnum:]./_-]
end
end
-When /^I successfully run ([[:alnum:]-]+)$/ do |command|
+When /^I successfully run "?([[:alnum:] -]+)"?$/ do |command|
@output = `#{File.expand_path("../../../auto/scripts/#{command}", __FILE__)}`
raise StandardError.new("#{command} failed. Exit code: #{$?}") if $? != 0
end
-When /^I run ([[:alnum:]-]+)$/ do |command|
+When /^I run "?([[:alnum:] -]+)"?$/ do |command|
@output = `#{File.expand_path("../../../auto/scripts/#{command}", __FILE__)}`
@exit_code = $?.exitstatus
end
@@ -113,3 +138,11 @@ end
Given(/^the config\/base_branch file is empty$/) do
File.truncate('config/base_branch', 0)
end
+
+Then(/^I should see the ([[:alnum:].-]+) tagged snapshot$/) do |tag|
+ @output.should have_tagged_snapshot(tag)
+end
+
+Then(/^I should see a time\-based snapshot$/) do
+ @output.should have_time_based_snapshot()
+end
diff --git a/features/support/env.rb b/features/support/env.rb
index f70f6b4..e3f039b 100644
--- a/features/support/env.rb
+++ b/features/support/env.rb
@@ -23,6 +23,10 @@ def create_git
Dir.mkdir 'config'
FileUtils.touch('config/base_branch')
Dir.mkdir('config/APT_overlays.d')
+ Dir.mkdir('config/APT_snapshots.d')
+ ['debian', 'debian-security', 'torproject'].map do |origin|
+ Dir.mkdir("config/APT_snapshots.d/#{origin}")
+ end
Dir.mkdir 'debian'
File.open('debian/changelog', 'w') do |changelog|
changelog.write(<<END_OF_CHANGELOG)
@@ -88,3 +92,35 @@ RSpec::Matchers.define :have_suite do |suite|
"expected an output with #{suite}"
end
end
+
+RSpec::Matchers.define :have_tagged_snapshot do |tag|
+ match do |string|
+ # e.g.: `http://tagged.snapshots.deb.tails.boum.org/0.10`
+ %r{^http://tagged\.snapshots\.deb\.tails\.boum\.org/#{Regexp.escape(tag)}/[a-z-]+$}.match(string)
+ end
+ failure_message_for_should do |string|
+ "expected the mirror to be #{tag}\nCurrent mirror: #{string}"
+ end
+ failure_message_for_should_not do |string|
+ "expected the mirror not to be #{tag}\nCurrent mirror: #{string}"
+ end
+ description do
+ "expected an output with #{tag}"
+ end
+end
+
+RSpec::Matchers.define :have_time_based_snapshot do |tag|
+ match do |string|
+ # e.g.: `http://time-based.snapshots.deb.tails.boum.org/debian/2016060602`
+ %r{^http://time\-based\.snapshots\.deb\.tails\.boum\.org/[^/]+/\d+}.match(string)
+ end
+ failure_message_for_should do |string|
+ "expected the mirror to be a time-based snapshot\nCurrent mirror: #{string}"
+ end
+ failure_message_for_should_not do |string|
+ "expected the mirror not to be a time-based snapshot\nCurrent mirror: #{string}"
+ end
+ description do
+ "expected a time-based snapshot"
+ end
+end
diff --git a/features/unsafe_browser.feature b/features/unsafe_browser.feature
index 5e1818a..f2f6291 100644
--- a/features/unsafe_browser.feature
+++ b/features/unsafe_browser.feature
@@ -4,7 +4,7 @@ Feature: Browsing the web using the Unsafe Browser
when I browse the web using the Unsafe Browser
I should have direct access to the web
- #10497, #11401
+ #10497, #11401, #11458
@fragile
Scenario: The Unsafe Browser can access the LAN
Given I have started Tails from DVD and logged in and the network is connected
@@ -13,7 +13,7 @@ Feature: Browsing the web using the Unsafe Browser
And I open a page on the LAN web server in the Unsafe Browser
Then I see "UnsafeBrowserHelloLANWebServer.png" after at most 20 seconds
- #10497
+ #10497, #11458
@fragile
Scenario: Starting the Unsafe Browser works as it should.
Given I have started Tails from DVD and logged in and the network is connected
@@ -27,7 +27,7 @@ Feature: Browsing the web using the Unsafe Browser
And the Unsafe Browser has no proxy configured
And the Unsafe Browser uses all expected TBB shared libraries
- #10497, #11457
+ #10497, #11457, #11458
@fragile
Scenario: Closing the Unsafe Browser shows a stop notification and properly tears down the chroot.
Given I have started Tails from DVD and logged in and the network is connected
@@ -36,7 +36,7 @@ Feature: Browsing the web using the Unsafe Browser
Then I see the Unsafe Browser stop notification
And the Unsafe Browser chroot is torn down
- #10497, #11401
+ #10497, #11401, #11458
@fragile
Scenario: Starting a second instance of the Unsafe Browser results in an error message being shown.
Given I have started Tails from DVD and logged in and the network is connected
@@ -44,14 +44,14 @@ Feature: Browsing the web using the Unsafe Browser
And I start the Unsafe Browser
Then I see a warning about another instance already running
- #10497, #11401
+ #10497, #11401, #11458
@fragile
Scenario: The Unsafe Browser cannot be configured to use Tor and other local proxies.
Given I have started Tails from DVD and logged in and the network is connected
When I successfully start the Unsafe Browser
Then I cannot configure the Unsafe Browser to use any local proxies
- #10497
+ #10497, #11458
@fragile
Scenario: The Unsafe Browser will not make any connections to the Internet which are not user initiated
Given I have started Tails from DVD and logged in and the network is connected
diff --git a/po/POTFILES.in b/po/POTFILES.in
index 9ff01b6..8a9aeff 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -22,8 +22,6 @@ config/chroot_local-includes/etc/skel/Desktop/tails-documentation.desktop.in
config/chroot_local-includes/usr/share/applications/tails-documentation.desktop.in
config/chroot_local-includes/usr/share/applications/i2p-browser.desktop.in
config/chroot_local-includes/usr/share/applications/tails-about.desktop.in
-config/chroot_local-includes/usr/share/applications/tails-reboot.desktop.in
-config/chroot_local-includes/usr/share/applications/tails-shutdown.desktop.in
config/chroot_local-includes/usr/share/applications/tor-browser.desktop.in
config/chroot_local-includes/usr/share/applications/unsafe-browser.desktop.in
config/chroot_local-includes/usr/share/desktop-directories/Tails.directory.in
diff --git a/vagrant/provision/assets/acng.conf b/vagrant/provision/assets/acng.conf
index 5ed3146..3d41fc8 100644
--- a/vagrant/provision/assets/acng.conf
+++ b/vagrant/provision/assets/acng.conf
@@ -8,5 +8,6 @@ Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete,
ReportPage: acng-report.html
ExTreshold: 50
VfilePattern = (^|.*?/)(Index|Packages(\.gz|\.bz2|\.lzma|\.xz)?|InRelease|Release|Release\.gpg|Sources(\.gz|\.bz2|\.lzma|\.xz)?|release|index\.db-.*\.gz|Contents-[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|pkglist[^/]*\.bz2|rclist[^/]*\.bz2|/meta-release[^/]*|Translation[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|MD5SUMS|SHA1SUMS|((setup|setup-legacy)(\.ini|\.bz2|\.hint)(\.sig)?)|mirrors\.lst|repo(index|md)\.xml(\.asc|\.key)?|directory\.yast|products|content(\.asc|\.key)?|media|filelists\.xml\.gz|filelists\.sqlite\.bz2|repomd\.xml|packages\.[a-zA-Z][a-zA-Z]\.gz|info\.txt|license\.tar\.gz|license\.zip|.*\.db(\.tar\.gz)?|.*\.files\.tar\.gz|.*\.abs\.tar\.gz|metalink\?repo|.*prestodelta\.xml\.gz)$|/dists/.*/installer-[^/]+/[^0-9][^/]+/images/.*
+VfilePatternEx = .*/project/trace/[a-z-]+$
PfilePattern = .*(\.d?deb|\.rpm|\.dsc|\.tar(\.gz|\.bz2|\.lzma|\.xz)(\.gpg)?|\.diff(\.gz|\.bz2|\.lzma|\.xz)|\.o|\.jigdo|\.template|changelog|copyright|\.udeb|\.debdelta|\.diff/.*\.gz|(Devel)?ReleaseAnnouncement(\?.*)?|[a-f0-9]+-(susedata|updateinfo|primary|deltainfo).xml.gz|fonts/(final/)?[a-z]+32.exe(\?download.*)?|/dists/.*/installer-[^/]+/[0-9][^/]+/images/.*)$
WfilePattern = (^|.*?/)(Release|InRelease|Release\.gpg|(Packages|Sources)(\.gz|\.bz2|\.lzma|\.xz)?|Translation[^/]*(\.gz|\.bz2|\.lzma|\.xz)?|MD5SUMS|SHA1SUMS|.*\.xml|.*\.db\.tar\.gz|.*\.files\.tar\.gz|.*\.abs\.tar\.gz|[a-z]+32.exe)$|/dists/.*/installer-.*/images/.*
diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails
index a414a2f..21fa924 100755
--- a/vagrant/provision/assets/build-tails
+++ b/vagrant/provision/assets/build-tails
@@ -86,6 +86,11 @@ git checkout --force "$REV"
git reset --hard "$COMMIT"
git submodule update --init
+if as_root_do systemctl --quiet is-active apt-cacher-ng.service ; then
+ as_root_do ./auto/scripts/update-acng-config
+ as_root_do systemctl restart apt-cacher-ng.service
+fi
+
if [ -n "$JENKINS_URL" ]; then
git clean --force -d -x
fi
diff --git a/wiki/src/blueprint/delete_obsolete_Git_branches.mdwn b/wiki/src/blueprint/delete_obsolete_Git_branches.mdwn
index 4895bd1..d6858d2 100644
--- a/wiki/src/blueprint/delete_obsolete_Git_branches.mdwn
+++ b/wiki/src/blueprint/delete_obsolete_Git_branches.mdwn
@@ -38,7 +38,8 @@ APT suite that they don't plan to use, but for that use case, so far
we've instead hard-coded the additional suite names we need in the
scripts that generate the reprepro configuration).
-Given how our [[contribute/APT_repository]] currently works, we sometimes push
+Given how our [[custom APT repository|contribute/APT_repository/custom]]
+currently works, we sometimes push
Git branches merely to have an APT suite created, and be able to upload Debian
packages to it. Such a branch will look like it has been merged (it has no
commit on top of current `master`), but still it should not be deleted:
diff --git a/wiki/src/contribute.mdwn b/wiki/src/contribute.mdwn
index 272b4ef..3ec969c 100644
--- a/wiki/src/contribute.mdwn
+++ b/wiki/src/contribute.mdwn
@@ -137,7 +137,9 @@ Tools for contributors
- [[Building a Tails image|contribute/build]]
- [[Build a local copy of the website|contribute/build/website]]
- [[Customize Tails|contribute/customize]]
- - [[APT repository|contribute/APT_repository]], to store our custom Debian packages
+ - [[APT repositories|contribute/APT_repository]], to store our
+ custom Debian packages and various snapshots of the upstream APT
+ repositories we use
- [[Glossary for contributors|contribute/glossary]]
<a id="release-cycle"></a>
diff --git a/wiki/src/contribute/APT_repository.mdwn b/wiki/src/contribute/APT_repository.mdwn
index ec87c2d..fc96122 100644
--- a/wiki/src/contribute/APT_repository.mdwn
+++ b/wiki/src/contribute/APT_repository.mdwn
@@ -1,371 +1,129 @@
[[!toc levels=2]]
-To avoid cluttering our main Git repository with Debian source and
-binary packages, we have set an APT repository up.
-
-[[!toc levels=2]]
-
-Overview
-========
-
-We use one single APT repository hosting multiple *suites*:
-
-* We have a (read-only) suite for every past release: `0.9`,
- `0.10.1`, etc.
-* We have a suite for each *main* branch: `stable`, `testing`,
- `devel`, `feature-jessie`
-* We have an overlay suite for each *topic* branch: `bugfix/*`,
- `feature/*`, etc.
- **Note**: the APT suite corresponding to a given Git topic
- branch contains *only* the packages this branch adds to the tag or
- *main* branch it diverged from. Think of it as an overlay.
-* We also have a less formal `unstable` suite, that should not be used
- by any Tails git branch; it can be used as hosting space for other
- packaging work we might do, e.g. acting as upstream or
- Debian maintainers.
-* We also have a `builder-wheezy` suite, used to provide additional
- packages needed on a Wheezy system to build Tails.
-
-The suite(s) to use as sources for APT, during the build and inside
-the resulting system, are determined by the content of the
-`config/base_branch` and `config/APT_overlays.d/*` files. See details in
-the *Build system* section below.
-
-We manage our APT repository with
-[reprepro](http://mirrorer.alioth.debian.org/).
-
-The Puppet modules used to manage this part of our infrastructure are
-listed on our [[contribute/Git]] page.
-
-Basically, a cronjob fetches and scans the Tails Git repository every
-few minutes, detects new branches, and accordingly:
-
-- generates `conf/distributions`
-- generates `conf/incoming`
-- create new suites in the APT repository
-
-Build system
-============
-
-The Tails ISO build system dynamically adds APT sources that will be
-used during the build, and inside the resulting ISO itself.
-
-If the last version in `debian/changelog` was released already (i.e.
-a matching tag exists), then the build system adds the suite
-corresponding to this release (e.g. `1.5` or `3.0`), and that's all.
-
-Else, it adds:
-
-* one APT source for the base branch of the one being built, as found
- in `config/base_branch`;
-* one APT source for each suite listed in
- `config/APT_overlays.d/*`; note that only the name of such
- files matters, and their content is ignored.
-
-In practice, `config/APT_overlays.d/` contains:
-
-* for a topic branch:
- - if needed, a file that is named like the branch's own overlay APT
- suite; e.g. for the `bugfix/12345-whatever` branch, it would be
- called `config/APT_overlays.d/bugfix-12345-whatever`
- - any file representing APT suites that came from merging its base
- branch into this topic branch, that is:
-* for a base branch (`stable`, `testing`, `devel` or
- `feature/jessie`): a file for each additional, overlay APT suite that
- came from topic branches that ship Debian packages and were merged
- into this base branch since last time it was used to prepare
- a release.
-
-The code that implements this is [[!tails_gitweb
-auto/scripts/tails-custom-apt-sources]]. It has [[!tails_gitweb
-features/build.feature desc="automated tests"]].
-
-At release time, the release manager:
-
-1. merges into the release branch's APT suite all APT overlay
- suites found in `config/APT_overlays.d/`;
-2. empties `config/APT_overlays.d/` in the release branch;
-3. merges the release branch into other base branches as needed, and
- ensures that all resulting `config/APT_overlays.d/`:s make sense.
-
-Note that a branch like `feature/jessie` needs to be a base branch: we want to be
-able to work on topic branches forked off `feature/jessie`.
-
-SSH access
-==========
-
-One must configure their SSH client to connect to the APT server:
-
- Host incoming.deb.tails.boum.org
- Port 3003
-
-HTTP access
-===========
-
-This is the http:// public APT repository used at Tails
-build time. The `tails::reprepro` Puppet class sets nginx up to
-serve that.
-
-Workflow
-========
-
-Creating a new branch
----------------------
-
-Push your branch to Git and wait a few minutes for the new APT suite
-to appear.
-
-Then you probably want to tell the build system, via
-`config/APT_overlays.d/` (documented above), that this new APT suite
-must be used when building that new branch.
-
-Importing a new package
------------------------
-
-### Building a package
-
-Make sure the `Distribution:` field in your `.changes` file matches
-the suite you want the package to land in (e.g.
-pass `--changes-option=-DDistribution=feature-torbrowser` to
-pdebuild's `--debbuildopts`).
-
-Make sure to have the `.changes` file include the original source
-archive (`.orig.tar.{gz,bz2,xz}`) if it is not already in our APT
-repository; this can be done by passing `-sa` to pdebuild's
-`--debbuildopts`.
-
-### Configuring an upload tool
-
-#### Configuring dupload
-
-Add this configuration snippet to your `dupload` configuration:
-
- $config::cfg{'tails'} = {
- fqdn => "incoming.deb.tails.boum.org",
- method => "scp",
- login => "reprepro",
- incoming => "/srv/reprepro/incoming/",
- dinstall_runs => 1,
- };
-
-#### Configuring dput
-
-Add this to `.dput.cf`:
-
- [tails]
- fqdn = incoming.deb.tails.boum.org
- method = scp
- login = reprepro
- incoming = /srv/reprepro/incoming/
- run_dinstall = 0
-
-
-### Uploading and importing process
-
-Carefully prepare and build your package. Usual precautions, (Lintian
-etc.) apply.
-
-Carefully check the `.changes` file (especially the `Distribution`
-control field, and the included files list; the former can be fixed
-with the `changestool(1)` command, from [[!debpkg reprepro]]).
-
-Sign the `.changes` file with a key that is in the uploaders list:
-
- $ debsign $CHANGES_FILE
-
-Upload the files to the incoming queue:
-
- $ dupload --to tails $CHANGES_FILE
-
-reprepro will automatically notice the new files and import them into
-the suite specified in your `.changes` file.
-
-Check the result:
-
- $ ssh reprepro@incoming.deb.tails.boum.org reprepro list $SUITE $PACKAGENAME
-
-<a id="workflow-merge-main-branch"></a>
-
-Merging a main branch
-----------------------
-
-When a Git *main* branch (`devel`, `testing`, `stable`,
-`feature/jessie`) is merged into another *main* branch, the corresponding
-operation must be done on the APT suites.
-
-1. Save the list of packages currently present in the APT suite we
- want to merge *into*, e.g. `reprepro list devel`.
-
-2. Make sure you are not going to overwrite newer packages with
- older ones (hint: use the `tails-diff-suites` script).
-
-3. Merge the APT suites:
-
- 1. Set some environment variables:
-
- # the branch you want to merge
- SRC=stable
- # the branch you want to merge _into_
- DST=devel
-
- 2. Merge in Git and APT:
-
- git checkout "$DST" && \
- git merge "$SRC" && \
- ssh reprepro@incoming.deb.tails.boum.org \
- tails-merge-suite "$SRC" "$DST"
-
- 3. Restore the `config/base_branch` if needed:
-
- echo "${DST}" > config/base_branch && \
- git commit config/base_branch -m "Restore ${DST}'s base branch." || :
-
- 4. Push:
-
- git push origin "${DST}:${DST}"
-
-4. Make sure not to re-add, into the branch we merge into, any package
- that was removed from it, but still is in the branch we merge from:
- e.g. when merging `stable` into `devel`, it may be that
- `devel` had some packages removed (e.g. due to previously
- merging a topic branch into it, whose purpose is to *remove* custom
- packages). To this end, compare the resulting list of (package,
- version) in the `devel` APT suite with the one saved before
- the merge, check Git
- merges history if needed, apply common sense, and remove from
- `devel` the packages that were removed from it a while ago,
- and were just erroneously re-added by the merge operation.
-
-<a id="workflow-reset"></a>
-
-Resetting a suite to the state of another one
----------------------------------------------
-
- a. First, set some environment variables:
-
- # the suite to reset
- OLD=testing
- # the final state it should be in
- NEW=devel
-
- b. Then, empty the `OLD` suite:
-
- ssh reprepro@incoming.deb.tails.boum.org \
- reprepro removematched $OLD '\*'
-
- c. Finally, merge `NEW` into `OLD`
-
- ssh reprepro@incoming.deb.tails.boum.org \
- tails-merge-suite $NEW $OLD
-
-<a id="workflow-merge-overlays"></a>
-
-Merging APT overlays
---------------------
-
-This operation merges all APT overlays listed in the given branch's
-`config/APT_overlays.d/` into its own APT suite, empties
-`config/APT_overlays.d/` accordingly, then commits and pushes to Git.
-
-1. Set some environment variables:
-
- # The branch that should have its overlays merged
- BRANCH=devel
-
-2. Merge the APT overlays in reprepro:
-
- git checkout "$BRANCH" && \
- for overlay in $(ls config/APT_overlays.d/) ; do
- if ! ssh reprepro@incoming.deb.tails.boum.org \
- tails-merge-suite "$overlay" "$BRANCH" ; then
- echo "Failed to merge '$overlay' into '$BRANCH': $?" >&2
- break
- fi
- done
-
-3. Empty `config/APT_overlays.d/`:
-
- git checkout "$BRANCH" && \
- git rm config/APT_overlays.d/* && \
- git commit config/APT_overlays.d/ \
- -m "Empty the list of APT overlays: they were merged"
-
-4. Push the Git branch:
-
- git push origin "${BRANCH}:${BRANCH}"
-
-<a id="workflow-post-tag"></a>
-
-Tagging a new Tails release
----------------------------
-
-Once the new release's Git tag is pushed, a cronjob creates
-a new APT suite on the APT repository's side within a few minutes.
-This new APT suite is called the same as the new release version.
-One may check it has appeared in `~reprepro/conf/distributions`.
-
-Then, the APT suite corresponding to the branch that was used to
-prepare the release must be copied to the new empty APT suite that
-just appeared:
-
- $ ssh reprepro@incoming.deb.tails.boum.org \
- tails-merge-suite "$RELEASE_BRANCH" "$TAG"
-
-<a id="workflow-post-release"></a>
-
-After a new Tails release is out
---------------------------------
-
-If you just put out a final release:
-
-* [[merge `stable` or `testing` into
- `devel`|APT_repository#workflow-merge-main-branch]]
-* increment the version number in devel's `debian/changelog` to match
- the next major release, so that
- next builds from the `devel` branch do not use the APT suite meant
- for the last release
-* increment the version number in stable's `debian/changelog` to match
- the next point release, so that
- next builds from the `stable` branch do not use the APT suite meant
- for the last release
-
-If you just released a RC:
-
-* add a dummy changelog entry (for the upcoming, non-RC version) in
- the branch used for the release (`stable` or `testing`), so that the
- next builds from it do not use the APT suite meant for the RC
-* add a dummy changelog entry (for the release *after* the one you
- released a RC for) in the branch used for the release (`stable` or
- `testing`), so that the next builds from it do not use the APT suite
- meant for the RC
-
-If the release was a major one, then:
-
-1. [[Hard reset the stable APT suite to
- the state of the testing one|APT_repository#workflow-reset]].
-
-2. Empty `config/APT_overlays.d` in the `stable` branch:
-
- git checkout stable && \
- git rm config/APT_overlays.d/* && \
- git commit config/APT_overlays.d/ \
- -m "Empty the list of APT overlays: they were merged"
-
-Giving access to a core developer
----------------------------------
-
-1. Give SSH access to the `reprepro` user on the system that hosts
- reprepro (using the `ssh_authorized_key` Puppet resource).
-2. Import the developer's public GnuPG key into the `reprepro` user's
- GnuPG keyring -- should be doable using Puppet, some day
-3. Add the developer's OpenPGP key ID to `$reprepro_uploaders` in our
- `tails::reprepro` Puppet module. Deploy.
-
-Contributing without privileged access
---------------------------------------
-
-Non-core developers without access to the "private" APT infrastructure
-would add the .deb they want to their Git branch as we have been
-doing until now, push the result on GitLab or whatever... and at
-merge time, we would rewrite their history to remove the .deb, and
-import it into our APT repo.
+# Our APT repositories
+
+We have three kinds of APT repositories:
+
+* our [[custom APT repository|contribute/APT_repository/custom]],
+ that stores our custom Debian packages;
+* (full) [[contribute/APT_repository/time-based snapshots]] of the
+ upstream APT repositories we use, so that we can use frozen
+ versions thereof when needed;
+* (partial) [[contribute/APT_repository/tagged snapshots]] of upstream
+ APT repositories we need, so that one can rebuild a released ISO in
+ the future, and we keep the corresponding source code around.
+
+# Snapshots and branches
+
+Here we discuss what APT snapshots of upstream repositories are used
+when building a Tails ISO image. This is a function of the branch we
+build from, we are building an ISO that is meant to be released (i.e.
+whether there is a tag in Git corresponding the last entry in
+debian/changelog).
+
+Note: when we say "building an ISO from the `B` branch" below, we only
+refer to the base branch, i.e. `B` is encoded in `config/base_branch`.
+Hence such statements also refers to topic branches with `B` as base
+branch.
+
+Building an ISO from the `devel` branch always uses the freshest set
+of APT repository snapshots available. Resolving what's the set of
+freshest APT repository snapshots is done at the beginning of the
+build ([[!tails_gitweb auto/config]],
+[[!tails_gitweb auto/scripts/apt-mirror]]), so that the entire build
+uses the exact same state of these
+repositories. This is needed for reproducible builds, and has a nice
+side effect: so long, `Hashsum mismatch`, and thanks for the fish.
+
+When building an ISO from the branch used to prepare the next major
+release (`testing`):
+
+ * **outside of the freeze period**: we use the latest set of APT
+ repository snapshots, just like when building from `devel`;
+ * **freeze period**: at freeze time, the RM encodes in the Git
+ `testing` branch the set of APT repository snapshots (via their
+ serial numbers) that shall be used during the freeze; the only
+ exception is security.debian.org, for which we always use our
+ latest snapshot;
+ * **at release time**: when building from a tagged branch, similarly to
+ what we do for our
+ [[custom APT repository|contribute/APT_repository/custom]], instead
+ of using time-based APT repository snapshots, we use snapshots
+ labeled with the Git tag (note that this is not needed, strictly speaking,
+ as the APT sources used at Tails runtime will anyway be the
+ official (and not frozen) Debian ones; this is mostly needed for
+ legal purposes (this allows to distribute for a long
+ time the source packages needed to build a given Tails ISO image),
+ and it will be useful when we want to be able to reproduce a given
+ Tails ISO build);
+ * **after releasing**, the RM encodes in the `testing` Git branch the
+ fact that it is not frozen anymore, that is: the RM removes the
+ indication that a specific set of APT repository snapshots must be
+ used; and then, we're back to the "outside of the freeze
+ period" case.
+
+When building an ISO from the branch used to prepare the next
+point-release (`stable`) we use snapshots labeled with the Git tag of
+the latest Tails release, except:
+
+ * we generally use our latest snapshot of security.debian.org;
+ * at release time: when building from a tagged branch, similarly to
+ what we do for our
+ [[custom APT repository|contribute/APT_repository/custom]], instead
+ of using time-based APT repository snapshots, we use snapshots
+ labeled with the Git tag
+ * if a set of APT repository snapshots is encoded directly in that
+ branch: use them, even for security.debian.org.
+
+# Design notes
+
+## Miscellaneous
+
+A given APT repository snapshot is immutable after it's been taken.
+We
+[[deal with freeze exceptions separately|contribute/APT_repository/time-based_snapshots#design-freeze-exceptions]].
+
+We want to have reproducible builds some day. Therefore, the APT
+`sources.list` shipped in the ISO must be stable across rebuilds from
+the same release Git tag.
+
+<a id="runtime-sources"></a>
+
+## APT sources used inside Tails
+
+A running Tails' APT must be pointed at the official, live Debian
+archive, and not to a Tails-specific and already obsolete snapshot.
+
+Say `kedit` is a package shipped in Debian, but not in Tails. Then,
+when run inside Tails, `apt install kedit` must fetch `kedit` from
+current Debian, as opposed to installing it from a Tails-specific, and
+generally obsolete, snapshot of the Debian APT repository.
+
+To achieve that we tweak `sources.list` in
+[[!tails_gitweb config/chroot_local-includes/lib/live/config/1500-reconfigure-APT]].
+
+## Upgrading to a new snapshot
+
+In other words: bumping, in Git, the pointers to the set of snapshots
+that shall be used by a given branch.
+
+Let's use, as an example of a situation in which we might want to do
+that, upgrading to a new Debian point-release.
+
+With this design:
+
+ * `devel` gets them automatically because it closely tracks the
+ Debian archive;
+ * for release branches (`stable`, `testing`): on a case-by-case
+ basis, depending on the respective Debian/Tails release schedule
+ timing, we can choose whether to switch to using a new snapshot of
+ the Debian archive for the next release. Note that this can be done
+ via a topic-branch since this information is encoded in Git. If we
+ choose not to manually pick the point release, which is the default
+ if we don't act at all, then:
+ - `testing` will start using the new Debian point-release as soon
+ as it is unfrozen, that is as soon as it has been used to release
+ a new major version of Tails;
+ - `stable` will start using the new Debian point-release once
+ a `testing` branch that uses that point-release is merged into
+ `stable`.
diff --git a/wiki/src/contribute/APT_repository/custom.mdwn b/wiki/src/contribute/APT_repository/custom.mdwn
new file mode 100644
index 0000000..13ebc67
--- /dev/null
+++ b/wiki/src/contribute/APT_repository/custom.mdwn
@@ -0,0 +1,366 @@
+[[!meta title="Custom APT repository"]]
+
+We use a custom APT repository to store our custom packages.
+
+[[!toc levels=2]]
+
+Overview
+========
+
+We use one single APT repository hosting multiple *suites*:
+
+* We have a (read-only) suite for every past release: `0.9`,
+ `0.10.1`, etc.
+* We have a suite for each *main* branch: `stable`, `testing`,
+ `devel`, `feature-jessie`
+* We have an overlay suite for each *topic* branch: `bugfix/*`,
+ `feature/*`, etc.
+ **Note**: the APT suite corresponding to a given Git topic
+ branch contains *only* the packages this branch adds to the tag or
+ *main* branch it diverged from. Think of it as an overlay.
+* We also have a less formal `unstable` suite, that should not be used
+ by any Tails git branch; it can be used as hosting space for other
+ packaging work we might do, e.g. acting as upstream or
+ Debian maintainers.
+* We also have a `builder-wheezy` suite, used to provide additional
+ packages needed on a Wheezy system to build Tails.
+
+The suite(s) to use as sources for APT, during the build and inside
+the resulting system, are determined by the content of the
+`config/base_branch` and `config/APT_overlays.d/*` files. See details in
+the *Build system* section below.
+
+We manage our APT repository with
+[reprepro](http://mirrorer.alioth.debian.org/).
+
+The Puppet modules used to manage this part of our infrastructure are
+listed on our [[contribute/Git]] page.
+
+Basically, a cronjob fetches and scans the Tails Git repository every
+few minutes, detects new branches, and accordingly:
+
+- generates `conf/distributions`
+- generates `conf/incoming`
+- create new suites in the APT repository
+
+Build system
+============
+
+The Tails ISO build system dynamically adds APT sources that will be
+used during the build, and inside the resulting ISO itself.
+
+If the last version in `debian/changelog` was released already (i.e.
+a matching tag exists), then the build system adds the suite
+corresponding to this release (e.g. `1.5` or `3.0`), and that's all.
+
+Else, it adds:
+
+* one APT source for the base branch of the one being built, as found
+ in `config/base_branch`;
+* one APT source for each suite listed in
+ `config/APT_overlays.d/*`; note that only the name of such
+ files matters, and their content is ignored.
+
+In practice, `config/APT_overlays.d/` contains:
+
+* for a topic branch:
+ - if needed, a file that is named like the branch's own overlay APT
+ suite; e.g. for the `bugfix/12345-whatever` branch, it would be
+ called `config/APT_overlays.d/bugfix-12345-whatever.suite`
+ - any file representing APT suites that came from merging its base
+ branch into this topic branch, that is:
+* for a base branch (`stable`, `testing`, `devel` or
+ `feature/jessie`): a file for each additional, overlay APT suite that
+ came from topic branches that ship Debian packages and were merged
+ into this base branch since last time it was used to prepare
+ a release.
+
+The code that implements this is [[!tails_gitweb
+auto/scripts/tails-custom-apt-sources]]. It has [[!tails_gitweb
+features/build.feature desc="automated tests"]].
+
+At release time, the release manager:
+
+1. merges into the release branch's APT suite all APT overlay
+ suites found in `config/APT_overlays.d/`;
+2. empties `config/APT_overlays.d/` in the release branch;
+3. merges the release branch into other base branches as needed, and
+ ensures that all resulting `config/APT_overlays.d/`:s make sense.
+
+Note that a branch like `feature/jessie` needs to be a base branch: we want to be
+able to work on topic branches forked off `feature/jessie`.
+
+SSH access
+==========
+
+One must configure their SSH client to connect to the APT server:
+
+ Host incoming.deb.tails.boum.org
+ Port 3003
+
+HTTP access
+===========
+
+This is the http:// public APT repository used at Tails
+build time. The `tails::reprepro` Puppet class sets nginx up to
+serve that.
+
+Workflow
+========
+
+Creating a new branch
+---------------------
+
+Push your branch to Git and wait a few minutes for the new APT suite
+to appear.
+
+Importing a new package
+-----------------------
+
+### Building a package
+
+Make sure the `Distribution:` field in your `.changes` file matches
+the suite you want the package to land in (e.g.
+pass `--changes-option=-DDistribution=feature-torbrowser` to
+pdebuild's `--debbuildopts`).
+
+Make sure to have the `.changes` file include the original source
+archive (`.orig.tar.{gz,bz2,xz}`) if it is not already in our APT
+repository; this can be done by passing `-sa` to pdebuild's
+`--debbuildopts`.
+
+### Configuring an upload tool
+
+#### Configuring dupload
+
+Add this configuration snippet to your `dupload` configuration:
+
+ $config::cfg{'tails'} = {
+ fqdn => "incoming.deb.tails.boum.org",
+ method => "scp",
+ login => "reprepro",
+ incoming => "/srv/reprepro/incoming/",
+ dinstall_runs => 1,
+ };
+
+#### Configuring dput
+
+Add this to `.dput.cf`:
+
+ [tails]
+ fqdn = incoming.deb.tails.boum.org
+ method = scp
+ login = reprepro
+ incoming = /srv/reprepro/incoming/
+ run_dinstall = 0
+
+
+### Uploading and importing process
+
+Carefully prepare and build your package. Usual precautions, (Lintian
+etc.) apply.
+
+Carefully check the `.changes` file (especially the `Distribution`
+control field, and the included files list; the former can be fixed
+with the `changestool(1)` command, from [[!debpkg reprepro]]).
+
+Sign the `.changes` file with a key that is in the uploaders list:
+
+ $ debsign $CHANGES_FILE
+
+Upload the files to the incoming queue:
+
+ $ dupload --to tails $CHANGES_FILE
+
+reprepro will automatically notice the new files and import them into
+the suite specified in your `.changes` file.
+
+Check the result:
+
+ $ ssh reprepro@incoming.deb.tails.boum.org reprepro list $SUITE $PACKAGENAME
+
+<a id="workflow-merge-main-branch"></a>
+
+Merging a main branch
+----------------------
+
+When a Git *main* branch (`devel`, `testing`, `stable`,
+`feature/jessie`) is merged into another *main* branch, the corresponding
+operation must be done on the APT suites.
+
+1. Save the list of packages currently present in the APT suite we
+ want to merge *into*, e.g. `reprepro list devel`.
+
+2. Make sure you are not going to overwrite newer packages with
+ older ones (hint: use the `tails-diff-suites` script).
+
+3. Merge the APT suites:
+
+ 1. Set some environment variables:
+
+ # the branch you want to merge
+ SRC=stable
+ # the branch you want to merge _into_
+ DST=devel
+
+ 2. Merge in Git and APT:
+
+ git checkout "$DST" && \
+ git merge "$SRC" && \
+ ssh reprepro@incoming.deb.tails.boum.org \
+ tails-merge-suite "$SRC" "$DST"
+
+ 3. Restore the `config/base_branch` if needed:
+
+ echo "${DST}" > config/base_branch && \
+ git commit config/base_branch -m "Restore ${DST}'s base branch." || :
+
+ 4. Push:
+
+ git push origin "${DST}:${DST}"
+
+4. Make sure not to re-add, into the branch we merge into, any package
+ that was removed from it, but still is in the branch we merge from:
+ e.g. when merging `stable` into `devel`, it may be that
+ `devel` had some packages removed (e.g. due to previously
+ merging a topic branch into it, whose purpose is to *remove* custom
+ packages). To this end, compare the resulting list of (package,
+ version) in the `devel` APT suite with the one saved before
+ the merge, check Git
+ merges history if needed, apply common sense, and remove from
+ `devel` the packages that were removed from it a while ago,
+ and were just erroneously re-added by the merge operation.
+
+<a id="workflow-reset"></a>
+
+Resetting a suite to the state of another one
+---------------------------------------------
+
+ a. First, set some environment variables:
+
+ # the suite to reset
+ OLD=testing
+ # the final state it should be in
+ NEW=devel
+
+ b. Then, empty the `OLD` suite:
+
+ ssh reprepro@incoming.deb.tails.boum.org \
+ reprepro removematched $OLD '\*'
+
+ c. Finally, merge `NEW` into `OLD`
+
+ ssh reprepro@incoming.deb.tails.boum.org \
+ tails-merge-suite $NEW $OLD
+
+<a id="workflow-merge-overlays"></a>
+
+Merging APT overlays
+--------------------
+
+This operation merges all APT overlays listed in the given branch's
+`config/APT_overlays.d/` into its own APT suite, empties
+`config/APT_overlays.d/` accordingly, then commits and pushes to Git.
+
+1. Set some environment variables:
+
+ # The branch that should have its overlays merged
+ BRANCH=devel
+
+2. Merge the APT overlays in reprepro:
+
+ git checkout "$BRANCH" && \
+ for overlay in $(ls config/APT_overlays.d/) ; do
+ if ! ssh reprepro@incoming.deb.tails.boum.org \
+ tails-merge-suite "$overlay" "$BRANCH" ; then
+ echo "Failed to merge '$overlay' into '$BRANCH': $?" >&2
+ break
+ fi
+ done
+
+3. Empty `config/APT_overlays.d/`:
+
+ git checkout "$BRANCH" && \
+ git rm config/APT_overlays.d/* && \
+ git commit config/APT_overlays.d/ \
+ -m "Empty the list of APT overlays: they were merged"
+
+4. Push the Git branch:
+
+ git push origin "${BRANCH}:${BRANCH}"
+
+<a id="workflow-post-tag"></a>
+
+Tagging a new Tails release
+---------------------------
+
+Once the new release's Git tag is pushed, a cronjob creates
+a new APT suite on the custom APT repository's side within a few minutes.
+This new APT suite is called the same as the new release version.
+One may check it has appeared in `~reprepro/conf/distributions`.
+
+Then, the APT suite corresponding to the branch that was used to
+prepare the release must be copied to the new empty APT suite that
+just appeared:
+
+ $ ssh reprepro@incoming.deb.tails.boum.org \
+ tails-merge-suite "$RELEASE_BRANCH" "$TAG"
+
+<a id="workflow-post-release"></a>
+
+After a new Tails release is out
+--------------------------------
+
+If you just put out a final release:
+
+* [[merge `stable` or `testing` into
+ `devel`|APT_repository/custom#workflow-merge-main-branch]]
+* increment the version number in devel's `debian/changelog` to match
+ the next major release, so that
+ next builds from the `devel` branch do not use the APT suite meant
+ for the last release
+* increment the version number in stable's `debian/changelog` to match
+ the next point release, so that
+ next builds from the `stable` branch do not use the APT suite meant
+ for the last release
+
+If you just released a RC:
+
+* add a dummy changelog entry (for the upcoming, non-RC version) in
+ the branch used for the release (`stable` or `testing`), so that the
+ next builds from it do not use the APT suite meant for the RC
+* add a dummy changelog entry (for the release *after* the one you
+ released a RC for) in the branch used for the release (`stable` or
+ `testing`), so that the next builds from it do not use the APT suite
+ meant for the RC
+
+If the release was a major one, then:
+
+1. [[Hard reset the stable APT suite to
+ the state of the testing one|APT_repository/custom#workflow-reset]].
+
+2. Empty `config/APT_overlays.d` in the `stable` branch:
+
+ git checkout stable && \
+ git rm config/APT_overlays.d/* && \
+ git commit config/APT_overlays.d/ \
+ -m "Empty the list of APT overlays: they were merged"
+
+Giving access to a core developer
+---------------------------------
+
+1. Give SSH access to the `reprepro` user on the system that hosts
+ reprepro (using the `ssh_authorized_key` Puppet resource).
+2. Import the developer's public GnuPG key into the `reprepro` user's
+ GnuPG keyring -- should be doable using Puppet, some day
+3. Add the developer's OpenPGP key ID to `$reprepro_uploaders` in our
+ `tails::reprepro` Puppet module. Deploy.
+
+Contributing without privileged access
+--------------------------------------
+
+Non-core developers without access to the "private" APT infrastructure
+would add the .deb they want to their Git branch as we have been
+doing until now, push the result on GitLab or whatever... and at
+merge time, we would rewrite their history to remove the .deb, and
+import it into our APT repo.
diff --git a/wiki/src/contribute/APT_repository/tagged_snapshots.mdwn b/wiki/src/contribute/APT_repository/tagged_snapshots.mdwn
new file mode 100644
index 0000000..7d80dca
--- /dev/null
+++ b/wiki/src/contribute/APT_repository/tagged_snapshots.mdwn
@@ -0,0 +1,154 @@
+[[!meta title="Tagged snapshots of upstream APT repositories"]]
+
+[[!toc levels=2]]
+
+# Overview
+
+Our tagged snapshots of upstream APT repositories are published on
+<http://tagged.snapshots.deb.tails.boum.org/>.
+
+These are _partial_, tagged snapshots of upstream APT repositories we
+need, so that one can rebuild a released ISO in the future, and we
+keep the corresponding source code around.
+
+The main goal here is having reproducible builds some day, and to
+comply with various licenses such as the GPL.
+
+These snapshots are partial: in a given snapshot, we import only the
+packages needed by a given build of Tails.
+
+The corresponding data shall be backup'ed, and expired very
+cautiously, if ever.
+
+# Source code
+
+* `tails::reprepro::snapshots::tagged` class in
+ [[!tails_gitweb_repo puppet-tails]]
+* bits scattered in the main Tails Git repository (details below)
+
+# Design notes
+
+## Listing needed packages
+
+To generate partial APT repositories, we need to know what to include
+in them. Therefore, we create a _build manifest_ at the end of an ISO
+build. It is generated by
+[[!tails_gitweb auto/scripts/generate-build-manifest]], thanks to
+[[!tails_gitweb data/wrappers/apt-get]] and
+[[!tails_gitweb data/debootstrap/scripts/jessie.patch]].
+
+Output:
+
+- for each APT repository we use time-based snapshots for: name, serial
+- for each binary package: name, version, architecture
+- for each source package: name, version
+
+In passing, here are some nice side-effects of having this build
+manifest:
+
+- It allows to inspect the diff between the subset of two different
+ snapshots that was used at build time; the benefit is quite small as
+ long as we're based on Debian stable (we also fetch packages from
+ testing, sid, backports, etc. though), but if/when we switch to
+ being based on Debian testing, then we will definitely want that.
+- Say a branch (topic one, or devel, etc.) introduces a regression,
+ and has changes in the set of packages used at build time, we may
+ want to check how exactly that set was changed. Think "check the
+ diff between `.packages`" as we do at release time, but done in
+ a more correct way.
+
+## Importing packages into partial snapshots
+
+### How it's done in practice
+
+* [[!tails_gitweb auto/scripts/tag-apt-snapshots]]
+* [tails-prepare-tagged-apt-snapshot-import](https://git-tails.immerda.ch/puppet-tails/tree/files/reprepro/snapshots/tagged/tails-prepare-tagged-apt-snapshot-import)
+* [tails-publish-tagged-apt-snapshot](https://git-tails.immerda.ch/puppet-tails/tree/files/reprepro/snapshots/time_based/tails-publish-tagged-apt-snapshot)
+
+### A corner case: APT pinning magics
+
+If a (package, version) is seen at build time in 2 or more APT
+sources, `tails-prepare-tagged-apt-snapshot-import` injects it
+into each of the tagged snapshots corresponding to these sources.
+
+The goal is to avoid this scenario, that could happen if we injected
+each package _only_ into the distribution it was downloaded from:
+
+ - version X of package P is available both in suite S1 on origin O1,
+ and in suite S2 on origin O2
+ - version Y of package P is available in suite S3 of origin O3
+ - our pinning makes us prefer version X of package P *because it's
+ available in O1/S1*; otherwise, if it wasn't in there, then our
+ pinning would make APT prefer version Y to version X
+ - at ISO build time, APT fetches package P version X from O2/S2
+ - given this build manifest, we import package P version X into our
+ tagged snapshot of O2/S2, but not into our tagged snapshot of O1/S1
+ - if we rebuild from the same source tree using that set of tagged
+ snapshots, then version Y of package P will be installed
+
+This scenario can happen in practice:
+
+ # cat /etc/apt/sources.list
+ deb http://security.debian.org wheezy/updates main
+ deb http://ftp.us.debian.org/debian/ wheezy main
+ deb http://ftp.us.debian.org/debian/ jessie main
+
+ # cat /etc/apt/preferences
+ Package: *
+ Pin: origin security.debian.org
+ Pin-Priority: -10
+
+ Package: *
+ Pin: release o=Debian,n=wheezy
+ Pin-Priority: 990
+
+ Package: *
+ Pin: release o=Debian,n=jessie
+ Pin-Priority: 700
+
+ # apt-cache madison a2ps
+ a2ps | 1:4.14-1.3 | http://ftp.us.debian.org/debian/ jessie/main amd64 Packages
+ a2ps | 1:4.14-1.1+deb7u1 | http://security.debian.org/ wheezy/updates/main amd64 Packages
+ a2ps | 1:4.14-1.1+deb7u1 | http://ftp.us.debian.org/debian/ wheezy/main amd64 Packages
+
+ # apt-cache policy a2ps
+ a2ps:
+ Installed: (none)
+ Candidate: 1:4.14-1.1+deb7u1
+ Version table:
+ 1:4.14-1.3 0
+ 700 http://ftp.us.debian.org/debian/ jessie/main amd64 Packages
+ 1:4.14-1.1+deb7u1 0
+ -10 http://security.debian.org/ wheezy/updates/main amd64 Packages
+ 990 http://ftp.us.debian.org/debian/ wheezy/main amd64 Packages
+
+And then, APT will download `a2ps` from security.d.o:
+
+ # apt-get download a2ps --print-uris
+ 'http://security.debian.org/pool/updates/main/a/a2ps/a2ps_4.14-1.1+deb7u1_amd64.deb' a2ps_4.14-1.1+deb7u1_amd64.deb 956298 sha256:e47d7fe9adb7aa62421108debf425830f4e2385e98151c5cb359d3eb8688eea8
+
+... but if `a2ps` was not available in the regular Wheezy archive,
+e.g. because we were using a tagged snapshot that imported `a2ps` into
+the security archive, then APT would prefer `a2ps` from Jessie, which
+demonstrates the problem.
+
+## Valid-Until
+
+A tagged APT repository snapshot that was used to build a given Tails
+release is immutable by design, so it does not need the protections
+provided by `Valid-Until`. Besides, not using `Valid-Until` for those
+makes it much easier to reproduce a given ISO build in the future.
+
+So, the `Release` files for tagged snapshots have no
+`Valid-Until` field.
+
+## Garbage collection
+
+We want to keep "forever" the tagged snapshots used by Tails releases.
+
+In practice, "forever" == min(3 years for GPL, how long we want to be
+able to reproduce the build of a released ISO) = 3 years.
+
+Depending on the growth rate of our tagged snapshots in practice, we
+may or may not need to implement expiration of these snapshots any
+time soon. Time will tell.
diff --git a/wiki/src/contribute/APT_repository/time-based_snapshots.mdwn b/wiki/src/contribute/APT_repository/time-based_snapshots.mdwn
new file mode 100644
index 0000000..147a040
--- /dev/null
+++ b/wiki/src/contribute/APT_repository/time-based_snapshots.mdwn
@@ -0,0 +1,431 @@
+[[!meta title="Time-based snapshots of upstream APT repositories"]]
+
+[[!toc levels=2]]
+
+Overview
+========
+
+Our time-based snapshots of upstream APT repositories are published on
+<http://time-based.snapshots.deb.tails.boum.org/>.
+
+These are _full_ snapshots of the upstream APT repositories we use for
+building Tails ISO images. They contain exactly the same set of
+packages as the mirrored repository. This has the advantage that some
+workflows are trivially handled, e.g. working on a topic branch that
+installs additional Debian packages; if such snapshots were not full
+ones, then to work on one such branch, one would need either that to
+have the credentials to import new packages from Debian into our own
+mirror or repositories (which raises the barrier for contributing), or
+that during some phases of Tails development the regular Debian
+archive is used instead of our own mirror, which feels prone to "time
+to QA vs. time to release" issues.
+
+We snapshot each upstream APT repository N times a day, and without
+further action, each snapshot is kept for D days.
+
+The main goal here is to be able to freeze the APT repositories used
+by a branch, whenever we freeze it.
+
+A time-based snapshot's name contains:
+
+ * an identifier of the APT repository this snapshot is about, e.g.
+ `debian`, `debian-security`, `torproject`;
+ * a `YYYYMMDD$ID` serial, `$ID` being an incremental decimal number
+ formatted on two digits (`01`, `02`, etc.).
+
+The APT repository mirroring infrastructure publishes the name of the
+latest snapshot for each mirrored repository over HTTP, in the
+`project/trace/$archive` file
+([example](http://time-based.snapshots.deb.tails.boum.org/debian-security/project/trace/debian-security)).
+Similarly, every ISO
+build exports the names of the APT repository snapshots it uses
+([example](http://nightly.tails.boum.org/build_Tails_ISO_devel/lastSuccessful/archive/latest.iso.apt-sources)).
+
+The corresponding data is not critical: we can restart the whole thing
+from scratch if needed, without too much pain ⇒ no need to synchronize
+this content to the failover server; no need to back it up.
+
+We don't bother merging mirrored APT repositories / suites into
+aggregated ones. It loses information, gives us more work, and brings
+little value.
+
+# Source code
+
+* `tails::reprepro::snapshots::time_based` class in
+ [[!tails_gitweb_repo puppet-tails]]
+* bits scattered in the main Tails Git repository (details below)
+
+SSH access
+==========
+
+One must configure their SSH client to connect to the APT server:
+
+ Host incoming.deb.tails.boum.org
+ Port 3003
+
+Workflow
+========
+
+<a id="freeze"></a>
+
+Freeze snapshots
+----------------
+
+For example, to encode in the `$RELEASE_BRANCH` branch the set of
+[[time-based APT repository snapshots|APT_repository/time-based snapshots]]
+that shall be used during the freeze:
+
+ git checkout "$RELEASE_BRANCH" && \
+ ./auto/scripts/apt-snapshots-serials freeze && \
+ git commit \
+ -m 'Freeze APT snapshots for ${VERSION}.' \
+ config/APT_snapshots.d/*/serial
+
+<a id="thaw"></a>
+
+Thaw snapshots
+--------------
+
+For example, to encode in the `$RELEASE_BRANCH` Git branch the fact
+that it is not frozen anymore, that is remove the indication that
+a specific set of APT repository snapshots must be used:
+
+ git checkout "$RELEASE_BRANCH" && \
+ ./auto/scripts/apt-snapshots-serials thaw && \
+ git commit \
+ -m 'Thaw APT snapshots after Tails $VERSION was released.' \
+ config/APT_snapshots.d/*/serial
+
+<a id="bump-expiration-date"></a>
+
+Bump expiration date
+--------------------
+
+We set `Valid-Until` of time-based snapshots 10 days after they are
+generated. In some cases, this can be too short, and we need to
+manually bump `Valid-Until` for a given time-based snapshot.
+
+Only release managers and sysadmins can do such operations.
+
+### Bump one specific snapshot's expiration date
+
+To bump `Valid-Until`, for a given snapshot (`$SERIAL`) of a given
+archive (`$ARCHIVE`), so that they are valid for `$DAYS_FROM_NOW` days
+from now:
+
+ ssh reprepro-time-based-snapshots@incoming.deb.tails.boum.org \
+ tails-bump-apt-snapshot-valid-until \
+ "$ARCHIVE" "$SERIAL" "$DAYS_FROM_NOW"
+
+<a id="bump-expiration-date-for-all-snapshots"></a>
+
+### Bump all snapshots' expiration date
+
+To bump `Valid-Until`, for every snapshot used by the current frozen
+`$RELEASE_BRANCH` branch, so that they are valid for `$DAYS_FROM_NOW`
+days from now:
+
+ git checkout "$RELEASE_BRANCH" && \
+ (
+ cd config/APT_snapshots.d && \
+ for ARCHIVE in * ; do
+ if ! grep -qs '^latest$' "$ARCHIVE"/serial; then
+ ssh reprepro-time-based-snapshots@incoming.deb.tails.boum.org \
+ tails-bump-apt-snapshot-valid-until \
+ "$ARCHIVE" "$(cat "$ARCHIVE"/serial)" \
+ "$DAYS_FROM_NOW"
+ fi
+ done
+ )
+
+Stop tracking a distribution
+----------------------------
+
+After we stop tracking a distribution, e.g. after we release Tails
+based on a new Debian, we need to manually remove all corresponding
+time-based snapshots, and the packages that are not referenced
+anymore.
+
+For example, when we stopped tracking Wheezy, we did:
+
+ reprepro dumpreferences \
+ | grep -E '^s=wheezy' \
+ | awk '{print $1}' \
+ | sort -u \
+ | xargs -n 1 reprepro _removereferences \
+ && reprepro deleteunreferenced
+
+Freeze exception
+----------------
+
+### Grant a freeze exception
+
+1. Import the package you want to upgrade into our own
+ [[custom APT repository|contribute/APT repository/custom]], in the
+ suite corresponding to the branch that we want to see this
+ package in.
+
+2. If the imported package comes from a Debian distribution whose
+ pinning value is at least 990 in `config/chroot_apt/preferences`:
+ you can stop right here. Otherwise, read on.
+
+3. Add a pinning entry in `config/chroot_apt/preferences` for the
+ package you imported:
+
+ Explanation: freeze exception
+ Package: XYZ
+ Pin: origin deb.tails.boum.org
+ Pin-Priority: 999
+
+4. Commit:
+
+ git commit config/chroot_apt/preferences \
+ -m "Add freeze exceptions for $(dpkg-parsechangelog -SVersion)"
+
+5. Push to Git.
+
+<a id="freeze-exceptions-post-release"></a>
+
+### Post-release
+
+Thaw the packages that were granted freeze exceptions, now that they
+can be fetched from a newer time-based snapshot of the repository
+we've initially pulled it from.
+
+1. For each entry in `config/chroot_apt/preferences` that has
+ `Explanation: freeze exception`: set `Pin-Priority` to `-1`.
+
+2. Commit:
+
+ git commit config/chroot_apt/preferences \
+ -m "Remove freeze exceptions added for $(dpkg-parsechangelog -SVersion)"
+
+3. Push to Git.
+
+# Design notes
+
+## gensnapshot
+
+We use reprepro's `gensnapshot` command, that basically copies
+a distribution, keeping references to the packages it contains.
+
+Compared to the "snapshots as full-blown distributions + `reprepro
+pull`" option we
+[used in our initial experiments](https://labs.riseup.net/code/issues/6295#note-14),
+we are saving _a lot_ on database size, and thus in performance,
+because reprepro does less tracking on snapshots, than what it does
+for real distributions.
+
+The counterpart of using snapshots created with `gensnapshot` is that:
+
+ * garbage collecting expired snapshots is a bit more involved, i.e.
+ we have to
+ [do it ourselves](https://git-tails.immerda.ch/puppet-tails/tree/files/reprepro/snapshots/time_based/tails-delete-expired-apt-snapshots);
+ * bumping `Valid-Until` for a given time-based snapshot has to be
+ done directly in `dist`, without any help from reprepro; so here
+ again, we
+ [do it ourselves](https://git-tails.immerda.ch/puppet-tails/tree/files/reprepro/snapshots/time_based/tails-bump-apt-snapshot-valid-until).
+
+None of these problems warrant going back to the other option... and
+having to deal with 80GB+ Berkeley DB databases.
+
+## Garbage collection and Valid-Until
+
+We expire snapshots older than 10 days in order to save disk space,
+and to avoid the reprepro database to grow too much.
+
+To ensure that garbage collection doesn't delete a snapshot we still
+need, e.g. the one currently referenced in the frozen `testing`
+branch, we rely on the `Valid-Until` field found in `Release` files:
+the way to express "I want to keep a given snapshot around" is to
+postpone its expiration date; i.e. we don't differentiate "keep
+a given snapshot around" from "keep a given snapshot usable", which
+seems to make sense.
+
+See [[above|time-based_snapshots#bump-expiration-date]] for how we
+can manage `Valid-Until` manually, whenever needed.
+
+One advantage of this design is that we don't have to regularly update
+`Valid-Until` fields, and the corresponding signatures: we only do
+that on a case-by-case basis, when needed. And thus, we can actually
+benefit from the protections offered by APT when `Valid-Until` fields
+are present, as any snapshot will expire unless we do something
+about it.
+
+In practice, the main use case for keeping a given time-based APT
+repository snapshot around and valid is when it's being used by
+a release branch:
+
+ - `testing`: while it's frozen, that is during 5-10 days most of the
+ time;
+ - `stable`: that's a corner case, since `stable` generally uses the
+ set of tagged snapshots of the latest Tails release; if and when we
+ decide to manually point `stable` to a different set of snapshots,
+ then we can as well deal with `Valid-Until` manually.
+
+In passing, note that we ship an empty `/var/cache/apt/lists/` in the
+ISO ⇒ modifying `Release` and `Release.gpg` files on our APT
+repository won't prevent the ISO build from being deterministic.
+
+## APT vs. reprepro: dist names
+
+We need to encode in the APT sources' base URL the exact snapshot we
+want to use, in order to be able to pass it to `lb config --mirror-*`.
+But this doesn't match reprepro's directory structure as-is.
+
+Thankfully this problem can be workaround'ed with some symlinks or
+HTTP rewrite rules. Here's how.
+
+Let's assume:
+
+ lb config --distribution jessie
+ lb config --mirror-chroot \
+ http://time-based.snapshots.deb.tails.boum.org/debian/2016031101/
+ lb config --mirror-chroot-security \
+ http://time-based.snapshots.deb.tails.boum.org/debian-security/2016031102/
+ [...]
+
+Which generates this APT `sources.list`:
+
+ deb http://time-based.snapshots.deb.tails.boum.org/debian/2016031101/ jessie main
+ deb http://time-based.snapshots.deb.tails.boum.org/debian-security/2016031102/ jessie/updates main
+ [...]
+
+As a result APT sends HTTP requests with URLs such as:
+
+ * <http://time-based.snapshots.deb.tails.boum.org/debian/2016032401/dists/jessie/Release>
+ * <http://time-based.snapshots.deb.tails.boum.org/debian/2016032401/pool/XYZ>
+ * <http://time-based.snapshots.deb.tails.boum.org/debian-security/2016032402/dists/jessie/updates/Release>
+ * <http://time-based.snapshots.deb.tails.boum.org/debian-security/2016032402/pool/XYZ>
+
+The corresponding files in reprepro's filesystem (given that we have
+one reprepro instance per mirrored archive) are:
+
+ * in Debian archive's reprepro:
+ - `/srv/apt-snapshots/time-based/repositories/debian/dists/jessie/snapshots/2016032401/Release`,
+ that contains `Suite: jessie/snapshots/2016032401` and `Codename: jessie`
+ - `/srv/apt-snapshots/time-based/repositories/debian/pool/XYZ`
+
+ * in Debian security archive's reprepro:
+ - `/srv/apt-snapshots/time-based/repositories/debian-security/dists/jessie/updates/snapshots/2016031102/Release`,
+ that contains `Suite: jessie/updates/snapshots/2016031102` and
+ `Codename: jessie/updates`
+ - `/srv/apt-snapshots/time-based/repositories/debian-security/pool/XYZ`
+
+To have the above HTTP requests translate to access to these files,
+we use
+[a set of HTTP rewrite rules](https://git-tails.immerda.ch/puppet-tails/tree/templates/reprepro/snapshots/time_based/nginx_site.erb).
+
+Note: this works because APT only warns when the codename in the
+`Release` file doesn't match the one requested in `sources.list`.
+There's a code comment around this check, dating back from 2004, that
+says something like "This might become fatal in the future". We bet that if it
+becomes fatal some day, it will be possible to turn it back into
+a warning via configuration. This affects only development builds
+since we're not going to configure APT _in the Tails ISO_ to point to
+our own snapshots of the Debian archive anyway.
+
+<a id="design-freeze-exceptions"></a>
+
+## Freeze exceptions
+
+This is a new problem brought by using "frozen" snapshot of APT
+repositories during a Tails code freeze: some bug, that we want to see
+fixed in the release we are preparing, would be resolved if we pulled
+an upgraded package as-is from a freshest Debian APT repository.
+Before we could freeze APT repositories, we would have got this bugfix
+for free. Now we need to grant freeze exceptions.
+
+This is similar to "Upgrading to a new snapshot", except that we want
+to upgrade one package only. By definition, this only affects *frozen*
+release branches (`stable`, `testing`), and topic branches based on
+them: all other branches use the freshest set of APT repository
+snapshots available.
+
+Most of the time, a bugfix branch we want to merge into a frozen
+release branch doesn't need to upgrade packages from Debian, so this
+is a corner case for the time being. Moreover, so far we have always
+dealt with this problem entirely by hand, so it's not critical to
+provide much improved tools. What makes it tempting to improve the
+situation here is mostly:
+
+ * even though freeze exceptions will remain exceptions, frozen will
+ add one use case:
+ * this will become a relatively common operation if we are based on
+ Debian testing some day, so let's check that it's not only
+ possible, but also reasonably easy to handle with this design
+ (otherwise we may have to switch to more powerful tools, such as
+ dak + britney).
+
+To grant a freeze exception to a given package, we simply import it
+into our own
+[[custom APT repository|contribute/APT repository/custom]], in the
+suite corresponding to the branch that we want to see this package in
+⇒ in the general case, the upgraded package will be installed in the
+next Tails release.
+
+This works because our APT pinning ranks Tails custom APT suites at
+the same level as the other APT sources corresponding to the current
+version of Debian Tails is based on, and higher than other Debian
+distribution (which, in passing, implies that we have to manually pin,
+in Git, the packages from our custom APT suites, that we want to
+override the ones found in other repositories regardless of version
+numbers):
+
+ * if the imported package comes from Debian stable: it will be
+ installed simply because its version is greater than the version of
+ the same package from Debian stable; and once we have thawed the
+ corresponding snapshot, the package can be pulled equally from any
+ of these two sources (Debian, and our custom APT repository), until
+ a newer version of this package is uploaded to Debian, and then the
+ newer one will supersede the package we have in our custom APT
+ repository;
+
+ * if the imported package comes from another Debian distribution,
+ that has a pinning value strictly lower than 990, such as Debian
+ unstable: if we did nothing more, the package would be installed
+ because its pinning (`origin deb.tails.boum.org`) is higher than
+ the one from the Debian distribution we're importing it from;
+ *however*, in this case we need to track this package, and to
+ remove it from our custom repository after we have thawed the
+ corresponding snapshot — otherwise, due to this pinning
+ configuration, we would stick to the version of the package we have
+ one day imported, while in most cases we want to resume tracking
+ the version from Debian; so, we do this that way:
+
+ 1. Import the package we want to upgrade into our own
+ [[custom APT repository|contribute/APT repository/custom]], in
+ the suite corresponding to the branch that we want to see this
+ package in.
+
+ 2. Explicitly pin, in `config/chroot_apt/preferences`, the upgraded
+ package we have just imported to a value higher than 990, with
+ a proper `Explanation:` field; this pinning is not required at
+ this stage, but it is one way to encode in Git the packages we
+ have imported, which simplifies the following (clean up) step.
+
+ 3. Once the corresponding APT snapshot has been thawed, that is
+ once the upgraded package can be fetched from a newer time-based
+ snapshot of the repository we've initially pulled it from: make
+ it so branches stop using the upgraded package, and resume
+ tracking the one available in Debian. To do that, we modify the
+ pinning entry added at the previous step, and give it a value of
+ `-1`. This should be done by the release manager, immediately
+ after a release, when they thaw the APT snapshots used for the
+ release, and merge it into other release branches.
+
+ Ideas for future improvements:
+
+ * At some point a helper tool can do this automatically,
+ assuming we always use the same `Explanation:` field to mark
+ these pinning entries. (Ideally we would simply use
+ a dedicated file under `apt/preferences.d/` for freeze
+ exceptions, but `live-build` 2.x doesn't support that.)
+
+ * Ideally we would remove these imported packages from our
+ custom APT repository at post-release time as well, so we can
+ get rid of the `-1` pinning entries, but it really needs to be
+ done in a 100% correct order, to ensure that after all the
+ merges we do post-release (and sometimes at other times)
+ between release branches, the imported packages are _not_
+ present anymore in any of the corresponding APT suites.
diff --git a/wiki/src/contribute/build.mdwn b/wiki/src/contribute/build.mdwn
index dfa5107..1085aca 100644
--- a/wiki/src/contribute/build.mdwn
+++ b/wiki/src/contribute/build.mdwn
@@ -337,11 +337,13 @@ The most common customizations are documented on this wiki:
* to avoid compressing the SquashFS using XZ (efficient, but very
slow), `export MKSQUASHFS_OPTIONS='-comp gzip'` in your
build environment;
-* [[using a custom Debian mirror to build Tails
- images|build/custom_mirror]];
-* [[using squid-deb-proxy to build Tails images|build/squid-deb-proxy]]
- (**Note**: most Tails contributors using the manual build method
- use [[!debpts apt-cacher-ng]] instead, nowadays.)
+* to avoid downloading lots of Debian packages during every build, you
+ can use [[!debpts apt-cacher-ng]]; however, the build system
+ constantly switches APT sources for our
+ [[APT repositories|contribute/APT_repository]], so some custom
+ configuration is needed to make `apt-cacher-ng` useful: see the
+ bits about `apt-cacher-ng` in
+ [[!tails_gitweb vagrant/provision/assets/build-tails]].
More documentation about this can be found in the [Debian Live
Manual](http://live.debian.net/manual-2.x/html/live-manual.en.html).
diff --git a/wiki/src/contribute/build/custom_mirror.mdwn b/wiki/src/contribute/build/custom_mirror.mdwn
deleted file mode 100644
index b55b3fb..0000000
--- a/wiki/src/contribute/build/custom_mirror.mdwn
+++ /dev/null
@@ -1,31 +0,0 @@
-[[!meta title="Using a custom Debian mirror to build Tails images"]]
-
-[[!toc levels=2]]
-
-# What?
-
-You may want to use a Debian mirror different from the one
-`live-build` uses by default.
-
-# Why?
-
-1. You want to use a mirror that is faster for you, e.g. one that is
- in the country you live in, or a local Debian mirror.
-2. You have [[configured squid-deb-proxy|build/squid-deb-proxy]] to
- only allow access to a given set of domains.
-
-# How?
-
-You need to set options in live-build's configuration file
-(`/etc/live/build.conf`), e.g.:
-
- LB_MIRROR_BOOTSTRAP="http://ftp.us.debian.org/debian/"
- LB_MIRROR_BOOTSTRAP_SECURITY="http://security.debian.org/"
-
- LB_MIRROR_CHROOT="http://ftp.us.debian.org/debian/"
- LB_MIRROR_CHROOT_SECURITY="http://security.debian.org/"
-
- LB_MIRROR_BINARY="http://ftp.us.debian.org/debian/"
- LB_MIRROR_BINARY_SECURITY="http://security.debian.org/"
-
-See the `lb_config(1)` manpage for details.
diff --git a/wiki/src/contribute/build/squid-deb-proxy.mdwn b/wiki/src/contribute/build/squid-deb-proxy.mdwn
deleted file mode 100644
index 37ad0ee..0000000
--- a/wiki/src/contribute/build/squid-deb-proxy.mdwn
+++ /dev/null
@@ -1,96 +0,0 @@
-[[!meta title="Using squid-deb-proxy to build Tails images"]]
-
-[[!toc levels=2]]
-
-# Why?
-
-The Debian mirrors live-build uses are very much configurable (see
-[[build/custom_mirror]]), but Tails uses some non-Debian APT
-repositories whose location cannot be adapted to your build
-environment using only live-build settings.
-
-Moreover, some packages fetch data online over HTTP at install time
-i.e. at Tails build time.
-
-Both of these factors make it hard to build Tails offline... unless
-you have a HTTP proxy with all necessary files cached on disk, that is
-able to switch to offline mode.
-
-# How?
-
-## Install and configure squid-deb-proxy
-
-squid-deb-proxy is a Debian proxy solution based on Squid with a
-tweaked configuration. It has a [homepage on
-Launchpad](https://launchpad.net/squid-deb-proxy) and packages in
-Ubuntu.
-
-On the other hand, we've found the upstream and Ubuntu packages to
-require many changes to be usable in Debian and especially for Tails
-purposes. Until we manage to turn our changes into a proper Debian
-package and make them generic enough to be pushed upstream, we
-recommend using the installation instructions below.
-
-1. Install necessary dependencies:
-
- apt-get install squid3
-
-2. Create the necessary directories:
-
- install -o root -g root -m 00755 -d /etc/squid-deb-proxy
- install -o proxy -g root -m 00750 -d /var/cache/squid-deb-proxy
- install -o proxy -g root -m 00750 -d /var/log/squid-deb-proxy
-
-3. Install the attached configuration files; all shall be owned by
- `root:root` and have `00644` permissions unless otherwise noted:
-
- * [[/etc/squid-deb-proxy/squid-deb-proxy.conf|squid-deb-proxy/squid-deb-proxy.conf]]
- is the main configuration file passed to Squid3.
- * [[/etc/squid-deb-proxy/allowed-networks-src.acl|squid-deb-proxy/allowed-networks-src.acl]]
- lists the network sources that are allowed to access the cache
- * [[/etc/squid-deb-proxy/mirror-dstdomain.acl|squid-deb-proxy/mirror-dstdomain.acl]]
- lists the network destinations the proxy is allowed to reach
- * [[/etc/init.d/squid-deb-proxy|squid-deb-proxy/squid-deb-proxy]] is
- the initscript; it needs to be made executable.
-
-4. Adapt the configuration for your needs:
-
- * The example `cache_mem` setting shall be changed depending on the
- amount of RAM your build machine has
- * The example `cache_dir` has a 10G limit that is a bit high if you
- only want to use squid-deb-proxy to build Tails.
-
-5. Have the proxy start on boot:
-
- update-rc.d squid-deb-proxy defaults
-
-6. Start the proxy:
-
- invoke-rc.d squid-deb-proxy start
-
-## Use the proxy for live-build and APT
-
-You need to set a few options in live-build's configuration file
-(`/etc/live/build.conf`).
-
-First, the following line asks live-build to use the configured proxy
-for any use of APT it makes:
-
- LB_APT_HTTP_PROXY="http://127.0.0.1:8000/"
-
-Second, you probably need to [[ask live-build to use a custom Debian
-mirror|build/custom_mirror]] to fit the destination domain
-restrictions you set previously up.
-
-## Use the proxy for non-APT purposes
-
-Export the `http_proxy='http://127.0.0.1:8000/'` environment variable
-in the (root) shell you want to run the build in. This way, HTTP
-requests that are made by anyone else than APT during the build will
-use the configured Squid proxy.
-
-## Build offline
-
-Un-comment the settings in the *Offline mode* section at the end of
-`/etc/squid-deb-proxy/squid-deb-proxy.conf`.
-
diff --git a/wiki/src/contribute/build/squid-deb-proxy/allowed-networks-src.acl b/wiki/src/contribute/build/squid-deb-proxy/allowed-networks-src.acl
deleted file mode 100644
index 9902fe5..0000000
--- a/wiki/src/contribute/build/squid-deb-proxy/allowed-networks-src.acl
+++ /dev/null
@@ -1,11 +0,0 @@
-### network sources that are allowed to access the cache
-
-### private RFC-1918 networks
-# 10.0.0.0/8
-# 172.16.0.0/12
-# 192.168.0.0/16
-
-### mDNS (.local)
-# 169.254.0.0/16
-
-### add your own here (if you need non-private network access)
diff --git a/wiki/src/contribute/build/squid-deb-proxy/mirror-dstdomain.acl b/wiki/src/contribute/build/squid-deb-proxy/mirror-dstdomain.acl
deleted file mode 100644
index e6ccd79..0000000
--- a/wiki/src/contribute/build/squid-deb-proxy/mirror-dstdomain.acl
+++ /dev/null
@@ -1,18 +0,0 @@
-### network destinations the proxy is allowed to reach
-
-### Debian
-cdimage.debian.org
-ftp.us.debian.org
-backports.debian.org
-security.debian.org
-volatile.debian.org
-
-### Other
-deb.tails.boum.org
-deb.torproject.org
-live.debian.net
-mozilla.debian.net
-
-### b43-fwcutter
-.openwrt.org
-www.lwfinger.com
diff --git a/wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy b/wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy
deleted file mode 100755
index 5ef5583..0000000
--- a/wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy
+++ /dev/null
@@ -1,174 +0,0 @@
-#! /bin/sh
-#
-# squid Startup script for the SQUID Deb proxy-cache.
-#
-# Version: @(#)squid.rc 2.20 01-Oct-2001 miquels@cistron.nl
-#
-### BEGIN INIT INFO
-# Provides: squid-deb-proxy
-# Required-Start: $local_fs $network $avahi
-# Required-Stop: $local_fs $network $avahi
-# Should-Start: $named
-# Should-Stop: $named
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: Squid Deb Proxy
-### END INIT INFO
-
-NAME=squid-deb-proxy
-DESC="Squid Debian Proxy"
-DAEMON=/usr/sbin/squid3
-PIDBASE=/var/run/squid-deb-proxy
-PIDFILE=$PIDBASE/$NAME.pid
-CONFIG=/etc/squid-deb-proxy/squid-deb-proxy.conf
-SQUID_ARGS="-YC -f $CONFIG"
-
-# [ ! -f /etc/default/squid ] || . /etc/default/squid
-
-. /lib/lsb/init-functions
-
-PATH=/bin:/usr/bin:/sbin:/usr/sbin
-
-[ -x $DAEMON ] || exit 0
-
-ulimit -n 65535
-
-find_cache_dir () {
- w=" " # space tab
- res=`sed -ne '
- s/^'$1'['"$w"']\+[^'"$w"']\+['"$w"']\+\([^'"$w"']\+\).*$/\1/p;
- t end;
- d;
- :end q' < $CONFIG`
- [ -n "$res" ] || res=$2
- echo "$res"
-}
-
-find_cache_type () {
- w=" " # space tab
- res=`sed -ne '
- s/^'$1'['"$w"']\+\([^'"$w"']\+\).*$/\1/p;
- t end;
- d;
- :end q' < $CONFIG`
- [ -n "$res" ] || res=$2
- echo "$res"
-}
-
-start () {
- cache_dir=`find_cache_dir cache_dir /var/cache/$NAME`
- cache_type=`find_cache_type cache_dir ufs`
-
- #
- # Create $PIDBASE if it doesn't exist.
- #
- if [ ! -d "$PIDBASE" ]; then
- mkdir --mode=00755 $PIDBASE
- chown proxy:proxy $PIDBASE
- fi
-
- #
- # Create spool dirs if they don't exist.
- #
- if [ "$cache_type" = "coss" -a -d "$cache_dir" -a ! -f "$cache_dir/stripe" ] || [ "$cache_type" != "coss" -a -d "$cache_dir" -a ! -d "$cache_dir/00" ]
- then
- log_warning_msg "Creating $DESC cache structure"
- $DAEMON $SQUID_ARGS -z
- fi
-
- umask 027
- ulimit -n 65535
- cd $cache_dir
- start-stop-daemon --quiet --start \
- --chuid proxy \
- --pidfile $PIDFILE \
- --exec $DAEMON -- $SQUID_ARGS < /dev/null
- res=$?
-
- if [ -x /usr/bin/avahi-publish-service ]; then
- http_port=$(grep --max-count=1 '^http_port' "$CONFIG"|cut -d' ' -f2)
- if [ -n "$http_port" ]; then
- if echo "$http_port" | grep -qs ':'; then
- PORT=$(echo "$http_port" | cut -d':' -f2)
- else
- PORT="$http_port"
- fi
- echo "Publishing Avahi service on port $PORT"
- avahi-publish-service "Squid deb proxy" _apt_proxy._tcp $PORT &
- fi
- fi
-
- return $res
-}
-
-stop () {
- PID=`cat $PIDFILE 2>/dev/null`
- start-stop-daemon --stop --quiet --pidfile $PIDFILE --exec $DAEMON
- #
- # Now we have to wait until squid has _really_ stopped.
- #
- sleep 2
- if test -n "$PID" && kill -0 $PID 2>/dev/null
- then
- log_action_begin_msg " Waiting"
- cnt=0
- while kill -0 $PID 2>/dev/null
- do
- cnt=`expr $cnt + 1`
- if [ $cnt -gt 24 ]
- then
- log_action_end_msg 1
- return 1
- fi
- sleep 5
- log_action_cont_msg ""
- done
- log_action_end_msg 0
- return 0
- else
- return 0
- fi
-}
-
-case "$1" in
- start)
- log_daemon_msg "Starting $DESC" "$NAME"
- if start ; then
- log_end_msg $?
- else
- log_end_msg $?
- fi
- ;;
- stop)
- log_daemon_msg "Stopping $DESC" "$NAME"
- if stop ; then
- log_end_msg $?
- else
- log_end_msg $?
- fi
- ;;
- reload|force-reload)
- log_action_msg "Reloading $DESC configuration files"
- $DAEMON $SQUID_ARGS -k reconfigure
- log_action_end_msg 0
- ;;
- restart)
- log_daemon_msg "Restarting $DESC" "$NAME"
- stop
- if start ; then
- log_end_msg $?
- else
- log_end_msg $?
- fi
- ;;
- status)
- status_of_proc -p "$PIDFILE" "$DAEMON" squid && exit 0 || exit $?
- ;;
- *)
- echo "Usage: /etc/init.d/$NAME {start|stop|reload|force-reload|restart}"
- exit 3
- ;;
-esac
-
-exit 0
-
diff --git a/wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy.conf b/wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy.conf
deleted file mode 100644
index 48489d2..0000000
--- a/wiki/src/contribute/build/squid-deb-proxy/squid-deb-proxy.conf
+++ /dev/null
@@ -1,124 +0,0 @@
-
-# WELCOME TO SQUID DEB PROXY
-# ------------------
-#
-# This config file is a version of a squid proxy file optimized
-# as a configuration for a caching proxy for Ubuntu systems.
-#
-# More information about squid and its configuration can be found here
-# http://www.squid-cache.org/ and in the FAQ
-
-# settings that you may want to customize
-# ---------------------------------------
-
-# this file contains private networks (10.0.0.0/8, 172.16.0.0/12,
-# 192.168.0.0/16) by default, you can add/remove additional allowed
-# source networks in it to customize it for your setup
-acl allowed_networks src "/etc/squid-deb-proxy/allowed-networks-src.acl"
-
-# this file contains the *archive.ubuntu.com mirrors by default,
-# if you use a different mirror, add it there
-acl to_ubuntu_mirrors dstdomain "/etc/squid-deb-proxy/mirror-dstdomain.acl"
-
-# default to a different port than stock squid
-http_port 127.0.0.1:8000
-
-# force outgoing IPv4
-# tcp_outgoing_address 192.168.1.17
-
-# -------------------------------------------------
-# settings below probably do not need customization
-
-# user visible name
-visible_hostname squid-deb-proxy
-
-# quicker - and non-graceful - shutdown
-shutdown_lifetime 1 seconds
-
-# we need a big cache, some debs are huge
-maximum_object_size 512 MB
-
-# use a different dir than stock squid and default to 10G
-cache_dir aufs /var/cache/squid-deb-proxy 10000 16 256
-
-# use different logs
-cache_access_log /var/log/squid-deb-proxy/access.log
-cache_log /var/log/squid-deb-proxy/cache.log
-cache_store_log /var/log/squid-deb-proxy/store.log
-
-# tweaks to speed things up
-cache_mem 1024 MB
-maximum_object_size_in_memory 10240 KB
-
-# write the PID in a directory writable by proxy user
-pid_filename /var/run/squid-deb-proxy/squid-deb-proxy.pid
-
-
-cache_replacement_policy heap LFUDA
-
-# refresh patterns
-refresh_pattern ^ftp: 1440 20% 10080
-refresh_pattern ^gopher: 1440 0% 1440
-refresh_pattern deb$ 129600 100% 129600
-refresh_pattern udeb$ 129600 100% 129600
-refresh_pattern tar.gz$ 129600 100% 129600
-refresh_pattern DiffIndex$ 0 20% 4320 refresh-ims
-refresh_pattern PackagesIndex$ 0 20% 4320 refresh-ims
-refresh_pattern Packages\.bz2$ 0 20% 4320 refresh-ims
-refresh_pattern Packages\.gz$ 0 20% 4320 refresh-ims
-refresh_pattern Packages\.lzma$ 0 20% 4320 refresh-ims
-refresh_pattern SourcesIndex$ 0 20% 4320 refresh-ims
-refresh_pattern Sources\.bz2$ 0 20% 4320 refresh-ims
-refresh_pattern Sources\.gz$ 0 20% 4320 refresh-ims
-refresh_pattern Sources\.lzma$ 0 20% 4320 refresh-ims
-refresh_pattern Release$ 0 20% 4320 refresh-ims
-refresh_pattern Release\.gpg$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-en\.bzip2$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-en\.bz2$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-en\.gz$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-en\.lzma$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-fr\.bzip2$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-fr\.bz2$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-fr\.gz$ 0 20% 4320 refresh-ims
-refresh_pattern Translation-fr\.lzma$ 0 20% 4320 refresh-ims
-refresh_pattern . 0 20% 4320
-
-# handle meta-release and changelogs.ubuntu.com special
-refresh_pattern changelogs.ubuntu.com/* 0 1% 1
-
-# see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=499379
-refresh_all_ims on
-
-# default acl
-acl all src all
-acl localhost src 127.0.0.1/32
-acl manager proto cache_object
-acl purge method PURGE
-
-# only allow connects to ports for http, https
-acl Safe_ports port 80
-acl Safe_ports port 443
-
-# Only allow cachemgr access from localhost
-http_access deny manager !localhost
-http_access deny purge !localhost
-
-# Password for the cachemgr
-# cachemgr_passwd secret all
-
-# only allow port we trust
-http_access deny !Safe_ports
-# and only to ubuntu
-http_access deny !to_ubuntu_mirrors
-
-# allow access from our network and localhost
-http_access allow allowed_networks
-http_access allow localhost
-
-# And finally deny all other access to this proxy
-http_access deny all
-
-## Offline mode
-# offline_mode on
-# connect_timeout 10 seconds
-# dns_timeout 10 seconds
diff --git a/wiki/src/contribute/design/I2P.mdwn b/wiki/src/contribute/design/I2P.mdwn
index d28e1a6..e6fbffc 100644
--- a/wiki/src/contribute/design/I2P.mdwn
+++ b/wiki/src/contribute/design/I2P.mdwn
@@ -147,7 +147,7 @@ Tails uses the I2P (and deps)
[Debian packages prepared by KillYourTV](http://deb.i2p2.no/), the official I2P
Linux package maintainer as listed on the [I2P Team page](https://geti2p.net/team).
The I2P source package and its binaries are imported into to our own
-[[APT repository|APT_repository]] into the devel or stable suite. The suite
+[[custom APT repository|APT_repository/custom]] into the devel or stable suite. The suite
will depend on whether a major- or point-release is being prepared.
## Prepare a Git topic branch
diff --git a/wiki/src/contribute/release_process.mdwn b/wiki/src/contribute/release_process.mdwn
index 5ea724f..b18db69 100644
--- a/wiki/src/contribute/release_process.mdwn
+++ b/wiki/src/contribute/release_process.mdwn
@@ -80,9 +80,8 @@ If we are at freeze time for a major release:
git checkout devel && git merge --no-ff origin/master
-2. [[Merge each APT overlay
- suite|contribute/APT_repository#workflow-merge-overlays]] listed in
- the `devel` branch's `config/APT_overlays.d/` into the `devel`
+2. [[Merge each APT overlay suite|APT_repository/custom#workflow-merge-overlays]]
+ listed in the `devel` branch's `config/APT_overlays.d/` into the `devel`
APT suite.
3. Merge the `devel` Git branch into the `testing` one:
@@ -92,8 +91,18 @@ If we are at freeze time for a major release:
... and check that the resulting `config/APT_overlays.d/` in the
`testing` branch is empty.
-4. [[Hard reset|APT_repository#workflow-reset]] the `testing` APT
- suite to the current state of the `devel` one.
+4. [[Hard reset|APT_repository/custom#workflow-reset]] the `testing`
+ custom APT suite to the current state of the `devel` one.
+
+5. [[Freeze|APT_repository/time-based snapshots#freeze]] the
+ time-based APT repository snapshots that shall be used
+ during the freeze.
+
+6. Make it so the time-based APT repository snapshots are kept around
+ long enough, by bumping their `Valid-Until` to max(now + 10 days,
+ expected release date + 5 days):
+ [[APT_repository/time-based_snapshots#bump-expiration-date-for-all-snapshots]]
+
Point-release
-------------
@@ -104,19 +113,18 @@ If we are at freeze time for a point-release:
git checkout stable && git merge --no-ff origin/master
-2. [[Merge each APT overlay
- suite|contribute/APT_repository#workflow-merge-overlays]] listed in
- the `stable` branch's `config/APT_overlays.d/` into the `stable`
+2. [[Merge each APT overlay suite|APT_repository/custom#workflow-merge-overlays]]
+ listed in the `stable` branch's `config/APT_overlays.d/` into the `stable`
APT suite.
Common steps for point and major releases
-----------------------------------------
-After either of the above sections' steps, reset the release branch's
-`config/base_branch`:
+Reset the release branch's `config/base_branch`:
- echo "${RELEASE_BRANCH}" > config/base_branch && \
- git commit config/base_branch -m "Restore ${RELEASE_BRANCH}'s base branch."
+ echo "${RELEASE_BRANCH}" > config/base_branch && \
+ git commit config/base_branch \
+ -m "Restore ${RELEASE_BRANCH}'s base branch."
Update included files
=====================
@@ -171,7 +179,7 @@ Then check the PO files:
Correct any displayed error, then commit the changes if any.
Then see the relevant release processes, and upload the packages to
-the release branch's APT suite:
+the release branch's custom APT suite:
* [[tails-installer]]
* [[tails-greeter]]
@@ -215,10 +223,9 @@ follow these instructions:
Major release
-------------
-[[Merge each APT overlay
-suite|contribute/APT_repository#workflow-merge-overlays]] listed in
-the `testing` branch's `config/APT_overlays.d/` into the `testing`
-APT suite.
+[[Merge each APT overlay suite|APT_repository/custom#workflow-merge-overlays]]
+listed in the `testing` branch's `config/APT_overlays.d/` into the `testing`
+custom APT suite.
Point-release
-------------
@@ -230,17 +237,31 @@ steps have already been done above, and this section is a noop in the
general case.
</div>
-[[Merge each APT overlay
-suite|contribute/APT_repository#workflow-merge-overlays]] listed in
-the `stable` branch's `config/APT_overlays.d/` into the `stable` APT suite.
+[[Merge each APT overlay suite|APT_repository/custom#workflow-merge-overlays]]
+listed in the `stable` branch's `config/APT_overlays.d/` into the `stable`
+custom APT suite.
Update other base branches
==========================
1. Merge the release branch into `devel` following the instructions for
- [[merging base branches|contribute/APT_repository#workflow-merge-main-branch]].
+ [[merging base branches|APT_repository/custom#workflow-merge-main-branch]].
+
+2. Merge `devel` into `feature/stretch` following the instructions for
+ [[merging base branches|APT_repository/custom#workflow-merge-main-branch]].
+ Given that these two branches' APT suites have diverged a lot, and
+ that `tails-merge-suite` currently happily overwrites newer
+ packages in the target with older packages from the source, it's
+ probably easier to just merge each individual APT overlay that was
+ just merged into the release branch into `feature/stretch`'s APT
+ suite. Also, most of our just upgraded bundled packages
+ (e.g. `tails-greeter`) may need to be rebuilt for Stretch.
-2. Push the modified branches to Git:
+3. Ensure that the release, `devel` and `feature/stretch` branches
+ have the expected content in `config/APT_overlays.d/`: e.g. it must
+ not list any overlay APT suite that has been merged already.
+
+4. Push the modified branches to Git:
git push origin "${RELEASE_BRANCH}:${RELEASE_BRANCH}" devel:devel
@@ -367,6 +388,19 @@ signatures, like the defaults we set in Tails:
cp config/chroot_local-includes/etc/skel/.gnupg/gpg.conf "$GNUPGHOME"
+Build the almost-final image
+============================
+
+1. [[Build an ISO image|contribute/build]] from the release branch.
+2. Carefully read the build logs to make sure nothing bad happened.
+3. Keep at least the resulting ISO image and the manifest of needed
+ packages until the end of this release process.
+4. Record where the manifest of needed packages is stored:
+
+ export PACKAGES_MANIFEST=XXX ; \
+ [ -f "$PACKAGES_MANIFEST" ] || echo "ERROR: PACKAGES_MANIFEST is incorrect"
+
+
Tag the release in Git
======================
@@ -380,11 +414,26 @@ premature, as testing might reveal critical issues, but this is
a signed tag, so it can be overridden later. Yes, there is room for
improvement here.)
-Prepare the versioned APT suite
-===============================
+Prepare the versioned APT suites
+================================
+
+* [[Prepare the versioned APT suite in our custom APT repository|APT_repository/custom#workflow-post-tag]].
+
+* Prepare tagged snapshots of upstream APT repositories:
+
+ ./bin/tag-apt-snapshots "$PACKAGES_MANIFEST" "$TAG"
-Follow the [[post-tag|contribute/APT_repository#workflow-post-tag]] APT
-repository documentation.
+ Note:
+
+ - This command can take a while (about a dozen minutes).
+ - It's expected that the packages that were pulled from our
+ [[custom APT repository|APT_repository/custom]] are
+ listed under "some packages were not found anywhere" (because we
+ are current not using time-based snapshots for our custom APT
+ repository). However, _no other package should be on that list_.
+ Now, we have a "safety" net, in case you don't notice such a problem: if
+ other packages are missing, the next build (that will use the
+ newly created partial, tagged APT repository) will fail.
Build images
============
@@ -403,25 +452,10 @@ A new tag may indicate that a new TBB release is imminent.
Better catch this before people spend time doing manual tests.
-Build the almost-final image
-----------------------------
-
-* Check out the release tag:
-
- git checkout "${TAG}"
-
-* [[Build images|contribute/build]] and carefully read the build logs to
- make sure nothing bad happened.
-
-* Check out the release branch again:
-
- git checkout "${RELEASE_BRANCH}"
-
SquashFS file order
-------------------
-1. Build an ISO image.
-1. Burn a DVD.
+1. Burn the almost final ISO image to a DVD.
1. Boot this DVD **on bare metal**.
1. Add `profile` to the kernel command-line.
1. Login.
@@ -454,10 +488,10 @@ suite should be ready, so it is time to:
* tag the release *again*, with all included files in:
- git tag -f -u "$TAILS_SIGNATURE_KEY" \
- -m "tagging version ${VERSION}" "${TAG}" && \
- git push origin "${RELEASE_BRANCH}" && \
- git push --tags --force
+ git tag -f -u "$TAILS_SIGNATURE_KEY" \
+ -m "tagging version ${VERSION}" "${TAG}" && \
+ git push origin "${RELEASE_BRANCH}" && \
+ git push --tags --force
* check out the release tag:
@@ -465,6 +499,9 @@ suite should be ready, so it is time to:
* build the final image!
+* compare the new build manifest with the one from the previous,
+ almost final build; they should be identical
+
* check out the release branch again:
git checkout "${RELEASE_BRANCH}"
@@ -757,10 +794,15 @@ of its name:
mv "$ARTIFACTS"/tails-i386-"$VERSION".iso.packages \
"$ARTIFACTS/tails-i386-$VERSION.packages"
-Copy the `.iso.sig`, `.packages` and `.torrent` files
-into the website repository:
+Rename the manifest of needed packages as well:
+
+ mv "$PACKAGES_MANIFEST" "$ARTIFACTS/tails-i386-$VERSION.build-manifest"
+
+Copy the `.iso.sig`, `.build-manifest`, `.packages`, `.torrent` and
+`.torrent.sig` files into the website repository:
cp "$ISOS/tails-i386-$VERSION/tails-i386-$VERSION.iso.sig" \
+ "$ARTIFACTS/tails-i386-$VERSION.build-manifest" \
"$ARTIFACTS/tails-i386-$VERSION.packages" \
"$ISOS/tails-i386-$VERSION.torrent" \
"$RELEASE_CHECKOUT/wiki/src/torrents/files/"
@@ -1051,11 +1093,14 @@ this, and skip what does not make sense for a RC.
never released. Explanation: the post-release APT repository steps
from the previous stable release will usually have had us prepare
for an emergency release that was never made.
+1. [[Thaw|APT_repository/time-based snapshots#thaw]] the time-based
+ APT repository snapshots that were used during the freeze, if any.
+1. [[Thaw the packages that were granted freeze exceptions|APT_repository/time-based snapshots#freeze-exceptions-post-release]].
1. Pull `master` back and merge it into `stable`, and in turn into
`devel`
1. Follow the
- [[post-release|contribute/APT_repository#workflow-post-release]] APT
- repository documentation. Make sure there are upgrade-description
+ [[post-release|APT_repository/custom#workflow-post-release]]
+ custom APT repository documentation. Make sure there are upgrade-description
files for any new versions that were added.
1. Push the resulting branches.
1. Make sure Jenkins manages to build all updated major branches fine:
diff --git a/wiki/src/contribute/release_schedule.mdwn b/wiki/src/contribute/release_schedule.mdwn
index fefbbfc..d949805 100644
--- a/wiki/src/contribute/release_schedule.mdwn
+++ b/wiki/src/contribute/release_schedule.mdwn
@@ -53,9 +53,6 @@ unexpected issues.
Reverting the faulty feature branch is an option too.
-Some day, a [[!tails_ticket 5926 desc="freezable APT repository"]]
-will remove quite some of the potential for last minute breakage.
-
Remaining issues
================
diff --git a/wiki/src/contribute/working_together/roles/sysadmins.mdwn b/wiki/src/contribute/working_together/roles/sysadmins.mdwn
index 57b741f..7738c5d 100644
--- a/wiki/src/contribute/working_together/roles/sysadmins.mdwn
+++ b/wiki/src/contribute/working_together/roles/sysadmins.mdwn
@@ -109,15 +109,39 @@ We use Redmine tickets for public discussion and tasks management:
# Services
-## APT repository
+## APT repositories
+
+### Custom APT repository
* purpose: host Tails-specific Debian packages
-* [[documentation|contribute/APT repository]]
+* [[documentation|contribute/APT repository/custom]]
* access: anyone can read, Tails core developers can write
* tools: [[!debpts reprepro]]
* configuration: `tails::reprepro::custom` class
in [[!tails_gitweb_repo puppet-tails]]
+### Time-based snapshots of APT repositories
+
+* purpose: host full snapshots of the upstream APT repositories we
+ need, which provides the freezable APT repositories feature needed
+ by the Tails development and QA processes
+* [[documentation|contribute/APT repository/time-based snapshots]]
+* access: anyone can read, release managers have some write access
+ (XXX: limited to the operations they need to perform? full?)
+* tools: [[!debpts reprepro]]
+* configuration: `tails::reprepro::snapshots::time_based` class
+ in [[!tails_gitweb_repo puppet-tails]]
+
+### Tagged snapshots of APT repositories
+
+* purpose: host partial snapshots of the upstream APT repositories we
+ need, for historical purposes and compliance with some licenses
+* [[documentation|contribute/APT repository/tagged snapshots]]
+* access: anyone can read, release managers can create new snapshots
+* tools: [[!debpts reprepro]]
+* configuration: `tails::reprepro::snapshots::tagged` class
+ in [[!tails_gitweb_repo puppet-tails]]
+
## Bitcoind
* purpose: handle the Tails Bitcoin wallet