WIP: split rake tasksfeature/11171-split-rakefile-build-task

1. Drop Jenkins-specific cruft

2. Will result in more efficient fetch since the iso won't have to be
   written to disk as an intermediate step.

2 files changed, 151 insertions, 134 deletions

diff --git a/Rakefile b/Rakefile
index 3345d25..617ca3b 100644
--- a/Rakefile
+++ b/Rakefile
@@ -175,6 +175,11 @@ task :parse_build_options do
   end
 end
 
+def exported_env
+  EXPORTED_VARIABLES.select { |k| ENV[k] }.
+    collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
+end
+
 task :ensure_clean_repository do
   git_status = `git status --porcelain`
   unless git_status.empty?
@@ -203,11 +208,6 @@ task :ensure_clean_repository do
   end
 end
 
-desc "Make sure the vagrant user's home directory has no undesired artifacts"
-task :ensure_clean_home_directory => ['vm:up'] do
-  run_vagrant('ssh', '-c', "sudo rm '/home/#{user}/'*.iso.*")
-end
-
 task :validate_http_proxy do
   if ENV['http_proxy']
     proxy_host = URI.parse(ENV['http_proxy']).host
@@ -228,9 +228,11 @@ task :validate_http_proxy do
   end
 end
 
-desc 'Build Tails'
-task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do
+desc 'Ensure that the VM is ready for action'
+task :ensure_ready_vm => ['parse_build_options', 'ensure_clean_repository', 'validate_http_proxy', 'vm:up']
 
+desc 'Prepare a suitable environment to build Tails in'
+task :prepare_build_dir => ['ensure_ready_vm'] do
   if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
     $stderr.puts <<-END_OF_MESSAGE.gsub(/^      /, '')
 
@@ -243,6 +245,24 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_
     abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.'
   end
 
+  # Must leave any build_dir prepared *this* run intact
+  $build_dir = capture_vagrant(
+    'ssh', '-c', "#{exported_env} build-tails prepare-build_dir"
+  ).first.chomp
+end
+
+desc 'Sync the Tails sources to the builder'
+task :build_website => ['ensure_ready_vm'] do
+  run_vagrant('ssh', '-c', "#{exported_env} build-tails sync-sources")
+end
+
+desc 'Build the Tails website'
+task :build_website => ['ensure_ready_vm', 'sync_sources'] do
+  run_vagrant('ssh', '-c', "#{exported_env} build-tails build-website")
+end
+
+desc 'Build a Tails image'
+task :build_image => ['ensure_ready_vm', 'sync_sources', 'prepare_build_dir'] do
   if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
     $stderr.puts <<-END_OF_MESSAGE.gsub(/^      /, '')
 
@@ -255,30 +275,41 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_
     abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
   end
 
-  exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
-                 collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
-  run_vagrant('ssh', '-c', "#{exported_env} build-tails")
-
-  artifacts = capture_vagrant('ssh', '-c', 'ls -1 tails-*.iso*').first.split("\n")
-  if not artifacts.empty?
-    ssh_info = capture_vagrant('ssh-config').first.split("\n") \
-               .map { |line| line.strip.split(/\s+/, 2) } .to_h
-    user = ssh_info['User']
-    hostname = ssh_info['HostName']
-    # The path in the ssh-config output is quoted, which is not what
-    # is expected outside of a shell, so let's get rid of the quotes.
-    key_file = ssh_info['IdentityFile'].gsub(/^"|"$/, '')
-    Net::SCP.start(hostname, user, :keys => [key_file]) do |scp|
-      artifacts.each do |artifact_name|
-        artifact_path = "/home/#{user}/#{artifact_name}"
-        run_vagrant('ssh', '-c', "sudo chown '#{user}' '#{artifact_path}'")
-        scp.download!(artifact_path, '.')
-        run_vagrant('ssh', '-c', "sudo rm -f '#{artifact_path}'")
-      end
+  run_vagrant('ssh', '-c', "#{exported_env} build-tails build-image #{$build_dir}")
+end
+
+desc 'Fetch all build artifacts'
+task :fetch_artifacts => ['ensure_ready_vm', 'prepare_build_dir'] do
+  artifacts = capture_vagrant(
+    'ssh', '-c', "find '#{$build_dir}' -maxdepth 1 -name 'tails-*.iso*'"
+  ).first.chomp.split("\n")
+  if artifacts.empty?
+    raise 'No build artifacts could be found'
+  end
+  ssh_info = capture_vagrant('ssh-config').first.split("\n") \
+             .map { |line| line.strip.split(/\s+/, 2) } .to_h
+  user = ssh_info['User']
+  hostname = ssh_info['HostName']
+  # The path in the ssh-config output is quoted, which is not what
+  # is expected outside of a shell, so let's get rid of the quotes.
+  key_file = ssh_info['IdentityFile'].gsub(/^"|"$/, '')
+  Net::SCP.start(hostname, user, :keys => [key_file]) do |scp|
+    artifacts.each do |artifact_path|
+      run_vagrant('ssh', '-c', "sudo chown '#{user}' '#{artifact_path}'")
+      scp.download!(artifact_path, '.')
+      run_vagrant('ssh', '-c', "sudo rm -f '#{artifact_path}'")
     end
   end
 end
 
+desc 'Teardown the build environment'
+task :teardown_build_dir => ['ensure_ready_vm'] do
+  run_vagrant('ssh', '-c', "#{exported_env} build-tails teardown-build_dir")
+end
+
+desc 'Build Tails and fetch the resulting artifacts'
+task :build => ['build_website', 'build_image', 'fetch_artifacts', 'teardown_build_dir']
+
 namespace :vm do
   desc 'Start the build virtual machine'
   task :up => ['parse_build_options', 'validate_http_proxy'] do
diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails
index a55cde8..4094770 100755
--- a/vagrant/provision/assets/build-tails
+++ b/vagrant/provision/assets/build-tails
@@ -5,121 +5,107 @@
 set -e
 
 as_root_do() {
-	sudo \
-		${RSYNC_PROXY:+RSYNC_PROXY="$RSYNC_PROXY"} \
-		${http_proxy:+http_proxy="$http_proxy"} \
-		${https_proxy:+https_proxy="$https_proxy"} \
-		${ftp_proxy:+ftp_proxy="$ftp_proxy"} \
-		${no_proxy:+no_proxy="$no_proxy"} \
-		${JENKINS_URL:+JENKINS_URL="$JENKINS_URL"} \
-		${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="$MKSQUASHFS_OPTIONS"} \
-		"$@"
+    sudo \
+	${RSYNC_PROXY:+RSYNC_PROXY="$RSYNC_PROXY"} \
+	${http_proxy:+http_proxy="$http_proxy"} \
+	${https_proxy:+https_proxy="$https_proxy"} \
+	${ftp_proxy:+ftp_proxy="$ftp_proxy"} \
+	${no_proxy:+no_proxy="$no_proxy"} \
+	${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="$MKSQUASHFS_OPTIONS"} \
+	"$@"
 }
 
-usable_memory() {
-	free -b | awk '/cache:/ { print $4 }'
+sync_sources() {
+    COMMIT="$(git --git-dir=/amnesia.git rev-parse --verify HEAD)"
+    if git --git-dir=/amnesia.git symbolic-ref HEAD >/dev/null 2>&1; then
+        # We are building from a branch
+        REV="${1:-$(git --git-dir=/amnesia.git name-rev --name-only HEAD)}"
+    else
+        # We are (hopefully) building from a tag
+        if ! REV="${1:-$(git --git-dir=/amnesia.git describe --tags --exact-match ${COMMIT})}"; then
+	    echo "It seems we are building from an untagged detached HEAD. Aborting." >&2
+	    exit 1
+        fi
+    fi
+    test -d "${GIT_SOURCES}" || git clone /amnesia.git "${GIT_SOURCES}"
+    cd "${GIT_SOURCES}"
+    git fetch --tags origin
+    git checkout --force "${REV}"
+    git reset --hard "${COMMIT}"
 }
 
-cleanup() {
-	[ -n "$BUILD_DIR" ] || return 0
-	cd /
-	mounts=$(mount | grep -E "^[^[:space:]]+ on $BUILD_DIR" | \
-		awk '{print $3}' | sort -r)
-	[ -n "$mounts" ] && sudo umount $mounts
-	[ -d "$TMPFS_DIR" ] && ( sudo umount $TMPFS_DIR ; rmdir $TMPFS_DIR )
-	[ -d "$BUILD_DIR" ] && sudo rm -rf $BUILD_DIR
+build_website() {
+    cd "${GIT_SOURCES}"
+    ./build-website
 }
 
-trap cleanup EXIT
-
-if [ -n "$JENKINS_URL" ]; then
-	if [ -z "$WORKSPACE" ]; then
-		echo "WORKSPACE environment variable is not set. Aborting." >&2
-		exit 2
-	fi
-	if [ -z "$GIT_BRANCH" ]; then
-		echo "GIT_BRANCH environment variable is not set. Aborting." >&2
-		exit 4
-	fi
-	if [ -z "$GIT_COMMIT" ]; then
-		echo "GIT_COMMIT environment variable is not set. Aborting." >&2
-		exit 5
-	fi
-	REV="${GIT_BRANCH##origin/}"
-	COMMIT="$GIT_COMMIT"
-	ARTIFACTS_DIR="$WORKSPACE/build-artifacts"
-else
-	# Build triggered by Vagrant
-	WORKSPACE=/home/vagrant/amnesia
-	ARTIFACTS_DIR=/home/vagrant
-	COMMIT="$(git --git-dir=/amnesia.git rev-parse --verify HEAD)"
-	if git --git-dir=/amnesia.git symbolic-ref HEAD >/dev/null 2>&1; then
-		# We are building from a branch
-		REV="${1:-$(git --git-dir=/amnesia.git name-rev --name-only HEAD)}"
-	else
-		# We are (hopefully) building from a tag
-		if ! REV="${1:-$(git --git-dir=/amnesia.git describe --tags --exact-match ${COMMIT})}"; then
-			echo "It seems we are building from an untagged detached HEAD. Aborting." >&2
-			exit 1
-		fi
-	fi
-	test -d "$WORKSPACE" || git clone /amnesia.git "$WORKSPACE"
-	cd "$WORKSPACE"
-	git fetch origin
-fi
-
-cd "$WORKSPACE"
-git checkout --force "$REV"
-git reset --hard "$COMMIT"
+get_build_dir() {
+    local build_dir
+    build_dir="$(find /tmp -name 'tails-build-dir.*')"
+    if [ -n "${build_dir}" ]; then
+        # XXX: sanity checks
+        # 1. there's only one dir
+        # 2. it's mounted according to TAILS_RAM_BUILD
+        :
+    fi
+    echo "${build_dir}"
+}
 
-if [ -n "$JENKINS_URL" ]; then
-	git clean --force -d -x
-fi
+prepare_build_dir() {
+    local build_dir tmpfs_dir
+    build_dir="$(get_build_dir)"
+    if [ -z "${build_dir}" ]; then
+        build_dir="$(mktemp -d /tmp/tails-build-dir.XXXXXXXX)"
+        if [ "${TAILS_RAM_BUILD}" ]; then
+            tmpfs_dir="$(mktemp -d /tmp/tails-build-tmpfs.XXXXXXXX)"
+            as_root_do mount -t tmpfs -o "noatime,size=100%,mode=0770,uid=root,gid=${USER}" tmpfs "${tmpfs_dir}"
+            as_root_do mount -t aufs -o "noatime,noxino,dirs=${tmpfs_dir}=rw:${GIT_SOURCES}/=rr+wh" aufs "${build_dir}"
+        else
+            as_root_do rsync -a "${GIT_SOURCES}"/ "${BUILD_DIR}"/
+        fi
+    fi
+    get_build_dir
+}
 
-install -m 0755 -d "$ARTIFACTS_DIR"
+teardown_build_dir() {
+    local build_dir tmpfs_dir
+    build_dir="$(get_build_dir)"
+    [ -z "${build_dir}" ] && return 0
+    build_dir_src="$(findmnt --noheadings --output SOURCE --target "${build_dir}")"
+
+    mounts=$(mount | grep -E "^[^[:space:]]+ on ${build_dir}" | \
+		    awk '{print $3}' | sort -r)
+    [ -n "${mounts}" ] && sudo umount ${mounts}
+
+    if [ "${build_dir_src}" = aufs ]; then
+        tmpfs_dir="$(find /tmp -name 'tails-build-dir.*')"
+        sudo umount "${tmpfs_dir}"
+        sudo rm -rf "${tmpfs_dir}"
+        sudo umount "${build_dir}"
+    fi
+    sudo rm -rf "${build_dir}"
+}
 
-if [ "$TAILS_CLEAN_BUILD" ]; then
+build_image() {
+    cd "$(get_build_dir)"
+    if [ "${TAILS_CLEAN_BUILD}" ]; then
 	as_root_do lb clean --all
-fi
-
-if [ -z "$JENKINS_URL" ]; then
-	./build-website
-fi
-
-BUILD_DIR=$(mktemp -d /tmp/tails-build.XXXXXXXX)
-if [ "$TAILS_RAM_BUILD" ]; then
-	TMPFS_DIR=$(mktemp -d /tmp/tmpfs.XXXXXXXX)
-	as_root_do mount -t tmpfs -o "noatime,size=100%,mode=0770,uid=root,gid=${USER}" tmpfs "$TMPFS_DIR"
-	as_root_do mount -t aufs -o "noatime,noxino,dirs=$TMPFS_DIR=rw:${WORKSPACE}/=rr+wh" aufs "$BUILD_DIR"
-else
-	as_root_do rsync -a "$WORKSPACE"/ "$BUILD_DIR"/
-fi
-
-cd "$BUILD_DIR"
-as_root_do lb config --cache false
-
-as_root_do lb build
-
-if [ -n "$JENKINS_URL" ]; then
-	ISO=$(ls *.iso)
-
-	USER_ON_ACNG_HOST=jenkins
-	ACNG_HOST=apt-proxy.lizard
-	ACNG_REMOTE_LOG=/var/log/apt-cacher-ng/apt-cacher.log
-	ACNG_LOCAL_LOG=$(basename "$ACNG_REMOTE_LOG")
-	sftp "${USER_ON_ACNG_HOST}@${ACNG_HOST}:${ACNG_REMOTE_LOG}"
-	BUILD_IP=$(ip a show dev eth0 | grep '^\s\+inet\s' | awk '{print $2}' | sed -e 's,/.*,,')
-	START_TIME=$(cat "$ISO.start.timestamp")
-	END_TIME=$(cat "$ISO.end.timestamp")
-	./auto/scripts/packages-from-acng-log "$ACNG_LOCAL_LOG" "$BUILD_IP" \
-					      "$START_TIME" "$END_TIME" \
-					      "${ISO}.binpkgs" "${ISO}.srcpkgs"
-	rm "$ISO".*.timestamp
-
-	for file in tails-*; do
-		sha512sum "$file" >> "$ISO.shasum"
-	done
-	gpg --batch --detach-sign --armor "$ISO.shasum"
-fi
+        git clean --force -d -x
+    fi
+    as_root_do lb config --cache false
+    as_root_do lb build
+}
 
-mv -f tails-* "$ARTIFACTS_DIR"
+GIT_SOURCES=/home/vagrant/amnesia
+
+case "${1}" in
+    build-website) build_website ;;
+    sync-sources) sync_sources ;;
+    prepare-build_dir) prepare_build_dir ;;
+    teardown-build_dir) teardown_build_dir ;;
+    build-image) build_image ;;
+    *)
+        echo "invalid action '${1}'" >&2
+        exit 1
+        ;;
+esac