summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoranonym <anonym@riseup.net>2016-02-23 13:16:32 +0100
committeranonym <anonym@riseup.net>2016-02-26 14:53:06 +0100
commite5b93f3ad5133a8f9bb32049aafd1eef30d4a155 (patch)
tree366ee2bfbef20578dd131fe172667e67d5a23a2b
parent1fb763d6fe2193b80ad4fe73f08a5b3b73f54f68 (diff)
1. Drop Jenkins-specific cruft 2. Will result in more efficient fetch since the iso won't have to be written to disk as an intermediate step.
-rw-r--r--Rakefile85
-rwxr-xr-xvagrant/provision/assets/build-tails200
2 files changed, 151 insertions, 134 deletions
diff --git a/Rakefile b/Rakefile
index 3345d25..617ca3b 100644
--- a/Rakefile
+++ b/Rakefile
@@ -175,6 +175,11 @@ task :parse_build_options do
end
end
+def exported_env
+ EXPORTED_VARIABLES.select { |k| ENV[k] }.
+ collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
+end
+
task :ensure_clean_repository do
git_status = `git status --porcelain`
unless git_status.empty?
@@ -203,11 +208,6 @@ task :ensure_clean_repository do
end
end
-desc "Make sure the vagrant user's home directory has no undesired artifacts"
-task :ensure_clean_home_directory => ['vm:up'] do
- run_vagrant('ssh', '-c', "sudo rm '/home/#{user}/'*.iso.*")
-end
-
task :validate_http_proxy do
if ENV['http_proxy']
proxy_host = URI.parse(ENV['http_proxy']).host
@@ -228,9 +228,11 @@ task :validate_http_proxy do
end
end
-desc 'Build Tails'
-task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do
+desc 'Ensure that the VM is ready for action'
+task :ensure_ready_vm => ['parse_build_options', 'ensure_clean_repository', 'validate_http_proxy', 'vm:up']
+desc 'Prepare a suitable environment to build Tails in'
+task :prepare_build_dir => ['ensure_ready_vm'] do
if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
@@ -243,6 +245,24 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_
abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.'
end
+ # Must leave any build_dir prepared *this* run intact
+ $build_dir = capture_vagrant(
+ 'ssh', '-c', "#{exported_env} build-tails prepare-build_dir"
+ ).first.chomp
+end
+
+desc 'Sync the Tails sources to the builder'
+task :build_website => ['ensure_ready_vm'] do
+ run_vagrant('ssh', '-c', "#{exported_env} build-tails sync-sources")
+end
+
+desc 'Build the Tails website'
+task :build_website => ['ensure_ready_vm', 'sync_sources'] do
+ run_vagrant('ssh', '-c', "#{exported_env} build-tails build-website")
+end
+
+desc 'Build a Tails image'
+task :build_image => ['ensure_ready_vm', 'sync_sources', 'prepare_build_dir'] do
if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
$stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
@@ -255,30 +275,41 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_
abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
end
- exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
- collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
- run_vagrant('ssh', '-c', "#{exported_env} build-tails")
-
- artifacts = capture_vagrant('ssh', '-c', 'ls -1 tails-*.iso*').first.split("\n")
- if not artifacts.empty?
- ssh_info = capture_vagrant('ssh-config').first.split("\n") \
- .map { |line| line.strip.split(/\s+/, 2) } .to_h
- user = ssh_info['User']
- hostname = ssh_info['HostName']
- # The path in the ssh-config output is quoted, which is not what
- # is expected outside of a shell, so let's get rid of the quotes.
- key_file = ssh_info['IdentityFile'].gsub(/^"|"$/, '')
- Net::SCP.start(hostname, user, :keys => [key_file]) do |scp|
- artifacts.each do |artifact_name|
- artifact_path = "/home/#{user}/#{artifact_name}"
- run_vagrant('ssh', '-c', "sudo chown '#{user}' '#{artifact_path}'")
- scp.download!(artifact_path, '.')
- run_vagrant('ssh', '-c', "sudo rm -f '#{artifact_path}'")
- end
+ run_vagrant('ssh', '-c', "#{exported_env} build-tails build-image #{$build_dir}")
+end
+
+desc 'Fetch all build artifacts'
+task :fetch_artifacts => ['ensure_ready_vm', 'prepare_build_dir'] do
+ artifacts = capture_vagrant(
+ 'ssh', '-c', "find '#{$build_dir}' -maxdepth 1 -name 'tails-*.iso*'"
+ ).first.chomp.split("\n")
+ if artifacts.empty?
+ raise 'No build artifacts could be found'
+ end
+ ssh_info = capture_vagrant('ssh-config').first.split("\n") \
+ .map { |line| line.strip.split(/\s+/, 2) } .to_h
+ user = ssh_info['User']
+ hostname = ssh_info['HostName']
+ # The path in the ssh-config output is quoted, which is not what
+ # is expected outside of a shell, so let's get rid of the quotes.
+ key_file = ssh_info['IdentityFile'].gsub(/^"|"$/, '')
+ Net::SCP.start(hostname, user, :keys => [key_file]) do |scp|
+ artifacts.each do |artifact_path|
+ run_vagrant('ssh', '-c', "sudo chown '#{user}' '#{artifact_path}'")
+ scp.download!(artifact_path, '.')
+ run_vagrant('ssh', '-c', "sudo rm -f '#{artifact_path}'")
end
end
end
+desc 'Teardown the build environment'
+task :teardown_build_dir => ['ensure_ready_vm'] do
+ run_vagrant('ssh', '-c', "#{exported_env} build-tails teardown-build_dir")
+end
+
+desc 'Build Tails and fetch the resulting artifacts'
+task :build => ['build_website', 'build_image', 'fetch_artifacts', 'teardown_build_dir']
+
namespace :vm do
desc 'Start the build virtual machine'
task :up => ['parse_build_options', 'validate_http_proxy'] do
diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails
index a55cde8..4094770 100755
--- a/vagrant/provision/assets/build-tails
+++ b/vagrant/provision/assets/build-tails
@@ -5,121 +5,107 @@
set -e
as_root_do() {
- sudo \
- ${RSYNC_PROXY:+RSYNC_PROXY="$RSYNC_PROXY"} \
- ${http_proxy:+http_proxy="$http_proxy"} \
- ${https_proxy:+https_proxy="$https_proxy"} \
- ${ftp_proxy:+ftp_proxy="$ftp_proxy"} \
- ${no_proxy:+no_proxy="$no_proxy"} \
- ${JENKINS_URL:+JENKINS_URL="$JENKINS_URL"} \
- ${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="$MKSQUASHFS_OPTIONS"} \
- "$@"
+ sudo \
+ ${RSYNC_PROXY:+RSYNC_PROXY="$RSYNC_PROXY"} \
+ ${http_proxy:+http_proxy="$http_proxy"} \
+ ${https_proxy:+https_proxy="$https_proxy"} \
+ ${ftp_proxy:+ftp_proxy="$ftp_proxy"} \
+ ${no_proxy:+no_proxy="$no_proxy"} \
+ ${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="$MKSQUASHFS_OPTIONS"} \
+ "$@"
}
-usable_memory() {
- free -b | awk '/cache:/ { print $4 }'
+sync_sources() {
+ COMMIT="$(git --git-dir=/amnesia.git rev-parse --verify HEAD)"
+ if git --git-dir=/amnesia.git symbolic-ref HEAD >/dev/null 2>&1; then
+ # We are building from a branch
+ REV="${1:-$(git --git-dir=/amnesia.git name-rev --name-only HEAD)}"
+ else
+ # We are (hopefully) building from a tag
+ if ! REV="${1:-$(git --git-dir=/amnesia.git describe --tags --exact-match ${COMMIT})}"; then
+ echo "It seems we are building from an untagged detached HEAD. Aborting." >&2
+ exit 1
+ fi
+ fi
+ test -d "${GIT_SOURCES}" || git clone /amnesia.git "${GIT_SOURCES}"
+ cd "${GIT_SOURCES}"
+ git fetch --tags origin
+ git checkout --force "${REV}"
+ git reset --hard "${COMMIT}"
}
-cleanup() {
- [ -n "$BUILD_DIR" ] || return 0
- cd /
- mounts=$(mount | grep -E "^[^[:space:]]+ on $BUILD_DIR" | \
- awk '{print $3}' | sort -r)
- [ -n "$mounts" ] && sudo umount $mounts
- [ -d "$TMPFS_DIR" ] && ( sudo umount $TMPFS_DIR ; rmdir $TMPFS_DIR )
- [ -d "$BUILD_DIR" ] && sudo rm -rf $BUILD_DIR
+build_website() {
+ cd "${GIT_SOURCES}"
+ ./build-website
}
-trap cleanup EXIT
-
-if [ -n "$JENKINS_URL" ]; then
- if [ -z "$WORKSPACE" ]; then
- echo "WORKSPACE environment variable is not set. Aborting." >&2
- exit 2
- fi
- if [ -z "$GIT_BRANCH" ]; then
- echo "GIT_BRANCH environment variable is not set. Aborting." >&2
- exit 4
- fi
- if [ -z "$GIT_COMMIT" ]; then
- echo "GIT_COMMIT environment variable is not set. Aborting." >&2
- exit 5
- fi
- REV="${GIT_BRANCH##origin/}"
- COMMIT="$GIT_COMMIT"
- ARTIFACTS_DIR="$WORKSPACE/build-artifacts"
-else
- # Build triggered by Vagrant
- WORKSPACE=/home/vagrant/amnesia
- ARTIFACTS_DIR=/home/vagrant
- COMMIT="$(git --git-dir=/amnesia.git rev-parse --verify HEAD)"
- if git --git-dir=/amnesia.git symbolic-ref HEAD >/dev/null 2>&1; then
- # We are building from a branch
- REV="${1:-$(git --git-dir=/amnesia.git name-rev --name-only HEAD)}"
- else
- # We are (hopefully) building from a tag
- if ! REV="${1:-$(git --git-dir=/amnesia.git describe --tags --exact-match ${COMMIT})}"; then
- echo "It seems we are building from an untagged detached HEAD. Aborting." >&2
- exit 1
- fi
- fi
- test -d "$WORKSPACE" || git clone /amnesia.git "$WORKSPACE"
- cd "$WORKSPACE"
- git fetch origin
-fi
-
-cd "$WORKSPACE"
-git checkout --force "$REV"
-git reset --hard "$COMMIT"
+get_build_dir() {
+ local build_dir
+ build_dir="$(find /tmp -name 'tails-build-dir.*')"
+ if [ -n "${build_dir}" ]; then
+ # XXX: sanity checks
+ # 1. there's only one dir
+ # 2. it's mounted according to TAILS_RAM_BUILD
+ :
+ fi
+ echo "${build_dir}"
+}
-if [ -n "$JENKINS_URL" ]; then
- git clean --force -d -x
-fi
+prepare_build_dir() {
+ local build_dir tmpfs_dir
+ build_dir="$(get_build_dir)"
+ if [ -z "${build_dir}" ]; then
+ build_dir="$(mktemp -d /tmp/tails-build-dir.XXXXXXXX)"
+ if [ "${TAILS_RAM_BUILD}" ]; then
+ tmpfs_dir="$(mktemp -d /tmp/tails-build-tmpfs.XXXXXXXX)"
+ as_root_do mount -t tmpfs -o "noatime,size=100%,mode=0770,uid=root,gid=${USER}" tmpfs "${tmpfs_dir}"
+ as_root_do mount -t aufs -o "noatime,noxino,dirs=${tmpfs_dir}=rw:${GIT_SOURCES}/=rr+wh" aufs "${build_dir}"
+ else
+ as_root_do rsync -a "${GIT_SOURCES}"/ "${BUILD_DIR}"/
+ fi
+ fi
+ get_build_dir
+}
-install -m 0755 -d "$ARTIFACTS_DIR"
+teardown_build_dir() {
+ local build_dir tmpfs_dir
+ build_dir="$(get_build_dir)"
+ [ -z "${build_dir}" ] && return 0
+ build_dir_src="$(findmnt --noheadings --output SOURCE --target "${build_dir}")"
+
+ mounts=$(mount | grep -E "^[^[:space:]]+ on ${build_dir}" | \
+ awk '{print $3}' | sort -r)
+ [ -n "${mounts}" ] && sudo umount ${mounts}
+
+ if [ "${build_dir_src}" = aufs ]; then
+ tmpfs_dir="$(find /tmp -name 'tails-build-dir.*')"
+ sudo umount "${tmpfs_dir}"
+ sudo rm -rf "${tmpfs_dir}"
+ sudo umount "${build_dir}"
+ fi
+ sudo rm -rf "${build_dir}"
+}
-if [ "$TAILS_CLEAN_BUILD" ]; then
+build_image() {
+ cd "$(get_build_dir)"
+ if [ "${TAILS_CLEAN_BUILD}" ]; then
as_root_do lb clean --all
-fi
-
-if [ -z "$JENKINS_URL" ]; then
- ./build-website
-fi
-
-BUILD_DIR=$(mktemp -d /tmp/tails-build.XXXXXXXX)
-if [ "$TAILS_RAM_BUILD" ]; then
- TMPFS_DIR=$(mktemp -d /tmp/tmpfs.XXXXXXXX)
- as_root_do mount -t tmpfs -o "noatime,size=100%,mode=0770,uid=root,gid=${USER}" tmpfs "$TMPFS_DIR"
- as_root_do mount -t aufs -o "noatime,noxino,dirs=$TMPFS_DIR=rw:${WORKSPACE}/=rr+wh" aufs "$BUILD_DIR"
-else
- as_root_do rsync -a "$WORKSPACE"/ "$BUILD_DIR"/
-fi
-
-cd "$BUILD_DIR"
-as_root_do lb config --cache false
-
-as_root_do lb build
-
-if [ -n "$JENKINS_URL" ]; then
- ISO=$(ls *.iso)
-
- USER_ON_ACNG_HOST=jenkins
- ACNG_HOST=apt-proxy.lizard
- ACNG_REMOTE_LOG=/var/log/apt-cacher-ng/apt-cacher.log
- ACNG_LOCAL_LOG=$(basename "$ACNG_REMOTE_LOG")
- sftp "${USER_ON_ACNG_HOST}@${ACNG_HOST}:${ACNG_REMOTE_LOG}"
- BUILD_IP=$(ip a show dev eth0 | grep '^\s\+inet\s' | awk '{print $2}' | sed -e 's,/.*,,')
- START_TIME=$(cat "$ISO.start.timestamp")
- END_TIME=$(cat "$ISO.end.timestamp")
- ./auto/scripts/packages-from-acng-log "$ACNG_LOCAL_LOG" "$BUILD_IP" \
- "$START_TIME" "$END_TIME" \
- "${ISO}.binpkgs" "${ISO}.srcpkgs"
- rm "$ISO".*.timestamp
-
- for file in tails-*; do
- sha512sum "$file" >> "$ISO.shasum"
- done
- gpg --batch --detach-sign --armor "$ISO.shasum"
-fi
+ git clean --force -d -x
+ fi
+ as_root_do lb config --cache false
+ as_root_do lb build
+}
-mv -f tails-* "$ARTIFACTS_DIR"
+GIT_SOURCES=/home/vagrant/amnesia
+
+case "${1}" in
+ build-website) build_website ;;
+ sync-sources) sync_sources ;;
+ prepare-build_dir) prepare_build_dir ;;
+ teardown-build_dir) teardown_build_dir ;;
+ build-image) build_image ;;
+ *)
+ echo "invalid action '${1}'" >&2
+ exit 1
+ ;;
+esac