diff options
author | anonym <anonym@riseup.net> | 2016-02-23 13:16:32 +0100 |
---|---|---|
committer | anonym <anonym@riseup.net> | 2016-02-26 14:53:06 +0100 |
commit | e5b93f3ad5133a8f9bb32049aafd1eef30d4a155 (patch) | |
tree | 366ee2bfbef20578dd131fe172667e67d5a23a2b | |
parent | 1fb763d6fe2193b80ad4fe73f08a5b3b73f54f68 (diff) |
WIP: split rake tasksfeature/11171-split-rakefile-build-task
1. Drop Jenkins-specific cruft
2. Will result in more efficient fetch since the iso won't have to be
written to disk as an intermediate step.
-rw-r--r-- | Rakefile | 85 | ||||
-rwxr-xr-x | vagrant/provision/assets/build-tails | 200 |
2 files changed, 151 insertions, 134 deletions
@@ -175,6 +175,11 @@ task :parse_build_options do end end +def exported_env + EXPORTED_VARIABLES.select { |k| ENV[k] }. + collect { |k| "#{k}='#{ENV[k]}'" }.join(' ') +end + task :ensure_clean_repository do git_status = `git status --porcelain` unless git_status.empty? @@ -203,11 +208,6 @@ task :ensure_clean_repository do end end -desc "Make sure the vagrant user's home directory has no undesired artifacts" -task :ensure_clean_home_directory => ['vm:up'] do - run_vagrant('ssh', '-c', "sudo rm '/home/#{user}/'*.iso.*") -end - task :validate_http_proxy do if ENV['http_proxy'] proxy_host = URI.parse(ENV['http_proxy']).host @@ -228,9 +228,11 @@ task :validate_http_proxy do end end -desc 'Build Tails' -task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do +desc 'Ensure that the VM is ready for action' +task :ensure_ready_vm => ['parse_build_options', 'ensure_clean_repository', 'validate_http_proxy', 'vm:up'] +desc 'Prepare a suitable environment to build Tails in' +task :prepare_build_dir => ['ensure_ready_vm'] do if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?) $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '') @@ -243,6 +245,24 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_ abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.' end + # Must leave any build_dir prepared *this* run intact + $build_dir = capture_vagrant( + 'ssh', '-c', "#{exported_env} build-tails prepare-build_dir" + ).first.chomp +end + +desc 'Sync the Tails sources to the builder' +task :build_website => ['ensure_ready_vm'] do + run_vagrant('ssh', '-c', "#{exported_env} build-tails sync-sources") +end + +desc 'Build the Tails website' +task :build_website => ['ensure_ready_vm', 'sync_sources'] do + run_vagrant('ssh', '-c', "#{exported_env} build-tails build-website") +end + +desc 'Build a Tails image' +task :build_image => ['ensure_ready_vm', 'sync_sources', 'prepare_build_dir'] do if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '') @@ -255,30 +275,41 @@ task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_ abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.' end - exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }. - collect { |k| "#{k}='#{ENV[k]}'" }.join(' ') - run_vagrant('ssh', '-c', "#{exported_env} build-tails") - - artifacts = capture_vagrant('ssh', '-c', 'ls -1 tails-*.iso*').first.split("\n") - if not artifacts.empty? - ssh_info = capture_vagrant('ssh-config').first.split("\n") \ - .map { |line| line.strip.split(/\s+/, 2) } .to_h - user = ssh_info['User'] - hostname = ssh_info['HostName'] - # The path in the ssh-config output is quoted, which is not what - # is expected outside of a shell, so let's get rid of the quotes. - key_file = ssh_info['IdentityFile'].gsub(/^"|"$/, '') - Net::SCP.start(hostname, user, :keys => [key_file]) do |scp| - artifacts.each do |artifact_name| - artifact_path = "/home/#{user}/#{artifact_name}" - run_vagrant('ssh', '-c', "sudo chown '#{user}' '#{artifact_path}'") - scp.download!(artifact_path, '.') - run_vagrant('ssh', '-c', "sudo rm -f '#{artifact_path}'") - end + run_vagrant('ssh', '-c', "#{exported_env} build-tails build-image #{$build_dir}") +end + +desc 'Fetch all build artifacts' +task :fetch_artifacts => ['ensure_ready_vm', 'prepare_build_dir'] do + artifacts = capture_vagrant( + 'ssh', '-c', "find '#{$build_dir}' -maxdepth 1 -name 'tails-*.iso*'" + ).first.chomp.split("\n") + if artifacts.empty? + raise 'No build artifacts could be found' + end + ssh_info = capture_vagrant('ssh-config').first.split("\n") \ + .map { |line| line.strip.split(/\s+/, 2) } .to_h + user = ssh_info['User'] + hostname = ssh_info['HostName'] + # The path in the ssh-config output is quoted, which is not what + # is expected outside of a shell, so let's get rid of the quotes. + key_file = ssh_info['IdentityFile'].gsub(/^"|"$/, '') + Net::SCP.start(hostname, user, :keys => [key_file]) do |scp| + artifacts.each do |artifact_path| + run_vagrant('ssh', '-c', "sudo chown '#{user}' '#{artifact_path}'") + scp.download!(artifact_path, '.') + run_vagrant('ssh', '-c', "sudo rm -f '#{artifact_path}'") end end end +desc 'Teardown the build environment' +task :teardown_build_dir => ['ensure_ready_vm'] do + run_vagrant('ssh', '-c', "#{exported_env} build-tails teardown-build_dir") +end + +desc 'Build Tails and fetch the resulting artifacts' +task :build => ['build_website', 'build_image', 'fetch_artifacts', 'teardown_build_dir'] + namespace :vm do desc 'Start the build virtual machine' task :up => ['parse_build_options', 'validate_http_proxy'] do diff --git a/vagrant/provision/assets/build-tails b/vagrant/provision/assets/build-tails index a55cde8..4094770 100755 --- a/vagrant/provision/assets/build-tails +++ b/vagrant/provision/assets/build-tails @@ -5,121 +5,107 @@ set -e as_root_do() { - sudo \ - ${RSYNC_PROXY:+RSYNC_PROXY="$RSYNC_PROXY"} \ - ${http_proxy:+http_proxy="$http_proxy"} \ - ${https_proxy:+https_proxy="$https_proxy"} \ - ${ftp_proxy:+ftp_proxy="$ftp_proxy"} \ - ${no_proxy:+no_proxy="$no_proxy"} \ - ${JENKINS_URL:+JENKINS_URL="$JENKINS_URL"} \ - ${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="$MKSQUASHFS_OPTIONS"} \ - "$@" + sudo \ + ${RSYNC_PROXY:+RSYNC_PROXY="$RSYNC_PROXY"} \ + ${http_proxy:+http_proxy="$http_proxy"} \ + ${https_proxy:+https_proxy="$https_proxy"} \ + ${ftp_proxy:+ftp_proxy="$ftp_proxy"} \ + ${no_proxy:+no_proxy="$no_proxy"} \ + ${MKSQUASHFS_OPTIONS:+MKSQUASHFS_OPTIONS="$MKSQUASHFS_OPTIONS"} \ + "$@" } -usable_memory() { - free -b | awk '/cache:/ { print $4 }' +sync_sources() { + COMMIT="$(git --git-dir=/amnesia.git rev-parse --verify HEAD)" + if git --git-dir=/amnesia.git symbolic-ref HEAD >/dev/null 2>&1; then + # We are building from a branch + REV="${1:-$(git --git-dir=/amnesia.git name-rev --name-only HEAD)}" + else + # We are (hopefully) building from a tag + if ! REV="${1:-$(git --git-dir=/amnesia.git describe --tags --exact-match ${COMMIT})}"; then + echo "It seems we are building from an untagged detached HEAD. Aborting." >&2 + exit 1 + fi + fi + test -d "${GIT_SOURCES}" || git clone /amnesia.git "${GIT_SOURCES}" + cd "${GIT_SOURCES}" + git fetch --tags origin + git checkout --force "${REV}" + git reset --hard "${COMMIT}" } -cleanup() { - [ -n "$BUILD_DIR" ] || return 0 - cd / - mounts=$(mount | grep -E "^[^[:space:]]+ on $BUILD_DIR" | \ - awk '{print $3}' | sort -r) - [ -n "$mounts" ] && sudo umount $mounts - [ -d "$TMPFS_DIR" ] && ( sudo umount $TMPFS_DIR ; rmdir $TMPFS_DIR ) - [ -d "$BUILD_DIR" ] && sudo rm -rf $BUILD_DIR +build_website() { + cd "${GIT_SOURCES}" + ./build-website } -trap cleanup EXIT - -if [ -n "$JENKINS_URL" ]; then - if [ -z "$WORKSPACE" ]; then - echo "WORKSPACE environment variable is not set. Aborting." >&2 - exit 2 - fi - if [ -z "$GIT_BRANCH" ]; then - echo "GIT_BRANCH environment variable is not set. Aborting." >&2 - exit 4 - fi - if [ -z "$GIT_COMMIT" ]; then - echo "GIT_COMMIT environment variable is not set. Aborting." >&2 - exit 5 - fi - REV="${GIT_BRANCH##origin/}" - COMMIT="$GIT_COMMIT" - ARTIFACTS_DIR="$WORKSPACE/build-artifacts" -else - # Build triggered by Vagrant - WORKSPACE=/home/vagrant/amnesia - ARTIFACTS_DIR=/home/vagrant - COMMIT="$(git --git-dir=/amnesia.git rev-parse --verify HEAD)" - if git --git-dir=/amnesia.git symbolic-ref HEAD >/dev/null 2>&1; then - # We are building from a branch - REV="${1:-$(git --git-dir=/amnesia.git name-rev --name-only HEAD)}" - else - # We are (hopefully) building from a tag - if ! REV="${1:-$(git --git-dir=/amnesia.git describe --tags --exact-match ${COMMIT})}"; then - echo "It seems we are building from an untagged detached HEAD. Aborting." >&2 - exit 1 - fi - fi - test -d "$WORKSPACE" || git clone /amnesia.git "$WORKSPACE" - cd "$WORKSPACE" - git fetch origin -fi - -cd "$WORKSPACE" -git checkout --force "$REV" -git reset --hard "$COMMIT" +get_build_dir() { + local build_dir + build_dir="$(find /tmp -name 'tails-build-dir.*')" + if [ -n "${build_dir}" ]; then + # XXX: sanity checks + # 1. there's only one dir + # 2. it's mounted according to TAILS_RAM_BUILD + : + fi + echo "${build_dir}" +} -if [ -n "$JENKINS_URL" ]; then - git clean --force -d -x -fi +prepare_build_dir() { + local build_dir tmpfs_dir + build_dir="$(get_build_dir)" + if [ -z "${build_dir}" ]; then + build_dir="$(mktemp -d /tmp/tails-build-dir.XXXXXXXX)" + if [ "${TAILS_RAM_BUILD}" ]; then + tmpfs_dir="$(mktemp -d /tmp/tails-build-tmpfs.XXXXXXXX)" + as_root_do mount -t tmpfs -o "noatime,size=100%,mode=0770,uid=root,gid=${USER}" tmpfs "${tmpfs_dir}" + as_root_do mount -t aufs -o "noatime,noxino,dirs=${tmpfs_dir}=rw:${GIT_SOURCES}/=rr+wh" aufs "${build_dir}" + else + as_root_do rsync -a "${GIT_SOURCES}"/ "${BUILD_DIR}"/ + fi + fi + get_build_dir +} -install -m 0755 -d "$ARTIFACTS_DIR" +teardown_build_dir() { + local build_dir tmpfs_dir + build_dir="$(get_build_dir)" + [ -z "${build_dir}" ] && return 0 + build_dir_src="$(findmnt --noheadings --output SOURCE --target "${build_dir}")" + + mounts=$(mount | grep -E "^[^[:space:]]+ on ${build_dir}" | \ + awk '{print $3}' | sort -r) + [ -n "${mounts}" ] && sudo umount ${mounts} + + if [ "${build_dir_src}" = aufs ]; then + tmpfs_dir="$(find /tmp -name 'tails-build-dir.*')" + sudo umount "${tmpfs_dir}" + sudo rm -rf "${tmpfs_dir}" + sudo umount "${build_dir}" + fi + sudo rm -rf "${build_dir}" +} -if [ "$TAILS_CLEAN_BUILD" ]; then +build_image() { + cd "$(get_build_dir)" + if [ "${TAILS_CLEAN_BUILD}" ]; then as_root_do lb clean --all -fi - -if [ -z "$JENKINS_URL" ]; then - ./build-website -fi - -BUILD_DIR=$(mktemp -d /tmp/tails-build.XXXXXXXX) -if [ "$TAILS_RAM_BUILD" ]; then - TMPFS_DIR=$(mktemp -d /tmp/tmpfs.XXXXXXXX) - as_root_do mount -t tmpfs -o "noatime,size=100%,mode=0770,uid=root,gid=${USER}" tmpfs "$TMPFS_DIR" - as_root_do mount -t aufs -o "noatime,noxino,dirs=$TMPFS_DIR=rw:${WORKSPACE}/=rr+wh" aufs "$BUILD_DIR" -else - as_root_do rsync -a "$WORKSPACE"/ "$BUILD_DIR"/ -fi - -cd "$BUILD_DIR" -as_root_do lb config --cache false - -as_root_do lb build - -if [ -n "$JENKINS_URL" ]; then - ISO=$(ls *.iso) - - USER_ON_ACNG_HOST=jenkins - ACNG_HOST=apt-proxy.lizard - ACNG_REMOTE_LOG=/var/log/apt-cacher-ng/apt-cacher.log - ACNG_LOCAL_LOG=$(basename "$ACNG_REMOTE_LOG") - sftp "${USER_ON_ACNG_HOST}@${ACNG_HOST}:${ACNG_REMOTE_LOG}" - BUILD_IP=$(ip a show dev eth0 | grep '^\s\+inet\s' | awk '{print $2}' | sed -e 's,/.*,,') - START_TIME=$(cat "$ISO.start.timestamp") - END_TIME=$(cat "$ISO.end.timestamp") - ./auto/scripts/packages-from-acng-log "$ACNG_LOCAL_LOG" "$BUILD_IP" \ - "$START_TIME" "$END_TIME" \ - "${ISO}.binpkgs" "${ISO}.srcpkgs" - rm "$ISO".*.timestamp - - for file in tails-*; do - sha512sum "$file" >> "$ISO.shasum" - done - gpg --batch --detach-sign --armor "$ISO.shasum" -fi + git clean --force -d -x + fi + as_root_do lb config --cache false + as_root_do lb build +} -mv -f tails-* "$ARTIFACTS_DIR" +GIT_SOURCES=/home/vagrant/amnesia + +case "${1}" in + build-website) build_website ;; + sync-sources) sync_sources ;; + prepare-build_dir) prepare_build_dir ;; + teardown-build_dir) teardown_build_dir ;; + build-image) build_image ;; + *) + echo "invalid action '${1}'" >&2 + exit 1 + ;; +esac |