summaryrefslogtreecommitdiffstats
path: root/run_test_suite
blob: 2848ec77a525a66e71377000e49c14fba892c74a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
#!/bin/bash

set -e
set -u
set -o pipefail

NAME=$(basename ${0})

GENERAL_DEPENDENCIES="
cucumber
devscripts
dnsmasq-base
gawk
git
i18nspector
libav-tools
libcap2-bin
libsikulixapi-java
libvirt-clients
libvirt-daemon-system
libvirt-dev
libvirt0
obfs4proxy
openssh-server
ovmf
pry
python-jabberbot
python-potr
qemu-kvm
qemu-system-x86
redir
ruby-guestfs
ruby-json
ruby-libvirt
ruby-net-irc
ruby-packetfu
ruby-rb-inotify
ruby-rjb
ruby-rspec
ruby-test-unit
seabios
tcpdump
tor
unclutter
virt-viewer
xvfb
"

usage() {
    echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
Sets up an appropriate environment and invokes cucumber. Note that this script
must be run from the Tails source directory root.

Options for '@product' features:
  --artifacts-base-uri URI
                     Pretend that the artifact is located at URI when printing
                     its location during a scenario failure. This is useful if
                     you intend to serve the artifacts via the web, for
                     instance.
  --capture          Captures failed scenarios into videos stored in the
                     temporary directory (see --tmpdir below) using x264
                     encoding. Requires x264.
  --capture-all      Keep videos for all scenarios, including those that
                     succeed (implies --capture).
  --interactive-debugging
                     On failure, pause test suite until pressing Enter. Also
                     offer the option to open an interactive Ruby shell (pry)
                     in the Cucumber world's context.
  --keep-snapshots   Don't ever delete any snapshots (including ones marked as
                     temporary). This can be a big time saver when debugging new
                     features.
  --retry-find       Print a warning whenever Sikuli fails to find an image
                     and allow *one* retry after pressing ENTER. This is useful
                     for updating outdated images.
  --fuzzy-image-matching
                     When Sikuli fails to find an image, let it retry with more
                     fuzziness (or \"lower similarity factor\" in Sikuli terms).
  --tmpdir           Directory where various temporary files are written
                     during a test, e.g. VM snapshots and memory dumps,
                     failure screenshots, pcap files and disk images
                     (default is TMPDIR in the environment, and if unset,
                     /tmp/TailsToaster).
  --view             Shows the test session in a windows. Requires x11vnc
                     and tigervnc-viewer.
  --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
  --iso IMAGE        Test '@product' features using IMAGE.
  --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
                     an older version of Tails, which this options sets to
                     IMAGE. If none is given, it defaults to the same IMAGE
                     given by --iso, which will be good enough for most testing
                     purposes.

Note that '@source' features has no relevant options.

CUCUMBER_ARGS can be used to specify which features to be run, but also any
cucumber option, although then you must pass \`--\` first to let this wrapper
script know that we're done with *its* options. For debugging purposes, a
'debug' formatter has been added so pretty debugging can be enabled with
\`--format debug\`. You could even combine the default (pretty) formatter with
pretty debugging printed to a file with \`--format pretty --format debug
--out debug.log\`.
"
}

error() {
    echo "${NAME}: error: ${*}" >&2
    usage
    exit 1
}

package_installed() {
    local ret
    set +o pipefail
    if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
        ret=0
    else
        ret=1
    fi
    set -o pipefail
    return ${ret}
}

check_dependencies() {
    while [ -n "${1:-}" ]; do
        if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
            error "'${1}' is missing, please install it and run again."
        fi
        shift
    done
}

display_in_use() {
    [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
}

next_free_display() {
    display_nr=0
    while display_in_use ":${display_nr}"; do
	display_nr=$((display_nr+1))
    done
    echo ":${display_nr}"
}

test_suite_cleanup() {
    (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true
}

start_xvfb() {
    Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 &
    XVFB_PID=$!
    # Wait for Xvfb to run on TARGET_DISPLAY
    until display_in_use $TARGET_DISPLAY; do
	sleep 1
    done
    echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
    # Hide the mouse cursor so it won't mess up Sikuli's screen scanning
    unclutter -display $TARGET_DISPLAY -root -idle 0.1 >/dev/null 2>&1 &
}

start_vnc_server() {
    check_dependencies x11vnc
    VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \
                              -bg -nopw -forever 2>&1 | \
                                  grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
    echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
}

start_vnc_viewer() {
    check_dependencies tigervnc-viewer
    xtigervncviewer -nojpeg -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 &
}

capture_session() {
    check_dependencies libvpx1
    echo "Capturing guest display into ${CAPTURE_FILE}"
    avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
        -vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 &
}

# main script

# Unset all environment variables used by this script to pass options
# to cucumber, except TMPDIR since we explicitly want to support
# setting it that way.
ARTIFACTS_BASE_URI=
CAPTURE=
CAPTURE_ALL=
LOG_FILE=
VNC_VIEWER=
VNC_SERVER=
INTERACTIVE_DEBUGGING=
KEEP_SNAPSHOTS=
SIKULI_RETRY_FINDFAILED=
SIKULI_FUZZY_IMAGE_MATCHING=
TAILS_ISO=
OLD_TAILS_ISO=

LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,fuzzy-image-matching,iso:,old-iso:,interactive-debugging"
OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
eval set -- "$OPTS"
while [ $# -gt 0 ]; do
    case $1 in
        --artifacts-base-uri)
            shift
            export ARTIFACTS_BASE_URI="${1}"
            ;;
        --view)
            VNC_VIEWER=yes
            VNC_SERVER=yes
            ;;
        --vnc-server-only)
            VNC_VIEWER=
            VNC_SERVER=yes
            ;;
        --capture)
            check_dependencies x264
            export CAPTURE="yes"
            ;;
        --capture-all)
            check_dependencies x264
            export CAPTURE="yes"
            export CAPTURE_ALL="yes"
            ;;
        --interactive-debugging)
            export INTERACTIVE_DEBUGGING="yes"
            ;;
        --keep-snapshots)
            export KEEP_SNAPSHOTS="yes"
            ;;
        --retry-find)
            export SIKULI_RETRY_FINDFAILED="yes"
            ;;
        --fuzzy-image-matching)
            export SIKULI_FUZZY_IMAGE_MATCHING="yes"
            ;;
        --tmpdir)
            shift
            export TMPDIR="$(readlink -f $1)"
            ;;
        --iso)
            shift
            export TAILS_ISO="$(readlink -f $1)"
            ;;
        --old-iso)
            shift
            export OLD_TAILS_ISO="$(readlink -f $1)"
            ;;
        --help)
	    usage
            exit 0
            ;;
        --)
            shift
            break
            ;;
    esac
    shift
done

trap "test_suite_cleanup" EXIT HUP INT QUIT TERM

check_dependencies ${GENERAL_DEPENDENCIES}

TARGET_DISPLAY=$(next_free_display)

start_xvfb

if [ -n "${VNC_SERVER:-}" ]; then
    start_vnc_server
fi
if [ -n "${VNC_VIEWER:-}" ]; then
    start_vnc_viewer
fi

export SIKULI_HOME="/usr/share/java"
export DISPLAY=${TARGET_DISPLAY}

cucumber ${@}