summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoranonym <anonym@riseup.net>2017-10-07 18:10:59 +0200
committeranonym <anonym@riseup.net>2017-10-08 11:33:37 +0200
commit145d0edf88d0a956286591e3f462e71479c6c9b0 (patch)
tree8ceacd9de17b3a0b2d9d50694492548ed3fcd851
parent0a9a90d06213436fb7d0e508577905f11753a114 (diff)
Test suite: make Sikuli attempt to find replacements on FindFailed.
... by employing fuzz, or "lowering the similarity factor". The replacements (if found) are saved among the artifacts, and serves as potential drop-in-replacements for outdated images). The main use case for this is when the font configuration in Tails changes, which normally invalidates a large part of our images given that our default high similarity factor. We also add the `--fuzzy-image-matching` where the replacements are used in case of FindFailed, so the tests can proceed beyond the first FindFailed. The idea is that a full test suite run will produce replacements for potentially *all* outdated images.
-rw-r--r--features/config/defaults.yml1
-rw-r--r--features/support/config.rb1
-rw-r--r--features/support/helpers/sikuli_helper.rb97
-rwxr-xr-xrun_test_suite9
4 files changed, 72 insertions, 36 deletions
diff --git a/features/config/defaults.yml b/features/config/defaults.yml
index 1e7f8ea..0c450ae 100644
--- a/features/config/defaults.yml
+++ b/features/config/defaults.yml
@@ -3,6 +3,7 @@ CAPTURE_ALL: false
INTERACTIVE_DEBUGGING: false
MAX_NEW_TOR_CIRCUIT_RETRIES: 10
SIKULI_RETRY_FINDFAILED: false
+SIKULI_FUZZY_IMAGE_MATCHING: false
TMPDIR: "/tmp/TailsToaster"
Unsafe_SSH_private_key: |
diff --git a/features/support/config.rb b/features/support/config.rb
index 6523a38..c654932 100644
--- a/features/support/config.rb
+++ b/features/support/config.rb
@@ -59,6 +59,7 @@ loop do
break
end
end
+SIKULI_CANDIDATES_DIR = "#{ARTIFACTS_DIR}/sikuli_candidates"
SIKULI_IMAGE_PATH = "#{Dir.pwd}/features/images/"
# Constants that are statically initialized.
diff --git a/features/support/helpers/sikuli_helper.rb b/features/support/helpers/sikuli_helper.rb
index 07fbf8c..f8a2b45 100644
--- a/features/support/helpers/sikuli_helper.rb
+++ b/features/support/helpers/sikuli_helper.rb
@@ -55,8 +55,45 @@ def bind_java_to_pseudo_fifo_logger
RJava::Lang::System.setOut(print_stream)
end
-def findfailed_hook(pic)
- pause("FindFailed for: '#{pic}'")
+def findfailed_hook(proxy, exception, override_method, signature, args)
+ picture = args.first
+ candidate_path = "#{SIKULI_CANDIDATES_DIR}/#{picture}"
+ if ! File.exist?(candidate_path)
+ [0.80, 0.70, 0.60, 0.50, 0.40].each do |similarity|
+ pattern = Sikuli::Pattern.new(picture)
+ pattern.similar(similarity)
+ match = proxy._invoke('exists', 'Ljava.lang.Object;', pattern)
+ if match
+ debug_log("Found fuzzy candidate picture for #{picture} with " +
+ "similarity #{similarity}")
+ capture = proxy._invoke('capture', 'Lorg.sikuli.script.Region;', match)
+ capture_path = capture.getFilename
+ # Let's verify that our screen capture actually matches
+ # with the default similarity
+ if proxy._invoke('exists', 'Ljava.lang.Object;', capture_path)
+ FileUtils.mkdir_p(SIKULI_CANDIDATES_DIR)
+ FileUtils.mv(capture_path, candidate_path)
+ break
+ end
+ end
+ end
+ if ! File.exist?(candidate_path)
+ debug_log("Failed to find fuzzy candidate picture for #{picture}")
+ end
+ end
+
+ if $config['SIKULI_FUZZY_IMAGE_MATCHING'] && File.exist?(candidate_path)
+ debug_log("Using fuzzy candidate picture for #{picture}")
+ args_with_candidate = [candidate_path] + args.drop(1)
+ return proxy._invoke(override_method, signature, *args_with_candidate)
+ end
+
+ if $config["SIKULI_RETRY_FINDFAILED"]
+ pause("FindFailed for: '#{picture}'")
+ return proxy._invoke(override_method, signature, *args)
+ else
+ raise exception
+ end
end
# Since rjb imports Java classes without creating a corresponding
@@ -85,39 +122,29 @@ end
def sikuli_script_proxy.new(*args)
s = $_original_sikuli_screen_new.call(*args)
- if $config["SIKULI_RETRY_FINDFAILED"]
- # The usage of `_invoke()` below exemplifies how one can wrap
- # around Java objects' methods when they're imported using RJB. It
- # isn't pretty. The seconds argument is the parameter signature,
- # which can be obtained by creating the intended Java object using
- # RJB, and then calling its `java_methods` method.
-
- def s.wait(pic, time)
- self._invoke('wait', 'Ljava.lang.Object;D', pic, time)
- rescue FindFailed => e
- findfailed_hook(pic)
- self._invoke('wait', 'Ljava.lang.Object;D', pic, time)
- end
-
- def s.find(pic)
- self._invoke('find', 'Ljava.lang.Object;', pic)
- rescue FindFailed => e
- findfailed_hook(pic)
- self._invoke('find', 'Ljava.lang.Object;', pic)
- end
-
- def s.waitVanish(pic, time)
- self._invoke('waitVanish', 'Ljava.lang.Object;D', pic, time)
- rescue FindFailed => e
- findfailed_hook(pic)
- self._invoke('waitVanish', 'Ljava.lang.Object;D', pic, time)
- end
-
- def s.click(pic)
- self._invoke('click', 'Ljava.lang.Object;', pic)
- rescue FindFailed => e
- findfailed_hook(pic)
- self._invoke('click', 'Ljava.lang.Object;', pic)
+ findfail_overrides = [
+ ['wait', 'Ljava.lang.Object;D'],
+ ['find', 'Ljava.lang.Object;'],
+ ['waitVanish', 'Ljava.lang.Object;D'],
+ ['click', 'Ljava.lang.Object;'],
+ ]
+
+ # The usage of `_invoke()` below exemplifies how one can wrap
+ # around Java objects' methods when they're imported using RJB. It
+ # isn't pretty. The seconds argument is the parameter signature,
+ # which can be obtained by creating the intended Java object using
+ # RJB, and then calling its `java_methods` method.
+ findfail_overrides.each do |method_name, signature|
+ s.define_singleton_method(method_name) do |*args|
+ begin
+ self._invoke(method_name, signature, *args)
+ rescue Exception => exception
+ # We really would like to only capture the FindFailed
+ # exceptions imported by rjb here, but that hasn't happened
+ # at the time this code is run. Yeah, meta-programming! :)
+ raise e unless exception.class.name == "FindFailed"
+ findfailed_hook(self, exception, method_name, signature, args)
+ end
end
end
diff --git a/run_test_suite b/run_test_suite
index 5e4655f..4f2f783 100755
--- a/run_test_suite
+++ b/run_test_suite
@@ -72,6 +72,9 @@ Options for '@product' features:
--retry-find Print a warning whenever Sikuli fails to find an image
and allow *one* retry after pressing ENTER. This is useful
for updating outdated images.
+ --fuzzy-image-matching
+ When Sikuli fails to find an image, let it retry with more
+ fuzziness (or \"lower similarity factor\" in Sikuli terms).
--tmpdir Directory where various temporary files are written
during a test, e.g. VM snapshots and memory dumps,
failure screenshots, pcap files and disk images
@@ -188,10 +191,11 @@ VNC_SERVER=
INTERACTIVE_DEBUGGING=
KEEP_SNAPSHOTS=
SIKULI_RETRY_FINDFAILED=
+SIKULI_FUZZY_IMAGE_MATCHING=
TAILS_ISO=
OLD_TAILS_ISO=
-LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,interactive-debugging"
+LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,fuzzy-image-matching,iso:,old-iso:,interactive-debugging"
OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
eval set -- "$OPTS"
while [ $# -gt 0 ]; do
@@ -226,6 +230,9 @@ while [ $# -gt 0 ]; do
--retry-find)
export SIKULI_RETRY_FINDFAILED="yes"
;;
+ --fuzzy-image-matching)
+ export SIKULI_FUZZY_IMAGE_MATCHING="yes"
+ ;;
--tmpdir)
shift
export TMPDIR="$(readlink -f $1)"