path: root/test/run-tests.sh

#!/usr/bin/env bash
#
# Tests for the devtools scripts.
#
# Each case drives a script into a specific failure mode and asserts it is
# detected. A script that only ever reports PASS is worse than no script, so the
# negative cases are the point — the positive ones just confirm it isn't
# failing everything.

set -uo pipefail

BIN=$(cd "$(dirname "${BASH_SOURCE[0]}")/../bin" && pwd)
pass=0; fail=0
tmproot=$(mktemp -d); trap 'rm -rf "$tmproot"' EXIT

ok()   { printf '  PASS  %s\n' "$1"; pass=$((pass+1)); }
bad()  { printf '  FAIL  %s\n     -> %s\n' "$1" "$2"; fail=$((fail+1)); }

# Builds a fake project dir containing a gradlew that behaves as told.
# $1 dir, $2 exit code for gradlew, $3 optional: "write-xml <tests> <failures>"
make_project() {
  local d=$1 code=$2; shift 2
  mkdir -p "$d"
  cat > "$d/gradlew" <<EOF
#!/usr/bin/env bash
echo "BUILD SUCCESSFUL in 1s"   # deliberately lies when exit code is nonzero
$*
exit $code
EOF
  chmod +x "$d/gradlew"
}

xml_writer() {
  local dir=$1 tests=$2 failures=$3
  printf 'mkdir -p "%s"; cat > "%s/TEST-Fake.xml" <<XML\n<testsuite name="Fake" tests="%s" failures="%s" errors="0" skipped="0"></testsuite>\nXML\n' \
    "$dir" "$dir" "$tests" "$failures"
}

printf '=== gradle-verify ===\n'

# 1. Honest failure: gradlew exits nonzero but prints BUILD SUCCESSFUL.
p="$tmproot/p1"; make_project "$p" 1
out=$("$BIN/gradle-verify" "$p" someTask 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "result  : FAIL"; then
  ok "reports FAIL on nonzero exit even when output says BUILD SUCCESSFUL"
else
  bad "grep-proof exit code" "rc=$rc out=$(printf '%s' "$out" | tail -2 | tr '\n' ' ')"
fi

# 2. Success path.
p="$tmproot/p2"; make_project "$p" 0
out=$("$BIN/gradle-verify" "$p" someTask 2>&1); rc=$?
if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "result  : PASS"; then
  ok "reports PASS on zero exit"
else
  bad "success path" "rc=$rc"
fi

# 3. Nonexistent project directory must be a hard error, not a silent fallback
#    to the current directory. This is the cwd-drift bug.
out=$("$BIN/gradle-verify" "$tmproot/does-not-exist" someTask 2>&1); rc=$?
if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "does not exist"; then
  ok "refuses a missing project dir instead of falling back to cwd"
else
  bad "missing dir is fatal" "rc=$rc out=$out"
fi

# 4. A directory with no gradlew is also fatal.
mkdir -p "$tmproot/p4"
out=$("$BIN/gradle-verify" "$tmproot/p4" someTask 2>&1); rc=$?
if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "no executable gradlew"; then
  ok "refuses a dir with no gradlew"
else
  bad "missing gradlew is fatal" "rc=$rc out=$out"
fi

# 5. The project used must be the one named, regardless of cwd.
p="$tmproot/p5"; make_project "$p" 0
out=$(cd /tmp && "$BIN/gradle-verify" "$p" someTask 2>&1); rc=$?
if printf '%s' "$out" | grep -q "project : $p"; then
  ok "names the resolved absolute project dir, independent of cwd"
else
  bad "cwd independence" "$out"
fi

printf '=== gradle-test ===\n'

# 6. Zero tests must fail even though the build succeeded.
p="$tmproot/t1"; results="$p/build/test-results/test"
make_project "$p" 0 "$(xml_writer "$results" 0 0)"
out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "zero tests executed"; then
  ok "fails when a green build executed no tests"
else
  bad "zero-test detection" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')"
fi

# 7. Real failures are reported.
p="$tmproot/t2"; results="$p/build/test-results/test"
make_project "$p" 1 "$(xml_writer "$results" 10 3)"
out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "failures: 3"; then
  ok "reports failure counts parsed from XML"
else
  bad "failure counts" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')"
fi

# 8. Passing suite passes.
p="$tmproot/t3"; results="$p/build/test-results/test"
make_project "$p" 0 "$(xml_writer "$results" 42 0)"
out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$?
if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "result  : PASS (42 tests"; then
  ok "passes and reports the parsed count"
else
  bad "pass path" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')"
fi

# 9. Stale results: XML exists but predates the run (the UP-TO-DATE case).
p="$tmproot/t4"; results="$p/build/test-results/test"
make_project "$p" 0                      # gradlew writes nothing this time
mkdir -p "$results"
cat > "$results/TEST-Old.xml" <<'XML'
<testsuite name="Old" tests="99" failures="0" errors="0" skipped="0"></testsuite>
XML
touch -d '2020-01-01 00:00:00' "$results/TEST-Old.xml"
out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "predates this run"; then
  ok "refuses to report stale results as current"
else
  bad "stale detection" "rc=$rc out=$(printf '%s' "$out" | tail -4 | tr '\n' ' ')"
fi

# 10. ...unless explicitly allowed.
out=$("$BIN/gradle-test" "$p" test --allow-stale 2>&1); rc=$?
if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "99"; then
  ok "--allow-stale opts in to counting old results"
else
  bad "allow-stale" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')"
fi

# 11. No XML at all.
p="$tmproot/t5"; make_project "$p" 0
out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "no JUnit XML found"; then
  ok "fails when no results were produced at all"
else
  bad "no-xml detection" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')"
fi

printf '=== android-launch-check ===\n'

# 12. Argument validation without a device.
out=$("$BIN/android-launch-check" 2>&1); rc=$?
if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "usage"; then
  ok "requires a package argument"
else
  bad "usage check" "rc=$rc out=$out"
fi

printf '=== gradle-report ===\n'

# 13. A config detekt rejects means the analysis never ran; the XML on disk is a previous
#     run's and looks entirely plausible. This is the trap that was read as real findings
#     three times in one session.
d="$tmproot/report-badconfig"
make_project "$d" 1 'echo "> Provided configuration file is invalid: Structure must be from type Map<String,Any>!" >&2'
mkdir -p "$d/app/build/reports/detekt"
cat > "$d/app/build/reports/detekt/detekt.xml" <<'XML'
<?xml version="1.0"?><checkstyle><file name="/x/Old.kt"><error line="1" source="detekt.Stale" message="from an earlier run"/></file></checkstyle>
XML
out=$("$BIN/gradle-report" "$d" detekt 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -qi "CONFIG INVALID"; then
  ok "names an invalid config instead of reading the stale report"
else
  bad "invalid config detection" "rc=$rc out=$out"
fi
if printf '%s' "$out" | grep -q "Stale"; then
  bad "stale findings leaked" "the previous run's findings were printed as current"
else
  ok "does not print findings from an earlier run"
fi

# 14. A report older than the run is refused even when the config is fine.
d="$tmproot/report-stale"
make_project "$d" 0
mkdir -p "$d/app/build/reports/detekt"
cat > "$d/app/build/reports/detekt/detekt.xml" <<'XML'
<?xml version="1.0"?><checkstyle></checkstyle>
XML
touch -d '2020-01-01' "$d/app/build/reports/detekt/detekt.xml"
out=$("$BIN/gradle-report" "$d" detekt 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -qi "STALE"; then
  ok "refuses a report that predates the run"
else
  bad "stale report detection" "rc=$rc out=$out"
fi

# 15. UP-TO-DATE is the common way a report goes stale, so say so rather than leaving the
#     reader to work out why nothing changed.
d="$tmproot/report-uptodate"
make_project "$d" 0 'echo "> Task :app:detekt UP-TO-DATE"'
mkdir -p "$d/app/build/reports/detekt"
printf '<?xml version="1.0"?><checkstyle></checkstyle>' > "$d/app/build/reports/detekt/detekt.xml"
touch -d '2020-01-01' "$d/app/build/reports/detekt/detekt.xml"
out=$("$BIN/gradle-report" "$d" detekt 2>&1); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -qi "UP-TO-DATE"; then
  ok "explains an UP-TO-DATE task rather than just saying stale"
else
  bad "up-to-date hint" "rc=$rc out=$out"
fi
out=$("$BIN/gradle-report" "$d" detekt --allow-stale 2>&1); rc=$?
if [ $rc -eq 0 ] && printf '%s' "$out" | grep -qi "allow-stale"; then
  ok "--allow-stale opts in to an older report"
else
  bad "allow-stale" "rc=$rc out=$out"
fi

# 16. A missing project directory is fatal, never a silent fallback to cwd.
out=$("$BIN/gradle-report" "$tmproot/nope" detekt 2>&1); rc=$?
if [ $rc -eq 2 ]; then ok "missing project dir is fatal"; else bad "missing dir" "rc=$rc"; fi

printf '=== android-ui ===\n'

# 16. Usage validation without a device.
out=$("$BIN/android-ui" 2>&1); rc=$?
if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "usage"; then
  ok "requires package and command"
else
  bad "usage check" "rc=$rc out=$out"
fi

# 17. The node parser is where a blind tap goes wrong, so it is tested directly.
LIB=$(cd "$BIN/../lib" && pwd)
dump='<hierarchy><node text="" content-desc="Play" bounds="[0,0][100,100]" clickable="true"/><node text="Settings" content-desc="" bounds="[200,0][300,100]" clickable="true"/></hierarchy>'
out=$(printf '%s' "$dump" | "$LIB/uinodes.py" find "Play")
if [ "$out" = "50 50" ]; then ok "finds a node by content-desc"; else bad "find by desc" "got '$out'"; fi
out=$(printf '%s' "$dump" | "$LIB/uinodes.py" find "Settings")
if [ "$out" = "250 50" ]; then ok "finds a node by text"; else bad "find by text" "got '$out'"; fi
out=$(printf '%s' "$dump" | "$LIB/uinodes.py" find "Nope")
if [ "$out" = "NONE" ]; then ok "reports a missing label"; else bad "missing label" "got '$out'"; fi

# 18. Several matches must refuse rather than tap the first -- that is how you act on the
#     wrong row in a list where every entry has the same button.
dup='<hierarchy><node content-desc="Play" bounds="[0,0][100,100]" clickable="true"/><node content-desc="Play" bounds="[0,200][100,300]" clickable="true"/></hierarchy>'
out=$(printf '%s' "$dup" | "$LIB/uinodes.py" find "Play")
case "$out" in AMBIGUOUS*) ok "refuses to guess between duplicate labels" ;;
  *) bad "ambiguity" "got '$out'" ;; esac

# 19. A label and the button carrying it are often separate nodes; tapping the label does
#     nothing, so a clickable node wins.
pair='<hierarchy><node text="Play" bounds="[0,0][100,100]" clickable="false"/><node content-desc="Play" bounds="[500,500][600,600]" clickable="true"/></hierarchy>'
out=$(printf '%s' "$pair" | "$LIB/uinodes.py" find "Play")
if [ "$out" = "550 550" ]; then ok "prefers the clickable node"; else bad "clickable preference" "got '$out'"; fi

printf '=== publish-apk ===\n'

# 20. Publishing from the wrong host is how a sandbox once got "deployed" to.
out=$(HOSTNAME=x "$BIN/publish-apk" /dev/null thing.apk --host definitely-not-this-host 2>&1); rc=$?
if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "expected"; then
  ok "refuses to publish from the wrong host"
else
  bad "host guard" "rc=$rc out=$out"
fi

# 21. A published name with a path would write outside the intended directory.
out=$("$BIN/publish-apk" /dev/null ../escape.apk --host "$(hostname)" 2>&1); rc=$?
if [ $rc -eq 2 ]; then ok "rejects a path in the published name"; else bad "name guard" "rc=$rc"; fi

# 22. An HTML error page saved as .apk is not an APK; publishing it would serve a file that
#     installs as nothing.
notapk="$tmproot/not.apk"; printf '<!DOCTYPE html><html>nope</html>' > "$notapk"
out=$("$BIN/publish-apk" "$notapk" thing.apk --host "$(hostname)" 2>&1); rc=$?
if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "not a zip"; then
  ok "rejects a non-APK"
else
  bad "apk validation" "rc=$rc out=$out"
fi

printf '\n=== summary: %d passed, %d failed ===\n' "$pass" "$fail"
[ "$fail" -eq 0 ]