From b51ba30f1c77bcadfbf96b2eb7b4917c20faa7b0 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Fri, 18 Sep 2026 21:04:06 +0000 Subject: Add devtools: build/test wrappers that can't be misread Three times in one session I reported a build or test result that was about something other than what I thought I was testing: - A failed `cd` meant Gradle ran in the previous directory; the "BUILD SUCCESSFUL" I reported was for the wrong tree. - `adb shell monkey` + a single `pidof` sample reported a healthy app as crashed and a crashed app as healthy, on the same emulator minutes apart. - `adb install` refused a filename and printed so to stdout, which was redirected to /dev/null; the old build stayed installed and got tested. All three share a cause: concluding from console text rather than from exit codes and structured artifacts. These wrappers make that mistake hard. gradle-verify pins the project with -p from an absolutely-resolved argument, treats a missing dir as fatal rather than falling back to cwd, and reports from the exit code. gradle-test parses JUnit XML for counts, and fails on zero tests executed, on no XML at all, and on results whose mtime predates the run (Gradle reporting UP-TO-DATE and skipping the task). android-launch-check resolves the launcher activity from the package manager rather than guessing, starts logcat capture before launching, uses `am start -W`, samples the pid across a settle window so a process that starts and then dies is a failure, and prints the fatal stack. Optionally installs an APK first and verifies the install actually reported Success. Two bugs of exactly this class were found and fixed in the scripts themselves while testing them, both of which broke the crash-detection path specifically: `... | grep -q` under pipefail SIGPIPEs its upstream and reports failure on a successful match, and `pid=$(pidof ...)` under `set -e` aborts the script at the moment the process is gone. Verified against the real projects: nav/android-app 241 tests, nav/test-runner 122, feedreader 39, and both the pass and crash paths of android-launch-check on a live emulator. gradle-test immediately caught nav reporting UP-TO-DATE with 33 stale result files, which naively read as "241 tests pass" without running one. 12 self-tests, mostly negative cases. --- test/run-tests.sh | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100755 test/run-tests.sh (limited to 'test') diff --git a/test/run-tests.sh b/test/run-tests.sh new file mode 100755 index 0000000..3a8117d --- /dev/null +++ b/test/run-tests.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# +# Tests for the devtools scripts. +# +# Each case drives a script into a specific failure mode and asserts it is +# detected. A script that only ever reports PASS is worse than no script, so the +# negative cases are the point — the positive ones just confirm it isn't +# failing everything. + +set -uo pipefail + +BIN=$(cd "$(dirname "${BASH_SOURCE[0]}")/../bin" && pwd) +pass=0; fail=0 +tmproot=$(mktemp -d); trap 'rm -rf "$tmproot"' EXIT + +ok() { printf ' PASS %s\n' "$1"; pass=$((pass+1)); } +bad() { printf ' FAIL %s\n -> %s\n' "$1" "$2"; fail=$((fail+1)); } + +# Builds a fake project dir containing a gradlew that behaves as told. +# $1 dir, $2 exit code for gradlew, $3 optional: "write-xml " +make_project() { + local d=$1 code=$2; shift 2 + mkdir -p "$d" + cat > "$d/gradlew" < "%s/TEST-Fake.xml" <\nXML\n' \ + "$dir" "$dir" "$tests" "$failures" +} + +printf '=== gradle-verify ===\n' + +# 1. Honest failure: gradlew exits nonzero but prints BUILD SUCCESSFUL. +p="$tmproot/p1"; make_project "$p" 1 +out=$("$BIN/gradle-verify" "$p" someTask 2>&1); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "result : FAIL"; then + ok "reports FAIL on nonzero exit even when output says BUILD SUCCESSFUL" +else + bad "grep-proof exit code" "rc=$rc out=$(printf '%s' "$out" | tail -2 | tr '\n' ' ')" +fi + +# 2. Success path. +p="$tmproot/p2"; make_project "$p" 0 +out=$("$BIN/gradle-verify" "$p" someTask 2>&1); rc=$? +if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "result : PASS"; then + ok "reports PASS on zero exit" +else + bad "success path" "rc=$rc" +fi + +# 3. Nonexistent project directory must be a hard error, not a silent fallback +# to the current directory. This is the cwd-drift bug. +out=$("$BIN/gradle-verify" "$tmproot/does-not-exist" someTask 2>&1); rc=$? +if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "does not exist"; then + ok "refuses a missing project dir instead of falling back to cwd" +else + bad "missing dir is fatal" "rc=$rc out=$out" +fi + +# 4. A directory with no gradlew is also fatal. +mkdir -p "$tmproot/p4" +out=$("$BIN/gradle-verify" "$tmproot/p4" someTask 2>&1); rc=$? +if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "no executable gradlew"; then + ok "refuses a dir with no gradlew" +else + bad "missing gradlew is fatal" "rc=$rc out=$out" +fi + +# 5. The project used must be the one named, regardless of cwd. +p="$tmproot/p5"; make_project "$p" 0 +out=$(cd /tmp && "$BIN/gradle-verify" "$p" someTask 2>&1); rc=$? +if printf '%s' "$out" | grep -q "project : $p"; then + ok "names the resolved absolute project dir, independent of cwd" +else + bad "cwd independence" "$out" +fi + +printf '=== gradle-test ===\n' + +# 6. Zero tests must fail even though the build succeeded. +p="$tmproot/t1"; results="$p/build/test-results/test" +make_project "$p" 0 "$(xml_writer "$results" 0 0)" +out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "zero tests executed"; then + ok "fails when a green build executed no tests" +else + bad "zero-test detection" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')" +fi + +# 7. Real failures are reported. +p="$tmproot/t2"; results="$p/build/test-results/test" +make_project "$p" 1 "$(xml_writer "$results" 10 3)" +out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "failures: 3"; then + ok "reports failure counts parsed from XML" +else + bad "failure counts" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')" +fi + +# 8. Passing suite passes. +p="$tmproot/t3"; results="$p/build/test-results/test" +make_project "$p" 0 "$(xml_writer "$results" 42 0)" +out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$? +if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "result : PASS (42 tests"; then + ok "passes and reports the parsed count" +else + bad "pass path" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')" +fi + +# 9. Stale results: XML exists but predates the run (the UP-TO-DATE case). +p="$tmproot/t4"; results="$p/build/test-results/test" +make_project "$p" 0 # gradlew writes nothing this time +mkdir -p "$results" +cat > "$results/TEST-Old.xml" <<'XML' + +XML +touch -d '2020-01-01 00:00:00' "$results/TEST-Old.xml" +out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "predates this run"; then + ok "refuses to report stale results as current" +else + bad "stale detection" "rc=$rc out=$(printf '%s' "$out" | tail -4 | tr '\n' ' ')" +fi + +# 10. ...unless explicitly allowed. +out=$("$BIN/gradle-test" "$p" test --allow-stale 2>&1); rc=$? +if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "99"; then + ok "--allow-stale opts in to counting old results" +else + bad "allow-stale" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')" +fi + +# 11. No XML at all. +p="$tmproot/t5"; make_project "$p" 0 +out=$("$BIN/gradle-test" "$p" test 2>&1); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "no JUnit XML found"; then + ok "fails when no results were produced at all" +else + bad "no-xml detection" "rc=$rc out=$(printf '%s' "$out" | tail -3 | tr '\n' ' ')" +fi + +printf '=== android-launch-check ===\n' + +# 12. Argument validation without a device. +out=$("$BIN/android-launch-check" 2>&1); rc=$? +if [ $rc -eq 2 ] && printf '%s' "$out" | grep -qi "usage"; then + ok "requires a package argument" +else + bad "usage check" "rc=$rc out=$out" +fi + +printf '\n=== summary: %d passed, %d failed ===\n' "$pass" "$fail" +[ "$fail" -eq 0 ] -- cgit v1.2.3