From b51ba30f1c77bcadfbf96b2eb7b4917c20faa7b0 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Fri, 18 Sep 2026 21:04:06 +0000 Subject: Add devtools: build/test wrappers that can't be misread Three times in one session I reported a build or test result that was about something other than what I thought I was testing: - A failed `cd` meant Gradle ran in the previous directory; the "BUILD SUCCESSFUL" I reported was for the wrong tree. - `adb shell monkey` + a single `pidof` sample reported a healthy app as crashed and a crashed app as healthy, on the same emulator minutes apart. - `adb install` refused a filename and printed so to stdout, which was redirected to /dev/null; the old build stayed installed and got tested. All three share a cause: concluding from console text rather than from exit codes and structured artifacts. These wrappers make that mistake hard. gradle-verify pins the project with -p from an absolutely-resolved argument, treats a missing dir as fatal rather than falling back to cwd, and reports from the exit code. gradle-test parses JUnit XML for counts, and fails on zero tests executed, on no XML at all, and on results whose mtime predates the run (Gradle reporting UP-TO-DATE and skipping the task). android-launch-check resolves the launcher activity from the package manager rather than guessing, starts logcat capture before launching, uses `am start -W`, samples the pid across a settle window so a process that starts and then dies is a failure, and prints the fatal stack. Optionally installs an APK first and verifies the install actually reported Success. Two bugs of exactly this class were found and fixed in the scripts themselves while testing them, both of which broke the crash-detection path specifically: `... | grep -q` under pipefail SIGPIPEs its upstream and reports failure on a successful match, and `pid=$(pidof ...)` under `set -e` aborts the script at the moment the process is gone. Verified against the real projects: nav/android-app 241 tests, nav/test-runner 122, feedreader 39, and both the pass and crash paths of android-launch-check on a live emulator. gradle-test immediately caught nav reporting UP-TO-DATE with 33 stale result files, which naively read as "241 tests pass" without running one. 12 self-tests, mostly negative cases. --- bin/android-launch-check | 159 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100755 bin/android-launch-check (limited to 'bin/android-launch-check') diff --git a/bin/android-launch-check b/bin/android-launch-check new file mode 100755 index 0000000..00dfcfd --- /dev/null +++ b/bin/android-launch-check @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# +# Determine whether an Android app actually launches and stays up, and surface +# the exception if it does not. +# +# Why this exists: `adb shell monkey -p -c android.intent.category.LAUNCHER 1` +# followed by polling `pidof` is not a reliable check. In practice it reported a +# healthy app as dead and a dead app as healthy, in the same session, on the same +# emulator — monkey's launch can land on whatever activity happens to be in the +# foreground, and a single `pidof` sample races app startup and restart. +# +# This script instead: +# * resolves the launcher activity from the package manager rather than guessing +# * starts a continuous logcat capture BEFORE launching, so the crash window is +# never missed (the `-b crash` buffer was empty even for real FATAL crashes) +# * launches with `am start -W` and checks its reported status +# * samples the pid repeatedly over a settle period, so a process that starts +# and then dies is reported as a failure, not a success +# * greps the captured log for FATAL EXCEPTION and prints the stack +# +# Usage: +# android-launch-check [--serial ] [--settle ] +# +# Exit: 0 = launched and stayed up with no fatal exception, 1 = did not. + +set -euo pipefail + +die() { printf '%s: %s\n' "$(basename "$0")" "$*" >&2; exit 2; } + +[ $# -ge 1 ] || die "usage: $(basename "$0") [--serial ] [--settle ]" + +pkg=$1; shift +serial="" +settle=20 +apk="" +while [ $# -gt 0 ]; do + case "$1" in + --serial) serial=${2:-}; shift 2 ;; + --settle) settle=${2:-20}; shift 2 ;; + --apk) apk=${2:-}; shift 2 ;; + *) die "unknown argument: $1" ;; + esac +done + +adb_bin=$(command -v adb || echo /opt/android-sdk/platform-tools/adb) +[ -x "$adb_bin" ] || die "adb not found" +ADB=("$adb_bin") +[ -n "$serial" ] && ADB+=(-s "$serial") + +devices=$("${ADB[@]}" devices | awk 'NR>1 && $2=="device" {print $1}') +[ -n "$devices" ] || die "no adb device in 'device' state" +[ "$(printf '%s\n' "$devices" | wc -l)" -eq 1 ] || [ -n "$serial" ] \ + || die "multiple devices attached; pass --serial. Found: $(echo $devices)" + +printf '=== android-launch-check ===\n' +printf 'package : %s\n' "$pkg" +printf 'device : %s\n' "$(echo $devices | tr '\n' ' ')" + +if [ -n "$apk" ]; then + [ -f "$apk" ] || die "apk not found: $apk" + # adb rejects any filename not ending .apk/.apex, and prints that refusal to + # stdout while still exiting 0 in some versions — so check the text as well as + # the status. Discarding this output is how a "reinstall" can silently not + # happen and leave the previous build under test. + case "$apk" in + *.apk|*.apex) : ;; + *) die "adb will refuse this filename (must end .apk/.apex): $apk" ;; + esac + printf 'apk : %s\n' "$apk" + install_out=$("${ADB[@]}" install -r "$apk" 2>&1 | tr -d '\r' || true) + if ! printf '%s' "$install_out" | grep -q '^Success$'; then + printf 'result : FAIL — install did not succeed\n' + printf '%s\n' "$install_out" | sed 's/^/ /' + exit 1 + fi + printf 'install : Success\n' +fi + +# Membership test done in pure shell, not `... | grep -q`. Under `set -o pipefail` +# a `grep -q` that exits on its first match can SIGPIPE the upstream command, and +# the pipeline then reports failure even though the match succeeded — which made +# this check report an installed package as missing. +pkg_list=$("${ADB[@]}" shell pm list packages 2>/dev/null | tr -d '\r' || true) +case $'\n'"$pkg_list"$'\n' in + *$'\n'"package:$pkg"$'\n'*) : ;; + *) die "package not installed: $pkg" ;; +esac + +# Resolve rather than assume. Guessing ".MainActivity" when the manifest says +# ".ui.MainActivity" produces "Activity class does not exist", which is an +# operator error that can easily be misread as an app crash. +activity=$("${ADB[@]}" shell cmd package resolve-activity --brief "$pkg" 2>/dev/null \ + | tr -d '\r' | tail -1) +case "$activity" in + "$pkg"/*) : ;; + *) die "could not resolve launcher activity for $pkg (got: '$activity')" ;; +esac +printf 'activity: %s\n' "$activity" + +logfile=$(mktemp); trap 'rm -f "$logfile"' EXIT + +"${ADB[@]}" shell am force-stop "$pkg" >/dev/null 2>&1 || true +sleep 2 +"${ADB[@]}" logcat -c >/dev/null 2>&1 || true + +# Capture starts first and runs across the whole window. +"${ADB[@]}" logcat -v threadtime > "$logfile" 2>&1 & +logcat_pid=$! +trap 'kill "$logcat_pid" 2>/dev/null || true; rm -f "$logfile"' EXIT +sleep 2 + +start_out=$("${ADB[@]}" shell am start -W -n "$activity" 2>&1 | tr -d '\r') +start_status=$(printf '%s\n' "$start_out" | awk -F': *' '/^Status:/ {print $2}') +printf 'am start: %s\n' "${start_status:-}" + +# Sample repeatedly. One sample cannot distinguish "not started yet" from +# "started and already died". +alive_samples=0; total_samples=0; last_pid="" +interval=5 +elapsed=0 +while [ "$elapsed" -lt "$settle" ]; do + sleep "$interval" + elapsed=$((elapsed + interval)) + # `|| true` is load-bearing: pidof exits non-zero when the process is gone, + # which under `set -e` would abort this script at exactly the moment it is + # supposed to be detecting a crash. + pid=$("${ADB[@]}" shell pidof "$pkg" 2>/dev/null | tr -d '\r' | awk '{print $1}' || true) + total_samples=$((total_samples + 1)) + if [ -n "$pid" ]; then alive_samples=$((alive_samples + 1)); last_pid=$pid; fi + printf ' t=%-3ss pid=%s\n' "$elapsed" "${pid:-}" +done + +sleep 1 +kill "$logcat_pid" 2>/dev/null || true +wait "$logcat_pid" 2>/dev/null || true + +fatal_count=$(grep -c "FATAL EXCEPTION" "$logfile" 2>/dev/null || true) +fatal_count=${fatal_count:-0} + +printf 'samples : %d/%d alive\n' "$alive_samples" "$total_samples" +printf 'fatal : %s\n' "$fatal_count" + +if [ "$fatal_count" -gt 0 ]; then + printf '\n--- first fatal exception ---\n' + grep -A 25 -m 1 "FATAL EXCEPTION" "$logfile" | sed 's/^/ /' + printf '\nresult : FAIL — app crashed\n' + exit 1 +fi + +if [ "$alive_samples" -ne "$total_samples" ]; then + printf '\nresult : FAIL — process was not alive for the whole settle window\n' + printf 'hint : started but died, or never started. No FATAL in logcat, so\n' + printf ' check for a native crash or an early System.exit.\n' + exit 1 +fi + +printf 'pid : %s\n' "$last_pid" +printf 'result : PASS — launched and stayed up %ss with no fatal exception\n' "$settle" +exit 0 -- cgit v1.2.3