From f3c033c92fc58053bfe9e11e98031b32d746fed1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 16:10:04 +0000 Subject: [PATCH 1/2] Print the emulator's state into the job log when device tests fail The API 36 device job still loses adb now and then mid-run (most recently run 37910180950 on 2026-10-09, in AccessibilityDeviceTest.shellAndPlayer). save_device_logs.sh saves the whole logcat to the artifact, but the artifact can't be downloaded from everywhere the job log can be read, so the cause is still unknown. On a failed run, the script now also prints the emulator's uptime and memory, and the logcat lines from adbd, the low memory killer, the test runner and crashes, in a collapsed group. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0181gRDMZBo6n2VbGaLqpZDt --- .github/workflows/ci.yml | 7 ++++--- scripts/save_device_logs.sh | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f419ade..fdd85e95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,13 +133,14 @@ jobs: # nothing runs, and Gradle still passes. Gradle uninstalls the app after each run, so # the local network test is run once more, on a fresh install again, if no test started. # The emulator's logs are saved after each run, by save_device_logs.sh, which reconnects - # first if adb lost the emulator during the run. + # first if adb lost the emulator during the run, and prints the lines that matter here + # when the run failed. script: | timeout 600 adb wait-for-device shell 'until pm path android > /dev/null 2>&1 && am get-current-user > /dev/null 2>&1; do sleep 2; done' scripts/smoke_test_release.sh app/build/outputs/apk/play/release/*.apk - for attempt in 1 2; do ./gradlew connectedFdroidDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=de.qspool.clementineremote.LocalNetworkPermissionDeviceTest; status=$?; if [ $status -ne 0 ] || find app/build/outputs/androidTest-results -name 'TEST-*.xml' -exec grep -q ' /dev/null 2>&1 && am get-current-user > /dev/null 2>&1; do sleep 2; done'; done; mkdir -p app/build/localNetworkPermission && cp -r app/build/outputs/androidTest-results app/build/reports/androidTests app/build/localNetworkPermission/; scripts/save_device_logs.sh app/build/localNetworkPermission; exit $status + for attempt in 1 2; do ./gradlew connectedFdroidDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=de.qspool.clementineremote.LocalNetworkPermissionDeviceTest; status=$?; if [ $status -ne 0 ] || find app/build/outputs/androidTest-results -name 'TEST-*.xml' -exec grep -q ' /dev/null 2>&1 && am get-current-user > /dev/null 2>&1; do sleep 2; done'; done; mkdir -p app/build/localNetworkPermission && cp -r app/build/outputs/androidTest-results app/build/reports/androidTests app/build/localNetworkPermission/; scripts/save_device_logs.sh app/build/localNetworkPermission $status; exit $status timeout 600 adb wait-for-device shell 'until pm path android > /dev/null 2>&1 && am get-current-user > /dev/null 2>&1; do sleep 2; done' - ./gradlew connectedFdroidDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.notClass=de.qspool.clementineremote.LocalNetworkPermissionDeviceTest; status=$?; scripts/save_device_logs.sh app/build/reports/androidTests; exit $status + ./gradlew connectedFdroidDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.notClass=de.qspool.clementineremote.LocalNetworkPermissionDeviceTest; status=$?; scripts/save_device_logs.sh app/build/reports/androidTests $status; exit $status # The test report is in the artifact below; the failures go in the log too, to be read # without downloading it. A skipped test is written down as a failure too (see below). - name: Show the failed tests diff --git a/scripts/save_device_logs.sh b/scripts/save_device_logs.sh index 66812390..60651600 100755 --- a/scripts/save_device_logs.sh +++ b/scripts/save_device_logs.sh @@ -7,10 +7,17 @@ # without logs: logcat can't be read from a device adb has lost. So it reconnects first, to save # the logs that say what happened. # -# Usage: save_device_logs.sh +# Given the run's exit status, and it isn't 0, it also prints what the logs say about the +# emulator's state into the job's log, in a group of its own: how long it has been up (a +# restart of Android shows there), its memory, and the lines from adbd, the low memory killer, +# the test runner and crashes. The artifact has the whole logcat, but can't be downloaded from +# everywhere the job's log can be read. +# +# Usage: save_device_logs.sh [exit status of the run] set -uo pipefail dir=$1 +status=${2:-0} mkdir -p "$dir" if [ "$(adb get-state 2> /dev/null)" != device ]; then echo "::warning::adb lost the emulator; reconnecting to save its logs" @@ -18,3 +25,11 @@ if [ "$(adb get-state 2> /dev/null)" != device ]; then timeout 120 adb wait-for-device || echo "::warning::adb couldn't reconnect to the emulator" fi adb logcat -d -b all > "$dir/logcat.txt" + +if [ "$status" -ne 0 ]; then + echo "::group::The emulator's state after the failed run" + adb shell 'uptime; grep -E "^(MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree):" /proc/meminfo' + grep -E -i 'adbd|lowmemorykiller|lmkd|out of memory|oom-kill|watchdog|FATAL EXCEPTION|AndroidRuntime|TestRunner|am_crash|am_anr|Zygote *:.*(died|crash)|system_server' \ + "$dir/logcat.txt" | tail -n 300 + echo "::endgroup::" +fi From 11d99c6e44b42dc09fb94787eaba670646e1d101 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 16:15:52 +0000 Subject: [PATCH 2/2] Say in the job log how far the saved device logs reach The saved logcat of the 2026-10-09 failure (run 37910180950) ends at 09:21:03, a minute after boot and 90 s before adb lost the emulator, so it can't show the cause. On a failed run, also print logd's buffers and the time the saved logs end, to show whether that happens again. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0181gRDMZBo6n2VbGaLqpZDt --- scripts/save_device_logs.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/save_device_logs.sh b/scripts/save_device_logs.sh index 60651600..98170523 100755 --- a/scripts/save_device_logs.sh +++ b/scripts/save_device_logs.sh @@ -9,9 +9,9 @@ # # Given the run's exit status, and it isn't 0, it also prints what the logs say about the # emulator's state into the job's log, in a group of its own: how long it has been up (a -# restart of Android shows there), its memory, and the lines from adbd, the low memory killer, -# the test runner and crashes. The artifact has the whole logcat, but can't be downloaded from -# everywhere the job's log can be read. +# restart of Android shows there), its memory, logd's buffers and when the saved logs end, and +# the lines from adbd, the low memory killer, the test runner and crashes. The artifact has the +# whole logcat, but can't be downloaded from everywhere the job's log can be read. # # Usage: save_device_logs.sh [exit status of the run] set -uo pipefail @@ -28,7 +28,10 @@ adb logcat -d -b all > "$dir/logcat.txt" if [ "$status" -ne 0 ]; then echo "::group::The emulator's state after the failed run" - adb shell 'uptime; grep -E "^(MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree):" /proc/meminfo' + adb shell 'uptime; grep -E "^(MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree):" /proc/meminfo; logcat -g' + # The saved logs of one failure ended 90 s before it, a minute after boot: logd's buffers + # (logcat -g, above) and this say whether the logs still reach the end of the run. + echo "The saved logs end at: $(grep -E '^[0-9]{2}-[0-9]{2} ' "$dir/logcat.txt" | cut -c1-18 | sort | tail -n 1)" grep -E -i 'adbd|lowmemorykiller|lmkd|out of memory|oom-kill|watchdog|FATAL EXCEPTION|AndroidRuntime|TestRunner|am_crash|am_anr|Zygote *:.*(died|crash)|system_server' \ "$dir/logcat.txt" | tail -n 300 echo "::endgroup::"