From d120c1c1171799635bb336939fc9ee63865dbb01 Mon Sep 17 00:00:00 2001 From: oech3 <79379754+oech3@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:22:26 +0900 Subject: [PATCH] android.yml: speed up CI (about 3 min) --- .github/workflows/android.yml | 240 ++------- src/uucore/build.rs | 18 +- src/uucore/src/lib/mods/locale.rs | 15 +- tests/by-util/test_cp.rs | 4 + tests/by-util/test_env.rs | 1 + tests/by-util/test_install.rs | 1 + tests/by-util/test_od.rs | 3 + tests/by-util/test_tail.rs | 3 + tests/by-util/test_test.rs | 1 + util/android-commands.sh | 711 --------------------------- util/android-scripts/collect-info.sh | 20 - util/android-scripts/run-tests.sh | 78 --- 12 files changed, 75 insertions(+), 1020 deletions(-) delete mode 100755 util/android-commands.sh delete mode 100755 util/android-scripts/collect-info.sh delete mode 100755 util/android-scripts/run-tests.sh diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 49f8c724649..04ab1b6e85c 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,8 +1,11 @@ name: Android - -# spell-checker:ignore (people) reactivecircus juliangruber -# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill cmdline commandlinetools sdkmanager -# spell-checker:ignore (misc) swiftshader playstore noaudio +# spell-checker:ignore (people) reactivecircus +# spell-checker:ignore (shell/tools) udevadm +# spell-checker:ignore (misc) noaudio +env: + CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER: "x86_64-linux-android28-clang" + CARGO_INCREMENTAL: "0" + RUST_BACKTRACE: "1" on: pull_request: @@ -10,7 +13,6 @@ on: branches: - '*' - permissions: contents: read # to fetch code (actions/checkout) @@ -19,200 +21,60 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} -env: - TERMUX: v0.118.0 - KEY_POSTFIX: nextest+rustc-hash+adb+sshd+upgrade+XGB+inc18 - COMMON_EMULATOR_OPTIONS: -no-metrics -no-window -noaudio -no-boot-anim -camera-back none -gpu off - EMULATOR_DISK_SIZE: 12GB - EMULATOR_HEAP_SIZE: 2048M - EMULATOR_BOOT_TIMEOUT: 1200 # 20min - jobs: - test_android: - name: ${{ matrix.test-suite == 'all' && 'Test builds' || 'Checksum and wc tests' }} (API ${{ matrix.api-level }}, ${{ matrix.arch }}) - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] # , macos-latest - cores: [4] # , 6 - ram: [4096] - api-level: [28, "37.0"] - target: [google_apis_playstore] - arch: [x86_64] # ,x86 ,arm64-v8a - include: - - api-level: 28 - test-suite: all - # Keep full-suite coverage on API 28. API 37 has separate filesystem - # and libc compatibility failures; cover the CPU-feature consumers here. - - api-level: "37.0" - test-suite: cksum-wc - runs-on: ${{ matrix.os }} - env: - EMULATOR_RAM_SIZE: ${{ matrix.ram }} - EMULATOR_CORES: ${{ matrix.cores }} - RUNNER_OS: ${{ matrix.os }} - AVD_CACHE_KEY: "set later due to limitations of github actions not able to concatenate env variables" + test: + name: test + runs-on: ubuntu-latest + timeout-minutes: 45 steps: - - name: Concatenate values to environment file - run: | - echo "AVD_CACHE_KEY=${{ matrix.os }}-${{ matrix.cores }}-${{ matrix.ram }}-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+${{ env.KEY_POSTFIX }}" >> $GITHUB_ENV - - name: Collect information about runner - if: always() - continue-on-error: true - run: | - hostname - uname -a - free -mh - df -Th - cat /proc/cpuinfo - - name: (Linux) create links from home to data partition - if: ${{ runner.os == 'Linux' }} - continue-on-error: true - run: | - ls -lah /mnt/ - sudo mkdir /mnt/data - sudo chmod a+rwx /mnt/data - mkdir /mnt/data/.android && ln -s /mnt/data/.android ~/.android - mkdir /mnt/data/work && ln -s /mnt/data/work ~/work - - name: Enable KVM group perms (linux hardware acceleration) - if: ${{ runner.os == 'Linux' }} - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - name: Install Android SDK command-line tools 22.0 - run: | - curl --fail --location --retry 3 --output "$RUNNER_TEMP/commandlinetools.zip" \ - https://dl.google.com/android/repository/commandlinetools-linux-15859902_latest.zip - unzip -q "$RUNNER_TEMP/commandlinetools.zip" -d "$RUNNER_TEMP/android-tools" - mv "$ANDROID_HOME/cmdline-tools/latest" "$RUNNER_TEMP/android-tools-previous" - mv "$RUNNER_TEMP/android-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" - "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --version - - name: Collect information about runner - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Restore AVD cache - uses: actions/cache/restore@v6 - id: avd-cache - continue-on-error: true + - uses: dtolnay/rust-toolchain@stable with: - path: | - ~/.android/avd/* - ~/.android/avd/*/snapshots/* - ~/.android/adb* - ~/__rustc_hash__ - key: avd-${{ env.AVD_CACHE_KEY }} - - name: Collect information about runner after AVD cache - if: always() + targets: x86_64-linux-android + - uses: Swatinem/rust-cache@v2 + - name: Run sccache-cache + id: sccache-setup + uses: mozilla-actions/sccache-action@v0.0.11 continue-on-error: true + - name: Export sccache + if: steps.sccache-setup.outcome == 'success' run: | - free -mh - df -Th - ls -lah /mnt/data - du -sch /mnt/data - - name: Delete AVD Lockfile when run from cache - if: steps.avd-cache.outputs.cache-hit == 'true' + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Set up Android NDK run: | - rm -f \ - ~/.android/avd/*.avd/*.lock \ - ~/.android/avd/*/*.lock - - name: Create and cache emulator image - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2.38.0 - with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - ram-size: ${{ env.EMULATOR_RAM_SIZE }} - heap-size: ${{ env.EMULATOR_HEAP_SIZE }} - disk-size: ${{ env.EMULATOR_DISK_SIZE }} - cores: ${{ env.EMULATOR_CORES }} - force-avd-creation: true - disable-animations: false - emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load - emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} - script: | - util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}" - - name: Save AVD cache - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: | - ~/.android/avd/* - ~/.android/avd/*/snapshots/* - ~/.android/adb* - ~/__rustc_hash__ - key: avd-${{ env.AVD_CACHE_KEY }} - - uses: juliangruber/read-file-action@v1 - id: read_rustc_hash - with: - # ~ expansion didn't work - path: ${{ runner.os == 'Linux' && '/home/runner/__rustc_hash__' || '/Users/runner/__rustc_hash__' }} - trim: true - - name: Restore rust cache - id: rust-cache - uses: actions/cache/restore@v6 - with: - path: ~/__rust_cache__ - # The version vX at the end of the key is just a development version to avoid conflicts in - # the github cache during the development of this workflow - key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 - - name: Collect information about runner resources - if: always() - continue-on-error: true + echo "${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin" >> $GITHUB_PATH + - name: prepare testing run: | - free -mh - df -Th - - name: Build and Test + set -euo pipefail + # FIXME: some ls::* depends on unask 022 + cargo test --target x86_64-linux-android --no-run --features feat_os_unix_android --message-format=json \ + | jq -rs ' + [ .[] | select(.executable != null) | .executable | select(test("/coreutils$") | not) ] + | unique + | .[] + ' | sed 's#^.*$#adb push & /data/local/tmp/tester\nadb shell "cd /data/local/tmp \&\& umask 022 \&\& TERM=xterm-256color \&\& /data/local/tmp/tester"#' > tests.sh + cat tests.sh + - name: Enable KVM (hardware acceleration) + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: tests uses: reactivecircus/android-emulator-runner@v2.38.0 with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - ram-size: ${{ env.EMULATOR_RAM_SIZE }} - heap-size: ${{ env.EMULATOR_HEAP_SIZE }} - disk-size: ${{ env.EMULATOR_DISK_SIZE }} - cores: ${{ env.EMULATOR_CORES }} - force-avd-creation: false - disable-animations: false - emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }} - emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} - # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If - # one of the lines returns with error the whole script is failed (like running a script with - # set -e) and in consequences the other lines (shells) are not executed. + api-level: 28 + disable-animations: true + emulator-options: -no-metrics -no-window -noaudio -no-boot-anim -camera-back none -gpu off + target: default + arch: x86_64 script: | - timeout --verbose --kill-after=30s 10m util/android-commands.sh sync_host - util/android-commands.sh build - util/android-commands.sh tests "${{ matrix.test-suite }}" - if [ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]; then util/android-commands.sh sync_image; fi; exit 0 - - name: Collect information about runner resources - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Save rust cache - if: steps.rust-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: ~/__rust_cache__ - key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 - - name: archive any output (error screenshots) - if: always() - uses: actions/upload-artifact@v7 - with: - name: test_output_${{ env.AVD_CACHE_KEY }} - path: output - - name: Collect information about runner resources - if: always() - continue-on-error: true - run: | - free -mh - df -Th + adb root + adb shell setenforce 0 + adb shell "mkdir -p /data/data/com.termux/files/usr/bin && ln -svf /system/bin/sh /data/data/com.termux/files/usr/bin/sh" # needed for test_stat + adb unroot + adb push tests/fixtures/. /data/local/tmp/tests/fixtures/ + adb push target/x86_64-linux-android/debug/coreutils /data/local/tmp/coreutils + sh -e ./tests.sh diff --git a/src/uucore/build.rs b/src/uucore/build.rs index d4753ca4b9f..9743b61caa7 100644 --- a/src/uucore/build.rs +++ b/src/uucore/build.rs @@ -402,20 +402,6 @@ fn embed_locale_file( Ok(()) } -/// Higher-level helper to embed locale files for a component with a path pattern. -/// -/// This eliminates the repetitive `for_each_locale` + `embed_locale_file` pattern. -/// -/// # Errors -/// -/// Returns an error if `for_each_locale` fails, which typically happens if -/// reading a locale file or writing to the `embedded_file` fails. -/// Check if we are cross-compiling for WASI (build.rs runs on the host, -/// so `#[cfg(target_os = "wasi")]` does not work here). -fn is_wasi_target() -> bool { - env::var("CARGO_CFG_TARGET_OS").is_ok_and(|os| os == "wasi") -} - /// For WASI/WASM builds, embed ALL available .ftl files in a locale /// directory so the playground can switch languages at runtime. fn embed_all_locales_for_component( @@ -456,7 +442,9 @@ fn embed_component_locales( where F: Fn(&str) -> PathBuf, { - if is_wasi_target() { + // we want to embedded locales on some targets to make publishing easier. + // cfg does not work for cross-building. + if env::var("CARGO_CFG_TARGET_OS").is_ok_and(|os| os == "wasi" || os == "android") { return embed_all_locales_for_component(embedded_file, component_name, &path_builder); } diff --git a/src/uucore/src/lib/mods/locale.rs b/src/uucore/src/lib/mods/locale.rs index 90b13d53b3c..d2f0fc2a6fd 100644 --- a/src/uucore/src/lib/mods/locale.rs +++ b/src/uucore/src/lib/mods/locale.rs @@ -430,11 +430,12 @@ fn create_english_bundle_from_embedded( } } -/// Create a bundle from embedded locale files for any locale on WASI. +/// Create a bundle from embedded locale files for any locale on platforms +/// to make publishing binaries easier. /// Resources are cached per key, so it can be called for multiple locales /// in the same process. -#[cfg(target_os = "wasi")] -fn create_wasi_bundle_from_embedded( +#[cfg(any(target_os = "android", target_os = "wasi"))] +fn create_bundle_from_embedded( locale: &LanguageIdentifier, util_name: &str, ) -> Result, LocalizationError> { @@ -642,19 +643,19 @@ pub fn setup_localization(p: &str) -> Result<(), LocalizationError> { let default_locale = LanguageIdentifier::from_str(DEFAULT_LOCALE) .expect("Default locale should always be valid"); - #[cfg(target_os = "wasi")] + #[cfg(any(target_os = "android", target_os = "wasi"))] let localizer = { - let english_bundle = create_wasi_bundle_from_embedded(&default_locale, p)?; + let english_bundle = create_bundle_from_embedded(&default_locale, p)?; if locale == default_locale { Localizer::new(english_bundle) - } else if let Ok(localized) = create_wasi_bundle_from_embedded(&locale, p) { + } else if let Ok(localized) = create_bundle_from_embedded(&locale, p) { Localizer::new(localized).with_fallback(english_bundle) } else { Localizer::new(english_bundle) } }; - #[cfg(not(target_os = "wasi"))] + #[cfg(not(any(target_os = "android", target_os = "wasi")))] let localizer = { let english_bundle = create_english_bundle_from_embedded(&default_locale, p)?; Localizer::new(english_bundle) diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index cf30fa8fccc..4eb6a7139e6 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -9274,6 +9274,10 @@ fn test_cp_preserve_xattr_failure_keeps_the_rest() { #[uutests::wasi_ignore(NoPermissionBits)] #[test] #[cfg(not(windows))] +#[cfg_attr( + target_os = "android", + ignore = "different result between adb and termux" +)] fn test_cp_preserve_directory_permissions_by_default() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; diff --git a/tests/by-util/test_env.rs b/tests/by-util/test_env.rs index b826de8a8ad..12a0dd87e06 100644 --- a/tests/by-util/test_env.rs +++ b/tests/by-util/test_env.rs @@ -2348,6 +2348,7 @@ fn test_non_utf8_env_vars() { #[test] #[cfg(unix)] +#[cfg_attr(target_os = "android", ignore = "different SIGPIPE handling on adb")] fn test_ignore_signal_pipe_broken_pipe_regression() { // Test that --ignore-signal=PIPE properly ignores SIGPIPE in child processes. // When SIGPIPE is ignored, processes should handle broken pipes gracefully diff --git a/tests/by-util/test_install.rs b/tests/by-util/test_install.rs index c87c588ed33..da14136ac7f 100644 --- a/tests/by-util/test_install.rs +++ b/tests/by-util/test_install.rs @@ -917,6 +917,7 @@ fn test_install_and_strip_with_program() { #[cfg(all(unix, feature = "chmod"))] #[test] +#[cfg_attr(target_os = "android", ignore = "printf ... | grep fails")] fn test_install_and_strip_with_program_hyphen() { let scene = TestScenario::new(util_name!()); diff --git a/tests/by-util/test_od.rs b/tests/by-util/test_od.rs index b665b8b99d1..d892ddc5054 100644 --- a/tests/by-util/test_od.rs +++ b/tests/by-util/test_od.rs @@ -260,6 +260,7 @@ fn test_hex32() { // Regression: 16-bit IEEE half should print with canonical precision (no spurious digits) #[test] +#[cfg_attr(target_os = "android", ignore = "failed with cross build")] fn test_float16_compact() { let input: [u8; 4] = [0x3c, 0x00, 0x3c, 0x00]; // two times 1.0 in big-endian half new_ucmd!() @@ -331,6 +332,7 @@ fn test_tf_explicit_float_still_uses_4_bytes() { } #[test] +#[cfg_attr(target_os = "android", ignore = "failed with cross build")] fn test_f16() { let input: [u8; 14] = [ 0x00, 0x3c, // 0x3C00 1.0 @@ -358,6 +360,7 @@ fn test_f16() { } #[test] +#[cfg_attr(target_os = "android", ignore = "failed with cross build")] fn test_fh() { let input: [u8; 14] = [ 0x00, 0x3c, // 0x3C00 1.0 diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index 8ad03a67494..dce271deb06 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -547,7 +547,10 @@ fn test_stdin_via_script_redirection_and_pipe() { at.write("file.txt", data); let mut script = at.make_file("test.sh"); + #[cfg(not(target_os = "android"))] writeln!(script, "#!/usr/bin/env sh").unwrap(); + #[cfg(target_os = "android")] + writeln!(script, "#!/system/bin/sh").unwrap(); writeln!(script, "tail").unwrap(); script .set_permissions(PermissionsExt::from_mode(0o755)) diff --git a/tests/by-util/test_test.rs b/tests/by-util/test_test.rs index d304186c394..dc7e20407b4 100644 --- a/tests/by-util/test_test.rs +++ b/tests/by-util/test_test.rs @@ -1034,6 +1034,7 @@ fn test_nonexistent_file_not_owned_by_egid() { #[uutests::wasi_ignore(NoUidGid)] #[test] #[cfg(not(windows))] +#[cfg_attr(target_os = "android", ignore = "no system binary needed for this test")] fn test_file_not_owned_by_egid() { let target_file = if cfg!(target_os = "freebsd") { // The coreutils test runner user has a primary group id of "wheel", diff --git a/util/android-commands.sh b/util/android-commands.sh deleted file mode 100755 index 0f0708d1bb9..00000000000 --- a/util/android-commands.sh +++ /dev/null @@ -1,711 +0,0 @@ -#!/usr/bin/env bash -# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest logfile -# spell-checker:ignore screencap reinit PIPESTATUS keygen sourceslist - -# There are four shells: the host's, adb, termux and termux via ssh. -# But only termux and termux via ssh provides a GNU environment on the -# emulated device (to e.g. run cargo). -# Initially, only adb lets us run commands directly on the emulated device. -# Thus we first establish a ssh connection which then can be used to access -# the termux shell directly, getting output and return code as usual. -# So we use adb to launch termux, then to send keystrokes to it while it's running. -# This way we install sshd and a public key from the host. After that we can -# use ssh to directly run commands in termux environment. - -# Before ssh, we need to consider some inconvenient, limiting specialties: -# The commands sent to termux via adb keystrokes are first parsed as arguments in -# this shell, then as arguments in the adb shell, before finally being used as -# text inputs to the app. Hence, the "'wrapping'" on those commands. -# Using this approach there's no way to get any direct feedback from termux, -# so every time we run a command on it, we make sure it creates a unique *.probe file -# which is polled every 30 seconds together with the current output of the -# command in a *.log file. The contents of the probe file are used as a return code: -# 0 on success, some other number for errors (an empty file is basically the same as 0). -# Note that the return codes are text, not raw bytes. - -# Additionally, we can use adb screenshot functionality to investigate issues -# when there is no feedback arriving from the android device. - -this_repo="$(dirname "$(dirname -- "$(readlink -- "${0}")")")" -cache_dir_name="__rust_cache__" -dev_probe_dir=/sdcard -dev_home_dir=/data/data/com.termux/files/home - -# This is a list of termux package mirrors approved to be used. -# The default mirror list contains entries that do not function properly anymore. -# To avoid failures due to broken mirrors, we use our own list. -# Choose only reliable mirrors here: -repo_url_list=( - "deb https://packages-cf.termux.org/apt/termux-main/ stable main" - "deb https://packages-cf.termux.dev/apt/termux-main/ stable main" -# "deb https://grimler.se/termux/termux-main stable main" # slow - "deb https://ftp.fau.de/termux/termux-main stable main" -) -number_repo_urls=${#repo_url_list[@]} -repo_url_round_robin=$RANDOM - -move_to_next_repo_url() { - repo_url_round_robin=$(((repo_url_round_robin + 1) % number_repo_urls)) - echo "next round robin repo_url: $repo_url_round_robin" -} -move_to_next_repo_url # first call needed for modulo - -get_current_repo_url() { - echo "${repo_url_list[$repo_url_round_robin]}" -} - -# dump some information about the runners system for debugging purposes: -echo "====== runner information ======" -echo "hostname: $(hostname)" -echo "uname -a: $(uname -a)" -echo "pwd: $(pwd)" -echo "\$*: $*" -echo "\$0: $0" -# shellcheck disable=SC2140 -echo "\$(readlink -- "\$\{0\}"): $(readlink -- "${0}")" -echo "\$this_repo: $this_repo" -echo "readlink -f \$this_repo: $(readlink -f "$this_repo")" -this_repo=$(readlink -f "$this_repo") -echo "====== runner info end =========" - -help() { - echo \ - "Usage: $0 COMMAND [ARG] - -where COMMAND is one of: - init download termux and initialize the emulator image - snapshot APK install APK and dependencies on an emulator to prep a snapshot - (you can, but probably don't want to, run this for physical - devices -- just set up termux and the dependencies yourself) - sync_host [REPO] - push the repo at REPO to the device, deleting and restoring all symlinks (locally) - in the process; The cached rust directories are restored, too; by default, REPO is: - $this_repo - sync_image [REPO] - copy the repo/target and the HOME/.cargo directories from the device back to the - host; by default, REPO is: $this_repo - build run \`cargo build --features feat_os_unix_android\` on the - device - tests [SUITE] run tests on the device (SUITE: all or cksum-wc; default: all) - -If you have multiple devices, use the ANDROID_SERIAL environment variable to -specify which to connect to." -} - -hit_enter() { - adb shell input keyevent 66 -} - -exit_termux() { - adb shell input text \"exit\" && hit_enter && hit_enter -} - -timestamp() { - date +"%H%M%S%Z" -} - -add_timestamp_to_lines() { - while IFS= read -r line; do printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$line"; done -} - -# takes a screenshot with given name from the android device. Filename is prefixed with timestamp. -# screenshots are collected at the end of the github workflow and provided as download link. -take_screen_shot() { - filename_prefix="$1" - filename="$this_repo/output/$(timestamp)_${filename_prefix}_screen.png" - echo "take screenshot: $filename" - mkdir -p "$this_repo/output" - adb exec-out screencap -p > "$filename" -} - -get_app_user() { - app="$1" - app_user="$(adb shell cmd package list packages -U "$app" | tr -d '\r' | - awk -v app="$app" '$1 == "package:" app { sub(/^uid:/, "", $2); print $2 }')" - if [[ ! "$app_user" =~ ^[0-9]+$ ]]; then - echo "Couldn't find user for app: $app">&2 - exit 1 - fi - echo "$app_user" -} - -termux_user() { - if [[ -z "$TERMUX_USER" ]]; then - TERMUX_USER="$(get_app_user com.termux)" - fi - echo "$TERMUX_USER" -} - -launch_termux() { - echo "launching termux" - take_screen_shot "launch_termux_enter" - - adb shell input tap 120 380 # close potential dialog "System UI isn't responding" with "wait". - # should not cause side effects when dialog is not there as there are - # no relevant GUI elements at this position otherwise. - - if ! adb shell 'am start -n com.termux/.HomeActivity'; then - echo "failed to launch termux" - exit 1 - fi - - take_screen_shot "launch_termux_after_start_activity" - - # the emulator can sometimes be a little slow to launch the app - loop_count=0 - while ! adb shell "dumpsys window windows" | \ - grep -E "imeInputTarget in display# 0 Window{[^}]+com.termux\/com\.termux\.HomeActivity}" - do - sleep 1 - loop_count=$((loop_count + 1)) - if [[ loop_count -ge 20 ]]; then - break - fi - done - - take_screen_shot "launch_termux_after_wait_activity" - - touch_cmd() { - adb shell input text "\"touch $dev_probe_dir/launch.probe\"" && hit_enter - sleep 1 - } - - local timeout_start=120 - local timeout=$timeout_start - touch_cmd - while ! adb shell "ls $dev_probe_dir/launch.probe" 2>/dev/null - do - echo "waiting for launch.probe - ($timeout / $timeout_start seconds)" - take_screen_shot "launch_termux_touch_probe" - sleep 4 - touch_cmd - - timeout=$((timeout - 4)) - if [[ timeout -le 0 ]]; then - take_screen_shot "error_launch_termux" - echo "timeout waiting for termux to start up" - return 1 - fi - - done - echo "found launch.probe" - take_screen_shot "launch_termux_found_probe" - adb shell "rm $dev_probe_dir/launch.probe" && echo "removed launch.probe" -} - -# Usage: run_termux_command -# -# Runs the command specified in $1 in a termux shell, polling for the probe specified in $2 (and the -# current output). If polling the probe succeeded the command is considered to have finished. This -# method prints the current stdout and stderr of the command every SLEEP_INTERVAL seconds and -# finishes a command run with a summary. It returns with the exit code of the probe if specified as -# file content of the probe. -# -# Positional arguments -# $1 The command to execute in the termux shell -# $2 The path to the probe. The file name must end with `.probe` -# -# It's possible to overwrite settings by specifying the setting the variable before calling this -# method (Default in parentheses): -# keep_log 0|1 Keeps the logs after running the command if set to 1. The log file name is -# derived from the probe file name (the last component of the path) and -# `.probe` replaced with `.log. (0) -# debug 0|1 Adds additional debugging output to the log file if set to 1. (1) -# timeout SECONDS The timeout in full SECONDS for the command to complete before giving up. (3600) -# retries RETRIES The number of retries for trying to fix possible issues when we're not receiving -# any progress from the emulator. (3) -# sleep_interval -# SECONDS The time interval in full SECONDS between polls for the probe and the current -# output. (5) -run_termux_command() { - # shellcheck disable=SC2155 - local command="$(echo "$1" | sed -E "s/^['](.*)[']$/\1/")" # text of the escaped command, including creating the probe! - local probe="$2" # unique file that indicates the command is complete - local keep_log=${keep_log:-0} - local debug=${debug:-1} - - log_name="$(basename -s .probe "${probe}").log" # probe name must have suffix .probe - log_file="$dev_probe_dir/${log_name}" - log_read="${log_name}.read" - echo 0 >"${log_read}" - if [[ $debug -eq 1 ]]; then - shell_command="'set -x; { ${command}; } &> ${log_file}; set +x'" - else - shell_command="'{ ${command}; } &> ${log_file}'" - fi - - launch_termux || return - - take_screen_shot "run_termux_command_before_input_of_shell_command" - - # remove artificial quoting - shell_command="${shell_command%\'}" - shell_command="${shell_command#\'}" - - echo "Running command: ${command}" - echo "Running shell-command: ${shell_command}" - start=$(date +%s) - adb_input_text_long "$shell_command" && sleep 1 && hit_enter - # just for safety wait a little bit before polling for the probe and the log file - sleep 1 - - take_screen_shot "run_termux_command_after_input_of_shell_command" - - local timeout=${timeout:-3600} - local retries=${retries:-10} - local sleep_interval=${sleep_interval:-10} - try_fix=3 - echo "run_termux_command with timeout=$timeout / retries=$retries / sleep_interval=$sleep_interval" - while ! adb shell "ls $probe" 2>/dev/null; do - echo -n "Waiting for $probe: " - - if [[ -e "$log_name" ]]; then - rm "$log_name" - fi - - adb pull "$log_file" . || try_fix=$((try_fix - 1)) - if [[ -e "$log_name" ]]; then - tail -n +"$(<"$log_read")" "$log_name" - echo - wc -l <"${log_name}" | tr -d "[:space:]" >"$log_read" - fi - - if [[ retries -le 0 ]]; then - echo "Maximum retries reached running command. Aborting ..." - take_screen_shot "run_termux_command_maximum_tries_reached" - return 1 - elif [[ try_fix -le 0 ]]; then - retries=$((retries - 1)) - try_fix=3 - # Since there is no output, there is no way to know what is happening inside. See if - # hitting the enter key solves the issue, sometimes the github runner is just a little - # bit slow. - echo "No output received. Trying to fix the issue ... (${retries} retries left)" - take_screen_shot "run_termux_command_before_trying_to_fix" - hit_enter - sleep 1 - take_screen_shot "run_termux_command_after_trying_to_fix" - fi - - sleep "$sleep_interval" - timeout=$((timeout - sleep_interval)) - - if [[ $timeout -le 0 ]]; then - echo "Timeout reached running command. Aborting ..." - take_screen_shot "run_termux_command_timeout_reached" - return 1 - fi - done - end=$(date +%s) - - return_code=$(adb shell "cat $probe") || return_code=0 - adb shell "rm ${probe}" - - adb shell "cat $log_file" > "$log_name" - echo "==================================== SUMMARY ===================================" - echo "Command: ${command}" - echo "Finished in $((end - start)) seconds." - echo "Output was:" - cat "$log_name" - echo "Return code: $return_code" - echo "================================================================================" - - adb shell "rm ${log_file}" - [[ $keep_log -ne 1 ]] && rm -f "$log_name" - rm -f "$log_read" "$probe" - - take_screen_shot "run_termux_command_finished_normally" - - # shellcheck disable=SC2086 - return $return_code -} - -init() { - arch="$1" - # shellcheck disable=SC2034 - api_level="$2" - termux="$3" - - snapshot_name="${AVD_CACHE_KEY}" - - # shellcheck disable=SC2015 - curl -sLO "https://github.com/termux/termux-app/releases/download/${termux}/termux-app_${termux}+github-debug_${arch}.apk" && - snapshot "termux-app_${termux}+github-debug_${arch}.apk" && - hash_rustc && - exit_termux && - adb -s emulator-5554 emu avd snapshot save "$snapshot_name" && - echo "Emulator image created. Name: $snapshot_name" || { - pkill -9 qemu-system-x86_64 - return 1 - } - pkill -9 qemu-system-x86_64 || true -} - -reinit_ssh_connection() { - setup_ssh_forwarding - test_ssh_connection && return - - start_sshd_via_adb_shell && ( - test_ssh_connection && return - generate_and_install_public_key && test_ssh_connection && return - ) || ( - install_packages_via_adb_shell openssh openssl - generate_and_install_public_key - start_sshd_via_adb_shell - test_ssh_connection && return - ) || ( - echo "failed to setup ssh connection" - return 1 - ) -} - -start_sshd_via_adb_shell() { - echo "start sshd via adb shell" - probe="$dev_probe_dir/sshd.probe" - command="'sshd; echo \$? > $probe'" - run_termux_command "$command" "$probe" -} - -setup_ssh_forwarding() { - echo "setup ssh forwarding" - adb forward tcp:9022 tcp:8022 -} - -copy_file_or_dir_to_device_via_ssh() { - scp -r "$1" "scp://$(termux_user)@127.0.0.1:9022/$2" -} - -copy_file_or_dir_from_device_via_ssh() { - scp -r "scp://$(termux_user)@127.0.0.1:9022/$1" "$2" -} - -# runs the in args provided command on android side via ssh. forwards return code. -# adds a timestamp to every line to be able to see where delays are -run_command_via_ssh() { - ssh -p 9022 "$(termux_user)@127.0.0.1" -o StrictHostKeyChecking=accept-new "$@" 2>&1 | add_timestamp_to_lines - return "${PIPESTATUS[0]}" -} - -test_ssh_connection() { - run_command_via_ssh echo ssh connection is working -} - -# takes a local (on runner side) script file and runs it via ssh on the virtual android device. forwards return code. -# adds a timestamp to every line to be able to see where delays are -run_script_file_via_ssh() { - local script=$1 - shift - local command="bash -s --" - local arg - for arg in "$@"; do - printf -v command '%s %q' "$command" "$arg" - done - ssh -p 9022 "$(termux_user)@127.0.0.1" -o StrictHostKeyChecking=accept-new "$command" < "$script" 2>&1 | add_timestamp_to_lines - return "${PIPESTATUS[0]}" -} - -# Experiments showed that the adb shell input text functionality has a limitation for the input length. -# If input length is too big, the input is not fully provided to the android device. -# To avoid this, we divide large inputs into smaller chunks and put them one-by-one. -adb_input_text_long() { - string=$1 - length=${#string} - step=20 - p=0 - for ((i = 0; i < length-step; i = i + step)); do - chunk="${string:i:$step}" - adb shell input text "'$chunk'" - p=$((i+step)) - done - - remaining="${string:p}" - adb shell input text "'$remaining'" -} - -generate_rsa_key_local() { - yes "" | ssh-keygen -t rsa -b 4096 -C "Github Action" -N "" -} - -install_rsa_pub() { - - run_command_via_ssh "echo hello" && return # if this works, we are already fine. Skipping - - # remove old host identity: - ssh-keygen -f ~/.ssh/known_hosts -R "[127.0.0.1]:9022" - - rsa_pub_key=$(cat ~/.ssh/id_rsa.pub) - echo "=====================================" - echo "$rsa_pub_key" - echo "=====================================" - - adb shell input text \"echo \" - - adb_input_text_long "$rsa_pub_key" - - adb shell input text "\" >> ~/.ssh/authorized_keys\"" && hit_enter - sleep 1 -} - -install_packages_via_adb_shell() { - install_package_list="$*" - - install_packages_via_adb_shell_using_apt "$install_package_list" - if [[ $? -ne 0 ]]; then - echo "apt failed. Now try install with pkg as fallback." - probe="$dev_probe_dir/pkg.probe" - command="'mkdir -vp ~/.cargo/bin; yes | pkg install $install_package_list -y; echo \$? > $probe'" - run_termux_command "$command" "$probe" || return 1 - fi - - return 0 -} - -# We use apt to install the packages as pkg doesn't respect any pre-defined mirror list. -# Its important to have a defined mirror list to avoid issues with broken mirrors. -install_packages_via_adb_shell_using_apt() { - install_package_list="$*" - - repo_url=$(get_current_repo_url) - move_to_next_repo_url - echo "set apt repository url: $repo_url" - probe="$dev_probe_dir/sourceslist.probe" - command="'echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list; echo \$? > $probe'" - run_termux_command "$command" "$probe" - - probe="$dev_probe_dir/adb_install.probe" - command="'mkdir -vp ~/.cargo/bin; apt update; yes | apt install $install_package_list -y; echo \$? > $probe'" - run_termux_command "$command" "$probe" -} - -install_packages_via_ssh_using_apt() { - install_package_list="$*" - - repo_url=$(get_current_repo_url) - move_to_next_repo_url - echo "set apt repository url: $repo_url" - run_command_via_ssh "echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list" - - run_command_via_ssh "apt update; yes | apt install $install_package_list -y" -} - -apt_upgrade_all_packages() { - repo_url=$(get_current_repo_url) - move_to_next_repo_url - echo "set apt repository url: $repo_url" - run_command_via_ssh "echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list" - - run_command_via_ssh "apt update; yes | apt upgrade -y" -} - -generate_and_install_public_key() { - echo "generate local public private key pair" - generate_rsa_key_local - echo "install public key via 'adb shell input'" - install_rsa_pub - echo "installed ssh public key on device" -} - -run_with_retry() { - tries=$1 - shift 1 - - for i in $(seq 1 $tries); do - echo "Try #$i of $tries: run $*" - "$@" && echo "Done in try#$i" && return 0 - done - - exit_code=$? - - echo "Still failing after $tries. Code: $exit_code" - - return $exit_code -} - -snapshot() { - apk="$1" - echo "Running snapshot" - adb install -g "$apk" - - echo "Prepare and install system packages" - - reinit_ssh_connection || return 1 - - apt_upgrade_all_packages - - install_packages_via_ssh_using_apt "rust binutils openssl tar mount-utils" - - echo "Read /proc/cpuinfo" - run_command_via_ssh "cat /proc/cpuinfo" - - echo "Installing cargo-nextest" - # We need to install nextest via cargo currently, since there is no pre-built binary for android x86 - # explicitly set CARGO_TARGET_DIR as otherwise a random generated tmp directory is used, - # which prevents incremental build for the retries. - command="export CARGO_TERM_COLOR=always && export CARGO_TARGET_DIR=\"cargo_install_target_dir\" && cargo install cargo-nextest --locked" - - run_with_retry 3 run_command_via_ssh "$command" - return_code=$? - - echo "Info about cargo and rust - via SSH Script" - run_script_file_via_ssh "$this_repo/util/android-scripts/collect-info.sh" - - echo "Snapshot complete" - # shellcheck disable=SC2086 - return $return_code -} - -sync_host() { - repo="$1" - cache_home="${HOME}/${cache_dir_name}" - cache_dest="$dev_home_dir/${cache_dir_name}" - - # Keep Termux in the foreground so Android does not freeze its SSH processes. - adb shell am start -n com.termux/.HomeActivity || return - reinit_ssh_connection || return - - echo "Running sync host -> image: ${repo}" - - # run_command_via_ssh "mkdir $dev_home_dir/coreutils" - - copy_file_or_dir_to_device_via_ssh "$repo" "$dev_home_dir" || return - if [[ -e "$cache_home" ]]; then - copy_file_or_dir_to_device_via_ssh "$cache_home" "$cache_dest" || return - fi - - echo "Finished sync host -> image: ${repo}" -} - -sync_image() { - repo="$1" - cache_home="${HOME}/${cache_dir_name}" - cache_dest="$dev_probe_dir/${cache_dir_name}" - - reinit_ssh_connection - - echo "Running sync image -> host: ${repo}" - - command="rm -rf $dev_probe_dir/coreutils ${cache_dest}; \ -mkdir -p ${cache_dest}; \ -cd ${cache_dest}; \ -tar czf cargo.tgz -C ~/ .cargo; \ -tar czf target.tgz -C ~/coreutils target; \ -ls -la ${cache_dest}" - run_command_via_ssh "$command" || return - - rm -rf "$cache_home" - copy_file_or_dir_from_device_via_ssh "$cache_dest" "$cache_home" || return - - echo "Finished sync image -> host: ${repo}" -} - -build() { - echo "Running build" - - reinit_ssh_connection - - run_script_file_via_ssh "$this_repo/util/android-scripts/collect-info.sh" - - command="export CARGO_TERM_COLOR=always; - export CARGO_INCREMENTAL=0; \ - cd ~/coreutils && cargo build --features feat_os_unix_android" - run_with_retry 3 run_command_via_ssh "$command" || return - - echo "Finished build" -} - -tests() { - echo "Running tests" - - reinit_ssh_connection - - run_script_file_via_ssh "$this_repo/util/android-scripts/collect-info.sh" - - run_script_file_via_ssh "$this_repo/util/android-scripts/run-tests.sh" "${1:-all}" || return - - echo "Finished tests" -} - -hash_rustc() { - tmp_hash="__rustc_hash__.tmp" - hash="__rustc_hash__" - - reinit_ssh_connection - - echo "Hashing rustc version: ${HOME}/${hash}" - - run_command_via_ssh "rustc -Vv" > rustc.log || return - rm -f "$tmp_hash" - mv "rustc.log" "$tmp_hash" || return - # sha256sum is not available. shasum is the macos native program. - shasum -a 256 "$tmp_hash" | cut -f 1 -d ' ' | tr -d '[:space:]' >"${HOME}/${hash}" || return - - rm -f "$tmp_hash" - - echo "Finished hashing rustc version: ${HOME}/${hash}" -} - -#adb logcat & -exit_code=0 - -if [ $# -eq 1 ]; then - case "$1" in - sync_host) - sync_host "$this_repo" - exit_code=$? - ;; - sync_image) - sync_image "$this_repo" - exit_code=$? - ;; - build) - build - exit_code=$? - ;; - tests) - tests - exit_code=$? - ;; - *) help ;; - esac -elif [ $# -eq 2 ]; then - case "$1" in - tests) - tests "$2" - exit_code=$? - ;; - snapshot) - snapshot "$2" - exit_code=$? - ;; - sync_host) - sync_host "$2" - exit_code=$? - ;; - sync_image) - sync_image "$2" - exit_code=$? - ;; - *) - help - exit 1 - ;; - esac -elif [ $# -eq 4 ]; then - case "$1" in - init) - shift - init "$@" - exit_code=$? - ;; - *) - help - exit 1 - ;; - esac -else - help - exit_code=1 -fi - -#pkill adb -exit $exit_code diff --git a/util/android-scripts/collect-info.sh b/util/android-scripts/collect-info.sh deleted file mode 100755 index 412423bd135..00000000000 --- a/util/android-scripts/collect-info.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# spell-checker:ignore nextest watchplus PIPESTATUS - -echo "system resources - RAM:" -free -hm -echo "system resources - CPU:" -lscpu -echo "system resources - file systems:" -mount - -echo "$HOME" -PATH=$HOME/.cargo/bin:$PATH -export PATH -echo "$PATH" -pwd -command -v rustc && rustc -Vv -ls -la ~/.cargo/bin -cargo --list -cargo nextest --version diff --git a/util/android-scripts/run-tests.sh b/util/android-scripts/run-tests.sh deleted file mode 100755 index 24337a4cdc9..00000000000 --- a/util/android-scripts/run-tests.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# spell-checker:ignore nextest watchplus PIPESTATUS - -echo "PATH: $PATH" - -export PATH=$HOME/.cargo/bin:$PATH -export RUST_BACKTRACE=full -export CARGO_TERM_COLOR=always -export CARGO_INCREMENTAL=0 -export CARGO_BUILD_JOBS=2 - -echo "PATH: $PATH" - -case "${1:-all}" in - all) test_filter=() ;; - cksum-wc) test_filter=(-E 'test(test_cksum::) | test(test_wc::)') ;; - *) echo "Unknown test suite: $1" >&2; exit 1 ;; -esac - -run_with_retry() { - local tries=$1 - shift 1 - local exit_code=0 - - for ((i = 1; i <= tries; i++)); do - echo "Try #$i of $tries: run $*" - if "$@"; then - echo "Done in try#$i" - return 0 - else - exit_code=$? - fi - done - - echo "Still failing after $tries. Code: $exit_code" - - return $exit_code -} - -run_tests_in_subprocess() ( - - watchplus() { - # call: watchplus - while true; do - "${@:2}" - sleep "$1" - done - } - - kill_all_background_jobs() { - jobs -p | xargs -I{} kill -- {} - } - - # observe (log) every 2 seconds the system resource usage to judge if we are at a limit - watchplus 2 df -h & - watchplus 2 free -hm & - - # Recent Android allocators reserve more virtual memory than a 3 GiB limit - # allows. Limit parallelism to reduce memory pressure without capping addresses. - nextest_params=(--profile ci --hide-progress-bar --test-threads 2 --features feat_os_unix_android "${test_filter[@]}") - - # run tests - cd ~/coreutils && \ - run_with_retry 3 timeout --preserve-status --verbose -k 1m 10m \ - cargo nextest run --no-run "${nextest_params[@]}" && - timeout --preserve-status --verbose -k 1m 60m \ - cargo nextest run "${nextest_params[@]}" - - result=$? - - kill_all_background_jobs - - return $result -) - -# Keep the resource monitors and their cleanup in a separate shell. -run_tests_in_subprocess