diff --git a/.github/workflows/tests_emulator.yml b/.github/workflows/tests_emulator.yml index 62bfb539df..7c014045e8 100644 --- a/.github/workflows/tests_emulator.yml +++ b/.github/workflows/tests_emulator.yml @@ -36,109 +36,112 @@ jobs: emulator_test: name: Android Emulator Test runs-on: macos-latest - timeout-minutes: 60 - env: - EMULATOR_COMMAND: "-avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 -timezone 'Europe/London' -cores 2" - EMULATOR_EXECUTABLE: qemu-system-x86_64-headless + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + # Refactor to make these dynamic with a low/high bracket only on schedule, not push + # For now this is just the fastest combo (api/arch/target/snapshot-warm-time) based on testing + api-level: [29] + arch: [x86_64] + target: [google_apis] + first-boot-delay: [600] + # This is useful for benchmarking, do 0, 1, 2, etc (up to 256 max job-per-matrix limit) for averages + iteration: [0] steps: - uses: actions/checkout@v3 with: fetch-depth: 50 - - name: Gradle Cache - uses: actions/cache@v3 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}-v1 - - name: Configure JDK 1.11 uses: actions/setup-java@v3 with: distribution: "adopt" java-version: "11" # ubuntu-latest is about to default to 11, force it everywhere - - name: Verify JDK11 - # Default JDK varies depending on different runner flavors, make sure we are on 11 - # Run a check that exits with error unless it is 11 version to future-proof against unexpected upgrades - run: java -fullversion 2>&1 | grep '11.0' - shell: bash + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + # Only write to the cache for builds on the 'main' branches, stops branches evicting main cache + # Builds on other branches will only read from main branch cache writes + # Comment this and the with: above out for performance testing on a branch + cache-read-only: ${{ github.ref != 'refs/heads/main' }} - name: Warm Gradle Cache # This makes sure we fetch gradle network resources with a retry uses: nick-invision/retry@v2 with: - timeout_minutes: 10 + timeout_minutes: 15 retry_wait_seconds: 60 max_attempts: 3 - command: ./gradlew assembleDebug assembleAndroidTest robolectricSdkDownload + command: ./gradlew packagePlayDebug packagePlayDebugAndroidTest - - name: Download Emulator Image - # This can fail on network request, wrap with retry - uses: nick-invision/retry@v2 + # This appears to be 'Cache Size: ~1230 MB (1290026823 B)' based on watching action logs + # Repo limit is 10GB; branch caches are independent; branches may read default branch cache. + # We don't want branches to evict main branch snapshot, so save on main, read-only all else + - name: AVD cache + uses: actions/cache@v3 + id: avd-cache with: - timeout_minutes: 10 - retry_wait_seconds: 60 - max_attempts: 3 - command: echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "system-images;android-30;google_apis;x86_64" + path: | + ~/.android/avd/* + ~/.android/adb* + key: avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1-${{ hashFiles('~/.android/avd/**/snapshots/**') }} + restore-keys: | + avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1 - - name: Create Emulator - run: echo "no" | $ANDROID_HOME/cmdline-tools/latest/bin//avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-30;google_apis;x86_64' -g google_apis + - name: AVD Boot and Snapshot Creation + # Only generate a snapshot for saving if we are on main branch with a cache miss + # Comment the if out to generate snapshots on branch for performance testing + if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: ${{ matrix.api-level }} + force-avd-creation: false + target: ${{ matrix.target }} + arch: ${{ matrix.arch }} + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + sdcard-path-or-size: 100M + disable-animations: true + # Give the emulator a little time to run and do first boot stuff before taking snapshot + script: echo "Generated AVD snapshot for caching." - # These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes - - name: Start Android Emulator - id: emu1 - timeout-minutes: 5 - continue-on-error: true - run: | - echo "Starting emulator" - nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND & - $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done' + # This step is separate so pure install time may be calculated as a step + - name: Emulator Snapshot After Firstboot Warmup + # Only generate a snapshot for saving if we are on main branch with a cache miss + # Switch the if statements via comment if generating snapshots for performance testing + # if: matrix.first-boot-delay != '0' + if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + env: + FIRST_BOOT_DELAY: ${{ matrix.first-boot-delay }} + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: ${{ matrix.api-level }} + force-avd-creation: false + target: ${{ matrix.target }} + arch: ${{ matrix.arch }} + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + sdcard-path-or-size: 100M + disable-animations: true + # Give the emulator a little time to run and do first boot stuff before taking snapshot + script: | + sleep $FIRST_BOOT_DELAY + echo "First boot warmup completed." - - name: Start Android Emulator Retry 1 - id: emu2 - if: steps.emu1.outcome=='failure' - timeout-minutes: 5 - continue-on-error: true - run: | - echo "Starting emulator, second attempt" - $ANDROID_HOME/platform-tools/adb devices - sudo killall -9 $EMULATOR_EXECUTABLE || true - sleep 2 - nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND & - $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done' - - - name: Start Android Emulator Retry 2 - id: emu3 - if: steps.emu2.outcome=='failure' - timeout-minutes: 5 - continue-on-error: true - run: | - echo "Starting emulator, third attempt" - $ANDROID_HOME/platform-tools/adb devices - sudo killall -9 $EMULATOR_EXECUTABLE || true - sleep 2 - nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND & - $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done' - - - name: Emulator Status - if: always() - run: | - if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then - echo "Emulator Started" - else - exit 1 - fi - - - name: Emulator Test - timeout-minutes: 40 - run: | - $ANDROID_HOME/platform-tools/adb devices - $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0 - $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0 - $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0 - nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" & - ./gradlew jacocoAndroidTestReport - shell: bash + - name: Run Emulator Tests + uses: reactivecircus/android-emulator-runner@v2 + timeout-minutes: 30 + with: + api-level: ${{ matrix.api-level }} + force-avd-creation: false + target: ${{ matrix.target }} + arch: ${{ matrix.arch }} + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + sdcard-path-or-size: 100M + disable-animations: true + script: | + $ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt & + ./gradlew uninstallAll jacocoAndroidTestReport - name: Compress Emulator Log if: always() @@ -149,7 +152,7 @@ jobs: uses: actions/upload-artifact@v3 if: always() with: - name: adb_logs + name: ${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-${{matrix.first-boot-delay}}-${{matrix.iteration}}-adb_logs path: adb-log.txt.gz - name: Submit Coverage diff --git a/tools/emulator_performance/.gitignore b/tools/emulator_performance/.gitignore new file mode 100644 index 0000000000..8bd6bcf4c5 --- /dev/null +++ b/tools/emulator_performance/.gitignore @@ -0,0 +1 @@ +emulator_perf_results* diff --git a/tools/emulator_performance/analyze_emulator_performance.js b/tools/emulator_performance/analyze_emulator_performance.js new file mode 100644 index 0000000000..121f80a57f --- /dev/null +++ b/tools/emulator_performance/analyze_emulator_performance.js @@ -0,0 +1,101 @@ +// Fetch results to parse like this, with the workflow run id you want: +// curl https://api.github.com/repos/mikehardy/Anki-Android/actions/runs/2210525974/jobs?per_page=100 > emulator_perf_results.json + +// Or if you have more than 100 results, you need to page through them and merge them, there is a script +// ./fetch_workflow_jobs_json.sh 2212862357 + +function main() { + // Read in the results + // console.log("Processing results in " + process.argv[2]); + var fs = require("fs"); + var runLog = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); + + console.log( + '"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"', + ); + + let averageTimings = {}; + + runLog.jobs.forEach(job => { + // console.log("analyzing job " + job.name); + const matrixVars = job.name.match(/.*\((.*)\)/)[1].split(", "); + // console.log("Job name: " + job.name); + // console.log(" Android API level: " + matrixVars[0]); + // console.log(" Emulator Architecture: " + matrixVars[1]); + // console.log(" Emulator Image: " + matrixVars[2]); + + const startTime = new Date(job.started_at); + const endTime = new Date(job.completed_at); + let jobElapsed = endTime - startTime; + jobElapsed = jobElapsed > 0 ? jobElapsed : 0; // some are negative !? + + // console.log(" conclusion: " + job.conclusion); + // console.log(" elapsed_time_seconds: " + jobElapsed / 1000); + + let AVDCreateBootElapsedSeconds = -1; + let AVDRebootTestElapsedSeconds = -1; + let stepFailed = false; + + job.steps.forEach(step => { + if (!["success", "skipped"].includes(step.conclusion)) { + stepFailed = true; + return; + } + const stepStart = new Date(step.started_at); + const stepEnd = new Date(step.completed_at); + let stepElapsedSeconds = (stepEnd - stepStart) / 1000; + stepElapsedSeconds = stepElapsedSeconds > 0 ? stepElapsedSeconds : 0; // some are negative !? + + switch (step.name) { + case "AVD Boot and Snapshot Creation": + AVDCreateBootElapsedSeconds = stepElapsedSeconds; + case "Run Emulator Tests": + AVDRebootTestElapsedSeconds = stepElapsedSeconds; + } + }); + + // Get or create aggregate timing entry + timingKey = `${matrixVars[0]}_${matrixVars[1]}_${matrixVars[2]}_${matrixVars[3]}`; + let currentAverageTiming = averageTimings[timingKey]; + if (currentAverageTiming === undefined) { + currentAverageTiming = { + api: matrixVars[0], + arch: matrixVars[1], + target: matrixVars[2], + warmtime: matrixVars[3], + totalCreateBootElapsedSecs: 0, + totalTestElapsedSecs: 0, + runs: 0, + failureCount: 0, + }; + averageTimings[timingKey] = currentAverageTiming; + } + + // If something failed, set status and skip timing aggregation + if (stepFailed) { + currentAverageTiming.failureCount++; + return; + } + + // Update our aggregate timings + currentAverageTiming.totalCreateBootElapsedSecs += AVDCreateBootElapsedSeconds; + currentAverageTiming.totalTestElapsedSecs += AVDRebootTestElapsedSeconds; + currentAverageTiming.runs++; + }); + + // Print out averages for each non-iteration combo + Object.keys(averageTimings).forEach(key => { + // console.log("printing timings for key " + key); + const timing = averageTimings[key]; + // console.log("entry is " + JSON.stringify(timing)); + console.log( + `"${timing.api}","${timing.arch}","${timing.target}","${timing.warmtime}","${ + timing.totalCreateBootElapsedSecs / timing.runs + }","${timing.totalTestElapsedSecs / timing.runs}","${ + (timing.totalCreateBootElapsedSecs + timing.totalTestElapsedSecs) / timing.runs + }","${timing.failureCount}"`, + ); + }); +} + +main(); diff --git a/tools/emulator_performance/fetch_workflow_jobs_json.sh b/tools/emulator_performance/fetch_workflow_jobs_json.sh new file mode 100755 index 0000000000..97bda3b33d --- /dev/null +++ b/tools/emulator_performance/fetch_workflow_jobs_json.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +echo "Fetching jobs JSON for workflow run $1" + +rm -f emulator_perf_results_page*.json + +REPO_URL=https://api.github.com/repos/mikehardy/Anki-Android +PER_PAGE=100 +PAGE=1 +curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json + +TOTAL_COUNT=$(jq '.total_count' emulator_perf_results.json) +LAST_PAGE=$((TOTAL_COUNT / PER_PAGE + 1)) +echo "$TOTAL_COUNT jobs so $LAST_PAGE pages" +for ((PAGE=2; PAGE <= LAST_PAGE; PAGE++)); do + echo "On iteration $PAGE" + curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json +done + +jq -s 'def deepmerge(a;b): + reduce b[] as $item (a; + reduce ($item | keys_unsorted[]) as $key (.; + $item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then + deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val]) + elif ($type == "array") then + (.[$key] + $val | unique) + else + $val + end) + ); + deepmerge({}; .)' emulator_perf_results_page*.json > emulator_perf_results.json + +rm -f emulator_perf_results_page*.json \ No newline at end of file