mirror/Anki-Android
0
0
Fork 0
mirror of https://github.com/ankidroid/Anki-Android.git synced 2024-09-19 19:42:17 +02:00
Code

perf(ci, emulator): community emulator action / AVD caching / gradle cachine

Browse Source 
Also adds an emulator run benchmarking suite that works in combo with matrix

If branch builds result in cache upload of emulator files the cache files there will
need to be scoped for just changes on the snapshot file(s)
This commit is contained in:
Mike Hardy 2022-04-22 11:29:58 -05:00
parent 14b3ac814b
commit b565d5b19c
4 changed files with 219 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
.github/workflows/tests_emulator.yml vendored
View File

@ -36,109 +36,112 @@ jobs:
emulator_test:
name: Android Emulator Test
runs-on: macos-latest
timeout-minutes: 60
env:
EMULATOR_COMMAND: "-avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 -timezone 'Europe/London' -cores 2"
EMULATOR_EXECUTABLE: qemu-system-x86_64-headless
timeout-minutes: 75
strategy:
fail-fast: false
matrix:
# Refactor to make these dynamic with a low/high bracket only on schedule, not push
# For now this is just the fastest combo (api/arch/target/snapshot-warm-time) based on testing
api-level: [29]
arch: [x86_64]
target: [google_apis]
first-boot-delay: [600]
# This is useful for benchmarking, do 0, 1, 2, etc (up to 256 max job-per-matrix limit) for averages
iteration: [0]
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 50
- name: Gradle Cache
uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}-v1
- name: Configure JDK 1.11
uses: actions/setup-java@v3
with:
distribution: "adopt"
java-version: "11" # ubuntu-latest is about to default to 11, force it everywhere
- name: Verify JDK11
# Default JDK varies depending on different runner flavors, make sure we are on 11
# Run a check that exits with error unless it is 11 version to future-proof against unexpected upgrades
run: java -fullversion 2>&1 | grep '11.0'
shell: bash
- name: Setup Gradle
uses: gradle/gradle-build-action@v2
with:
# Only write to the cache for builds on the 'main' branches, stops branches evicting main cache
# Builds on other branches will only read from main branch cache writes
# Comment this and the with: above out for performance testing on a branch
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
- name: Warm Gradle Cache
# This makes sure we fetch gradle network resources with a retry
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
timeout_minutes: 15
retry_wait_seconds: 60
max_attempts: 3
command: ./gradlew assembleDebug assembleAndroidTest robolectricSdkDownload
command: ./gradlew packagePlayDebug packagePlayDebugAndroidTest
- name: Download Emulator Image
# This can fail on network request, wrap with retry
uses: nick-invision/retry@v2
# This appears to be 'Cache Size: ~1230 MB (1290026823 B)' based on watching action logs
# Repo limit is 10GB; branch caches are independent; branches may read default branch cache.
# We don't want branches to evict main branch snapshot, so save on main, read-only all else
- name: AVD cache
uses: actions/cache@v3
id: avd-cache
with:
timeout_minutes: 10
retry_wait_seconds: 60
max_attempts: 3
command: echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "system-images;android-30;google_apis;x86_64"
path: |
~/.android/avd/*
~/.android/adb*
key: avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1-${{ hashFiles('~/.android/avd/**/snapshots/**') }}
restore-keys: |
avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1
- name: Create Emulator
run: echo "no" | $ANDROID_HOME/cmdline-tools/latest/bin//avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-30;google_apis;x86_64' -g google_apis
- name: AVD Boot and Snapshot Creation
# Only generate a snapshot for saving if we are on main branch with a cache miss
# Comment the if out to generate snapshots on branch for performance testing
if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
uses: reactivecircus/android-emulator-runner@v2
with:
api-level: ${{ matrix.api-level }}
force-avd-creation: false
target: ${{ matrix.target }}
arch: ${{ matrix.arch }}
emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
sdcard-path-or-size: 100M
disable-animations: true
# Give the emulator a little time to run and do first boot stuff before taking snapshot
script: echo "Generated AVD snapshot for caching."
# These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes
- name: Start Android Emulator
id: emu1
timeout-minutes: 5
continue-on-error: true
run: |
echo "Starting emulator"
nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
$ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
# This step is separate so pure install time may be calculated as a step
- name: Emulator Snapshot After Firstboot Warmup
# Only generate a snapshot for saving if we are on main branch with a cache miss
# Switch the if statements via comment if generating snapshots for performance testing
# if: matrix.first-boot-delay != '0'
if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
env:
FIRST_BOOT_DELAY: ${{ matrix.first-boot-delay }}
uses: reactivecircus/android-emulator-runner@v2
with:
api-level: ${{ matrix.api-level }}
force-avd-creation: false
target: ${{ matrix.target }}
arch: ${{ matrix.arch }}
emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
sdcard-path-or-size: 100M
disable-animations: true
# Give the emulator a little time to run and do first boot stuff before taking snapshot
script: |
sleep $FIRST_BOOT_DELAY
echo "First boot warmup completed."
- name: Start Android Emulator Retry 1
id: emu2
if: steps.emu1.outcome=='failure'
timeout-minutes: 5
continue-on-error: true
run: |
echo "Starting emulator, second attempt"
$ANDROID_HOME/platform-tools/adb devices
sudo killall -9 $EMULATOR_EXECUTABLE || true
sleep 2
nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
$ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
- name: Start Android Emulator Retry 2
id: emu3
if: steps.emu2.outcome=='failure'
timeout-minutes: 5
continue-on-error: true
run: |
echo "Starting emulator, third attempt"
$ANDROID_HOME/platform-tools/adb devices
sudo killall -9 $EMULATOR_EXECUTABLE || true
sleep 2
nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
$ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
- name: Emulator Status
if: always()
run: |
if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then
echo "Emulator Started"
else
exit 1
fi
- name: Emulator Test
timeout-minutes: 40
run: |
$ANDROID_HOME/platform-tools/adb devices
$ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
$ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
$ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" &
./gradlew jacocoAndroidTestReport
shell: bash
- name: Run Emulator Tests
uses: reactivecircus/android-emulator-runner@v2
timeout-minutes: 30
with:
api-level: ${{ matrix.api-level }}
force-avd-creation: false
target: ${{ matrix.target }}
arch: ${{ matrix.arch }}
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
sdcard-path-or-size: 100M
disable-animations: true
script: |
$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt &
./gradlew uninstallAll jacocoAndroidTestReport
- name: Compress Emulator Log
if: always()
@ -149,7 +152,7 @@ jobs:
uses: actions/upload-artifact@v3
if: always()
with:
name: adb_logs
name: ${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-${{matrix.first-boot-delay}}-${{matrix.iteration}}-adb_logs
path: adb-log.txt.gz
- name: Submit Coverage

1
tools/emulator_performance/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
emulator_perf_results*

101
tools/emulator_performance/analyze_emulator_performance.js Normal file
View File

@ -0,0 +1,101 @@
// Fetch results to parse like this, with the workflow run id you want:
// curl https://api.github.com/repos/mikehardy/Anki-Android/actions/runs/2210525974/jobs?per_page=100 > emulator_perf_results.json
// Or if you have more than 100 results, you need to page through them and merge them, there is a script
// ./fetch_workflow_jobs_json.sh 2212862357
function main() {
// Read in the results
// console.log("Processing results in " + process.argv[2]);
var fs = require("fs");
var runLog = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
console.log(
'"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"',
);
let averageTimings = {};
runLog.jobs.forEach(job => {
// console.log("analyzing job " + job.name);
const matrixVars = job.name.match(/.*\((.*)\)/)[1].split(", ");
// console.log("Job name: " + job.name);
// console.log(" Android API level: " + matrixVars[0]);
// console.log(" Emulator Architecture: " + matrixVars[1]);
// console.log(" Emulator Image: " + matrixVars[2]);
const startTime = new Date(job.started_at);
const endTime = new Date(job.completed_at);
let jobElapsed = endTime - startTime;
jobElapsed = jobElapsed > 0 ? jobElapsed : 0; // some are negative !?
// console.log(" conclusion: " + job.conclusion);
// console.log(" elapsed_time_seconds: " + jobElapsed / 1000);
let AVDCreateBootElapsedSeconds = -1;
let AVDRebootTestElapsedSeconds = -1;
let stepFailed = false;
job.steps.forEach(step => {
if (!["success", "skipped"].includes(step.conclusion)) {
stepFailed = true;
return;
}
const stepStart = new Date(step.started_at);
const stepEnd = new Date(step.completed_at);
let stepElapsedSeconds = (stepEnd - stepStart) / 1000;
stepElapsedSeconds = stepElapsedSeconds > 0 ? stepElapsedSeconds : 0; // some are negative !?
switch (step.name) {
case "AVD Boot and Snapshot Creation":
AVDCreateBootElapsedSeconds = stepElapsedSeconds;
case "Run Emulator Tests":
AVDRebootTestElapsedSeconds = stepElapsedSeconds;
}
});
// Get or create aggregate timing entry
timingKey = `${matrixVars[0]}_${matrixVars[1]}_${matrixVars[2]}_${matrixVars[3]}`;
let currentAverageTiming = averageTimings[timingKey];
if (currentAverageTiming === undefined) {
currentAverageTiming = {
api: matrixVars[0],
arch: matrixVars[1],
target: matrixVars[2],
warmtime: matrixVars[3],
totalCreateBootElapsedSecs: 0,
totalTestElapsedSecs: 0,
runs: 0,
failureCount: 0,
};
averageTimings[timingKey] = currentAverageTiming;
}
// If something failed, set status and skip timing aggregation
if (stepFailed) {
currentAverageTiming.failureCount++;
return;
}
// Update our aggregate timings
currentAverageTiming.totalCreateBootElapsedSecs += AVDCreateBootElapsedSeconds;
currentAverageTiming.totalTestElapsedSecs += AVDRebootTestElapsedSeconds;
currentAverageTiming.runs++;
});
// Print out averages for each non-iteration combo
Object.keys(averageTimings).forEach(key => {
// console.log("printing timings for key " + key);
const timing = averageTimings[key];
// console.log("entry is " + JSON.stringify(timing));
console.log(
`"${timing.api}","${timing.arch}","${timing.target}","${timing.warmtime}","${
timing.totalCreateBootElapsedSecs / timing.runs
}","${timing.totalTestElapsedSecs / timing.runs}","${
(timing.totalCreateBootElapsedSecs + timing.totalTestElapsedSecs) / timing.runs
}","${timing.failureCount}"`,
);
});
}
main();

33
tools/emulator_performance/fetch_workflow_jobs_json.sh Executable file
View File

@ -0,0 +1,33 @@
#!/bin/bash
echo "Fetching jobs JSON for workflow run $1"
rm -f emulator_perf_results_page*.json
REPO_URL=https://api.github.com/repos/mikehardy/Anki-Android
PER_PAGE=100
PAGE=1
curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
TOTAL_COUNT=$(jq '.total_count' emulator_perf_results.json)
LAST_PAGE=$((TOTAL_COUNT / PER_PAGE + 1))
echo "$TOTAL_COUNT jobs so $LAST_PAGE pages"
for ((PAGE=2; PAGE <= LAST_PAGE; PAGE++)); do
echo "On iteration $PAGE"
curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
done
jq -s 'def deepmerge(a;b):
reduce b[] as $item (a;
reduce ($item | keys_unsorted[]) as $key (.;
$item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then
deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val])
elif ($type == "array") then
(.[$key] + $val | unique)
else
$val
end)
);
deepmerge({}; .)' emulator_perf_results_page*.json > emulator_perf_results.json
rm -f emulator_perf_results_page*.json