diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 3446049c6..cd225915c 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -79,6 +79,12 @@ jobs: - name: Run gradle tests run: ./gradlew test + # Slow JVM unit tests (tagged @Category(NightlyOnlyTest::class)) that are excluded from + # the regular `test` task run in run-tests.yaml, e.g. because they just replay every GPX + # fixture through the full grid/callout engine to check nothing crashes. + - name: Run nightly-only unit tests + run: ./gradlew :app:nightlyUnitTest + # Setup release build keys - name: Decode Keystore uses: ./.github/actions/decode-base64-secret @@ -104,79 +110,73 @@ jobs: name: release-apk path: ${{ env.main_project_module }}/build/outputs/apk/releaseTest/ - # Generate android instrumentation tests APK -# - name: Assemble Android Instrumentation Tests -# env: -# SIGNING_KEY_STORE_PATH: ${{ secrets.SIGNING_KEY_STORE_PATH }} -# SIGNING_KEY_ALIAS: ${{ secrets.SIGNING_KEY_ALIAS }} -# SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }} -# SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }} -# run: ./gradlew assembleAndroidTest -DtestBuildType=debug -# - name: Upload Android Test APK -# uses: actions/upload-artifact@v6 -# with: -# name: release-androidTest -# path: ${{ env.main_project_module }}/build/outputs/apk/androidTest/debug/app-release-androidTest.apk - -# instrumentation-tests: -# name: Run matrix of instrumentation tests -# needs: build -# environment: development -# runs-on: ubuntu-latest -# strategy: -# matrix: -# api-level: [30, 31, 34] -# fail-fast: false -# steps: -# - uses: actions/checkout@v6 -# with: -# fetch-depth: 0 -# -# - name: Setup tile provider -# env: -# TILE_PROVIDER_API_KEY: ${{ secrets.TILE_PROVIDER_API_KEY }} -# TILE_PROVIDER_URL: ${{ secrets.TILE_PROVIDER_URL }} -# run: | -# echo tileProviderUrl=$TILE_PROVIDER_URL > local.properties -# echo tileProviderApiKey=$TILE_PROVIDER_API_KEY >> local.properties -# -# - name: Decode Google services -# env: -# ENCODED_STRING: ${{ secrets.GOOGLE_SERVICES }} -# GOOGLE_SERVICES_PATH: ${{ secrets.GOOGLE_SERVICES_PATH }} -# -# run: | -# echo $ENCODED_STRING > google-services-b64.txt -# base64 -d google-services-b64.txt > ${{ env.main_project_module }}/$GOOGLE_SERVICES_PATH -# -# - name: Setup Gradle -# uses: gradle/actions/setup-gradle@v5 -# -# - name: Set Up JDK -# uses: actions/setup-java@v5 -# with: -# distribution: 'zulu' # See 'Supported distributions' for available options -# java-version: '19' -# -# - name: Enable KVM -# run: | -# echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules -# sudo udevadm control --reload-rules -# sudo udevadm trigger --name-match=kvm -# -# - name: Instrumentation Tests -# id: instrumentation-tests -# uses: reactivecircus/android-emulator-runner@v2 -# with: -# api-level: ${{ matrix.api-level }} -# target: default -# arch: x86_64 -# script: | -# adb logcat -c # clear logs -# touch app/emulator.log # create log file -# chmod 777 app/emulator.log # allow writing to log file -# adb logcat >> app/emulator.log & # pipe all logcat messages into log file as a background process -# ./gradlew connectedCheck + instrumentation-tests-nightly: + name: Run full instrumentation test suite + runs-on: ubuntu-latest + environment: development + needs: check_date + if: ${{ needs.check_date.outputs.should_run != 'false' }} + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup offline maps for tests + uses: ./.github/actions/setup-offline-maps + with: + output_dir: ${{ env.main_project_module }}/src/test/res/org/scottishtecharmy/soundscape + + - name: Setup JDK + Gradle + uses: ./.github/actions/setup-jdk-gradle + + - name: Setup tile and search providers + uses: ./.github/actions/write-provider-config + with: + platform: android + tile_provider_url: ${{ secrets.TILE_PROVIDER_URL }} + tile_provider_api_key: ${{ secrets.TILE_PROVIDER_API_KEY }} + search_provider_url: ${{ secrets.SEARCH_PROVIDER_URL }} + search_provider_api_key: ${{ secrets.SEARCH_PROVIDER_API_KEY }} + extract_provider_url: ${{ secrets.EXTRACT_PROVIDER_URL }} + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + # Runs the full instrumented suite unfiltered - both the tests that also run on every PR + # (run-tests.yaml excludes @NightlyOnly there via notAnnotation) and the @NightlyOnly ones + # (real-time audio soak tests, live-network map fetches) that are too slow for PRs. + - name: Full instrumentation test suite + id: instrumentation-tests-nightly + uses: reactivecircus/android-emulator-runner@v2 + continue-on-error: true # IMPORTANT: allow pipeline to continue to log upload step + with: + api-level: 35 + target: google_apis + arch: x86_64 + script: | + adb root + adb shell mkdir -p /storage/emulated/0/Android/data/org.scottishtecharmy.soundscape/files/Download + adb push ${{ env.main_project_module }}/src/test/res/org/scottishtecharmy/soundscape/glasgow-gb.pmtiles /storage/emulated/0/Android/data/org.scottishtecharmy.soundscape/files/Download/ + adb logcat -c # clear logs + touch app/emulator.log # create log file + chmod 777 app/emulator.log # allow writing to log file + adb logcat >> app/emulator.log & # pipe all logcat messages into log file as a background process + ./gradlew connectedCheck + + - name: Upload Failing Test Report Log + if: steps.instrumentation-tests-nightly.outcome != 'success' + uses: actions/upload-artifact@v6 + with: + name: nightly-instrumentation-logs + path: app/emulator.log + + - name: Raise error on test fail + if: steps.instrumentation-tests-nightly.outcome != 'success' + run: exit 1 firebase: name: Run UI tests with Firebase Test Lab @@ -194,11 +194,6 @@ jobs: with: name: release-apk -# - name: Download Android test APK -# uses: actions/download-artifact@v8 -# with: -# name: release-androidTest - - name: Auth uses: google-github-actions/auth@v3 with: @@ -210,10 +205,6 @@ jobs: - name: Set current project run: gcloud config set project ${{ secrets.FIREBASE_PROJECT_ID }} -# These tests now need the pmtiles file, skip for now as we already test locally -# - name: Run Instrumentation Tests in Firebase Test Lab -# run: gcloud firebase test android run --type instrumentation --use-orchestrator --app app-release.apk --test app-release-androidTest.apk --device model=shiba,version=34,locale=en,orientation=portrait - - name: Run Robo test in Firebase Test Lab run: gcloud firebase test android run --type robo --app=app-releaseTest.apk --robo-script firebase/robo_script.json --timeout=1800s --device model=shiba,version=34,locale=en,orientation=portrait diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 35b1a2ea0..45ac46166 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -79,7 +79,9 @@ jobs: touch app/emulator.log # create log file chmod 777 app/emulator.log # allow writing to log file adb logcat >> app/emulator.log & # pipe all logcat messages into log file as a background process - ./gradlew connectedCheck + # NightlyOnly-annotated tests (real-time audio soak tests, live-network map fetches) + # are excluded here and run separately in nightly.yaml instead. + ./gradlew connectedCheck -Pandroid.testInstrumentationRunnerArguments.notAnnotation=org.scottishtecharmy.soundscape.NightlyOnly - name: Upload Failing Test Report Log if: steps.instrumentation-tests.outcome != 'success' # upload the generated log on failure of the tests job diff --git a/app/build.gradle.kts b/app/build.gradle.kts index e6317baf8..957eb1075 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -468,6 +468,33 @@ dependencies { testImplementation(libs.json) } +// Some JVM unit tests (e.g. MvtTileTest.testCallouts, which replays every GPX fixture through +// the full grid/callout engine) are too slow to run on every PR. They're tagged with the +// NightlyOnlyTest JUnit4 category so they can be excluded from the regular `test` task and run +// separately, e.g. overnight - see .github/workflows/nightly.yaml. +val nightlyOnlyTestCategory = "org.scottishtecharmy.soundscape.NightlyOnlyTest" + +tasks.withType().configureEach { + // Matches the AGP-generated testDebugUnitTest/testReleaseUnitTest/testReleaseTestUnitTest + // tasks, but not nightlyUnitTest below (that one includes the category instead). + if (name.startsWith("test") && name.endsWith("UnitTest")) { + useJUnit { + excludeCategories(nightlyOnlyTestCategory) + } + } +} + +tasks.register("nightlyUnitTest") { + group = "verification" + description = "Runs the slow JVM unit tests excluded from the regular test task (nightly build only)." + val debugUnitTest = tasks.named("testDebugUnitTest").get() + testClassesDirs = debugUnitTest.testClassesDirs + classpath = debugUnitTest.classpath + useJUnit { + includeCategories(nightlyOnlyTestCategory) + } +} + dokka { dokkaSourceSets.configureEach { if (name == "main") { diff --git a/app/src/androidTest/java/org/scottishtecharmy/soundscape/AudioEngineTest.kt b/app/src/androidTest/java/org/scottishtecharmy/soundscape/AudioEngineTest.kt index 26cb01bf9..847f9edab 100644 --- a/app/src/androidTest/java/org/scottishtecharmy/soundscape/AudioEngineTest.kt +++ b/app/src/androidTest/java/org/scottishtecharmy/soundscape/AudioEngineTest.kt @@ -66,6 +66,7 @@ class AudioEngineTest { audioEngine.destroy() } + @NightlyOnly @Test fun soundBeacon() { val audioEngine = initializeAudioEngine() @@ -84,6 +85,7 @@ class AudioEngineTest { tidyUp(audioEngine) } + @NightlyOnly @Test fun allBeacons() { val audioEngine = initializeAudioEngine() @@ -103,6 +105,7 @@ class AudioEngineTest { tidyUp(audioEngine) } + @NightlyOnly @Test fun queuedSpeech() { val audioEngine = initializeAudioEngine() @@ -115,6 +118,7 @@ class AudioEngineTest { tidyUp(audioEngine) } + @NightlyOnly @Test fun earcon() { val audioEngine = initializeAudioEngine() @@ -141,6 +145,7 @@ class AudioEngineTest { tidyUp(audioEngine) } + @NightlyOnly @Test fun textAndEarcon() { val audioEngine = initializeAudioEngine() @@ -151,6 +156,7 @@ class AudioEngineTest { tidyUp(audioEngine) } + @NightlyOnly @Test fun textWithShutdownRestart() { var audioEngine = initializeAudioEngine() @@ -185,6 +191,7 @@ class AudioEngineTest { // tidyUp(audioEngine) // } + @NightlyOnly @Test fun earconPosition() { val audioEngine = initializeAudioEngine() @@ -222,6 +229,7 @@ class AudioEngineTest { tidyUp(audioEngine) } + @NightlyOnly @Test fun textPosition() { val audioEngine = initializeAudioEngine() diff --git a/app/src/androidTest/java/org/scottishtecharmy/soundscape/MvtPerformanceTest.kt b/app/src/androidTest/java/org/scottishtecharmy/soundscape/MvtPerformanceTest.kt index ac3a89dfc..f565ab2c3 100644 --- a/app/src/androidTest/java/org/scottishtecharmy/soundscape/MvtPerformanceTest.kt +++ b/app/src/androidTest/java/org/scottishtecharmy/soundscape/MvtPerformanceTest.kt @@ -316,6 +316,7 @@ class MvtPerformanceTest { Thread.sleep(10000) } + @NightlyOnly @Test fun testMapAreas() { if (!tileProviderAvailable()) diff --git a/app/src/androidTest/java/org/scottishtecharmy/soundscape/NightlyOnly.kt b/app/src/androidTest/java/org/scottishtecharmy/soundscape/NightlyOnly.kt new file mode 100644 index 000000000..b2c05cfb3 --- /dev/null +++ b/app/src/androidTest/java/org/scottishtecharmy/soundscape/NightlyOnly.kt @@ -0,0 +1,11 @@ +package org.scottishtecharmy.soundscape + +/** + * Marks an instrumented test that is too slow (real-time audio playback, live network tile + * fetches, etc.) to run on every PR and mostly just checks that nothing crashes. Excluded from + * run-tests.yaml's `connectedCheck` via `-e notAnnotation`, and run on its own in + * nightly.yaml via `-e annotation`. + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.FUNCTION, AnnotationTarget.CLASS) +annotation class NightlyOnly diff --git a/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt b/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt index 8e903a46b..b736170d7 100644 --- a/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt +++ b/app/src/test/java/org/scottishtecharmy/soundscape/MvtTileTest.kt @@ -8,6 +8,7 @@ import junit.framework.TestCase.assertTrue import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.runBlocking import org.junit.Test +import org.junit.experimental.categories.Category import org.scottishtecharmy.soundscape.MainActivity.Companion.MOBILITY_KEY import org.scottishtecharmy.soundscape.MainActivity.Companion.PLACES_AND_LANDMARKS_KEY import org.scottishtecharmy.soundscape.geoengine.GRID_SIZE @@ -1977,6 +1978,7 @@ class MvtTileTest { } @OptIn(ExperimentalCoroutinesApi::class) + @Category(NightlyOnlyTest::class) @Test fun testCallouts() { val directoryPath = Path("src/test/res/org/scottishtecharmy/soundscape/gpxFiles/") @@ -2141,6 +2143,7 @@ class MvtTileTest { } @OptIn(ExperimentalCoroutinesApi::class) + @Category(NightlyOnlyTest::class) @Test fun replayStreetNumbers() { val directoryPath = Path("src/test/res/org/scottishtecharmy/soundscape/gpxFiles/") diff --git a/app/src/test/java/org/scottishtecharmy/soundscape/NightlyOnlyTest.kt b/app/src/test/java/org/scottishtecharmy/soundscape/NightlyOnlyTest.kt new file mode 100644 index 000000000..8f43b4672 --- /dev/null +++ b/app/src/test/java/org/scottishtecharmy/soundscape/NightlyOnlyTest.kt @@ -0,0 +1,9 @@ +package org.scottishtecharmy.soundscape + +/** + * JUnit4 category marker for JVM unit tests that are too slow to run on every PR (e.g. they + * replay every fixture through the full grid/callout engine rather than asserting against a + * single case). Tests annotated `@Category(NightlyOnlyTest::class)` are excluded from the + * regular `test` task and only run via the `nightlyUnitTest` task, see app/build.gradle.kts. + */ +interface NightlyOnlyTest