From 6ffb7d9e728d7730abbeedc465e24ef333583010 Mon Sep 17 00:00:00 2001 From: Marcel Schnelle Date: Wed, 31 Dec 2025 17:54:49 +0100 Subject: [PATCH 1/3] Allow control over instrumentation tests on unsupported devices via configparam & DSL --- .../.idea/runConfigurations/Format_Code.xml | 31 +++++++++++ instrumentation/CHANGELOG.md | 1 + instrumentation/compose/build.gradle.kts | 4 ++ instrumentation/core/build.gradle.kts | 4 ++ .../junit5/AndroidJUnitFrameworkBuilder.kt | 5 +- .../internal/ConfigurationParameters.kt | 10 ++++ .../internal/runners/AndroidJUnitFramework.kt | 4 +- .../internal/runners/DummyJUnitFramework.kt | 55 ++++++++++++++----- .../runners/JUnitFrameworkRunnerFactory.kt | 6 +- .../runners/JUnitFrameworkRunnerParams.kt | 23 +++++--- .../dummy/JupiterTestMethodFinderTests.kt | 4 +- instrumentation/sample/build.gradle.kts | 4 ++ .../.idea/runConfigurations/Format_Code.xml | 27 +++++++++ .../Plugin__Publish_Release_Manually.xml | 7 ++- .../Plugin__Update_Public_API_File.xml | 10 +++- plugin/CHANGELOG.md | 5 ++ .../dsl/AndroidJUnitPlatformExtension.kt | 1 + .../junit5/dsl/InstrumentationTestOptions.kt | 15 +++-- .../junit5/dsl/UnsupportedDeviceBehavior.kt | 11 ++++ .../plugins/junit5/internal/configure.kt | 27 ++++++--- 20 files changed, 206 insertions(+), 48 deletions(-) create mode 100644 instrumentation/.idea/runConfigurations/Format_Code.xml create mode 100644 instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/ConfigurationParameters.kt create mode 100644 plugin/.idea/runConfigurations/Format_Code.xml create mode 100644 plugin/android-junit5/src/main/kotlin/de/mannodermaus/gradle/plugins/junit5/dsl/UnsupportedDeviceBehavior.kt diff --git a/instrumentation/.idea/runConfigurations/Format_Code.xml b/instrumentation/.idea/runConfigurations/Format_Code.xml new file mode 100644 index 00000000..a99e49f5 --- /dev/null +++ b/instrumentation/.idea/runConfigurations/Format_Code.xml @@ -0,0 +1,31 @@ + + + + + + + true + true + false + + false + false + + false + false + false + false + + + \ No newline at end of file diff --git a/instrumentation/CHANGELOG.md b/instrumentation/CHANGELOG.md index 3b65fd95..819895d2 100644 --- a/instrumentation/CHANGELOG.md +++ b/instrumentation/CHANGELOG.md @@ -8,6 +8,7 @@ Change Log - Update to Compose 1.10 - Support instrumentation with JUnit 5 and 6 (the plugin will choose the correct runtime accordingly) - Avoid error when a client doesn't include junit-jupiter-params on the runtime classpath +- New: Instead of silently skipping tests when running on unsupported devices, fail test execution via configuration parameter `de.mannodermaus.junit.unsupported.behavior` ## 1.9.0 (2025-10-10) diff --git a/instrumentation/compose/build.gradle.kts b/instrumentation/compose/build.gradle.kts index 04a32d7a..e08cb35e 100644 --- a/instrumentation/compose/build.gradle.kts +++ b/instrumentation/compose/build.gradle.kts @@ -21,6 +21,10 @@ android { junitPlatform { // Using local dependency instead of Maven coordinates instrumentationTests.enabled = false + + // Fail test execution when running on unsupported device + // (TODO: Change this to the proper instrumentationTests API once released as stable) + configurationParameter("de.mannodermaus.junit.unsupported.behavior", "fail") } dependencies { diff --git a/instrumentation/core/build.gradle.kts b/instrumentation/core/build.gradle.kts index bc39abe8..18e87a2e 100644 --- a/instrumentation/core/build.gradle.kts +++ b/instrumentation/core/build.gradle.kts @@ -18,6 +18,10 @@ junitPlatform { // See TaggedTests.kt for usage of this tag excludeTags("nope") } + + // Fail test execution when running on unsupported device + // (TODO: Change this to the proper instrumentationTests API once released as stable) + configurationParameter("de.mannodermaus.junit.unsupported.behavior", "fail") } // Use local project dependencies on android-test instrumentation libraries diff --git a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/AndroidJUnitFrameworkBuilder.kt b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/AndroidJUnitFrameworkBuilder.kt index 243c5e42..626e4510 100644 --- a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/AndroidJUnitFrameworkBuilder.kt +++ b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/AndroidJUnitFrameworkBuilder.kt @@ -48,13 +48,12 @@ public open class AndroidJUnitFrameworkBuilder internal constructor() : RunnerBu } // One-time parsing setup for runner params, taken from instrumentation arguments - private val params by lazy { + private val params = JUnitFrameworkRunnerParams.create().also { params -> // Apply all environment variables & system properties to the running process params.registerEnvironmentVariables() params.registerSystemProperties() } - } @Throws(Throwable::class) override fun runnerForClass(testClass: Class<*>): Runner? { @@ -63,7 +62,7 @@ public open class AndroidJUnitFrameworkBuilder internal constructor() : RunnerBu try { return if (junitFrameworkAvailable) { - tryCreateJUnitFrameworkRunner(testClass) { params } + tryCreateJUnitFrameworkRunner(testClass, params) } else { null } diff --git a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/ConfigurationParameters.kt b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/ConfigurationParameters.kt new file mode 100644 index 00000000..1279a030 --- /dev/null +++ b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/ConfigurationParameters.kt @@ -0,0 +1,10 @@ +package de.mannodermaus.junit5.internal + +public object ConfigurationParameters { + /** + * How to behave when executing instrumentation tests on an unsupported device (i.e. too old). + * Accepted values: "skip", "fail" + */ + public const val BEHAVIOR_FOR_UNSUPPORTED_DEVICES: String = + "de.mannodermaus.junit.unsupported.behavior" +} diff --git a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFramework.kt b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFramework.kt index f82f8e26..228bdc6c 100644 --- a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFramework.kt +++ b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitFramework.kt @@ -18,10 +18,10 @@ import org.junit.runner.notification.RunNotifier @VisibleForTesting(otherwise = VisibleForTesting.PACKAGE_PRIVATE) internal class AndroidJUnitFramework( private val testClass: Class<*>, - paramsSupplier: () -> JUnitFrameworkRunnerParams = JUnitFrameworkRunnerParams::create, + params: JUnitFrameworkRunnerParams, ) : Runner() { private val launcher = LauncherFactory.create() - private val testTree by lazy { generateTestTree(paramsSupplier()) } + private val testTree by lazy { generateTestTree(params) } override fun getDescription() = testTree.suiteDescription diff --git a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/DummyJUnitFramework.kt b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/DummyJUnitFramework.kt index db7a3706..22bc583f 100644 --- a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/DummyJUnitFramework.kt +++ b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/DummyJUnitFramework.kt @@ -2,10 +2,11 @@ package de.mannodermaus.junit5.internal.runners import android.os.Build import android.util.Log +import de.mannodermaus.junit5.internal.ConfigurationParameters import de.mannodermaus.junit5.internal.JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION import de.mannodermaus.junit5.internal.LOG_TAG import de.mannodermaus.junit5.internal.dummy.JupiterTestMethodFinder -import java.lang.reflect.Method +import org.junit.platform.commons.JUnitException import org.junit.runner.Description import org.junit.runner.Runner import org.junit.runner.notification.RunNotifier @@ -14,22 +15,28 @@ import org.junit.runner.notification.RunNotifier * Fake Runner that marks all JUnit Framework methods as ignored, used for old devices without the * required Java capabilities. */ -internal class DummyJUnitFramework(private val testClass: Class<*>) : Runner() { +internal class DummyJUnitFramework( + private val testClass: Class<*>, + params: JUnitFrameworkRunnerParams, +) : Runner() { - private val testMethods: Set = JupiterTestMethodFinder.find(testClass) + private val testMethods = JupiterTestMethodFinder.find(testClass) + private val behaviorForUnsupportedDevices = params.behaviorForUnsupportedDevices override fun run(notifier: RunNotifier) { - Log.w( - LOG_TAG, - "JUnit Framework is not supported on this device: " + - "API level ${Build.VERSION.SDK_INT} is less than " + - "${JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION}, the minimum requirement. " + - "All Jupiter tests for ${testClass.name} will be disabled.", - ) - - for (testMethod in testMethods) { - val description = Description.createTestDescription(testClass, testMethod.name) - notifier.fireTestIgnored(description) + when (behaviorForUnsupportedDevices) { + "skip" -> skipTests(notifier) + "fail" -> failExecution(unsupportedDeviceMessage) + else -> { + Log.w( + LOG_TAG, + "Unknown value found for configuration parameter " + + "'${ConfigurationParameters.BEHAVIOR_FOR_UNSUPPORTED_DEVICES}': " + + "$behaviorForUnsupportedDevices. Apply default behavior " + + "and skip tests for this class.", + ) + skipTests(notifier) + } } } @@ -39,4 +46,24 @@ internal class DummyJUnitFramework(private val testClass: Class<*>) : Runner() { it.addChild(Description.createTestDescription(testClass, method.name)) } } + + private val unsupportedDeviceMessage by lazy { + "JUnit Framework is not supported on this device: " + + "API level ${Build.VERSION.SDK_INT} is less than " + + "${JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION}, the minimum requirement. " + + "All Jupiter tests for ${testClass.name} will be disabled." + } + + private fun skipTests(notifier: RunNotifier) { + Log.w(LOG_TAG, unsupportedDeviceMessage) + + for (testMethod in testMethods) { + val description = Description.createTestDescription(testClass, testMethod.name) + notifier.fireTestIgnored(description) + } + } + + private fun failExecution(message: String): Nothing { + throw JUnitException(message) + } } diff --git a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerFactory.kt b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerFactory.kt index 779b0458..5be97c50 100644 --- a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerFactory.kt +++ b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerFactory.kt @@ -13,13 +13,13 @@ import org.junit.runner.Runner */ internal fun tryCreateJUnitFrameworkRunner( klass: Class<*>, - paramsSupplier: () -> JUnitFrameworkRunnerParams, + params: JUnitFrameworkRunnerParams, ): Runner? { val runner = if (Build.VERSION.SDK_INT >= JUNIT_FRAMEWORK_MINIMUM_SDK_VERSION) { - AndroidJUnitFramework(klass, paramsSupplier) + AndroidJUnitFramework(klass, params) } else { - DummyJUnitFramework(klass) + DummyJUnitFramework(klass, params) } // It's still possible for the runner to not be relevant to the test run, diff --git a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerParams.kt b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerParams.kt index 693634a9..233f157c 100644 --- a/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerParams.kt +++ b/instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/JUnitFrameworkRunnerParams.kt @@ -2,6 +2,7 @@ package de.mannodermaus.junit5.internal.runners import android.os.Bundle import androidx.test.platform.app.InstrumentationRegistry +import de.mannodermaus.junit5.internal.ConfigurationParameters import de.mannodermaus.junit5.internal.discovery.GeneratedFilters import de.mannodermaus.junit5.internal.discovery.ParsedSelectors import de.mannodermaus.junit5.internal.discovery.PropertiesParser @@ -13,7 +14,7 @@ import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder internal data class JUnitFrameworkRunnerParams( private val arguments: Bundle = Bundle(), - private val filters: List> = emptyList(), + private val filtersSupplier: () -> List> = { emptyList() }, val environmentVariables: Map = emptyMap(), val systemProperties: Map = emptyMap(), private val configurationParameters: Map = emptyMap(), @@ -25,7 +26,7 @@ internal data class JUnitFrameworkRunnerParams( fun createDiscoveryRequest(selectors: List): LauncherDiscoveryRequest { return LauncherDiscoveryRequestBuilder.request() .selectors(selectors) - .filters(*this.filters.toTypedArray()) + .filters(*this.filtersSupplier().toTypedArray()) .configurationParameters(this.configurationParameters) .build() } @@ -33,6 +34,9 @@ internal data class JUnitFrameworkRunnerParams( val isParallelExecutionEnabled: Boolean get() = configurationParameters["junit.jupiter.execution.parallel.enabled"] == "true" + val behaviorForUnsupportedDevices: String? + get() = configurationParameters[ConfigurationParameters.BEHAVIOR_FOR_UNSUPPORTED_DEVICES] + val isUsingOrchestrator: Boolean get() = arguments.getString("orchestratorService") != null @@ -64,16 +68,19 @@ internal data class JUnitFrameworkRunnerParams( // which aren't subject to the filtering imposed through adb. // A special resource file may be looked up at runtime, containing // the filters to apply by the AndroidJUnit5 runner. - val filters = + // This requires lazy access because it reaches into JUnit internals, + // which may need Java functionality not supported by the current device + val filtersSupplier = { GeneratedFilters.fromContext(instrumentation.context) + listOfNotNull(ShardingFilter.fromArguments(arguments)) + } return JUnitFrameworkRunnerParams( - arguments, - filters, - environmentVariables, - systemProperties, - configurationParameters, + arguments = arguments, + filtersSupplier = filtersSupplier, + environmentVariables = environmentVariables, + systemProperties = systemProperties, + configurationParameters = configurationParameters, ) } } diff --git a/instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/internal/dummy/JupiterTestMethodFinderTests.kt b/instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/internal/dummy/JupiterTestMethodFinderTests.kt index 439b19dd..72c9bcbf 100644 --- a/instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/internal/dummy/JupiterTestMethodFinderTests.kt +++ b/instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/internal/dummy/JupiterTestMethodFinderTests.kt @@ -96,8 +96,8 @@ class JupiterTestMethodFinderTests { val listener = CountingRunListener() notifier.addListener(listener) - val params = JUnitFrameworkRunnerParams(filters = listOfNotNull(filter)) - AndroidJUnitFramework(cls) { params }.run(notifier) + val params = JUnitFrameworkRunnerParams(filtersSupplier = { listOfNotNull(filter) }) + AndroidJUnitFramework(cls, params).run(notifier) return listener } diff --git a/instrumentation/sample/build.gradle.kts b/instrumentation/sample/build.gradle.kts index e70be489..761adee1 100644 --- a/instrumentation/sample/build.gradle.kts +++ b/instrumentation/sample/build.gradle.kts @@ -40,6 +40,10 @@ junitPlatform { // Using local dependency instead of Maven coordinates instrumentationTests.enabled = false + + // Fail test execution when running on unsupported device + // (TODO: Change this to the proper instrumentationTests API once released as stable) + configurationParameter("de.mannodermaus.junit.unsupported.behavior", "fail") } dependencies { diff --git a/plugin/.idea/runConfigurations/Format_Code.xml b/plugin/.idea/runConfigurations/Format_Code.xml new file mode 100644 index 00000000..a8e85921 --- /dev/null +++ b/plugin/.idea/runConfigurations/Format_Code.xml @@ -0,0 +1,27 @@ + + + + + + + true + true + false + false + false + false + false + + + \ No newline at end of file diff --git a/plugin/.idea/runConfigurations/Plugin__Publish_Release_Manually.xml b/plugin/.idea/runConfigurations/Plugin__Publish_Release_Manually.xml index c56b79ee..9b365bd2 100644 --- a/plugin/.idea/runConfigurations/Plugin__Publish_Release_Manually.xml +++ b/plugin/.idea/runConfigurations/Plugin__Publish_Release_Manually.xml @@ -10,16 +10,19 @@