Skip to content

Continue tests after a crash when using isolated = false #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
runningcode merged 5 commits into from
Jul 31, 2025
Merged

Continue tests after a crash when using isolated = false #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
da93ba5
Add ability to continue test execution after failure with isolated=true
runningcode Jul 21, 2025
c43b62d
Only execute 500 tests at a time because of argument limit
runningcode Jul 22, 2025
fd8c7ee
Fix bug where isolated=false would execute all the tests.
runningcode Jul 25, 2025
a12be84
Write tests to a file instead of passing them as arguments avoiding the
runningcode Jul 31, 2025
213fade
Fix TestRunnableTest constructor to match new TestRunnable signature
runningcode Jul 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions build_and_install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash

# Script to build AndroidX Test m2repository and install orchestrator APKs
# Based on CONTRIBUTING.md instructions

set -e # Exit on any error

echo "🔨 Building AndroidX Test m2repository..."
bazelisk build :axt_m2repository

echo "📦 Unpacking m2repository to ~/.m2/"
unzip -o bazel-bin/axt_m2repository.zip -d ~/.m2/

echo "📱 Installing orchestrator and services APKs..."

# Find and install test services APK
SERVICES_APK=~/.m2/repository/androidx/test/services/test-services/1.7.0-alpha01/test-services-1.7.0-alpha01.apk
if [ -n "$SERVICES_APK" ]; then
echo "Installing test services APK: $SERVICES_APK"
adb install --force-queryable -r "$SERVICES_APK"
else
echo "❌ Test services APK not found in ~/.m2/repository"
exit 1
fi

# Find and install orchestrator APK
ORCHESTRATOR_APK=~/.m2/repository/androidx/test/orchestrator/1.7.0-alpha01/orchestrator-1.7.0-alpha01.apk
if [ -n "$ORCHESTRATOR_APK" ]; then
echo "Installing orchestrator APK: $ORCHESTRATOR_APK"
adb install --force-queryable -r "$ORCHESTRATOR_APK"
else
echo "❌ Orchestrator APK not found in ~/.m2/repository"
exit 1
fi

echo "✅ Build and installation complete!"
105 changes: 101 additions & 4 deletions ...er/android_test_orchestrator/java/androidx/test/orchestrator/AndroidTestOrchestrator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,26 +40,37 @@
import android.os.Build;
import android.os.Bundle;
import android.os.Debug;
import android.os.Environment;
import android.os.IBinder;
import android.os.RemoteException;
import android.text.TextUtils;
import android.util.Log;

import androidx.annotation.VisibleForTesting;
import androidx.core.content.ContextCompat;
import androidx.test.orchestrator.TestRunnable.RunFinishedListener;
import androidx.test.orchestrator.junit.ParcelableDescription;
import androidx.test.orchestrator.junit.ParcelableFailure;
import androidx.test.orchestrator.listeners.OrchestrationListenerManager;
import androidx.test.orchestrator.listeners.OrchestrationResult;
import androidx.test.orchestrator.listeners.OrchestrationResultPrinter;
import androidx.test.orchestrator.listeners.OrchestrationRunListener;
import androidx.test.services.shellexecutor.ClientNotConnected;
import androidx.test.services.shellexecutor.ShellExecSharedConstants;
import androidx.test.services.shellexecutor.ShellExecutor;
import androidx.test.services.shellexecutor.ShellExecutorFactory;

import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
Expand Down Expand Up @@ -127,6 +138,27 @@
public final class AndroidTestOrchestrator extends android.app.Instrumentation
implements RunFinishedListener {

private class OrchestratorListener extends OrchestrationRunListener {
@Override
public void testStarted(ParcelableDescription description) {
if (!runsInIsolatedMode(arguments)) {
test = description.getClassName() + "#" + description.getMethodName();
testsToExecuteNoIsolation.remove(test);
}
}

@Override
public void testFailure(ParcelableFailure failure) {
isRecoveringFromCrash = true;
}

@Override
public void testProcessFinished(String message) {
super.testProcessFinished(message);
Log.i(TAG, "Test process finished: " + message);
}
}

private static final String TAG = "AndroidTestOrchestrator";
// As defined in the AndroidManifest of the Orchestrator app.
private static final String ORCHESTRATOR_SERVICE_LOCATION = "OrchestratorService";
Expand All @@ -145,6 +177,7 @@ public final class AndroidTestOrchestrator extends android.app.Instrumentation
private final OrchestrationResultPrinter resultPrinter = new OrchestrationResultPrinter();
private final OrchestrationListenerManager listenerManager =
new OrchestrationListenerManager(this);
private final OrchestratorListener orchestratorListener = new OrchestratorListener();

private final ExecutorService executorService;

Expand All @@ -158,6 +191,9 @@ public final class AndroidTestOrchestrator extends android.app.Instrumentation
private String test;
private Iterator<String> testIterator;

private List<String> testsToExecuteNoIsolation;
private boolean isRecoveringFromCrash = false;

public AndroidTestOrchestrator() {
super();
// We never want to execute multiple tests in parallel.
Expand Down Expand Up @@ -315,6 +351,7 @@ public void runFinished() {
if (null == test) {
List<String> allTests = callbackLogic.provideCollectedTests();
testIterator = allTests.iterator();
testsToExecuteNoIsolation = new ArrayList<>(allTests);
addListeners(allTests.size());

if (allTests.isEmpty()) {
Expand All @@ -333,17 +370,76 @@ public void runFinished() {
}

private void executeEntireTestSuite() {
Log.i(TAG, "Executing entire test suite...");
// If we're recovering from a crash, continue with remaining tests
if (isRecoveringFromCrash && testsToExecuteNoIsolation != null && !testsToExecuteNoIsolation.isEmpty()) {
isRecoveringFromCrash = false;
executeRemainingTests();
return;
}

if (null != test) {
finish(Activity.RESULT_OK, createResultBundle());
return;
}

// We don't actually need test to have any particular value,
// just to indicate we've started execution.
executeRemainingTests();
}

private void executeRemainingTests() {
Log.i(TAG, "Executing remaining tests...");
// Create a list of remaining tests from the current iterator position
List<String> remainingTests = new ArrayList<>(testsToExecuteNoIsolation);

if (remainingTests.isEmpty()) {
Log.i(TAG, "No remaining tests to execute.");
finish(Activity.RESULT_OK, createResultBundle());
return;
}

// Set test to indicate execution has started
test = "";
// Execute remaining tests using a subset TestRunnable, we need to prevent the argument list from being too long
// Run 500 tests at a time
Log.i(TAG, "Executing subset of remaining tests: " + remainingTests.size() + " tests.");

// Write tests to file to avoid argument length limits
String testFilePath = writeTestsToFile(remainingTests);
Log.i(TAG, "Test file path: " + testFilePath);

executorService.execute(
TestRunnable.legacyTestRunnable(
getContext(), getSecret(arguments), arguments, getOutputStream(), this));
TestRunnable.testSubsetRunnable(
getContext(), getSecret(arguments), arguments, getOutputStream(), this, testFilePath));
}

private String writeTestsToFile(List<String> tests) {
String fileName = "test_subset.txt";
String filePath = "/data/local/tmp/" + fileName;
Context context = getContext();
String secret = getSecret(arguments);

try {
// Create the file using shell command to ensure proper permissions
execShellCommandSync(context, secret, "touch", Arrays.asList(filePath));

// Make the file world readable/writable so both processes can access it
execShellCommandSync(context, secret, "chmod", Arrays.asList("666", filePath));

// Write the test content to the file using shell commands
StringBuilder content = new StringBuilder();
for (String test : tests) {
content.append(test).append("\n");
}

// Use echo to write content (escape any special characters)
String escapedContent = content.toString().replace("\"", "\\\"").replace("$", "\\$");
execShellCommandSync(context, secret, "sh", Arrays.asList("-c", "echo \"" + escapedContent + "\" > " + filePath));

return filePath;
} catch (Exception e) {
Log.e(TAG, "Failed to write tests to file using shell commands", e);
throw new RuntimeException("Could not write tests to file", e);
}
}

private void executeNextTest() {
Expand Down Expand Up @@ -434,6 +530,7 @@ private String getOutputFile(String testName) {
private void addListeners(int testSize) {
listenerManager.addListener(resultBuilder);
listenerManager.addListener(resultPrinter);
listenerManager.addListener(orchestratorListener);
listenerManager.orchestrationRunStarted(testSize);
}

Expand Down
40 changes: 24 additions & 16 deletions runner/android_test_orchestrator/java/androidx/test/orchestrator/TestRunnable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,65 +45,68 @@ public class TestRunnable implements Runnable {
private final RunFinishedListener listener;
private final OutputStream outputStream;
private final String test;
private final String testFilePath;
private final boolean collectTests;
private final Context context;
private final String secret;

/**
* Constructs a TestRunnable executes all tests in arguments.
* Constructs a TestRunnable which will run a single test.
*
* @param context A context
* @param secret A string representing the speakeasy binder key
* @param arguments contains arguments to be passed to the target instrumentation
* @param outputStream the stream to write the results of the test process
* @param listener a callback listener to know when the run has completed
* @param test contains a specific test#method to run. Will override whatever is specified in the
* bundle.
*/
public static TestRunnable legacyTestRunnable(
public static TestRunnable singleTestRunnable(
Context context,
String secret,
Bundle arguments,
OutputStream outputStream,
RunFinishedListener listener) {
return new TestRunnable(context, secret, arguments, outputStream, listener, null, false);
RunFinishedListener listener,
String test) {
return new TestRunnable(context, secret, arguments, outputStream, listener, test, null, false);
}

/**
* Constructs a TestRunnable which will run a single test.
* Constructs a TestRunnable which will ask the instrumentation to list out its tests.
*
* @param context A context
* @param secret A string representing the speakeasy binder key
* @param arguments contains arguments to be passed to the target instrumentation
* @param outputStream the stream to write the results of the test process
* @param listener a callback listener to know when the run has completed
* @param test contains a specific test#method to run. Will override whatever is specified in the
* bundle.
*/
public static TestRunnable singleTestRunnable(
public static TestRunnable testCollectionRunnable(
Context context,
String secret,
Bundle arguments,
OutputStream outputStream,
RunFinishedListener listener,
String test) {
return new TestRunnable(context, secret, arguments, outputStream, listener, test, false);
RunFinishedListener listener) {
return new TestRunnable(context, secret, arguments, outputStream, listener, null, null, true);
}

/**
* Constructs a TestRunnable which will ask the instrumentation to list out its tests.
* Constructs a TestRunnable which will run a specific subset of tests from a file.
*
* @param context A context
* @param secret A string representing the speakeasy binder key
* @param arguments contains arguments to be passed to the target instrumentation
* @param outputStream the stream to write the results of the test process
* @param listener a callback listener to know when the run has completed
* @param testFilePath the path to a file containing the tests to run
*/
public static TestRunnable testCollectionRunnable(
public static TestRunnable testSubsetRunnable(
Context context,
String secret,
Bundle arguments,
OutputStream outputStream,
RunFinishedListener listener) {
return new TestRunnable(context, secret, arguments, outputStream, listener, null, true);
RunFinishedListener listener,
String testFilePath) {
return new TestRunnable(context, secret, arguments, outputStream, listener, null, testFilePath, false);
}

@VisibleForTesting
Expand All @@ -114,13 +117,15 @@ public static TestRunnable testCollectionRunnable(
OutputStream outputStream,
RunFinishedListener listener,
String test,
String testFilePath,
boolean collectTests) {
this.context = context;
this.secret = secret;
this.arguments = new Bundle(arguments);
this.outputStream = outputStream;
this.listener = listener;
this.test = test;
this.testFilePath = testFilePath;
this.collectTests = collectTests;
}

Expand Down Expand Up @@ -178,9 +183,12 @@ private Bundle getTargetInstrumentationArguments() {
targetArgs.remove("testFile");
}

// Override the class parameter with the current test target.
// Override the class parameter with the current test target or use testFile for test subset.
if (test != null) {
targetArgs.putString(AJUR_CLASS_ARGUMENT, test);
} else if (testFilePath != null && !testFilePath.isEmpty()) {
// For test subset, use testFile parameter to read tests from file
targetArgs.putString("testFile", testFilePath);
}

return targetArgs;
Expand Down
2 changes: 1 addition & 1 deletion runner/android_test_orchestrator/javatests/androidx/test/orchestrator/TestRunnableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ private static class FakeTestRunnable extends TestRunnable {
RunFinishedListener listener,
String test,
boolean collectTests) {
super(context, secret, arguments, outputStream, listener, test, collectTests);
super(context, secret, arguments, outputStream, listener, test, null, collectTests);
}

@Override
Expand Down
Loading