diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.Retry/PACKAGE.md index ac00f1458c..3275f0bc39 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/PACKAGE.md @@ -16,9 +16,10 @@ This package extends Microsoft.Testing.Platform with: - **Automatic retry**: automatically re-runs failed tests up to a configurable number of times - **Retry guards**: can stop retries when failure thresholds are exceeded (`--retry-failed-tests-max-percentage`, `--retry-failed-tests-max-tests`) +- **Retry delay**: optionally wait between retry attempts (`--retry-failed-tests-delay`) - **Integration-test focus**: intended for scenarios where transient environment issues can cause intermittent failures -Configure retry using `--retry-failed-tests `, and optionally limit retries with `--retry-failed-tests-max-percentage` or `--retry-failed-tests-max-tests`. +Configure retry using `--retry-failed-tests `, and optionally limit retries with `--retry-failed-tests-max-percentage` or `--retry-failed-tests-max-tests`, or add a delay between retries with `--retry-failed-tests-delay` (e.g. `1s`, `2.5m`, `1h`). ## Documentation diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/ExtensionResources.resx b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/ExtensionResources.resx index 8179e3290f..46b79b89e5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/ExtensionResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/ExtensionResources.resx @@ -164,6 +164,12 @@ Moving last attempt asset files to the default result directory Retry failed tests feature is not supported in server mode + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Retry failed tests the given number of times diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.cs.xlf index 0ec43e125a..b9c43ebcdb 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.cs.xlf @@ -50,6 +50,16 @@ Přesouvání souborů prostředků posledního pokusu do výchozího adresáře Nepovedlo se spustit proces {0}. + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. Funkce opakování neúspěšných testů umožňuje po selhání znovu spustit test. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.de.xlf index 54f48e40b8..d4fce7d287 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.de.xlf @@ -50,6 +50,16 @@ Medienobjektdateien des letzten Versuchs werden in das Standardergebnisverzeichn Fehler beim Starten von Prozess "{0}". + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. Das Feature zum Wiederholen fehlerhafter Tests ermöglicht es, die Testausführung bei einem Fehler neu zu starten. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.es.xlf index 55edc18d71..a8916e044e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.es.xlf @@ -50,6 +50,16 @@ Moviendo los archivos de recursos del último intento al directorio de resultado No se pudo iniciar el proceso '{0}' + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. La característica Reintentar pruebas con errores permite reiniciar la ejecución de pruebas en caso de error. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.fr.xlf index 2fc7921866..6cdedcef69 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.fr.xlf @@ -50,6 +50,16 @@ Déplacement des fichiers de ressources de la dernière tentative vers le réper Échec de démarrage du processus « {0} » + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. La fonctionnalité de nouvelles tentatives de tests ayant échoué permet de redémarrer l’exécution des tests en cas d’échec. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.it.xlf index a55bf502da..a527f4594b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.it.xlf @@ -50,6 +50,16 @@ Spostamento dei file di asset dell'ultimo tentativo nella directory dei risultat Impossibile avviare il processo '{0}' + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. La funzionalità di ripetizione dei test non riusciti consente di riavviare l'esecuzione dei test in caso di errore. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ja.xlf index 23d77b2232..24d3ea5710 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ja.xlf @@ -50,6 +50,16 @@ Moving last attempt asset files to the default result directory 処理 '{0}' を開始できませんでした + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. 失敗したテストの再試行機能を使用すると、失敗時にテストの実行を再開できます。 diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ko.xlf index df4c3290dd..0cd33ebc93 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ko.xlf @@ -50,6 +50,16 @@ Moving last attempt asset files to the default result directory 프로세스 '{0}'을(를) 시작하지 못했습니다. + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. 실패한 테스트 다시 시도 기능을 사용하면 실패 시 테스트 실행을 다시 시작할 수 있습니다. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pl.xlf index cde119184e..4f3e1c6148 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pl.xlf @@ -50,6 +50,16 @@ Przeniesienie plików zasobów ostatniej próby do domyślnego katalogu wyników Nie można uruchomić procesu „{0}” + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. Funkcja ponawiania testów zakończonych niepowodzeniem umożliwia ponowne uruchomienie wykonywania testu po niepowodzeniu. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pt-BR.xlf index e7b749064a..3617bb4a4d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.pt-BR.xlf @@ -50,6 +50,16 @@ Movendo arquivos de ativo da última tentativa para o diretório de resultados p Falha ao iniciar o processo '{0}' + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. O recurso repetir testes com falha permite reiniciar a execução de teste após falha. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ru.xlf index eecf000105..48bf2fb6f5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.ru.xlf @@ -50,6 +50,16 @@ Moving last attempt asset files to the default result directory Не удалось запустить процесс "{0}". + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. Функция повтора неудачных тестов позволяет перезапустить выполнение теста после сбоя. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.tr.xlf index 8879edbaee..0bc4b41d5b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.tr.xlf @@ -50,6 +50,16 @@ Son deneme varlık dosyaları, varsayılan sonuç dizinine taşınıyor '{0}' işlemi başlatılamadı + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. Başarısız testleri yeniden dene özelliği, başarısızlık durumunda test yürütmenin yeniden başlatılmasına olanak tanır. diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hans.xlf index 643ff90857..c19d72df8b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hans.xlf @@ -50,6 +50,16 @@ Moving last attempt asset files to the default result directory 无法启动进程“{0}” + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. 重试失败的测试功能允许在失败时重新启动测试执行。 diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hant.xlf index 417569a8bc..ee48cb5e68 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/Resources/xlf/ExtensionResources.zh-Hant.xlf @@ -50,6 +50,16 @@ Moving last attempt asset files to the default result directory 無法啟動處理程序 '{0}' + + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. + + + + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + Option '--retry-failed-tests-delay' requires a valid time span value that is non-negative and no greater than 2147483647ms (~24.20:31:23.647) (e.g. 200, 1s, 2.5m, 1h) + + Retry failed tests feature allows to restart test execution upon failure. 重試失敗的測試功能允許在失敗時重新啟動測試執行。 diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryCommandLineOptionsProvider.cs index 360e814519..d44c1e3917 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryCommandLineOptionsProvider.cs @@ -5,6 +5,7 @@ using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.CommandLine; +using Microsoft.Testing.Platform.Helpers; namespace Microsoft.Testing.Extensions.Policy; @@ -13,6 +14,7 @@ internal sealed class RetryCommandLineOptionsProvider : ICommandLineOptionsProvi public const string RetryFailedTestsOptionName = "retry-failed-tests"; public const string RetryFailedTestsMaxPercentageOptionName = "retry-failed-tests-max-percentage"; public const string RetryFailedTestsMaxTestsOptionName = "retry-failed-tests-max-tests"; + public const string RetryFailedTestsDelayOptionName = "retry-failed-tests-delay"; public const string RetryFailedTestsPipeNameOptionName = "internal-retry-pipename"; public string Uid => nameof(RetryCommandLineOptionsProvider); @@ -30,6 +32,7 @@ public IReadOnlyCollection GetCommandLineOptions() => new(RetryFailedTestsOptionName, ExtensionResources.RetryFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(RetryFailedTestsMaxPercentageOptionName, ExtensionResources.RetryFailedTestsMaxPercentageOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(RetryFailedTestsMaxTestsOptionName, ExtensionResources.RetryFailedTestsMaxTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), + new(RetryFailedTestsDelayOptionName, ExtensionResources.RetryFailedTestsDelayOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), // Hidden internal args new(RetryFailedTestsPipeNameOptionName, "Communication between the test host and the retry infra.", ArgumentArity.ExactlyOne, isHidden: true, isBuiltIn: true) @@ -54,6 +57,11 @@ public Task ValidateCommandLineOptionsAsync(ICommandLineOption return ValidationResult.InvalidTask(string.Format(CultureInfo.CurrentCulture, ExtensionResources.RetryFailedTestsOptionIsMissingErrorMessage, RetryFailedTestsMaxTestsOptionName, RetryFailedTestsOptionName)); } + if (commandLineOptions.IsOptionSet(RetryFailedTestsDelayOptionName) && !commandLineOptions.IsOptionSet(RetryFailedTestsOptionName)) + { + return ValidationResult.InvalidTask(string.Format(CultureInfo.CurrentCulture, ExtensionResources.RetryFailedTestsOptionIsMissingErrorMessage, RetryFailedTestsDelayOptionName, RetryFailedTestsOptionName)); + } + // No problem found return ValidationResult.ValidTask; } @@ -75,6 +83,14 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com return ValidationResult.InvalidTask(string.Format(CultureInfo.CurrentCulture, ExtensionResources.RetryFailedTestsOptionSingleIntegerArgumentErrorMessage, RetryFailedTestsMaxTestsOptionName)); } + if (commandOption.Name == RetryFailedTestsDelayOptionName + && (!TimeSpanParser.TryParse(arguments[0], out TimeSpan delay) + || delay < TimeSpan.Zero + || delay.TotalMilliseconds > int.MaxValue)) + { + return ValidationResult.InvalidTask(ExtensionResources.RetryFailedTestsDelayOptionInvalidArgument); + } + // No problem found return ValidationResult.ValidTask; } diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs index 18a53c689b..6d40011cde 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs @@ -124,6 +124,16 @@ public async Task OrchestrateTestHostExecutionAsync(CancellationToken cance indexToCleanup.Add(argIndex + 1); } + argIndex = GetOptionArgumentIndex(RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName, executableArguments); + if (argIndex > -1) + { + indexToCleanup.Add(argIndex); + if (argIndex + 1 < executableArguments.Length) + { + indexToCleanup.Add(argIndex + 1); + } + } + argIndex = GetOptionArgumentIndex(PlatformCommandLineProvider.ResultDirectoryOptionKey, executableArguments); if (argIndex > -1) { @@ -145,10 +155,26 @@ public async Task OrchestrateTestHostExecutionAsync(CancellationToken cance bool thresholdPolicyKickedIn = false; string retryRootFolder = CreateRetriesDirectory(resultDirectory); bool retryInterrupted = false; + + // Parse the delay once before the loop since command-line options don't change. + TimeSpan? retryDelay = null; + if (_commandLineOptions.TryGetOptionArgumentList(RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName, out string[]? retryDelayArgs) + && retryDelayArgs is { Length: > 0 } + && TimeSpanParser.TryParse(retryDelayArgs[0], out TimeSpan parsedDelay)) + { + retryDelay = parsedDelay; + } + while (attemptCount < userMaxRetryCount + 1) { attemptCount++; + if (attemptCount > 1 && retryDelay is { } delay) + { + await logger.LogDebugAsync($"Waiting {delay:c} before retry attempt {attemptCount}").ConfigureAwait(false); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + // Cleanup the arguments for (int i = 0; i < executableArguments.Length; i++) { diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 5151f2f827..ea2d4b260e 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -60,6 +60,8 @@ The directory where the test results are going to be placed. The default is TestResults in the directory that contains the test application. --retry-failed-tests Retry failed tests the given number of times + --retry-failed-tests-delay + Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. --retry-failed-tests-max-percentage Disable retry mechanism if the percentage of failed tests is greater than the specified value --retry-failed-tests-max-tests @@ -338,6 +340,10 @@ Default type is 'Full' Arity: 1 Hidden: False Description: Disable retry mechanism if the number of failed tests is greater than the specified value + --retry-failed-tests-delay + Arity: 1 + Hidden: False + Description: Add a delay between retries. The delay is expressed as a time value, e.g. 200, 1s, 2.5m, 1h. Default unit is milliseconds. TerminalTestReporterCommandLineOptionsProvider Name: Terminal test reporter Version: * diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs index b53aa9ab19..70460982c4 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs @@ -67,6 +67,31 @@ public async Task RetryFailedTests_OnlyRetryTimes_Succeeds(string tfm, bool fail } } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task RetryFailedTests_WithDelay_StripsDelayFromChildArgs(string tfm) + { + // The retry asset has AddRetryProvider() registered. If --retry-failed-tests-delay is NOT stripped from + // child-process arguments, the child will receive --retry-failed-tests-delay without --retry-failed-tests + // (the orchestrator strips the latter), causing validation to fail. A successful run therefore proves + // arg-stripping is working. + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + string resultDirectory = Path.Combine(testHost.DirectoryName, Guid.NewGuid().ToString("N")); + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--retry-failed-tests 3 --retry-failed-tests-delay 0 --results-directory {resultDirectory}", + new() + { + { EnvironmentVariableConstants.TESTINGPLATFORM_TELEMETRY_OPTOUT, "1" }, + { "METHOD1", "1" }, + { "FAIL", "0" }, + { "RESULTDIR", resultDirectory }, + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertOutputContains("Tests suite completed successfully in 2 attempts"); + } + [TestMethod] [DynamicData(nameof(GetMatrix))] public async Task RetryFailedTests_MaxPercentage_Succeeds(string tfm, bool fail) diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/RetryTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/RetryTests.cs index 53f3b8f3d3..17768c5468 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/RetryTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/RetryTests.cs @@ -110,4 +110,65 @@ public async Task IsValid_When_TestOption_Provided_With_Either_MaxPercentage_Max Assert.IsTrue(validateOptionsResult.IsValid); Assert.IsTrue(string.IsNullOrEmpty(validateOptionsResult.ErrorMessage)); } + + [DataRow("0")] + [DataRow("0s")] + [DataRow("200")] + [DataRow("1s")] + [DataRow("2.5m")] + [DataRow("1h")] + [TestMethod] + public async Task IsValid_If_CorrectTimeSpan_Is_Provided_For_DelayOption(string delay) + { + var provider = new RetryCommandLineOptionsProvider(); + CommandLineOption option = provider.GetCommandLineOptions().First(x => x.Name == RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName); + + ValidationResult validateOptionsResult = await provider.ValidateOptionArgumentsAsync(option, [delay]).ConfigureAwait(false); + Assert.IsTrue(validateOptionsResult.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(validateOptionsResult.ErrorMessage)); + } + + [DataRow("invalid")] + [DataRow("")] + [DataRow(" ")] + [DataRow("25d")] + [TestMethod] + public async Task IsInvalid_If_InvalidTimeSpan_Is_Provided_For_DelayOption(string delay) + { + var provider = new RetryCommandLineOptionsProvider(); + CommandLineOption option = provider.GetCommandLineOptions().First(x => x.Name == RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName); + + ValidationResult validateOptionsResult = await provider.ValidateOptionArgumentsAsync(option, [delay]).ConfigureAwait(false); + Assert.IsFalse(validateOptionsResult.IsValid); + Assert.AreEqual(Policy.Resources.ExtensionResources.RetryFailedTestsDelayOptionInvalidArgument, validateOptionsResult.ErrorMessage); + } + + [TestMethod] + public async Task IsInvalid_When_DelayOption_Provided_But_RetryOption_Missing() + { + var provider = new RetryCommandLineOptionsProvider(); + var options = new Dictionary + { + { RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName, ["1s"] }, + }; + + ValidationResult validateOptionsResult = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + Assert.IsFalse(validateOptionsResult.IsValid); + Assert.AreEqual(string.Format(CultureInfo.CurrentCulture, Policy.Resources.ExtensionResources.RetryFailedTestsOptionIsMissingErrorMessage, RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName, RetryCommandLineOptionsProvider.RetryFailedTestsOptionName), validateOptionsResult.ErrorMessage); + } + + [TestMethod] + public async Task IsValid_When_DelayOption_Provided_With_RetryOption() + { + var provider = new RetryCommandLineOptionsProvider(); + var options = new Dictionary + { + { RetryCommandLineOptionsProvider.RetryFailedTestsOptionName, ["3"] }, + { RetryCommandLineOptionsProvider.RetryFailedTestsDelayOptionName, ["1s"] }, + }; + + ValidationResult validateOptionsResult = await provider.ValidateCommandLineOptionsAsync(new TestCommandLineOptions(options)).ConfigureAwait(false); + Assert.IsTrue(validateOptionsResult.IsValid); + Assert.IsTrue(string.IsNullOrEmpty(validateOptionsResult.ErrorMessage)); + } }