diff --git a/lib/src/backend/declarer.dart b/lib/src/backend/declarer.dart
index 8adeacb4329f27772b211996c90bb9cd1aba7673..881a5dca393ef3efed968800468d5a546cb17419 100644
--- a/lib/src/backend/declarer.dart
+++ b/lib/src/backend/declarer.dart
@@ -101,7 +101,8 @@ class Declarer {
Timeout timeout,
skip,
Map<String, dynamic> onPlatform,
- tags}) {
+ tags,
+ int retry}) {
_checkNotBuilt("test");
var metadata = _metadata.merge(new Metadata.parse(
@@ -109,7 +110,8 @@ class Declarer {
timeout: timeout,
skip: skip,
onPlatform: onPlatform,
- tags: tags));
+ tags: tags,
+ retry: retry));
_entries.add(new LocalTest(_prefix(name), metadata, () async {
var parents = <Declarer>[];
diff --git a/lib/src/backend/invoker.dart b/lib/src/backend/invoker.dart
index f7fcb4892e9b299ec6a149bb0af32fc273e4df08..cb6ff4582d73893b774da341ce3cbc54469fa032 100644
--- a/lib/src/backend/invoker.dart
+++ b/lib/src/backend/invoker.dart
@@ -107,6 +107,9 @@ class Invoker {
/// on one anothers' toes.
final _counterKey = new Object();
+ /// The number of times this [liveTest] has been run.
+ int _runCount = 0;
+
/// The current invoker, or `null` if none is defined.
///
/// An invoker is only set within the zone scope of a running test.
@@ -275,6 +278,8 @@ class Invoker {
/// Notifies the invoker of an asynchronous error.
void _handleError(error, [StackTrace stackTrace]) {
+ // Ignore errors propagated from previous test runs
+ if (_runCount != Zone.current[#runCount]) return;
if (stackTrace == null) stackTrace = new Chain.current();
// Store these here because they'll change when we set the state below.
@@ -324,6 +329,7 @@ class Invoker {
var outstandingCallbacksForBody = new OutstandingCallbackCounter();
+ _runCount++;
Chain.capture(() {
runZonedWithValues(() async {
_invokerZone = Zone.current;
@@ -348,6 +354,14 @@ class Invoker {
await _outstandingCallbacks.noOutstandingCallbacks;
if (_timeoutTimer != null) _timeoutTimer.cancel();
+ if (liveTest.state.result != Result.success &&
+ _runCount < liveTest.test.metadata.retry + 1) {
+ _controller
+ .message(new Message.print("Retry: ${liveTest.test.name}"));
+ _onRun();
+ return;
+ }
+
_controller.setState(new State(Status.complete, liveTest.state.result));
_controller.completer.complete();
@@ -357,7 +371,8 @@ class Invoker {
// Use the invoker as a key so that multiple invokers can have different
// outstanding callback counters at once.
_counterKey: outstandingCallbacksForBody,
- _closableKey: true
+ _closableKey: true,
+ #runCount: _runCount
},
zoneSpecification: new ZoneSpecification(
print: (self, parent, zone, line) =>
diff --git a/lib/src/backend/metadata.dart b/lib/src/backend/metadata.dart
index 8fa6bc426b083b76f87070e2c703231b7a58a41b..92a8ac81114b149df9a53ae7bce4bd34b0824571 100644
--- a/lib/src/backend/metadata.dart
+++ b/lib/src/backend/metadata.dart
@@ -49,6 +49,10 @@ class Metadata {
/// The user-defined tags attached to the test or suite.
final Set<String> tags;
+ /// The number of times to re-run a test before being marked as a failure.
+ int get retry => _retry ?? 0;
+ final int _retry;
+
/// Platform-specific metadata.
///
/// Each key identifies a platform, and its value identifies the specific
@@ -140,6 +144,7 @@ class Metadata {
bool skip,
bool verboseTrace,
bool chainStackTraces,
+ int retry,
String skipReason,
Iterable<String> tags,
Map<PlatformSelector, Metadata> onPlatform,
@@ -151,6 +156,7 @@ class Metadata {
skip: skip,
verboseTrace: verboseTrace,
chainStackTraces: chainStackTraces,
+ retry: retry,
skipReason: skipReason,
tags: tags,
onPlatform: onPlatform,
@@ -185,6 +191,7 @@ class Metadata {
this.skipReason,
bool verboseTrace,
bool chainStackTraces,
+ int retry,
Iterable<String> tags,
Map<PlatformSelector, Metadata> onPlatform,
Map<BooleanSelector, Metadata> forTag})
@@ -193,10 +200,12 @@ class Metadata {
_skip = skip,
_verboseTrace = verboseTrace,
_chainStackTraces = chainStackTraces,
+ _retry = retry,
tags = new UnmodifiableSetView(tags == null ? new Set() : tags.toSet()),
onPlatform =
onPlatform == null ? const {} : new UnmodifiableMapView(onPlatform),
forTag = forTag == null ? const {} : new UnmodifiableMapView(forTag) {
+ if (retry != null) RangeError.checkNotNegative(retry, "retry");
_validateTags();
}
@@ -210,6 +219,7 @@ class Metadata {
skip,
bool verboseTrace,
bool chainStackTraces,
+ int retry,
Map<String, dynamic> onPlatform,
tags})
: testOn = testOn == null
@@ -219,6 +229,7 @@ class Metadata {
_skip = skip == null ? null : skip != false,
_verboseTrace = verboseTrace,
_chainStackTraces = chainStackTraces,
+ _retry = retry,
skipReason = skip is String ? skip : null,
onPlatform = _parseOnPlatform(onPlatform),
tags = _parseTags(tags),
@@ -228,6 +239,8 @@ class Metadata {
'"skip" must be a String or a bool, was "$skip".');
}
+ if (retry != null) RangeError.checkNotNegative(retry, "retry");
+
_validateTags();
}
@@ -241,6 +254,7 @@ class Metadata {
skipReason = serialized['skipReason'],
_verboseTrace = serialized['verboseTrace'],
_chainStackTraces = serialized['chainStackTraces'],
+ _retry = serialized['retry'],
tags = new Set.from(serialized['tags']),
onPlatform = new Map.fromIterable(serialized['onPlatform'],
key: (pair) => new PlatformSelector.parse(pair.first),
@@ -284,6 +298,7 @@ class Metadata {
skipReason: other.skipReason ?? skipReason,
verboseTrace: other._verboseTrace ?? _verboseTrace,
chainStackTraces: other._chainStackTraces ?? _chainStackTraces,
+ retry: other._retry ?? _retry,
tags: tags.union(other.tags),
onPlatform: mergeMaps(onPlatform, other.onPlatform,
value: (metadata1, metadata2) => metadata1.merge(metadata2)),
@@ -297,6 +312,7 @@ class Metadata {
bool skip,
bool verboseTrace,
bool chainStackTraces,
+ int retry,
String skipReason,
Map<PlatformSelector, Metadata> onPlatform,
Set<String> tags,
@@ -306,6 +322,7 @@ class Metadata {
skip ??= this._skip;
verboseTrace ??= this._verboseTrace;
chainStackTraces ??= this._chainStackTraces;
+ retry ??= this._retry;
skipReason ??= this.skipReason;
onPlatform ??= this.onPlatform;
tags ??= this.tags;
@@ -319,7 +336,8 @@ class Metadata {
skipReason: skipReason,
onPlatform: onPlatform,
tags: tags,
- forTag: forTag);
+ forTag: forTag,
+ retry: retry);
}
/// Returns a copy of [this] with all platform-specific metadata from
@@ -351,6 +369,7 @@ class Metadata {
'skipReason': skipReason,
'verboseTrace': _verboseTrace,
'chainStackTraces': _chainStackTraces,
+ 'retry': _retry,
'tags': tags.toList(),
'onPlatform': serializedOnPlatform,
'forTag': mapMap(forTag,
diff --git a/lib/src/runner/live_suite_controller.dart b/lib/src/runner/live_suite_controller.dart
index ae5a93347fd43afbbc9106fc872cb4942164d72b..266ddc42eb30d8c0112e8671a86fc62b9495e0dc 100644
--- a/lib/src/runner/live_suite_controller.dart
+++ b/lib/src/runner/live_suite_controller.dart
@@ -128,6 +128,8 @@ class LiveSuiteController {
_failed.add(liveTest);
} else if (countSuccess) {
_passed.add(liveTest);
+ // A passing test that was once failing was retried
+ _failed.remove(liveTest);
}
});
diff --git a/lib/src/runner/parse_metadata.dart b/lib/src/runner/parse_metadata.dart
index 7b86166093618f16848703b3c3214da3945603d3..8ac8014b3d78fef4763239dc89151e0106f4d616 100644
--- a/lib/src/runner/parse_metadata.dart
+++ b/lib/src/runner/parse_metadata.dart
@@ -60,6 +60,7 @@ class _Parser {
var skip;
Map<PlatformSelector, Metadata> onPlatform;
Set<String> tags;
+ int retry;
for (var annotation in _annotations) {
var pair =
@@ -82,6 +83,8 @@ class _Parser {
} else if (name == 'Tags') {
_assertSingle(tags, 'Tags', annotation);
tags = _parseTags(annotation, constructorName);
+ } else if (name == 'Retry') {
+ retry = _parseRetry(annotation, constructorName);
}
}
@@ -91,7 +94,8 @@ class _Parser {
skip: skip == null ? null : true,
skipReason: skip is String ? skip : null,
onPlatform: onPlatform,
- tags: tags);
+ tags: tags,
+ retry: retry);
}
/// Parses a `@TestOn` annotation.
@@ -106,6 +110,16 @@ class _Parser {
literal, () => new PlatformSelector.parse(literal.stringValue));
}
+ /// Parses a `@Retry` annotation.
+ ///
+ /// [annotation] is the annotation. [constructorName] is the name of the named
+ /// constructor for the annotation, if any.
+ int _parseRetry(Annotation annotation, String constructorName) {
+ _assertConstructorName(constructorName, 'Retry', annotation);
+ _assertArguments(annotation.arguments, 'Retry', annotation, positional: 1);
+ return _parseInt(annotation.arguments.arguments.first);
+ }
+
/// Parses a `@Timeout` annotation.
///
/// [annotation] is the annotation. [constructorName] is the name of the named
diff --git a/lib/test.dart b/lib/test.dart
index 13d182f8fde530a6951727d48585280c58c408f7..200aab16248bf061be9bda74dd35c605969d6296 100644
--- a/lib/test.dart
+++ b/lib/test.dart
@@ -101,6 +101,9 @@ Declarer get _declarer {
/// [package configuration file][configuring tags]. The parameter can be an
/// [Iterable] of tag names, or a [String] representing a single tag.
///
+/// If [retry] is passed, the test will be retried the provided number of times
+/// before being marked as a failure.
+///
/// [configuring tags]: https://github.com/dart-lang/test/blob/master/doc/package_config.md#configuring-tags
///
/// [onPlatform] allows tests to be configured on a platform-by-platform
@@ -127,13 +130,15 @@ void test(description, body(),
Timeout timeout,
skip,
tags,
- Map<String, dynamic> onPlatform}) {
+ Map<String, dynamic> onPlatform,
+ int retry}) {
_declarer.test(description.toString(), body,
testOn: testOn,
timeout: timeout,
skip: skip,
onPlatform: onPlatform,
- tags: tags);
+ tags: tags,
+ retry: retry);
// Force dart2js not to inline this function. We need it to be separate from
// `main()` in JS stack traces in order to properly determine the line and
diff --git a/test/runner/retry_test.dart b/test/runner/retry_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..e260471faef4ff9a0396d1edfc9bc46c724faea8
--- /dev/null
+++ b/test/runner/retry_test.dart
@@ -0,0 +1,182 @@
+// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_stream.dart';
+import 'package:scheduled_test/scheduled_test.dart';
+
+import '../io.dart';
+
+void main() {
+ useSandbox();
+
+ test("respects top-level @Retry declarations", () {
+ d
+ .file(
+ "test.dart",
+ """
+ @Retry(3)
+
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ var attempt = 0;
+ void main() {
+ test("failure", () {
+ attempt++;
+ if(attempt <= 3) {
+ throw new TestFailure("oh no");
+ }
+ });
+ }
+ """)
+ .create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("+1: All tests passed!")));
+ test.shouldExit(0);
+ });
+
+ test("Tests are not retried after they have already been reported successful",
+ () {
+ d
+ .file(
+ "test.dart",
+ """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ void main() {
+ var completer1 = new Completer();
+ var completer2 = new Completer();
+ test("first", () {
+ completer1.future.then((_) {
+ completer2.complete();
+ throw "oh no";
+ });
+ }, retry: 2);
+
+ test("second", () async {
+ completer1.complete();
+ await completer2.future;
+ });
+ }
+ """)
+ .create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(
+ contains("This test failed after it had already completed")));
+ test.shouldExit(1);
+ });
+
+ group("retries tests", () {
+ test("and eventually passes for valid tests", () {
+ d
+ .file(
+ "test.dart",
+ """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ var attempt = 0;
+ void main() {
+ test("eventually passes", () {
+ attempt++;
+ if(attempt <= 2) {
+ throw new TestFailure("oh no");
+ }
+ }, retry: 2);
+ }
+ """)
+ .create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("+1: All tests passed!")));
+ test.shouldExit(0);
+ });
+
+ test("and ignores previous errors", () {
+ d
+ .file(
+ "test.dart",
+ """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ var attempt = 0;
+ Completer completer = new Completer();
+ void main() {
+ test("failure", () async {
+ attempt++;
+ if (attempt == 1) {
+ completer.future.then((_) => throw 'some error');
+ throw new TestFailure("oh no");
+ }
+ completer.complete(null);
+ await new Future((){});
+ }, retry: 1);
+ }
+ """)
+ .create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("+1: All tests passed!")));
+ test.shouldExit(0);
+ });
+
+ test("and eventually fails for invalid tests", () {
+ d
+ .file(
+ "test.dart",
+ """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ void main() {
+ test("failure", () {
+ throw new TestFailure("oh no");
+ }, retry: 2);
+ }
+ """)
+ .create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("-1: Some tests failed.")));
+ test.shouldExit(1);
+ });
+
+ test("only after a failure", () {
+ d
+ .file(
+ "test.dart",
+ """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ var attempt = 0;
+ void main() {
+ test("eventually passes", () {
+ attempt++;
+ if (attempt != 2){
+ throw new TestFailure("oh no");
+ }
+ }, retry: 5);
+ }
+ """)
+ .create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("+1: All tests passed!")));
+ test.shouldExit(0);
+ });
+ });
+}
diff --git a/test/runner/timeout_test.dart b/test/runner/timeout_test.dart
index 3d9375a7811a55e29a2f33ead0b8208782c8b9a9..c28ee8772c7b73cdc80bfe0be62840616e59bc6a 100644
--- a/test/runner/timeout_test.dart
+++ b/test/runner/timeout_test.dart
@@ -59,6 +59,36 @@ void main() {
test.shouldExit(1);
});
+ test("timeout is reset with each retry", () {
+ d
+ .file(
+ "test.dart",
+ '''
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+ var runCount = 0;
+ test("timeout", () async {
+ runCount++;
+ if (runCount <=2) {
+ await new Future.delayed(new Duration(milliseconds: 1000));
+ }
+ }, retry: 3);
+}
+''')
+ .create();
+
+ var test = runTest(["--timeout=400ms", "test.dart"]);
+ test.stdout.expect(containsInOrder([
+ "Test timed out after 0.4 seconds.",
+ "Test timed out after 0.4 seconds.",
+ "+1: All tests passed!"
+ ]));
+ test.shouldExit(0);
+ });
+
test("the --timeout flag applies on top of the default 30s timeout", () {
d
.file(