diff --git a/pkgs/test/test/runner/configuration/randomize_order_test.dart b/pkgs/test/test/runner/configuration/randomize_order_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..537db9cf3ba2f5919418c694580fc26e6810d29e
--- /dev/null
+++ b/pkgs/test/test/runner/configuration/randomize_order_test.dart
@@ -0,0 +1,187 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import 'package:test/test.dart';
+
+import '../../io.dart';
+
+void main() {
+ test("shuffles test order when passed a seed", () async {
+ await d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1", () {});
+ test("test 2", () {});
+ test("test 3", () {});
+ test("test 4", () {});
+ }
+ """).create();
+
+ // Test with a given seed
+ var test =
+ await runTest(["test.dart", "--test-randomize-ordering-seed=987654"]);
+ expect(
+ test.stdout,
+ containsInOrder([
+ "+0: test 4",
+ "+1: test 3",
+ "+2: test 1",
+ "+3: test 2",
+ "+4: All tests passed!"
+ ]));
+ await test.shouldExit(0);
+
+ // Do not shuffle when passed 0
+ test = await runTest(["test.dart", "--test-randomize-ordering-seed=0"]);
+ expect(
+ test.stdout,
+ containsInOrder([
+ "+0: test 1",
+ "+1: test 2",
+ "+2: test 3",
+ "+3: test 4",
+ "+4: All tests passed!"
+ ]));
+ await test.shouldExit(0);
+
+ // Do not shuffle when passed nothing
+ test = await runTest(["test.dart"]);
+ expect(
+ test.stdout,
+ containsInOrder([
+ "+0: test 1",
+ "+1: test 2",
+ "+2: test 3",
+ "+3: test 4",
+ "+4: All tests passed!"
+ ]));
+ await test.shouldExit(0);
+
+ // Shuffle when passed random
+ test =
+ await runTest(["test.dart", "--test-randomize-ordering-seed=random"]);
+ expect(
+ test.stdout,
+ emitsInAnyOrder([
+ contains("Shuffling test order with --test-randomize-ordering-seed"),
+ isNot(contains(
+ "Shuffling test order with --test-randomize-ordering-seed=0"))
+ ]));
+ await test.shouldExit(0);
+ });
+
+ test("shuffles each suite with the same seed", () async {
+ await d.file("1_test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1.1", () {});
+ test("test 1.2", () {});
+ test("test 1.3", () {});
+ }
+ """).create();
+
+ await d.file("2_test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 2.1", () {});
+ test("test 2.2", () {});
+ test("test 2.3", () {});
+ }
+ """).create();
+
+ var test = await runTest([".", "--test-randomize-ordering-seed=12345"]);
+ expect(
+ test.stdout,
+ emitsInAnyOrder([
+ containsInOrder([
+ "./1_test.dart: test 1.2",
+ "./1_test.dart: test 1.3",
+ "./1_test.dart: test 1.1"
+ ]),
+ containsInOrder([
+ "./2_test.dart: test 2.2",
+ "./2_test.dart: test 2.3",
+ "./2_test.dart: test 2.1"
+ ]),
+ contains("+6: All tests passed!")
+ ]));
+ await test.shouldExit(0);
+ });
+
+ test("shuffles groups as well as tests in groups", () async {
+ await d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ group("Group 1", () {
+ test("test 1.1", () {});
+ test("test 1.2", () {});
+ test("test 1.3", () {});
+ test("test 1.4", () {});
+ });
+ group("Group 2", () {
+ test("test 2.1", () {});
+ test("test 2.2", () {});
+ test("test 2.3", () {});
+ test("test 2.4", () {});
+ });
+ }
+ """).create();
+
+ // Test with a given seed
+ var test =
+ await runTest(["test.dart", "--test-randomize-ordering-seed=123"]);
+ expect(
+ test.stdout,
+ containsInOrder([
+ "+0: Group 2 test 2.4",
+ "+1: Group 2 test 2.2",
+ "+2: Group 2 test 2.1",
+ "+3: Group 2 test 2.3",
+ "+4: Group 1 test 1.4",
+ "+5: Group 1 test 1.2",
+ "+6: Group 1 test 1.1",
+ "+7: Group 1 test 1.3",
+ "+8: All tests passed!"
+ ]));
+ await test.shouldExit(0);
+ });
+
+ test("shuffles nested groups", () async {
+ await d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ group("Group 1", () {
+ test("test 1.1", () {});
+ test("test 1.2", () {});
+ group("Group 2", () {
+ test("test 2.3", () {});
+ test("test 2.4", () {});
+ });
+ });
+ }
+ """).create();
+
+ var test =
+ await runTest(["test.dart", "--test-randomize-ordering-seed=123"]);
+ expect(
+ test.stdout,
+ containsInOrder([
+ "+0: Group 1 test 1.1",
+ "+1: Group 1 Group 2 test 2.4",
+ "+2: Group 1 Group 2 test 2.3",
+ "+3: Group 1 test 1.2",
+ "+4: All tests passed!"
+ ]));
+ await test.shouldExit(0);
+ });
+}
diff --git a/pkgs/test/test/runner/configuration/suite_test.dart b/pkgs/test/test/runner/configuration/suite_test.dart
index 7d923dc5b6f78603c4e413fd43c7ffd775e81366..bc9f3a73dc64065a2d92f5047340b6af441bc2ad 100644
--- a/pkgs/test/test/runner/configuration/suite_test.dart
+++ b/pkgs/test/test/runner/configuration/suite_test.dart
@@ -20,6 +20,7 @@ void main() {
expect(merged.runSkipped, isFalse);
expect(merged.precompiledPath, isNull);
expect(merged.runtimes, equals([Runtime.vm.identifier]));
+ expect(merged.testRandomizeOrderingSeed, 0);
});
test("if only the old configuration's is defined, uses it", () {
@@ -41,11 +42,13 @@ void main() {
jsTrace: true,
runSkipped: true,
precompiledPath: "/tmp/js",
+ testRandomizeOrderingSeed: 1234,
runtimes: [RuntimeSelection(Runtime.chrome.identifier)]));
expect(merged.jsTrace, isTrue);
expect(merged.runSkipped, isTrue);
expect(merged.precompiledPath, equals("/tmp/js"));
+ expect(merged.testRandomizeOrderingSeed, 1234);
expect(merged.runtimes, equals([Runtime.chrome.identifier]));
});
diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart
index 2359b0b035c1f4f6725e4cd307f8555e6908976d..0856380a4e5adf75de3f353f3b832c6a349eefd8 100644
--- a/pkgs/test/test/runner/runner_test.dart
+++ b/pkgs/test/test/runner/runner_test.dart
@@ -49,76 +49,80 @@ void main() {
final _defaultConcurrency = math.max(1, Platform.numberOfProcessors ~/ 2);
-final _browsers = "[vm (default), chrome, phantomjs, firefox" +
- (Platform.isMacOS ? ", safari" : "") +
- (Platform.isWindows ? ", ie" : "") +
- ", node]";
-
final _usage = """
Usage: pub run test [files or directories...]
--h, --help Shows this usage information.
- --version Shows the package's version.
+-h, --help Shows this usage information.
+ --version Shows the package's version.
======== Selecting Tests
--n, --name A substring of the name of the test to run.
- Regular expression syntax is supported.
- If passed multiple times, tests must match all substrings.
+-n, --name A substring of the name of the test to run.
+ Regular expression syntax is supported.
+ If passed multiple times, tests must match all substrings.
--N, --plain-name A plain-text substring of the name of the test to run.
- If passed multiple times, tests must match all substrings.
+-N, --plain-name A plain-text substring of the name of the test to run.
+ If passed multiple times, tests must match all substrings.
--t, --tags Run only tests with all of the specified tags.
- Supports boolean selector syntax.
+-t, --tags Run only tests with all of the specified tags.
+ Supports boolean selector syntax.
--x, --exclude-tags Don't run tests with any of the specified tags.
- Supports boolean selector syntax.
+-x, --exclude-tags Don't run tests with any of the specified tags.
+ Supports boolean selector syntax.
- --[no-]run-skipped Run skipped tests instead of skipping them.
+ --[no-]run-skipped Run skipped tests instead of skipping them.
======== Running Tests
--p, --platform The platform(s) on which to run the tests.
- $_browsers
-
--P, --preset The configuration preset(s) to use.
--j, --concurrency=<threads> The number of concurrent test suites run.
- (defaults to "$_defaultConcurrency")
-
- --total-shards The total number of invocations of the test runner being run.
- --shard-index The index of this test runner invocation (of --total-shards).
- --pub-serve=<port> The port of a pub serve instance serving "test/".
- --timeout The default test timeout. For example: 15s, 2x, none
- (defaults to "30s")
-
- --pause-after-load Pauses for debugging before any tests execute.
- Implies --concurrency=1, --debug, and --timeout=none.
- Currently only supported for browser tests.
-
- --debug Runs the VM and Chrome tests in debug mode.
- --coverage=<directory> Gathers coverage and outputs it to the specified directory.
- Implies --debug.
-
- --[no-]chain-stack-traces Chained stack traces to provide greater exception details
- especially for asynchronous code. It may be useful to disable
- to provide improved test performance but at the cost of
- debuggability.
- (defaults to on)
-
- --no-retry Don't re-run tests that have retry set.
+-p, --platform The platform(s) on which to run the tests.
+ $_browsers
+
+-P, --preset The configuration preset(s) to use.
+-j, --concurrency=<threads> The number of concurrent test suites run.
+ (defaults to "$_defaultConcurrency")
+
+ --total-shards The total number of invocations of the test runner being run.
+ --shard-index The index of this test runner invocation (of --total-shards).
+ --pub-serve=<port> The port of a pub serve instance serving "test/".
+ --timeout The default test timeout. For example: 15s, 2x, none
+ (defaults to "30s")
+
+ --pause-after-load Pauses for debugging before any tests execute.
+ Implies --concurrency=1, --debug, and --timeout=none.
+ Currently only supported for browser tests.
+
+ --debug Runs the VM and Chrome tests in debug mode.
+ --coverage=<directory> Gathers coverage and outputs it to the specified directory.
+ Implies --debug.
+
+ --[no-]chain-stack-traces Chained stack traces to provide greater exception details
+ especially for asynchronous code. It may be useful to disable
+ to provide improved test performance but at the cost of
+ debuggability.
+ (defaults to on)
+
+ --no-retry Don't re-run tests that have retry set.
+ --test-randomize-ordering-seed If positive, use this as a seed to randomize the execution
+ of test cases (must be a 32bit unsigned integer).
+ If "random", pick a random seed to use.
+ If 0 or not set, do not randomize test case execution order.
======== Output
--r, --reporter The runner used to print test results.
+-r, --reporter The runner used to print test results.
- [compact] A single line, updated continuously.
- [expanded] (default) A separate line for each update.
- [json] A machine-readable format (see https://goo.gl/gBsV1a).
+ [compact] A single line, updated continuously.
+ [expanded] (default) A separate line for each update.
+ [json] A machine-readable format (see https://goo.gl/gBsV1a).
- --verbose-trace Whether to emit stack traces with core library frames.
- --js-trace Whether to emit raw JavaScript stack traces for browser tests.
- --[no-]color Whether to use terminal colors.
- (auto-detected by default)
+ --verbose-trace Whether to emit stack traces with core library frames.
+ --js-trace Whether to emit raw JavaScript stack traces for browser tests.
+ --[no-]color Whether to use terminal colors.
+ (auto-detected by default)
""";
+final _browsers = "[vm (default), chrome, phantomjs, firefox" +
+ (Platform.isMacOS ? ", safari" : "") +
+ (Platform.isWindows ? ", ie" : "") +
+ ", node]";
+
void main() {
test("prints help information", () async {
var test = await runTest(["--help"]);
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index 01601e24998716fafd8ebc9f92cad8ecc37bd478..ad302826b8a78e0721c3deb644273c4398803a26 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,6 +1,8 @@
## 0.2.15-dev
* Add a `StringSink` argument to reporters to prepare for reporting to a file.
+* Add --test-randomize-ordering-seed` argument to randomize test
+execution order based on a provided seed
## 0.2.14
diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart
index f1df78f466a53fd1047b1ba8042234772b88008d..26f0c62407c9be352ba82b58857bf39abf04f07c 100644
--- a/pkgs/test_core/lib/src/runner/configuration.dart
+++ b/pkgs/test_core/lib/src/runner/configuration.dart
@@ -249,6 +249,7 @@ class Configuration {
BooleanSelector excludeTags,
Map<BooleanSelector, SuiteConfiguration> tags,
Map<PlatformSelector, SuiteConfiguration> onPlatform,
+ int testRandomizeOrderingSeed,
// Test-level configuration
Timeout timeout,
@@ -294,6 +295,7 @@ class Configuration {
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
// Test-level configuration
timeout: timeout,
@@ -539,6 +541,7 @@ class Configuration {
BooleanSelector excludeTags,
Map<BooleanSelector, SuiteConfiguration> tags,
Map<PlatformSelector, SuiteConfiguration> onPlatform,
+ int testRandomizeOrderingSeed,
// Test-level configuration
Timeout timeout,
@@ -580,6 +583,7 @@ class Configuration {
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
timeout: timeout,
verboseTrace: verboseTrace,
chainStackTraces: chainStackTraces,
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index e0150f36763e236c13a2ccd5603c96e2ff4979df..0d04fdbac8ac91e3c75d49a897cb6f43f959a036 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
+import 'dart:math';
import 'package:args/args.dart';
import 'package:boolean_selector/boolean_selector.dart';
@@ -105,6 +106,11 @@ final ArgParser _parser = (() {
help: "Don't re-run tests that have retry set.",
defaultsTo: false,
negatable: false);
+ parser.addOption("test-randomize-ordering-seed",
+ help: 'If positive, use this as a seed to randomize the execution\n'
+ 'of test cases (must be a 32bit unsigned integer).\n'
+ 'If "random", pick a random seed to use.\n'
+ 'If 0 or not set, do not randomize test case execution order.\n');
var reporterDescriptions = <String, String>{};
for (var reporter in allReporters.keys) {
@@ -204,6 +210,17 @@ class _Parser {
}
}
+ var testRandomizeOrderingSeed =
+ _parseOption('test-randomize-ordering-seed', (value) {
+ var seed = value == 'random'
+ ? Random().nextInt(4294967295)
+ : int.parse(value).toUnsigned(32);
+ if (seed != null && seed > 0) {
+ print('Shuffling test order with --test-randomize-ordering-seed=$seed');
+ }
+ return seed;
+ });
+
return Configuration(
help: _ifParsed('help'),
version: _ifParsed('version'),
@@ -233,7 +250,8 @@ class _Parser {
paths: _options.rest.isEmpty ? null : _options.rest,
includeTags: includeTags,
excludeTags: excludeTags,
- noRetry: _ifParsed('no-retry'));
+ noRetry: _ifParsed('no-retry'),
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed);
}
/// Returns the parsed option for [name], or `null` if none was parsed.
diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart
index c0f922de4cd40c5f22a4dcbc873b74aa073375c2..812d7a84c27737b182a1bb29466a6097c43dfb2d 100644
--- a/pkgs/test_core/lib/src/runner/engine.dart
+++ b/pkgs/test_core/lib/src/runner/engine.dart
@@ -4,6 +4,7 @@
import 'dart:async';
import 'dart:collection';
+import 'dart:math';
import 'package:async/async.dart' hide Result;
import 'package:collection/collection.dart';
@@ -320,7 +321,13 @@ class Engine {
}
if (!_closed && setUpAllSucceeded) {
- for (var entry in group.entries) {
+ // shuffle the group entries
+ var entries = group.entries.toList();
+ if (suiteConfig.testRandomizeOrderingSeed > 0) {
+ entries.shuffle(Random(suiteConfig.testRandomizeOrderingSeed));
+ }
+
+ for (var entry in entries) {
if (_closed) return;
if (entry is Group) {
diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart
index 0421605e8bfae582f594a83505185df25882c922..08c6ab45ccc878e15fa4614c0cf621b8d1ced7d9 100644
--- a/pkgs/test_core/lib/src/runner/suite.dart
+++ b/pkgs/test_core/lib/src/runner/suite.dart
@@ -84,6 +84,12 @@ class SuiteConfiguration {
/// configuration fields, but that isn't enforced.
final Map<PlatformSelector, SuiteConfiguration> onPlatform;
+ /// The seed with which to shuffle the test order.
+ /// Default value is 0 if not provided and will not change the test order.
+ /// The same seed will shuffle the tests in the same way every time.
+ int get testRandomizeOrderingSeed => _testRandomizeOrderingSeed ?? 0;
+ final int _testRandomizeOrderingSeed;
+
/// The global test metadata derived from this configuration.
Metadata get metadata {
if (tags.isEmpty && onPlatform.isEmpty) return _metadata;
@@ -135,6 +141,7 @@ class SuiteConfiguration {
BooleanSelector excludeTags,
Map<BooleanSelector, SuiteConfiguration> tags,
Map<PlatformSelector, SuiteConfiguration> onPlatform,
+ int testRandomizeOrderingSeed,
// Test-level configuration
Timeout timeout,
@@ -156,6 +163,7 @@ class SuiteConfiguration {
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
metadata: Metadata(
timeout: timeout,
verboseTrace: verboseTrace,
@@ -183,6 +191,7 @@ class SuiteConfiguration {
BooleanSelector excludeTags,
Map<BooleanSelector, SuiteConfiguration> tags,
Map<PlatformSelector, SuiteConfiguration> onPlatform,
+ int testRandomizeOrderingSeed,
Metadata metadata})
: _jsTrace = jsTrace,
_runSkipped = runSkipped,
@@ -193,6 +202,7 @@ class SuiteConfiguration {
excludeTags = excludeTags ?? BooleanSelector.none,
tags = _map(tags),
onPlatform = _map(onPlatform),
+ _testRandomizeOrderingSeed = testRandomizeOrderingSeed,
_metadata = metadata ?? Metadata.empty;
/// Creates a new [SuiteConfiguration] that takes its configuration from
@@ -241,6 +251,8 @@ class SuiteConfiguration {
excludeTags: excludeTags.union(other.excludeTags),
tags: _mergeConfigMaps(tags, other.tags),
onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform),
+ testRandomizeOrderingSeed:
+ other._testRandomizeOrderingSeed ?? _testRandomizeOrderingSeed,
metadata: metadata.merge(other.metadata));
return config._resolveTags();
}
@@ -260,6 +272,7 @@ class SuiteConfiguration {
BooleanSelector excludeTags,
Map<BooleanSelector, SuiteConfiguration> tags,
Map<PlatformSelector, SuiteConfiguration> onPlatform,
+ int testRandomizeOrderingSeed,
// Test-level configuration
Timeout timeout,
@@ -281,6 +294,8 @@ class SuiteConfiguration {
excludeTags: excludeTags ?? this.excludeTags,
tags: tags ?? this.tags,
onPlatform: onPlatform ?? this.onPlatform,
+ testRandomizeOrderingSeed:
+ testRandomizeOrderingSeed ?? _testRandomizeOrderingSeed,
metadata: _metadata.change(
timeout: timeout,
verboseTrace: verboseTrace,