diff --git a/lib/src/runner.dart b/lib/src/runner.dart
index 3a5ce38c3f3e676ff762839027522054401c1d52..f6a66d44380b6acb1fded72b4da28842d1c2a7c2 100644
--- a/lib/src/runner.dart
+++ b/lib/src/runner.dart
@@ -244,7 +244,7 @@ class Runner {
return loadSuite.changeSuite((suite) {
_warnForUnknownTags(suite);
- return suite.filter((test) {
+ return _shardSuite(suite.filter((test) {
// Skip any tests that don't match all the given patterns.
if (!_config.patterns.every(test.name.contains)) {
return false;
@@ -257,7 +257,7 @@ class Runner {
if (_config.excludeTags.evaluate(test.metadata.tags)) return false;
return true;
- });
+ }));
});
});
}
@@ -337,6 +337,29 @@ class Runner {
return 'the suite itself';
}
+ /// If sharding is enabled, filters [suite] to only include the tests that
+ /// should be run in this shard.
+ ///
+ /// We just take a slice of the tests in each suite corresponding to the shard
+ /// index. This makes the tests pretty tests across shards, and since the
+ /// tests are continuous, makes us more likely to be able to re-use
+ /// `setUpAll()` logic.
+ Suite _shardSuite(Suite suite) {
+ if (_config.totalShards == null) return suite;
+
+ var shardSize = suite.group.testCount / _config.totalShards;
+ var shardStart = (shardSize * _config.shardIndex).round();
+ var shardEnd = (shardSize * (_config.shardIndex + 1)).round();
+
+ var count = -1;
+ var filtered = suite.filter((test) {
+ count++;
+ return count >= shardStart && count < shardEnd;
+ });
+
+ return filtered;
+ }
+
/// Loads each suite in [suites] in order, pausing after load for platforms
/// that support debugging.
Future<bool> _loadThenPause(Stream<LoadSuite> suites) async {
diff --git a/lib/src/runner/configuration.dart b/lib/src/runner/configuration.dart
index bf21b0932a4f8888e37fd34ecb344bc013958e47..5f9689a8ebdb37d609bfbc431d670ef4ef5ede3c 100644
--- a/lib/src/runner/configuration.dart
+++ b/lib/src/runner/configuration.dart
@@ -92,6 +92,27 @@ class Configuration {
pauseAfterLoad ? 1 : (_concurrency ?? defaultConcurrency);
final int _concurrency;
+ /// The index of the current shard, if sharding is in use, or `null` if it's
+ /// not.
+ ///
+ /// Sharding is a technique that allows the Google internal test framework to
+ /// easily split a test run across multiple workers without requiring the
+ /// tests to be modified by the user. When sharding is in use, the runner gets
+ /// a shard index (this field) and a total number of shards, and is expected
+ /// to provide the following guarantees:
+ ///
+ /// * Running the same invocation of the runner, with the same shard index and
+ /// total shards, will run the same set of tests.
+ /// * Across all shards, each test must be run exactly once.
+ ///
+ /// In addition, tests should be balanced across shards as much as possible.
+ final int shardIndex;
+
+ /// The total number of shards, if sharding is in use, or `null` if it's not.
+ ///
+ /// See [shardIndex] for details.
+ final int totalShards;
+
/// The paths from which to load tests.
List<String> get paths => _paths ?? ["test"];
final List<String> _paths;
@@ -249,6 +270,8 @@ class Configuration {
String reporter,
int pubServePort,
int concurrency,
+ int shardIndex,
+ int totalShards,
Timeout timeout,
Iterable<Pattern> patterns,
Iterable<TestPlatform> platforms,
@@ -275,6 +298,8 @@ class Configuration {
reporter: reporter,
pubServePort: pubServePort,
concurrency: concurrency,
+ shardIndex: shardIndex,
+ totalShards: totalShards,
timeout: timeout,
patterns: patterns,
platforms: platforms,
@@ -339,6 +364,8 @@ class Configuration {
String reporter,
int pubServePort,
int concurrency,
+ this.shardIndex,
+ this.totalShards,
Timeout timeout,
Iterable<Pattern> patterns,
Iterable<TestPlatform> platforms,
@@ -385,6 +412,14 @@ class Configuration {
"filename's context must match the current operating system, was "
"${_filename.context.style}.");
}
+
+ if ((shardIndex == null) != (totalShards == null)) {
+ throw new ArgumentError(
+ "shardIndex and totalShards may only be passed together.");
+ } else if (shardIndex != null) {
+ RangeError.checkValueInInterval(
+ shardIndex, 0, totalShards - 1, "shardIndex");
+ }
}
/// Returns a [input] as an unmodifiable list or `null`.
@@ -427,6 +462,8 @@ class Configuration {
reporter: other._reporter ?? _reporter,
pubServePort: (other.pubServeUrl ?? pubServeUrl)?.port,
concurrency: other._concurrency ?? _concurrency,
+ shardIndex: other.shardIndex ?? shardIndex,
+ totalShards: other.totalShards ?? totalShards,
timeout: timeout.merge(other.timeout),
patterns: patterns.union(other.patterns),
platforms: other._platforms ?? _platforms,
@@ -464,6 +501,8 @@ class Configuration {
String reporter,
int pubServePort,
int concurrency,
+ int shardIndex,
+ int totalShards,
Timeout timeout,
Iterable<Pattern> patterns,
Iterable<TestPlatform> platforms,
@@ -490,6 +529,8 @@ class Configuration {
reporter: reporter ?? _reporter,
pubServePort: pubServePort ?? pubServeUrl?.port,
concurrency: concurrency ?? _concurrency,
+ shardIndex: shardIndex ?? this.shardIndex,
+ totalShards: totalShards ?? this.totalShards,
timeout: timeout ?? this.timeout,
patterns: patterns ?? this.patterns,
platforms: platforms ?? _platforms,
diff --git a/lib/src/runner/configuration/args.dart b/lib/src/runner/configuration/args.dart
index 38a200d3a68f8530bd57f6c01aa56ba04be6191c..3f6b66f48d1e51bcab2809ca7cfc7272af5dcdfa 100644
--- a/lib/src/runner/configuration/args.dart
+++ b/lib/src/runner/configuration/args.dart
@@ -85,6 +85,12 @@ final ArgParser _parser = (() {
'Currently only supported for browser tests.',
negatable: false);
+ // These are used by the internal Google test runner, so they're hidden from
+ // the --help output but still supported as stable API surface. See
+ // [Configuration.shardIndex] for details on their semantics.
+ parser.addOption("shard-index", hide: true);
+ parser.addOption("total-shards", hide: true);
+
parser.addSeparator("======== Output");
parser.addOption("reporter",
abbr: 'r',
@@ -146,6 +152,20 @@ class _Parser {
return selector.union(tagSelector);
});
+ var shardIndex = _parseOption('shard-index', int.parse);
+ var totalShards = _parseOption('total-shards', int.parse);
+ if ((shardIndex == null) != (totalShards == null)) {
+ throw new FormatException(
+ "--shard-index and --total-shards may only be passed together.");
+ } else if (shardIndex != null) {
+ if (shardIndex < 0) {
+ throw new FormatException("--shard-index may not be negative.");
+ } else if (shardIndex >= totalShards) {
+ throw new FormatException(
+ "--shard-index must be less than --total-shards.");
+ }
+ }
+
return new Configuration(
help: _ifParsed('help'),
version: _ifParsed('version'),
@@ -157,6 +177,8 @@ class _Parser {
reporter: _ifParsed('reporter'),
pubServePort: _parseOption('pub-serve', int.parse),
concurrency: _parseOption('concurrency', int.parse),
+ shardIndex: shardIndex,
+ totalShards: totalShards,
timeout: _parseOption('timeout', (value) => new Timeout.parse(value)),
patterns: patterns,
platforms: _ifParsed('platform')?.map(TestPlatform.find),
diff --git a/test/runner/configuration/configuration_test.dart b/test/runner/configuration/configuration_test.dart
index 72b9834cf251eaa08dd415df94765c70d140e28a..cb0e0e0cfc7b635cd7ddb9a3e820ef317290978a 100644
--- a/test/runner/configuration/configuration_test.dart
+++ b/test/runner/configuration/configuration_test.dart
@@ -27,6 +27,8 @@ void main() {
expect(merged.skipReason, isNull);
expect(merged.pauseAfterLoad, isFalse);
expect(merged.color, equals(canUseSpecialChars));
+ expect(merged.shardIndex, isNull);
+ expect(merged.totalShards, isNull);
expect(merged.packageRoot, equals(p.join(p.current, 'packages')));
expect(merged.reporter, equals(defaultReporter));
expect(merged.pubServeUrl, isNull);
@@ -44,6 +46,8 @@ void main() {
skipReason: "boop",
pauseAfterLoad: true,
color: true,
+ shardIndex: 3,
+ totalShards: 10,
packageRoot: "root",
reporter: "json",
pubServePort: 1234,
@@ -59,6 +63,8 @@ void main() {
expect(merged.skipReason, equals("boop"));
expect(merged.pauseAfterLoad, isTrue);
expect(merged.color, isTrue);
+ expect(merged.shardIndex, equals(3));
+ expect(merged.totalShards, equals(10));
expect(merged.packageRoot, equals("root"));
expect(merged.reporter, equals("json"));
expect(merged.pubServeUrl.port, equals(1234));
@@ -76,6 +82,8 @@ void main() {
skipReason: "boop",
pauseAfterLoad: true,
color: true,
+ shardIndex: 3,
+ totalShards: 10,
packageRoot: "root",
reporter: "json",
pubServePort: 1234,
@@ -90,6 +98,8 @@ void main() {
expect(merged.skipReason, equals("boop"));
expect(merged.pauseAfterLoad, isTrue);
expect(merged.color, isTrue);
+ expect(merged.shardIndex, equals(3));
+ expect(merged.totalShards, equals(10));
expect(merged.packageRoot, equals("root"));
expect(merged.reporter, equals("json"));
expect(merged.pubServeUrl.port, equals(1234));
@@ -108,6 +118,8 @@ void main() {
skipReason: "foo",
pauseAfterLoad: true,
color: false,
+ shardIndex: 2,
+ totalShards: 4,
packageRoot: "root",
reporter: "json",
pubServePort: 1234,
@@ -122,6 +134,8 @@ void main() {
skipReason: "bar",
pauseAfterLoad: false,
color: true,
+ shardIndex: 3,
+ totalShards: 10,
packageRoot: "boot",
reporter: "compact",
pubServePort: 5678,
@@ -136,6 +150,8 @@ void main() {
expect(merged.skipReason, equals("bar"));
expect(merged.pauseAfterLoad, isFalse);
expect(merged.color, isTrue);
+ expect(merged.shardIndex, equals(3));
+ expect(merged.totalShards, equals(10));
expect(merged.packageRoot, equals("boot"));
expect(merged.reporter, equals("compact"));
expect(merged.pubServeUrl.port, equals(5678));
diff --git a/test/runner/shard_test.dart b/test/runner/shard_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..541687c0447827cd058c0da3667bcbc86a408b61
--- /dev/null
+++ b/test/runner/shard_test.dart
@@ -0,0 +1,169 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_stream.dart';
+import 'package:scheduled_test/scheduled_test.dart';
+
+import 'package:test/src/util/exit_codes.dart' as exit_codes;
+
+import '../io.dart';
+
+void main() {
+ useSandbox();
+
+ test("divides all the tests among the available shards", () {
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1", () {});
+ test("test 2", () {});
+ test("test 3", () {});
+ test("test 4", () {});
+ test("test 5", () {});
+ test("test 6", () {});
+ test("test 7", () {});
+ test("test 8", () {});
+ test("test 9", () {});
+ test("test 10", () {});
+ }
+ """).create();
+
+ var test = runTest(["test.dart", "--shard-index=0", "--total-shards=3"]);
+ test.stdout.expect(containsInOrder([
+ "+0: test 1",
+ "+1: test 2",
+ "+2: test 3",
+ "+3: All tests passed!"
+ ]));
+ test.shouldExit(0);
+
+ test = runTest(["test.dart", "--shard-index=1", "--total-shards=3"]);
+ test.stdout.expect(containsInOrder([
+ "+0: test 4",
+ "+1: test 5",
+ "+2: test 6",
+ "+3: test 7",
+ "+4: All tests passed!"
+ ]));
+ test.shouldExit(0);
+
+ test = runTest(["test.dart", "--shard-index=2", "--total-shards=3"]);
+ test.stdout.expect(containsInOrder([
+ "+0: test 8",
+ "+1: test 9",
+ "+2: test 10",
+ "+3: All tests passed!"
+ ]));
+ test.shouldExit(0);
+ });
+
+ test("shards each suite", () {
+ d.file("1_test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1.1", () {});
+ test("test 1.2", () {});
+ test("test 1.3", () {});
+ }
+ """).create();
+
+ d.file("2_test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 2.1", () {});
+ test("test 2.2", () {});
+ test("test 2.3", () {});
+ }
+ """).create();
+
+ var test = runTest([".", "--shard-index=0", "--total-shards=3"]);
+ test.stdout.expect(inOrder([
+ either(containsInOrder([
+ "+0: ./1_test.dart: test 1.1",
+ "+1: ./2_test.dart: test 2.1"
+ ]), containsInOrder([
+ "+0: ./2_test.dart: test 2.1",
+ "+1: ./1_test.dart: test 1.1"
+ ])),
+ contains("+2: All tests passed!")
+ ]));
+ test.shouldExit(0);
+
+
+ test = runTest([".", "--shard-index=1", "--total-shards=3"]);
+ test.stdout.expect(inOrder([
+ either(containsInOrder([
+ "+0: ./1_test.dart: test 1.2",
+ "+1: ./2_test.dart: test 2.2"
+ ]), containsInOrder([
+ "+0: ./2_test.dart: test 2.2",
+ "+1: ./1_test.dart: test 1.2"
+ ])),
+ contains("+2: All tests passed!")
+ ]));
+ test.shouldExit(0);
+
+ test = runTest([".", "--shard-index=2", "--total-shards=3"]);
+ test.stdout.expect(inOrder([
+ either(containsInOrder([
+ "+0: ./1_test.dart: test 1.3",
+ "+1: ./2_test.dart: test 2.3"
+ ]), containsInOrder([
+ "+0: ./2_test.dart: test 2.3",
+ "+1: ./1_test.dart: test 1.3"
+ ])),
+ contains("+2: All tests passed!")
+ ]));
+ test.shouldExit(0);
+ });
+
+ test("an empty shard reports success", () {
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1", () {});
+ test("test 2", () {});
+ }
+ """).create();
+
+ var test = runTest(["test.dart", "--shard-index=1", "--total-shards=3"]);
+ test.stdout.expect(consumeThrough("No tests ran."));
+ test.shouldExit(0);
+ });
+
+ group("reports an error if", () {
+ test("--shard-index is provided alone", () {
+ var test = runTest(["--shard-index=1"]);
+ test.stderr.expect(
+ "--shard-index and --total-shards may only be passed together.");
+ test.shouldExit(exit_codes.usage);
+ });
+
+ test("--total-shards is provided alone", () {
+ var test = runTest(["--total-shards=5"]);
+ test.stderr.expect(
+ "--shard-index and --total-shards may only be passed together.");
+ test.shouldExit(exit_codes.usage);
+ });
+
+ test("--shard-index is negative", () {
+ var test = runTest(["--shard-index=-1", "--total-shards=5"]);
+ test.stderr.expect("--shard-index may not be negative.");
+ test.shouldExit(exit_codes.usage);
+ });
+
+ test("--shard-index is equal to --total-shards", () {
+ var test = runTest(["--shard-index=5", "--total-shards=5"]);
+ test.stderr.expect("--shard-index must be less than --total-shards.");
+ test.shouldExit(exit_codes.usage);
+ });
+ });
+}