From 50b7d68b48a96a8b8896d35a8b7324f3cd93f944 Mon Sep 17 00:00:00 2001
From: kathrinschuler <kathrinschuler@users.noreply.github.com>
Date: Wed, 22 Jan 2020 18:17:34 +0000
Subject: [PATCH] Differentiate between test-randomize-ordering-seed=0 and
null. (#1150)
* Differentiate between seed=0 and not set
* Add note to readme about shuffling tests
---
pkgs/test/README.md | 13 +++++++++++++
pkgs/test/test/runner/configuration/suite_test.dart | 2 +-
pkgs/test_core/CHANGELOG.md | 2 ++
.../lib/src/runner/configuration/args.dart | 2 +-
pkgs/test_core/lib/src/runner/engine.dart | 3 ++-
pkgs/test_core/lib/src/runner/suite.dart | 11 +++++------
6 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/pkgs/test/README.md b/pkgs/test/README.md
index 7c91633d..a661323d 100644
--- a/pkgs/test/README.md
+++ b/pkgs/test/README.md
@@ -171,6 +171,19 @@ pub run test --total-shards 3 --shard-index 1 path/to/test.dart
pub run test --total-shards 3 --shard-index 2 path/to/test.dart
```
+### Shuffling Tests
+Test order can be shuffled with the `--test-randomize-ordering-seed` argument.
+This allows you to shuffle your tests with a specific seed (deterministic) or
+a random seed for each run. For example, consider the following test runs:
+
+```bash
+pub run test --test-randomize-ordering-seed=12345
+pub run test --test-randomize-ordering-seed=random
+```
+
+Setting `--test-randomize-ordering-seed=0` will have the same effect as not
+specifying it at all, meaning the test order will remain as-is.
+
### Collecting Code Coverage
To collect code coverage, you can run tests with the `--coverage <directory>`
argument. The directory specified can be an absolute or relative path.
diff --git a/pkgs/test/test/runner/configuration/suite_test.dart b/pkgs/test/test/runner/configuration/suite_test.dart
index 56b8119e..a1b4eb75 100644
--- a/pkgs/test/test/runner/configuration/suite_test.dart
+++ b/pkgs/test/test/runner/configuration/suite_test.dart
@@ -20,7 +20,7 @@ void main() {
expect(merged.runSkipped, isFalse);
expect(merged.precompiledPath, isNull);
expect(merged.runtimes, equals([Runtime.vm.identifier]));
- expect(merged.testRandomizeOrderingSeed, 0);
+ expect(merged.testRandomizeOrderingSeed, null);
});
test("if only the old configuration's is defined, uses it", () {
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index e43f0ab5..f28a3727 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -3,6 +3,8 @@
* Bump minimum SDK to `2.4.0` for safer usage of for-loop elements.
* Deprecate `PhantomJS` and provide warning when used. Support for `PhantomJS`
will be removed in version `2.0.0`.
+* Differentiate between test-randomize-ordering-seed not set and 0 being chosen
+ as the random seed.
## 0.2.18
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index ed7dd263..d4d8ab8e 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -220,7 +220,7 @@ class _Parser {
var seed = value == 'random'
? Random().nextInt(4294967295)
: int.parse(value).toUnsigned(32);
- if (seed != null && seed > 0) {
+ if (seed != null) {
print('Shuffling test order with --test-randomize-ordering-seed=$seed');
}
return seed;
diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart
index cf7dbd66..e4a7fa43 100644
--- a/pkgs/test_core/lib/src/runner/engine.dart
+++ b/pkgs/test_core/lib/src/runner/engine.dart
@@ -323,7 +323,8 @@ class Engine {
if (!_closed && setUpAllSucceeded) {
// shuffle the group entries
var entries = group.entries.toList();
- if (suiteConfig.testRandomizeOrderingSeed > 0) {
+ if (suiteConfig.testRandomizeOrderingSeed != null &&
+ suiteConfig.testRandomizeOrderingSeed > 0) {
entries.shuffle(Random(suiteConfig.testRandomizeOrderingSeed));
}
diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart
index 233dc72c..c95494ad 100644
--- a/pkgs/test_core/lib/src/runner/suite.dart
+++ b/pkgs/test_core/lib/src/runner/suite.dart
@@ -85,10 +85,9 @@ class SuiteConfiguration {
final Map<PlatformSelector, SuiteConfiguration> onPlatform;
/// The seed with which to shuffle the test order.
- /// Default value is 0 if not provided and will not change the test order.
+ /// Default value is null if not provided and will not change the test order.
/// The same seed will shuffle the tests in the same way every time.
- int get testRandomizeOrderingSeed => _testRandomizeOrderingSeed ?? 0;
- final int _testRandomizeOrderingSeed;
+ final int testRandomizeOrderingSeed;
/// The global test metadata derived from this configuration.
Metadata get metadata {
@@ -202,7 +201,7 @@ class SuiteConfiguration {
excludeTags = excludeTags ?? BooleanSelector.none,
tags = _map(tags),
onPlatform = _map(onPlatform),
- _testRandomizeOrderingSeed = testRandomizeOrderingSeed,
+ testRandomizeOrderingSeed = testRandomizeOrderingSeed,
_metadata = metadata ?? Metadata.empty;
/// Creates a new [SuiteConfiguration] that takes its configuration from
@@ -252,7 +251,7 @@ class SuiteConfiguration {
tags: _mergeConfigMaps(tags, other.tags),
onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform),
testRandomizeOrderingSeed:
- other._testRandomizeOrderingSeed ?? _testRandomizeOrderingSeed,
+ other.testRandomizeOrderingSeed ?? testRandomizeOrderingSeed,
metadata: metadata.merge(other.metadata));
return config._resolveTags();
}
@@ -295,7 +294,7 @@ class SuiteConfiguration {
tags: tags ?? this.tags,
onPlatform: onPlatform ?? this.onPlatform,
testRandomizeOrderingSeed:
- testRandomizeOrderingSeed ?? _testRandomizeOrderingSeed,
+ testRandomizeOrderingSeed ?? testRandomizeOrderingSeed,
metadata: _metadata.change(
timeout: timeout,
verboseTrace: verboseTrace,
--
GitLab