From fe0b4ce81fdcad810ecd07855afb0893b1257dcd Mon Sep 17 00:00:00 2001
From: Natalie Weizenbaum <nweiz@google.com>
Date: Wed, 3 Feb 2016 16:07:30 -0800
Subject: [PATCH] Add basic support for a configuration file.
This just loads a subset of the config that can be pased on the command line
from the configuration file. The command line takes precedence over the config.
See #46
R=kevmoo@google.com
Review URL: https://codereview.chromium.org//1649663003 .
---
README.md | 26 ++
dart_test.yaml | 2 +
doc/package_config.md | 116 +++++++
lib/src/executable.dart | 20 ++
lib/src/runner/configuration.dart | 302 ++++++------------
lib/src/runner/configuration/args.dart | 177 ++++++++++
lib/src/runner/configuration/load.dart | 156 +++++++++
lib/src/runner/configuration/values.dart | 22 ++
lib/src/util/io.dart | 6 +-
lib/src/utils.dart | 7 +
test/io.dart | 10 +-
.../configuration/configuration_test.dart | 213 ++++++++++++
.../configuration/top_level_error_test.dart | 161 ++++++++++
test/runner/configuration/top_level_test.dart | 234 ++++++++++++++
test/runner/pub_serve_test.dart | 2 -
test/runner/runner_test.dart | 4 +-
16 files changed, 1248 insertions(+), 210 deletions(-)
create mode 100644 dart_test.yaml
create mode 100644 doc/package_config.md
create mode 100644 lib/src/runner/configuration/args.dart
create mode 100644 lib/src/runner/configuration/load.dart
create mode 100644 lib/src/runner/configuration/values.dart
create mode 100644 test/runner/configuration/configuration_test.dart
create mode 100644 test/runner/configuration/top_level_error_test.dart
create mode 100644 test/runner/configuration/top_level_test.dart
diff --git a/README.md b/README.md
index beda9ef1..cabbd1f7 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,7 @@
* [Skipping Tests](#skipping-tests)
* [Timeouts](#timeouts)
* [Platform-Specific Configuration](#platform-specific-configuration)
+ * [Whole-Package Configuration](#whole-package-configuration)
* [Debugging](#debugging)
* [Testing with `barback`](#testing-with-barback)
* [Further Reading](#further-reading)
@@ -486,6 +487,31 @@ If multiple platforms match, the configuration is applied in order from first to
last, just as they would in nested groups. This means that for configuration
like duration-based timeouts, the last matching value wins.
+### Whole-Package Configuration
+
+For configuration that applies across multiple files, or even the entire
+package, `test` supports a configuration file called `dart_test.yaml`. At its
+simplest, this file can contain the same sort of configuration that can be
+passed as command-line arguments:
+
+```yaml
+# This package's tests are very slow. Double the default timeout.
+timeout: 2x
+
+# This is a browser-only package, so test on content shell by default.
+platforms: [content-shell]
+```
+
+The configuration file sets new defaults. These defaults can still be overridden
+by command-line arguments, just like the built-in defaults. In the example
+above, you could pass `--platform chrome` to run on Chrome instead of the
+Dartium content shell.
+
+A configuration file can do much more than just set global defaults. See
+[the full documentation][package config] for more details.
+
+[package config]: https://github.com/dart-lang/test/blob/master/doc/package_config.md
+
## Debugging
Tests can be debugged interactively using browsers' built-in development tools,
diff --git a/dart_test.yaml b/dart_test.yaml
new file mode 100644
index 00000000..1df83980
--- /dev/null
+++ b/dart_test.yaml
@@ -0,0 +1,2 @@
+# Terse traces hide frames from test itself, which is bad.
+verbose_trace: true
diff --git a/doc/package_config.md b/doc/package_config.md
new file mode 100644
index 00000000..b0fb74e4
--- /dev/null
+++ b/doc/package_config.md
@@ -0,0 +1,116 @@
+Each package may include a configuration file that applies to the package as a
+whole. This file can be used to provide custom defaults for various options, to
+define configuration for multiple files, and more.
+
+The file is named `dart_test.yaml` and lives at the root of the package, next to
+the package's pubspec. Like the pubspec, it's a [YAML][] file. Here's an
+example:
+
+[YAML]: http://yaml.org/
+
+```yaml
+# This package's tests are very slow. Double the default timeout.
+timeout: 2x
+
+# This is a browser-only package, so test on content shell by default.
+platforms: [content-shell]
+```
+
+* [Config Fields](#config-fields)
+
+## Config Fields
+
+These fields directly affect the behavior of the test runner. They go at the
+root of the configuration file.
+
+### `platforms`
+
+This field indicates which platforms tests should run on by default. It allows
+the same paltform identifiers that can be passed to `--platform`. If multiple
+platforms are included, the test runner will default to running tests on all of
+them. This defaults to `[vm]`.
+
+```yaml
+platforms: [content_shell]
+
+platforms:
+- chrome
+- firefox
+```
+
+### `concurrency`
+
+This field indicates the default number of test suites to run in parallel. More
+parallelism can improve the overall speed of running tests up to a point, but
+eventually it just adds more memory overhead without any performance gain. This
+defaults to approximately half the number of processors on the current machine.
+If it's set to 1, only one test suite will run at a time.
+
+```yaml
+concurrency: 3
+```
+
+### `pub_serve`
+
+This field indicates that the test runner should run against a `pub serve`
+instance by default, and provides the port number for that instance. Note that
+if there is no `pub serve` instance running at that port, running the tests will
+fail by default.
+
+```yaml
+pub_serve: 8081
+```
+
+### `timeout`
+
+This field indicates how much time the test runner should allow a test to remain
+inactive before it considers that test to have failed. It has three possible
+formats:
+
+* The string "none" indicates that tests should never time out.
+
+* A number followed by a unit abbreviation indicates an exact time. For example,
+ "1m" means a timeout of one minute, and "30s" means a timeout of thirty
+ seconds. Multiple numbers can be combined, as in "1m 30s".
+
+* A number followed by "x" indicates a multiple. This is applied to the default
+ value of 30s.
+
+```yaml
+timeout: 1m
+```
+
+### `reporter`
+
+This field indicates the default reporter to use. It may be set to "compact",
+"expanded", or "json" (although why anyone would want to default to JSON is
+beyond me). It defaults to "expanded" on Windows and "compact" everywhere else.
+
+```yaml
+reporter: expanded
+```
+
+### `verbose_trace`
+
+This boolean field controls whether or not stack traces caused by errors are
+trimmed to remove internal stack frames. This includes frames from the Dart core
+libraries, the [`stack_trace`][stack_trace] package, and the `test` package
+itself. It defaults to `false`.
+
+[stack_trace]: https://pub.dartlang.org/packages/stack_trace
+
+```yaml
+verbose_trace: true
+```
+
+### `js_trace`
+
+This boolean field controls whether or not stack traces caused by errors that
+occur while running Dart compiled to JS are converted back to Dart style. This
+conversion uses the source map generated by `dart2js` to approximate the
+original Dart line, column, and in some cases member name for each stack frame.
+It defaults to `false`.
+
+```yaml
+js_trace: true
+```
diff --git a/lib/src/executable.dart b/lib/src/executable.dart
index fa9e2f19..8dd546ec 100644
--- a/lib/src/executable.dart
+++ b/lib/src/executable.dart
@@ -7,6 +7,7 @@
// bin.
import 'dart:io';
+import 'package:source_span/source_span.dart';
import 'package:stack_trace/stack_trace.dart';
import 'package:yaml/yaml.dart';
@@ -84,6 +85,25 @@ main(List<String> args) async {
return;
}
+ try {
+ if (new File("dart_test.yaml").existsSync()) {
+ var fileConfiguration = new Configuration.load("dart_test.yaml");
+ configuration = fileConfiguration.merge(configuration);
+ }
+ } on SourceSpanFormatException catch (error) {
+ stderr.writeln(error.toString(color: configuration.color));
+ exitCode = exit_codes.data;
+ return;
+ } on FormatException catch (error) {
+ stderr.writeln(error.message);
+ exitCode = exit_codes.data;
+ return;
+ } on IOException catch (error) {
+ stderr.writeln(error.toString());
+ exitCode = exit_codes.noInput;
+ return;
+ }
+
if (configuration.pubServeUrl != null && !_usesTransformer) {
stderr.write('''
When using --pub-serve, you must include the "test/pub_serve" transformer in
diff --git a/lib/src/runner/configuration.dart b/lib/src/runner/configuration.dart
index b9e84945..28f42788 100644
--- a/lib/src/runner/configuration.dart
+++ b/lib/src/runner/configuration.dart
@@ -3,165 +3,99 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
-import 'dart:math' as math;
-import 'package:args/args.dart';
import 'package:path/path.dart' as p;
import '../frontend/timeout.dart';
import '../backend/metadata.dart';
import '../backend/test_platform.dart';
-import '../utils.dart';
import '../util/io.dart';
-
-/// The default number of test suites to run at once.
-///
-/// This defaults to half the available processors, since presumably some of
-/// them will be used for the OS and other processes.
-final _defaultConcurrency = math.max(1, Platform.numberOfProcessors ~/ 2);
+import 'configuration/args.dart' as args;
+import 'configuration/load.dart';
+import 'configuration/values.dart';
/// A class that encapsulates the command-line configuration of the test runner.
class Configuration {
- /// The parser used to parse the command-line arguments.
- static final ArgParser _parser = (() {
- var parser = new ArgParser(allowTrailingOptions: true);
-
- var allPlatforms = TestPlatform.all.toList();
- if (!Platform.isMacOS) allPlatforms.remove(TestPlatform.safari);
- if (!Platform.isWindows) allPlatforms.remove(TestPlatform.internetExplorer);
-
- parser.addFlag("help", abbr: "h", negatable: false,
- help: "Shows this usage information.");
- parser.addFlag("version", negatable: false,
- help: "Shows the package's version.");
- parser.addOption("package-root", hide: true);
-
- parser.addSeparator("======== Selecting Tests");
- parser.addOption("name",
- abbr: 'n',
- help: 'A substring of the name of the test to run.\n'
- 'Regular expression syntax is supported.');
- parser.addOption("plain-name",
- abbr: 'N',
- help: 'A plain-text substring of the name of the test to run.');
- // TODO(nweiz): Support the full platform-selector syntax for choosing which
- // tags to run. In the shorter term, disallow non-"identifier" tags.
- parser.addOption("tags",
- abbr: 't',
- help: 'Run only tests with all of the specified tags.',
- allowMultiple: true);
- parser.addOption("tag", hide: true, allowMultiple: true);
- parser.addOption("exclude-tags",
- abbr: 'x',
- help: "Don't run tests with any of the specified tags.",
- allowMultiple: true);
- parser.addOption("exclude-tag", hide: true, allowMultiple: true);
-
- parser.addSeparator("======== Running Tests");
- parser.addOption("platform",
- abbr: 'p',
- help: 'The platform(s) on which to run the tests.',
- allowed: allPlatforms.map((platform) => platform.identifier).toList(),
- defaultsTo: 'vm',
- allowMultiple: true);
- parser.addOption("concurrency",
- abbr: 'j',
- help: 'The number of concurrent test suites run.\n'
- '(defaults to $_defaultConcurrency)',
- valueHelp: 'threads');
- parser.addOption("pub-serve",
- help: 'The port of a pub serve instance serving "test/".',
- valueHelp: 'port');
-
- // Note: although we list the 30s default timeout as though it were a
- // default value for this argument, it's actually encoded in the [Invoker]'s
- // call to [Timeout.apply].
- parser.addOption("timeout",
- help: 'The default test timeout. For example: 15s, 2x, none\n'
- '(defaults to 30s)');
- parser.addFlag("pause-after-load",
- help: 'Pauses for debugging before any tests execute.\n'
- 'Implies --concurrency=1 and --timeout=none.\n'
- 'Currently only supported for browser tests.',
- negatable: false);
-
- parser.addSeparator("======== Output");
- parser.addOption("reporter",
- abbr: 'r',
- help: 'The runner used to print test results.',
- allowed: ['compact', 'expanded', 'json'],
- defaultsTo: Platform.isWindows ? 'expanded' : 'compact',
- allowedHelp: {
- 'compact': 'A single line, updated continuously.',
- 'expanded': 'A separate line for each update.',
- 'json': 'A machine-readable format (see https://goo.gl/0HRhdZ).'
- });
- parser.addFlag("verbose-trace", negatable: false,
- help: 'Whether to emit stack traces with core library frames.');
- parser.addFlag("js-trace", negatable: false,
- help: 'Whether to emit raw JavaScript stack traces for browser tests.');
- parser.addFlag("color", defaultsTo: null,
- help: 'Whether to use terminal colors.\n(auto-detected by default)');
-
- return parser;
- })();
-
/// The usage string for the command-line arguments.
- static String get usage => _parser.usage;
+ static String get usage => args.usage;
/// Whether `--help` was passed.
- final bool help;
+ bool get help => _help ?? false;
+ final bool _help;
/// Whether `--version` was passed.
- final bool version;
+ bool get version => _version ?? false;
+ final bool _version;
/// Whether stack traces should be presented as-is or folded to remove
/// irrelevant packages.
- final bool verboseTrace;
+ bool get verboseTrace => _verboseTrace ?? false;
+ final bool _verboseTrace;
/// Whether JavaScript stack traces should be left as-is or converted to
/// Dart-like traces.
- final bool jsTrace;
+ bool get jsTrace => _jsTrace ?? false;
+ final bool _jsTrace;
/// Whether to pause for debugging after loading each test suite.
- final bool pauseAfterLoad;
+ bool get pauseAfterLoad => _pauseAfterLoad ?? false;
+ final bool _pauseAfterLoad;
/// The package root for resolving "package:" URLs.
- final String packageRoot;
+ String get packageRoot => _packageRoot ?? p.join(p.current, 'packages');
+ final String _packageRoot;
/// The name of the reporter to use to display results.
- final String reporter;
+ String get reporter => _reporter ?? defaultReporter;
+ final String _reporter;
/// The URL for the `pub serve` instance from which to load tests, or `null`
/// if tests should be loaded from the filesystem.
final Uri pubServeUrl;
/// The default test timeout.
+ ///
+ /// When [merge]d, this combines with the other configuration's timeout using
+ /// [Timeout.merge].
final Timeout timeout;
/// Whether to use command-line color escapes.
- final bool color;
+ bool get color => _color ?? canUseSpecialChars;
+ final bool _color;
/// How many tests to run concurrently.
- final int concurrency;
+ int get concurrency =>
+ pauseAfterLoad ? 1 : (_concurrency ?? defaultConcurrency);
+ final int _concurrency;
- /// The from which to load tests.
- final List<String> paths;
+ /// The paths from which to load tests.
+ List<String> get paths => _paths ?? ["test"];
+ final List<String> _paths;
/// Whether the load paths were passed explicitly or the default was used.
- final bool explicitPaths;
+ bool get explicitPaths => _paths != null;
/// The pattern to match against test names to decide which to run, or `null`
/// if all tests should be run.
final Pattern pattern;
/// The set of platforms on which to run tests.
- final List<TestPlatform> platforms;
+ List<TestPlatform> get platforms => _platforms ?? [TestPlatform.vm];
+ final List<TestPlatform> _platforms;
- /// Restricts the set of tests to a set of tags
+ /// Restricts the set of tests to a set of tags.
+ ///
+ /// If this is empty, it applies no restrictions.
+ ///
+ /// When [merge]d, this is unioned with the other configuration's tags.
final Set<String> tags;
- /// Does not run tests with tags from this set
+ /// Does not run tests with tags from this set.
+ ///
+ /// If this is empty, it applies no restrictions.
+ ///
+ /// When [merge]d, this is unioned with the other configuration's excluded
+ /// tags.
final Set<String> excludeTags;
/// The global test metadata derived from this configuration.
@@ -171,100 +105,70 @@ class Configuration {
/// Parses the configuration from [args].
///
/// Throws a [FormatException] if [args] are invalid.
- factory Configuration.parse(List<String> args) {
- var options = _parser.parse(args);
-
- var pattern;
- if (options['name'] != null) {
- if (options["plain-name"] != null) {
- throw new FormatException(
- "--name and --plain-name may not both be passed.");
- }
+ factory Configuration.parse(List<String> arguments) => args.parse(arguments);
- pattern = _wrapFormatException(
- options, 'name', (value) => new RegExp(value));
- } else if (options['plain-name'] != null) {
- pattern = options['plain-name'];
- }
-
- var tags = new Set();
- tags.addAll(options['tags'] ?? []);
- tags.addAll(options['tag'] ?? []);
-
- var excludeTags = new Set();
- excludeTags.addAll(options['exclude-tags'] ?? []);
- excludeTags.addAll(options['exclude-tag'] ?? []);
-
- var tagIntersection = tags.intersection(excludeTags);
- if (tagIntersection.isNotEmpty) {
- throw new FormatException(
- 'The ${pluralize('tag', tagIntersection.length)} '
- '${toSentence(tagIntersection)} '
- '${pluralize('was', tagIntersection.length, plural: 'were')} '
- 'both included and excluded.');
- }
-
- return new Configuration(
- help: options['help'],
- version: options['version'],
- verboseTrace: options['verbose-trace'],
- jsTrace: options['js-trace'],
- pauseAfterLoad: options['pause-after-load'],
- color: options['color'],
- packageRoot: options['package-root'],
- reporter: options['reporter'],
- pubServePort: _wrapFormatException(options, 'pub-serve', int.parse),
- concurrency: _wrapFormatException(options, 'concurrency', int.parse,
- orElse: () => _defaultConcurrency),
- timeout: _wrapFormatException(options, 'timeout',
- (value) => new Timeout.parse(value),
- orElse: () => new Timeout.factor(1)),
- pattern: pattern,
- platforms: options['platform'].map(TestPlatform.find),
- paths: options.rest.isEmpty ? null : options.rest,
- tags: tags,
- excludeTags: excludeTags);
- }
-
- /// Runs [parse] on the value of the option [name], and wraps any
- /// [FormatException] it throws with additional information.
- static _wrapFormatException(ArgResults options, String name, parse(value),
- {orElse()}) {
- var value = options[name];
- if (value == null) return orElse == null ? null : orElse();
-
- try {
- return parse(value);
- } on FormatException catch (error) {
- throw new FormatException('Couldn\'t parse --$name "${options[name]}": '
- '${error.message}');
- }
- }
-
- Configuration({this.help: false, this.version: false,
- this.verboseTrace: false, this.jsTrace: false,
- bool pauseAfterLoad: false, bool color, String packageRoot,
- String reporter, int pubServePort, int concurrency, Timeout timeout,
- this.pattern, Iterable<TestPlatform> platforms,
- Iterable<String> paths, Set<String> tags, Set<String> excludeTags})
- : pauseAfterLoad = pauseAfterLoad,
- color = color == null ? canUseSpecialChars : color,
- packageRoot = packageRoot == null
- ? p.join(p.current, 'packages')
- : packageRoot,
- reporter = reporter == null ? 'compact' : reporter,
+ /// Loads the configuration from [path].
+ ///
+ /// Throws an [IOException] if [path] does not exist or cannot be read. Throws
+ /// a [FormatException] if its contents are invalid.
+ factory Configuration.load(String path) => load(path);
+
+ Configuration({bool help, bool version, bool verboseTrace, bool jsTrace,
+ bool pauseAfterLoad, bool color, String packageRoot, String reporter,
+ int pubServePort, int concurrency, Timeout timeout, this.pattern,
+ Iterable<TestPlatform> platforms, Iterable<String> paths,
+ Iterable<String> tags, Iterable<String> excludeTags})
+ : _help = help,
+ _version = version,
+ _verboseTrace = verboseTrace,
+ _jsTrace = jsTrace,
+ _pauseAfterLoad = pauseAfterLoad,
+ _color = color,
+ _packageRoot = packageRoot,
+ _reporter = reporter,
pubServeUrl = pubServePort == null
? null
: Uri.parse("http://localhost:$pubServePort"),
- concurrency = pauseAfterLoad
- ? 1
- : (concurrency == null ? _defaultConcurrency : concurrency),
- timeout = pauseAfterLoad
+ _concurrency = concurrency,
+ timeout = (pauseAfterLoad ?? false)
? Timeout.none
: (timeout == null ? new Timeout.factor(1) : timeout),
- platforms = platforms == null ? [TestPlatform.vm] : platforms.toList(),
- paths = paths == null ? ["test"] : paths.toList(),
- explicitPaths = paths != null,
- this.tags = tags,
- this.excludeTags = excludeTags;
+ _platforms = _list(platforms),
+ _paths = _list(paths),
+ tags = tags?.toSet() ?? new Set(),
+ excludeTags = excludeTags?.toSet() ?? new Set();
+
+ /// Returns a [input] as a list or `null`.
+ ///
+ /// If [input] is `null` or empty, this returns `null`. Otherwise, it returns
+ /// `input.toList()`.
+ static List _list(Iterable input) {
+ if (input == null) return null;
+ input = input.toList();
+ if (input.isEmpty) return null;
+ return input;
+ }
+
+ /// Merges this with [other].
+ ///
+ /// For most fields, if both configurations have values set, [other]'s value
+ /// takes precedence. However, certain fields are merged together instead.
+ /// This is indicated in those fields' documentation.
+ Configuration merge(Configuration other) => new Configuration(
+ help: other._help ?? _help,
+ version: other._version ?? _version,
+ verboseTrace: other._verboseTrace ?? _verboseTrace,
+ jsTrace: other._jsTrace ?? _jsTrace,
+ pauseAfterLoad: other._pauseAfterLoad ?? _pauseAfterLoad,
+ color: other._color ?? _color,
+ packageRoot: other._packageRoot ?? _packageRoot,
+ reporter: other._reporter ?? _reporter,
+ pubServePort: (other.pubServeUrl ?? pubServeUrl)?.port,
+ concurrency: other._concurrency ?? _concurrency,
+ timeout: timeout.merge(other.timeout),
+ pattern: other.pattern ?? pattern,
+ platforms: other._platforms ?? _platforms,
+ paths: other._paths ?? _paths,
+ tags: other.tags.union(tags),
+ excludeTags: other.excludeTags.union(excludeTags));
}
diff --git a/lib/src/runner/configuration/args.dart b/lib/src/runner/configuration/args.dart
new file mode 100644
index 00000000..47df545a
--- /dev/null
+++ b/lib/src/runner/configuration/args.dart
@@ -0,0 +1,177 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+
+import '../../frontend/timeout.dart';
+import '../../backend/test_platform.dart';
+import '../../utils.dart';
+import '../configuration.dart';
+import 'values.dart';
+
+/// The parser used to parse the command-line arguments.
+final ArgParser _parser = (() {
+ var parser = new ArgParser(allowTrailingOptions: true);
+
+ var allPlatforms = TestPlatform.all.toList();
+ if (!Platform.isMacOS) allPlatforms.remove(TestPlatform.safari);
+ if (!Platform.isWindows) allPlatforms.remove(TestPlatform.internetExplorer);
+
+ parser.addFlag("help", abbr: "h", negatable: false,
+ help: "Shows this usage information.");
+ parser.addFlag("version", negatable: false,
+ help: "Shows the package's version.");
+ parser.addOption("package-root", hide: true);
+
+ // Note that defaultsTo declarations here are only for documentation purposes.
+ // We pass null values rather than defaults to [new Configuration] so that it
+ // merges properly with the config file.
+
+ parser.addSeparator("======== Selecting Tests");
+ parser.addOption("name",
+ abbr: 'n',
+ help: 'A substring of the name of the test to run.\n'
+ 'Regular expression syntax is supported.');
+ parser.addOption("plain-name",
+ abbr: 'N',
+ help: 'A plain-text substring of the name of the test to run.');
+ // TODO(nweiz): Support the full platform-selector syntax for choosing which
+ // tags to run. In the shorter term, disallow non-"identifier" tags.
+ parser.addOption("tags",
+ abbr: 't',
+ help: 'Run only tests with all of the specified tags.',
+ allowMultiple: true);
+ parser.addOption("tag", hide: true, allowMultiple: true);
+ parser.addOption("exclude-tags",
+ abbr: 'x',
+ help: "Don't run tests with any of the specified tags.",
+ allowMultiple: true);
+ parser.addOption("exclude-tag", hide: true, allowMultiple: true);
+
+ parser.addSeparator("======== Running Tests");
+ parser.addOption("platform",
+ abbr: 'p',
+ help: 'The platform(s) on which to run the tests.',
+ defaultsTo: 'vm',
+ allowed: allPlatforms.map((platform) => platform.identifier).toList(),
+ allowMultiple: true);
+ parser.addOption("concurrency",
+ abbr: 'j',
+ help: 'The number of concurrent test suites run.',
+ defaultsTo: defaultConcurrency.toString(),
+ valueHelp: 'threads');
+ parser.addOption("pub-serve",
+ help: 'The port of a pub serve instance serving "test/".',
+ valueHelp: 'port');
+ parser.addOption("timeout",
+ help: 'The default test timeout. For example: 15s, 2x, none',
+ defaultsTo: '30s');
+ parser.addFlag("pause-after-load",
+ help: 'Pauses for debugging before any tests execute.\n'
+ 'Implies --concurrency=1 and --timeout=none.\n'
+ 'Currently only supported for browser tests.',
+ negatable: false);
+
+ parser.addSeparator("======== Output");
+ parser.addOption("reporter",
+ abbr: 'r',
+ help: 'The runner used to print test results.',
+ defaultsTo: defaultReporter,
+ allowed: allReporters,
+ allowedHelp: {
+ 'compact': 'A single line, updated continuously.',
+ 'expanded': 'A separate line for each update.',
+ 'json': 'A machine-readable format (see https://goo.gl/0HRhdZ).'
+ });
+ parser.addFlag("verbose-trace", negatable: false,
+ help: 'Whether to emit stack traces with core library frames.');
+ parser.addFlag("js-trace", negatable: false,
+ help: 'Whether to emit raw JavaScript stack traces for browser tests.');
+ parser.addFlag("color",
+ help: 'Whether to use terminal colors.\n(auto-detected by default)');
+
+ return parser;
+})();
+
+/// The usage string for the command-line arguments.
+String get usage => _parser.usage;
+
+/// Parses the configuration from [args].
+///
+/// Throws a [FormatException] if [args] are invalid.
+Configuration parse(List<String> args) {
+ var options = _parser.parse(args);
+
+ var pattern;
+ if (options['name'] != null) {
+ if (options["plain-name"] != null) {
+ throw new FormatException(
+ "--name and --plain-name may not both be passed.");
+ }
+
+ pattern = _wrapFormatException(
+ options, 'name', (value) => new RegExp(value));
+ } else if (options['plain-name'] != null) {
+ pattern = options['plain-name'];
+ }
+
+ var tags = new Set();
+ tags.addAll(options['tags'] ?? []);
+ tags.addAll(options['tag'] ?? []);
+
+ var excludeTags = new Set();
+ excludeTags.addAll(options['exclude-tags'] ?? []);
+ excludeTags.addAll(options['exclude-tag'] ?? []);
+
+ var tagIntersection = tags.intersection(excludeTags);
+ if (tagIntersection.isNotEmpty) {
+ throw new FormatException(
+ 'The ${pluralize('tag', tagIntersection.length)} '
+ '${toSentence(tagIntersection)} '
+ '${pluralize('was', tagIntersection.length, plural: 'were')} '
+ 'both included and excluded.');
+ }
+
+ // If the user hasn't explicitly chosen a value, we want to pass null values
+ // to [new Configuration] so that it considers those fields unset when merging
+ // with configuration from the config file.
+ ifParsed(name) => options.wasParsed(name) ? options[name] : null;
+
+ return new Configuration(
+ help: ifParsed('help'),
+ version: ifParsed('version'),
+ verboseTrace: ifParsed('verbose-trace'),
+ jsTrace: ifParsed('js-trace'),
+ pauseAfterLoad: ifParsed('pause-after-load'),
+ color: ifParsed('color'),
+ packageRoot: ifParsed('package-root'),
+ reporter: ifParsed('reporter'),
+ pubServePort: _wrapFormatException(options, 'pub-serve', int.parse),
+ concurrency: _wrapFormatException(options, 'concurrency', int.parse),
+ timeout: _wrapFormatException(options, 'timeout',
+ (value) => new Timeout.parse(value)),
+ pattern: pattern,
+ platforms: ifParsed('platform')?.map(TestPlatform.find),
+ paths: options.rest.isEmpty ? null : options.rest,
+ tags: tags,
+ excludeTags: excludeTags);
+}
+
+/// Runs [parse] on the value of the option [name], and wraps any
+/// [FormatException] it throws with additional information.
+_wrapFormatException(ArgResults options, String name, parse(value)) {
+ if (!options.wasParsed(name)) return null;
+
+ var value = options[name];
+ if (value == null) return null;
+
+ try {
+ return parse(value);
+ } on FormatException catch (error) {
+ throw new FormatException('Couldn\'t parse --$name "${options[name]}": '
+ '${error.message}');
+ }
+}
diff --git a/lib/src/runner/configuration/load.dart b/lib/src/runner/configuration/load.dart
new file mode 100644
index 00000000..d2def184
--- /dev/null
+++ b/lib/src/runner/configuration/load.dart
@@ -0,0 +1,156 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+import 'package:source_span/source_span.dart';
+import 'package:yaml/yaml.dart';
+
+import '../../utils.dart';
+import '../../frontend/timeout.dart';
+import '../../backend/test_platform.dart';
+import '../configuration.dart';
+import 'values.dart';
+
+/// Loads configuration information from a YAML file at [path].
+///
+/// Throws a [FormatException] if the configuration is invalid, and a
+/// [FileSystemException] if it can't be read.
+Configuration load(String path) {
+ var source = new File(path).readAsStringSync();
+ var document = loadYamlNode(source, sourceUrl: p.toUri(path));
+
+ if (document.value == null) return new Configuration();
+
+ if (document is! Map) {
+ throw new SourceSpanFormatException(
+ "The configuration must be a YAML map.", document.span, source);
+ }
+
+ var loader = new _ConfigurationLoader(document, source);
+ return loader.load();
+}
+
+/// A helper for [load] that tracks the YAML document.
+class _ConfigurationLoader {
+ /// The parsed configuration document.
+ final YamlMap _document;
+
+ /// The source string for [_document].
+ ///
+ /// Used for error reporting.
+ final String _source;
+
+ _ConfigurationLoader(this._document, this._source);
+
+ /// Loads the configuration in [_document].
+ Configuration load() {
+ var verboseTrace = _getBool("verbose_trace");
+ var jsTrace = _getBool("js_trace");
+
+ var reporter = _getString("reporter");
+ if (reporter != null && !allReporters.contains(reporter)) {
+ _error('Unknown reporter "$reporter".', "reporter");
+ }
+
+ var pubServePort = _getInt("pub_serve");
+ var concurrency = _getInt("concurrency");
+ var timeout = _parseValue("timeout", (value) => new Timeout.parse(value));
+
+ var allPlatformIdentifiers =
+ TestPlatform.all.map((platform) => platform.identifier).toSet();
+ var platforms = _getList("platforms", (platformNode) {
+ _validate(platformNode, "Platforms must be strings.",
+ (value) => value is String);
+ _validate(platformNode, 'Unknown platform "${platformNode.value}".',
+ allPlatformIdentifiers.contains);
+
+ return TestPlatform.find(platformNode.value);
+ });
+
+ // TODO(nweiz): Add support for using globs to define defaults paths to run.
+
+ return new Configuration(
+ verboseTrace: verboseTrace,
+ jsTrace: jsTrace,
+ reporter: reporter,
+ pubServePort: pubServePort,
+ concurrency: concurrency,
+ timeout: timeout,
+ platforms: platforms);
+ }
+
+ /// Throws an exception with [message] if [test] returns `false` when passed
+ /// [node]'s value.
+ void _validate(YamlNode node, String message, bool test(value)) {
+ if (test(node.value)) return;
+ throw new SourceSpanFormatException(message, node.span, _source);
+ }
+
+ /// Returns the value of the node at [field].
+ ///
+ /// If [typeTest] returns `false` for that value, instead throws an error
+ /// complaining that the field is not a [typeName].
+ _getValue(String field, String typeName, bool typeTest(value)) {
+ var value = _document[field];
+ if (value == null || typeTest(value)) return value;
+ _error("$field must be ${a(typeName)}.", field);
+ }
+
+ /// Returns the YAML node at [field].
+ ///
+ /// If [typeTest] returns `false` for that node's value, instead throws an
+ /// error complaining that the field is not a [typeName].
+ YamlNode _getNode(String field, String typeName, bool typeTest(value)) {
+ var node = _document.nodes[field];
+ if (node == null) return null;
+ _validate(node, "$field must be ${a(typeName)}.", typeTest);
+ return node;
+ }
+
+ /// Asserts that [field] is an int and returns its value.
+ int _getInt(String field) =>
+ _getValue(field, "int", (value) => value is int);
+
+ /// Asserts that [field] is a boolean and returns its value.
+ bool _getBool(String field) =>
+ _getValue(field, "boolean", (value) => value is bool);
+
+ /// Asserts that [field] is a string and returns its value.
+ String _getString(String field) =>
+ _getValue(field, "string", (value) => value is String);
+
+ /// Asserts that [field] is a list and runs [forElement] for each element it
+ /// contains.
+ ///
+ /// Returns a list of values returned by [forElement].
+ List _getList(String field, forElement(YamlNode elementNode)) {
+ var node = _getNode(field, "list", (value) => value is List);
+ if (node == null) return [];
+ return node.nodes.map(forElement).toList();
+ }
+
+ /// Asserts that [field] is a string, passes it to [parse], and returns the
+ /// result.
+ ///
+ /// If [parse] throws a [FormatException], it's wrapped to include [field]'s
+ /// span.
+ _parseValue(String field, parse(value)) {
+ var value = _getString(field);
+ if (value == null) return null;
+
+ try {
+ return parse(value);
+ } on FormatException catch (error) {
+ _error('Invalid $field: ${error.message}', field);
+ }
+ }
+
+ /// Throws a [SourceSpanFormatException] with [message] about [field].
+ void _error(String message, String field) {
+ throw new SourceSpanFormatException(
+ message, _document.nodes[field].span, _source);
+ }
+}
diff --git a/lib/src/runner/configuration/values.dart b/lib/src/runner/configuration/values.dart
new file mode 100644
index 00000000..a266f090
--- /dev/null
+++ b/lib/src/runner/configuration/values.dart
@@ -0,0 +1,22 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:math' as math;
+
+import '../../util/io.dart';
+
+/// The default number of test suites to run at once.
+///
+/// This defaults to half the available processors, since presumably some of
+/// them will be used for the OS and other processes.
+final defaultConcurrency = math.max(1, Platform.numberOfProcessors ~/ 2);
+
+/// The reporters supported by the test runner.
+const allReporters = const ["compact", "expanded", "json"];
+
+/// The default reporter.
+final defaultReporter = inTestTests
+ ? 'expanded'
+ : (Platform.isWindows ? 'expanded' : 'compact');
diff --git a/lib/src/util/io.dart b/lib/src/util/io.dart
index 4a9f6a82..ab408b60 100644
--- a/lib/src/util/io.dart
+++ b/lib/src/util/io.dart
@@ -42,6 +42,9 @@ final OperatingSystem currentOS = (() {
final stdinLines = new StreamQueue(
UTF8.decoder.fuse(const LineSplitter()).bind(stdin));
+/// Whether this is being run as a subprocess in the test package's own tests.
+bool inTestTests = Platform.environment["_DART_TEST_TESTING"] == "true";
+
/// The root directory below which to nest temporary directories created by the
/// test runner.
///
@@ -64,8 +67,7 @@ String libDir({String packageRoot}) {
/// On Windows or when not printing to a terminal, only printable ASCII
/// characters should be used.
bool get canUseSpecialChars =>
- Platform.operatingSystem != 'windows' &&
- Platform.environment["_UNITTEST_USE_COLOR"] != "false";
+ Platform.operatingSystem != 'windows' && !inTestTests;
/// Creates a temporary directory and returns its path.
String createTempDir() =>
diff --git a/lib/src/utils.dart b/lib/src/utils.dart
index 4b554ee5..74675907 100644
--- a/lib/src/utils.dart
+++ b/lib/src/utils.dart
@@ -34,6 +34,9 @@ final lineSplitter = UTF8.decoder.fuse(const LineSplitter());
/// [Object.toString] values contain.
final _exceptionPrefix = new RegExp(r'^([A-Z][a-zA-Z]*)?(Exception|Error): ');
+/// A regular expression matching a single vowel.
+final _vowel = new RegExp('[aeiou]');
+
/// Directories that are specific to OS X.
///
/// This is used to try to distinguish OS X and Linux in [currentOSGuess].
@@ -118,6 +121,10 @@ String pluralize(String name, int number, {String plural}) {
return '${name}s';
}
+/// Returns [noun] with an indefinite article ("a" or "an") added, based on
+/// whether its first letter is a vowel.
+String a(String noun) => noun.startsWith(_vowel) ? "an $noun" : "a $noun";
+
/// Wraps [text] so that it fits within [lineLength], which defaults to 100
/// characters.
///
diff --git a/test/io.dart b/test/io.dart
index b3477430..1db00ed7 100644
--- a/test/io.dart
+++ b/test/io.dart
@@ -102,18 +102,18 @@ StreamMatcher containsInOrder(Iterable<String> strings) =>
ScheduledProcess runTest(List args, {String reporter,
int concurrency, Map<String, String> environment,
bool forwardStdio: false}) {
- reporter ??= "expanded";
concurrency ??= 1;
var allArgs = [
p.absolute(p.join(packageDir, 'bin/test.dart')),
"--package-root=${p.join(packageDir, 'packages')}",
- "--concurrency=$concurrency",
- "--reporter=$reporter"
- ]..addAll(args);
+ "--concurrency=$concurrency"
+ ];
+ if (reporter != null) allArgs.add("--reporter=$reporter");
+ allArgs.addAll(args);
if (environment == null) environment = {};
- environment.putIfAbsent("_UNITTEST_USE_COLOR", () => "false");
+ environment.putIfAbsent("_DART_TEST_TESTING", () => "true");
var process = runDart(allArgs,
environment: environment,
diff --git a/test/runner/configuration/configuration_test.dart b/test/runner/configuration/configuration_test.dart
new file mode 100644
index 00000000..1fa22b54
--- /dev/null
+++ b/test/runner/configuration/configuration_test.dart
@@ -0,0 +1,213 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'package:path/path.dart' as p;
+import 'package:test/test.dart';
+
+import 'package:test/src/backend/test_platform.dart';
+import 'package:test/src/runner/configuration.dart';
+import 'package:test/src/runner/configuration/values.dart';
+import 'package:test/src/util/io.dart';
+
+void main() {
+ group("merge", () {
+ group("for most fields", () {
+ test("if neither is defined, preserves the default", () {
+ var merged = new Configuration().merge(new Configuration());
+ expect(merged.help, isFalse);
+ expect(merged.version, isFalse);
+ expect(merged.verboseTrace, isFalse);
+ expect(merged.jsTrace, isFalse);
+ expect(merged.pauseAfterLoad, isFalse);
+ expect(merged.color, equals(canUseSpecialChars));
+ expect(merged.packageRoot, equals(p.join(p.current, 'packages')));
+ expect(merged.reporter, equals(defaultReporter));
+ expect(merged.pubServeUrl, isNull);
+ expect(merged.pattern, isNull);
+ expect(merged.platforms, equals([TestPlatform.vm]));
+ expect(merged.paths, equals(["test"]));
+ });
+
+ test("if only the old configuration's is defined, uses it", () {
+ var merged = new Configuration(
+ help: true,
+ version: true,
+ verboseTrace: true,
+ jsTrace: true,
+ pauseAfterLoad: true,
+ color: true,
+ packageRoot: "root",
+ reporter: "json",
+ pubServePort: 1234,
+ pattern: "foo",
+ platforms: [TestPlatform.chrome],
+ paths: ["bar"])
+ .merge(new Configuration());
+
+ expect(merged.help, isTrue);
+ expect(merged.version, isTrue);
+ expect(merged.verboseTrace, isTrue);
+ expect(merged.jsTrace, isTrue);
+ expect(merged.pauseAfterLoad, isTrue);
+ expect(merged.color, isTrue);
+ expect(merged.packageRoot, equals("root"));
+ expect(merged.reporter, equals("json"));
+ expect(merged.pubServeUrl.port, equals(1234));
+ expect(merged.pattern, equals("foo"));
+ expect(merged.platforms, equals([TestPlatform.chrome]));
+ expect(merged.paths, equals(["bar"]));
+ });
+
+ test("if only the new configuration's is defined, uses it", () {
+ var merged = new Configuration().merge(new Configuration(
+ help: true,
+ version: true,
+ verboseTrace: true,
+ jsTrace: true,
+ pauseAfterLoad: true,
+ color: true,
+ packageRoot: "root",
+ reporter: "json",
+ pubServePort: 1234,
+ pattern: "foo",
+ platforms: [TestPlatform.chrome],
+ paths: ["bar"]));
+
+ expect(merged.help, isTrue);
+ expect(merged.version, isTrue);
+ expect(merged.verboseTrace, isTrue);
+ expect(merged.jsTrace, isTrue);
+ expect(merged.pauseAfterLoad, isTrue);
+ expect(merged.color, isTrue);
+ expect(merged.packageRoot, equals("root"));
+ expect(merged.reporter, equals("json"));
+ expect(merged.pubServeUrl.port, equals(1234));
+ expect(merged.pattern, equals("foo"));
+ expect(merged.platforms, equals([TestPlatform.chrome]));
+ expect(merged.paths, equals(["bar"]));
+ });
+
+ test("if the two configurations conflict, uses the new configuration's "
+ "values", () {
+ var older = new Configuration(
+ help: true,
+ version: false,
+ verboseTrace: true,
+ jsTrace: false,
+ pauseAfterLoad: true,
+ color: false,
+ packageRoot: "root",
+ reporter: "json",
+ pubServePort: 1234,
+ pattern: "foo",
+ platforms: [TestPlatform.chrome],
+ paths: ["bar"]);
+ var newer = new Configuration(
+ help: false,
+ version: true,
+ verboseTrace: false,
+ jsTrace: true,
+ pauseAfterLoad: false,
+ color: true,
+ packageRoot: "boot",
+ reporter: "compact",
+ pubServePort: 5678,
+ pattern: "gonk",
+ platforms: [TestPlatform.dartium],
+ paths: ["blech"]);
+ var merged = older.merge(newer);
+
+ expect(merged.help, isFalse);
+ expect(merged.version, isTrue);
+ expect(merged.verboseTrace, isFalse);
+ expect(merged.jsTrace, isTrue);
+ expect(merged.pauseAfterLoad, isFalse);
+ expect(merged.color, isTrue);
+ expect(merged.packageRoot, equals("boot"));
+ expect(merged.reporter, equals("compact"));
+ expect(merged.pubServeUrl.port, equals(5678));
+ expect(merged.pattern, equals("gonk"));
+ expect(merged.platforms, equals([TestPlatform.dartium]));
+ expect(merged.paths, equals(["blech"]));
+ });
+ });
+
+ group("for tags", () {
+ test("if neither is defined, preserves the default", () {
+ var merged = new Configuration().merge(new Configuration());
+ expect(merged.tags, isEmpty);
+ expect(merged.excludeTags, isEmpty);
+ });
+
+ test("if only the old configuration's is defined, uses it", () {
+ var merged = new Configuration(
+ tags: ["foo", "bar"],
+ excludeTags: ["baz", "bang"])
+ .merge(new Configuration());
+
+ expect(merged.tags, unorderedEquals(["foo", "bar"]));
+ expect(merged.excludeTags, unorderedEquals(["baz", "bang"]));
+ });
+
+ test("if only the new configuration's is defined, uses it", () {
+ var merged = new Configuration().merge(new Configuration(
+ tags: ["foo", "bar"],
+ excludeTags: ["baz", "bang"]));
+
+ expect(merged.tags, unorderedEquals(["foo", "bar"]));
+ expect(merged.excludeTags, unorderedEquals(["baz", "bang"]));
+ });
+
+ test("if both are defined, unions them", () {
+ var older = new Configuration(
+ tags: ["foo", "bar"],
+ excludeTags: ["baz", "bang"]);
+ var newer = new Configuration(
+ tags: ["bar", "blip"],
+ excludeTags: ["bang", "qux"]);
+ var merged = older.merge(newer);
+
+ expect(merged.tags, unorderedEquals(["foo", "bar", "blip"]));
+ expect(merged.excludeTags, unorderedEquals(["baz", "bang", "qux"]));
+ });
+ });
+
+ group("for timeout", () {
+ test("if neither is defined, preserves the default", () {
+ var merged = new Configuration().merge(new Configuration());
+ expect(merged.timeout, equals(new Timeout.factor(1)));
+ });
+
+ test("if only the old configuration's is defined, uses it", () {
+ var merged = new Configuration(timeout: new Timeout.factor(2))
+ .merge(new Configuration());
+ expect(merged.timeout, equals(new Timeout.factor(2)));
+ });
+
+ test("if only the new configuration's is defined, uses it", () {
+ var merged = new Configuration()
+ .merge(new Configuration(timeout: new Timeout.factor(2)));
+ expect(merged.timeout, equals(new Timeout.factor(2)));
+ });
+
+ test("if both are defined, merges them", () {
+ var older = new Configuration(timeout: new Timeout.factor(2));
+ var newer = new Configuration(timeout: new Timeout.factor(3));
+ var merged = older.merge(newer);
+ expect(merged.timeout, equals(new Timeout.factor(6)));
+ });
+
+ test("if the merge conflicts, prefers the new value", () {
+ var older = new Configuration(
+ timeout: new Timeout(new Duration(seconds: 1)));
+ var newer = new Configuration(
+ timeout: new Timeout(new Duration(seconds: 2)));
+ var merged = older.merge(newer);
+ expect(merged.timeout, equals(new Timeout(new Duration(seconds: 2))));
+ });
+ });
+ });
+}
diff --git a/test/runner/configuration/top_level_error_test.dart b/test/runner/configuration/top_level_error_test.dart
new file mode 100644
index 00000000..84ba460b
--- /dev/null
+++ b/test/runner/configuration/top_level_error_test.dart
@@ -0,0 +1,161 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'dart:convert';
+
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_test.dart';
+import 'package:test/src/util/exit_codes.dart' as exit_codes;
+
+import '../../io.dart';
+
+void main() {
+ useSandbox();
+
+ test("rejects an invalid verbose_trace", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "verbose_trace": "flup"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "verbose_trace must be a boolean",
+ "^^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid js_trace",
+ () {
+ d.file("dart_test.yaml", JSON.encode({
+ "js_trace": "flup"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "js_trace must be a boolean",
+ "^^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid reporter type", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "reporter": 12
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "reporter must be a string",
+ "^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid reporter name", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "reporter": "non-existent"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ 'Unknown reporter "non-existent"',
+ "^^^^^^^^^^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid pub serve port", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "pub_serve": "foo"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "pub_serve must be an int",
+ "^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid concurrency", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "concurrency": "foo"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "concurrency must be an int",
+ "^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid timeout type", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "timeout": 12
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "timeout must be a string",
+ "^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid timeout format", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "timeout": "12p"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "Invalid timeout: expected unit",
+ "^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid platforms list type", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "platforms": "vm"
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "platforms must be a list",
+ "^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid platforms member type", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "platforms": [12]
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ "Platforms must be strings",
+ "^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+
+ test("rejects an invalid platforms member name", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "platforms": ["foo"]
+ })).create();
+
+ var test = runTest(["test.dart"]);
+ test.stderr.expect(containsInOrder([
+ 'Unknown platform "foo"',
+ "^^^^^"
+ ]));
+ test.shouldExit(exit_codes.data);
+ });
+}
diff --git a/test/runner/configuration/top_level_test.dart b/test/runner/configuration/top_level_test.dart
new file mode 100644
index 00000000..25104452
--- /dev/null
+++ b/test/runner/configuration/top_level_test.dart
@@ -0,0 +1,234 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'dart:convert';
+
+import 'package:path/path.dart' as p;
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_stream.dart';
+import 'package:scheduled_test/scheduled_test.dart';
+
+import '../../io.dart';
+
+void main() {
+ useSandbox();
+
+ test("ignores an empty file", () {
+ d.file("dart_test.yaml", "").create();
+
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () {});
+ }
+ """).create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("+1: All tests passed!")));
+ test.shouldExit(0);
+ });
+
+ test("includes the full stack with verbose_trace: true", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "verbose_trace": true
+ })).create();
+
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("failure", () => throw "oh no");
+ }
+ """).create();
+
+ var test = runTest(["test.dart"], reporter: "compact");
+ test.stdout.expect(consumeThrough(contains("dart:isolate-patch")));
+ test.shouldExit(1);
+ });
+
+ test("doesn't dartify stack traces for JS-compiled tests with js_trace: true",
+ () {
+ d.file("dart_test.yaml", JSON.encode({
+ "js_trace": true
+ })).create();
+
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("failure", () => throw "oh no");
+ }
+ """).create();
+
+ var test = runTest(["-p", "chrome", "--verbose-trace", "test.dart"]);
+ test.stdout.fork().expect(never(endsWith(" main.<fn>")));
+ test.stdout.fork().expect(never(contains("package:test")));
+ test.stdout.fork().expect(never(contains("dart:async/zone.dart")));
+ test.stdout.expect(consumeThrough(contains("-1: Some tests failed.")));
+ test.shouldExit(1);
+ });
+
+ test("uses the specified reporter", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "reporter": "json"
+ })).create();
+
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () {});
+ }
+ """).create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains('"testStart"')));
+ test.shouldExit(0);
+ });
+
+ test("uses the specified pub serve port", () {
+ d.file("pubspec.yaml", """
+name: myapp
+dependencies:
+ barback: any
+ test: {path: ${p.current}}
+transformers:
+- myapp:
+ \$include: test/**_test.dart
+- test/pub_serve:
+ \$include: test/**_test.dart
+""").create();
+
+ d.dir("lib", [
+ d.file("myapp.dart", """
+ import 'package:barback/barback.dart';
+
+ class MyTransformer extends Transformer {
+ final allowedExtensions = '.dart';
+
+ MyTransformer.asPlugin();
+
+ Future apply(Transform transform) async {
+ var contents = await transform.primaryInput.readAsString();
+ transform.addOutput(new Asset.fromString(
+ transform.primaryInput.id,
+ contents.replaceAll("isFalse", "isTrue")));
+ }
+ }
+ """)
+ ]).create();
+
+ runPub(['get']).shouldExit(0);
+
+ d.dir("test", [
+ d.file("my_test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () => expect(true, isFalse));
+ }
+ """)
+ ]).create();
+
+ var pub = runPubServe();
+
+ d.async(pubServePort.then((port) {
+ return d.file("dart_test.yaml", JSON.encode({
+ "pub_serve": port
+ }));
+ })).create();
+
+ var test = runTest([]);
+ test.stdout.expect(consumeThrough(contains('+1: All tests passed!')));
+ test.shouldExit(0);
+ pub.kill();
+ });
+
+ test("uses the specified concurrency", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "concurrency": 2
+ })).create();
+
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () {});
+ }
+ """).create();
+
+ // We can't reliably test cthe concurrency, but this at least ensures that
+ // it doesn't fail to parse.
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(consumeThrough(contains("+1: All tests passed!")));
+ test.shouldExit(0);
+ });
+
+ test("uses the specified timeout", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "timeout": "0s"
+ })).create();
+
+ d.file("test.dart", """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () => new Future.delayed(Duration.ZERO));
+ }
+ """).create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(containsInOrder([
+ "Test timed out after 0 seconds.",
+ "-1: Some tests failed."
+ ]));
+ test.shouldExit(1);
+ });
+
+ test("runs on the specified platforms", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "platforms": ["vm", "content-shell"]
+ })).create();
+
+ d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () {});
+ }
+ """).create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(containsInOrder([
+ "[VM] success",
+ "[Dartium Content Shell] success"
+ ]));
+ test.shouldExit(0);
+ });
+
+ test("command line args take precedence", () {
+ d.file("dart_test.yaml", JSON.encode({
+ "timeout": "0s"
+ })).create();
+
+ d.file("test.dart", """
+ import 'dart:async';
+
+ import 'package:test/test.dart';
+
+ void main() {
+ test("success", () => new Future.delayed(Duration.ZERO));
+ }
+ """).create();
+
+ var test = runTest(["--timeout=none", "test.dart"]);
+ test.stdout.expect(consumeThrough(contains("All tests passed!")));
+ test.shouldExit(0);
+ });
+}
diff --git a/test/runner/pub_serve_test.dart b/test/runner/pub_serve_test.dart
index b2e6ad99..4d84f622 100644
--- a/test/runner/pub_serve_test.dart
+++ b/test/runner/pub_serve_test.dart
@@ -31,8 +31,6 @@ transformers:
\$include: test/**_test.dart
- test/pub_serve:
\$include: test/**_test.dart
-dependency_overrides:
- matcher: '0.12.0-alpha.0'
""").create();
d.dir("test", [
diff --git a/test/runner/runner_test.dart b/test/runner/runner_test.dart
index 18ecd570..1268b73a 100644
--- a/test/runner/runner_test.dart
+++ b/test/runner/runner_test.dart
@@ -60,11 +60,11 @@ Usage: pub run test:test [files or directories...]
$_browsers
-j, --concurrency=<threads> The number of concurrent test suites run.
- (defaults to $_defaultConcurrency)
+ (defaults to "$_defaultConcurrency")
--pub-serve=<port> The port of a pub serve instance serving "test/".
--timeout The default test timeout. For example: 15s, 2x, none
- (defaults to 30s)
+ (defaults to "30s")
--pause-after-load Pauses for debugging before any tests execute.
Implies --concurrency=1 and --timeout=none.
--
GitLab