From 533b58ce1de2a3a7fe31635a709b42e88e4588c1 Mon Sep 17 00:00:00 2001
From: Natalie Weizenbaum <nweiz@google.com>
Date: Thu, 19 Feb 2015 12:01:31 -0800
Subject: [PATCH] Add a test runner executable.
This is still extremely bare-bones and won't work with tests that actually
import "package:unittest/unittest.dart", but it's something.
R=kevmoo@google.com
See #2
Review URL: https://codereview.chromium.org//933083002
---
.status | 1 +
bin/unittest.dart | 110 +++++++++++++++++
lib/src/console_reporter.dart | 38 +++++-
lib/src/engine.dart | 5 +
lib/src/exit_codes.dart | 58 +++++++++
lib/src/invoker.dart | 5 +-
lib/src/load_exception.dart | 42 +++++++
lib/src/loader.dart | 28 +++--
lib/src/remote_exception.dart | 55 ++++++---
lib/src/utils.dart | 30 +++++
lib/src/vm_listener.dart | 50 +++++++-
pubspec.yaml | 3 +-
test/console_reporter_test.dart | 179 ++++++++++++++++++++++++++++
test/io.dart | 16 ++-
test/loader_test.dart | 2 +-
test/runner_test.dart | 203 ++++++++++++++++++++++++++++++++
test/utils.dart | 6 +
test/vm_listener_test.dart | 89 ++++++++++++--
18 files changed, 867 insertions(+), 53 deletions(-)
create mode 100644 bin/unittest.dart
create mode 100644 lib/src/exit_codes.dart
create mode 100644 lib/src/load_exception.dart
create mode 100644 test/console_reporter_test.dart
create mode 100644 test/runner_test.dart
diff --git a/.status b/.status
index 0d5889ef..b1e4dbea 100644
--- a/.status
+++ b/.status
@@ -26,6 +26,7 @@ lib/*/*/*/*: SkipByDesign
# dart:io-specific tests.
[ $browser ]
test/loader_test: SkipByDesign
+test/runner_test: SkipByDesign
test/vm_listener_test: SkipByDesign
[ $runtime == safari ]
diff --git a/bin/unittest.dart b/bin/unittest.dart
new file mode 100644
index 00000000..32379360
--- /dev/null
+++ b/bin/unittest.dart
@@ -0,0 +1,110 @@
+// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library unittest.unittest;
+
+import 'dart:async';
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:args/args.dart';
+import 'package:stack_trace/stack_trace.dart';
+
+import 'package:unittest/src/console_reporter.dart';
+import 'package:unittest/src/exit_codes.dart' as exit_codes;
+import 'package:unittest/src/load_exception.dart';
+import 'package:unittest/src/loader.dart';
+import 'package:unittest/src/utils.dart';
+
+/// The argument parser used to parse the executable arguments.
+final _parser = new ArgParser();
+
+void main(List<String> args) {
+ _parser.addFlag("help", abbr: "h", negatable: false,
+ help: "Shows this usage information.");
+ _parser.addOption("package-root", hide: true);
+
+ var options;
+ try {
+ options = _parser.parse(args);
+ } on FormatException catch (error) {
+ _printUsage(error.message);
+ exitCode = exit_codes.usage;
+ return;
+ }
+
+ if (options["help"]) {
+ _printUsage();
+ return;
+ }
+
+ var loader = new Loader(packageRoot: options["package-root"]);
+ new Future.sync(() {
+ var paths = options.rest;
+ if (paths.isEmpty) {
+ if (!new Directory("test").existsSync()) {
+ throw new LoadException("test",
+ "No test files were passed and the default directory doesn't "
+ "exist.");
+ }
+ paths = ["test"];
+ }
+
+ return Future.wait(paths.map((path) {
+ if (new Directory(path).existsSync()) return loader.loadDir(path);
+ if (new File(path).existsSync()) return loader.loadFile(path);
+ throw new LoadException(path, 'Does not exist.');
+ }));
+ }).then((suites) {
+ var reporter = new ConsoleReporter(flatten(suites));
+ return reporter.run().then((success) {
+ exitCode = success ? 0 : 1;
+ }).whenComplete(() => reporter.close());
+ }).catchError((error, stackTrace) {
+ if (error is LoadException) {
+ // TODO(nweiz): color this message?
+ stderr.writeln(getErrorMessage(error));
+
+ // Only print stack traces for load errors that come from the user's
+ if (error.innerError is! IOException &&
+ error.innerError is! IsolateSpawnException &&
+ error.innerError is! String) {
+ stderr.write(terseChain(stackTrace));
+ }
+
+ exitCode = error.innerError is IOException
+ ? exit_codes.io
+ : exit_codes.data;
+ } else {
+ stderr.writeln(getErrorMessage(error));
+ stderr.writeln(new Trace.from(stackTrace).terse);
+ stderr.writeln(
+ "This is an unexpected error. Please file an issue at "
+ "http://github.com/dart-lang/unittest\n"
+ "with the stack trace and instructions for reproducing the error.");
+ exitCode = exit_codes.software;
+ }
+ }).whenComplete(() => loader.close());
+}
+
+/// Print usage information for this command.
+///
+/// If [error] is passed, it's used in place of the usage message and the whole
+/// thing is printed to stderr instead of stdout.
+void _printUsage([String error]) {
+ var output = stdout;
+
+ var message = "Runs tests in this package.";
+ if (error != null) {
+ message = error;
+ output = stderr;
+ }
+
+ output.write("""$message
+
+Usage: pub run unittest:unittest [files or directories...]
+
+${_parser.usage}
+""");
+}
diff --git a/lib/src/console_reporter.dart b/lib/src/console_reporter.dart
index 3faa1b47..7d208e0c 100644
--- a/lib/src/console_reporter.dart
+++ b/lib/src/console_reporter.dart
@@ -49,6 +49,15 @@ class ConsoleReporter {
/// The set of tests that have completed and been marked as failing or error.
final _failed = new Set<LiveTest>();
+ /// The size of [_passed] last time a progress notification was printed.
+ int _lastProgressPassed;
+
+ /// The size of [_failed] last time a progress notification was printed.
+ int _lastProgressFailed;
+
+ /// The message printed for the last progress notification.
+ String _lastProgressMessage;
+
/// Creates a [ConsoleReporter] that will run all tests in [suites].
ConsoleReporter(Iterable<Suite> suites)
: _multipleSuites = suites.length > 1,
@@ -70,11 +79,10 @@ class ConsoleReporter {
liveTest.onError.listen((error) {
if (liveTest.state.status != Status.complete) return;
- // TODO(nweiz): don't re-print the progress line if a test has multiple
- // errors in a row.
_progressLine(_description(liveTest));
print('');
- print(indent("${error.error}\n${error.stackTrace}"));
+ print(indent(error.error.toString()));
+ print(indent(terseChain(error.stackTrace).toString()));
});
});
}
@@ -89,11 +97,14 @@ class ConsoleReporter {
"once.");
}
+ if (_engine.liveTests.isEmpty) {
+ print("No tests ran.");
+ return new Future.value(true);
+ }
+
_stopwatch.start();
return _engine.run().then((success) {
- if (_engine.liveTests.isEmpty) {
- print("\nNo tests ran.");
- } else if (success) {
+ if (success) {
_progressLine("All tests passed!");
print('');
} else {
@@ -105,12 +116,27 @@ class ConsoleReporter {
});
}
+ /// Signals that the caller is done with any test output and the reporter
+ /// should release any resources it has allocated.
+ Future close() => _engine.close();
+
/// Prints a line representing the current state of the tests.
///
/// [message] goes after the progress report, and may be truncated to fit the
/// entire line within [_lineLength]. If [color] is passed, it's used as the
/// color for [message].
void _progressLine(String message, {String color}) {
+ // Print nothing if nothing has changed since the last progress line.
+ if (_passed.length == _lastProgressPassed &&
+ _failed.length == _lastProgressFailed &&
+ message == _lastProgressMessage) {
+ return;
+ }
+
+ _lastProgressPassed = _passed.length;
+ _lastProgressFailed = _failed.length;
+ _lastProgressMessage = message;
+
if (color == null) color = '';
var duration = _stopwatch.elapsed;
var buffer = new StringBuffer();
diff --git a/lib/src/engine.dart b/lib/src/engine.dart
index 7bd69588..5e2b6508 100644
--- a/lib/src/engine.dart
+++ b/lib/src/engine.dart
@@ -64,4 +64,9 @@ class Engine {
}).then((_) =>
liveTests.every((liveTest) => liveTest.state.result == Result.success));
}
+
+ /// Signals that the caller is done paying attention to test results and the
+ /// engine should release any resources it has allocated.
+ Future close() =>
+ Future.wait(liveTests.map((liveTest) => liveTest.close()));
}
diff --git a/lib/src/exit_codes.dart b/lib/src/exit_codes.dart
new file mode 100644
index 00000000..ce39d1e3
--- /dev/null
+++ b/lib/src/exit_codes.dart
@@ -0,0 +1,58 @@
+// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// Exit code constants.
+///
+/// From [the BSD sysexits manpage][manpage]. Not every constant here is used.
+///
+/// [manpage]: http://www.freebsd.org/cgi/man.cgi?query=sysexits
+library unittest.exit_codes;
+
+/// The command completely successfully.
+const success = 0;
+
+/// The command was used incorrectly.
+const usage = 64;
+
+/// The input data was incorrect.
+const data = 65;
+
+/// An input file did not exist or was unreadable.
+const noInput = 66;
+
+/// The user specified did not exist.
+const noUser = 67;
+
+/// The host specified did not exist.
+const noHost = 68;
+
+/// A service is unavailable.
+const unavailable = 69;
+
+/// An internal software error has been detected.
+const software = 70;
+
+/// An operating system error has been detected.
+const os = 71;
+
+/// Some system file did not exist or was unreadable.
+const osFile = 72;
+
+/// A user-specified output file cannot be created.
+const cantCreate = 73;
+
+/// An error occurred while doing I/O on some file.
+const io = 74;
+
+/// Temporary failure, indicating something that is not really an error.
+const tempFail = 75;
+
+/// The remote system returned something invalid during a protocol exchange.
+const protocol = 76;
+
+/// The user did not have sufficient permissions.
+const noPerm = 77;
+
+/// Something was unconfigured or mis-configured.
+const config = 78;
diff --git a/lib/src/invoker.dart b/lib/src/invoker.dart
index 4d14b48d..edc6d622 100644
--- a/lib/src/invoker.dart
+++ b/lib/src/invoker.dart
@@ -172,7 +172,10 @@ class Invoker {
timer.cancel();
_controller.setState(
new State(Status.complete, liveTest.state.result));
- _controller.completer.complete();
+
+ // Use [Timer.run] here to avoid starving the DOM or other
+ // non-microtask events.
+ Timer.run(_controller.completer.complete);
});
}, zoneValues: {#unittest.invoker: this}, onError: handleError);
});
diff --git a/lib/src/load_exception.dart b/lib/src/load_exception.dart
new file mode 100644
index 00000000..8c6dff0d
--- /dev/null
+++ b/lib/src/load_exception.dart
@@ -0,0 +1,42 @@
+// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library unittest.load_exception;
+
+import 'dart:isolate';
+
+import 'package:path/path.dart' as p;
+
+import 'utils.dart';
+
+class LoadException implements Exception {
+ final String path;
+
+ final innerError;
+
+ LoadException(this.path, this.innerError);
+
+ String toString() {
+ var buffer = new StringBuffer('Failed to load "$path":');
+
+ var innerString = getErrorMessage(innerError);
+ if (innerError is IsolateSpawnException) {
+ // If this is a parse error, get rid of the noisy preamble.
+ innerString = innerString
+ .replaceFirst("'${p.toUri(p.absolute(path))}': error: ", "");
+
+ // If this is a file system error, get rid of both the preamble and the
+ // useless stack trace.
+ innerString = innerString.replaceFirst(
+ "Unhandled exception:\n"
+ "Uncaught Error: Load Error: FileSystemException: ",
+ "");
+ innerString = innerString.split("Stack Trace:\n").first.trim();
+ }
+
+ buffer.write(innerString.contains("\n") ? "\n" : " ");
+ buffer.write(innerString);
+ return buffer.toString();
+ }
+}
diff --git a/lib/src/loader.dart b/lib/src/loader.dart
index e5b843c5..2df9bfd1 100644
--- a/lib/src/loader.dart
+++ b/lib/src/loader.dart
@@ -12,6 +12,8 @@ import 'package:path/path.dart' as p;
import 'dart.dart';
import 'isolate_test.dart';
+import 'load_exception.dart';
+import 'remote_exception.dart';
import 'suite.dart';
/// A class for finding test files and loading them into a runnable form.
@@ -49,9 +51,7 @@ class Loader {
/// Loads a test suite from the file at [path].
///
- /// This wil throw a [FileSystemException] if there's no `packages/` directory
- /// available for [path]. Any other load error will cause an
- /// [IsolateSpawnException] or a [RemoteException].
+ /// This will throw a [LoadException] if the file fails to load.
Future<Suite> loadFile(String path) {
// TODO(nweiz): Support browser tests.
var packageRoot = _packageRoot == null
@@ -59,7 +59,7 @@ class Loader {
: _packageRoot;
if (!new Directory(packageRoot).existsSync()) {
- throw new FileSystemException("Directory $packageRoot does not exist.");
+ throw new LoadException(path, "Directory $packageRoot does not exist.");
}
var receivePort = new ReceivePort();
@@ -70,15 +70,27 @@ import "${p.toUri(p.absolute(path))}" as test;
void main(_, Map message) {
var sendPort = message['reply'];
- VmListener.start(sendPort, test.main);
+ VmListener.start(sendPort, () => test.main);
}
''', {
'reply': receivePort.sendPort
- }, packageRoot: packageRoot).then((isolate) {
+ }, packageRoot: packageRoot).catchError((error, stackTrace) {
+ receivePort.close();
+ return new Future.error(new LoadException(path, error), stackTrace);
+ }).then((isolate) {
_isolates.add(isolate);
return receivePort.first;
- }).then((tests) {
- return new Suite(path, tests.map((test) {
+ }).then((response) {
+ if (response["type"] == "loadException") {
+ return new Future.error(new LoadException(path, response["message"]));
+ } else if (response["type"] == "error") {
+ var asyncError = RemoteException.deserialize(response["error"]);
+ return new Future.error(
+ new LoadException(path, asyncError.error),
+ asyncError.stackTrace);
+ }
+
+ return new Suite(path, response["tests"].map((test) {
return new IsolateTest(test['name'], test['sendPort']);
}));
});
diff --git a/lib/src/remote_exception.dart b/lib/src/remote_exception.dart
index ba62cf71..683b19a1 100644
--- a/lib/src/remote_exception.dart
+++ b/lib/src/remote_exception.dart
@@ -5,6 +5,7 @@
library unittest.remote_exception;
import 'dart:async';
+import 'dart:isolate';
import 'package:stack_trace/stack_trace.dart';
@@ -43,10 +44,20 @@ class RemoteException implements Exception {
}
}
+ // It's possible (although unlikely) for a user-defined class to have
+ // multiple of these supertypes. That's fine, though, since we only care
+ // about core-library-raised IsolateSpawnExceptions anyway.
+ var supertype;
+ if (error is TestFailure) {
+ supertype = 'TestFailure';
+ } else if (error is IsolateSpawnException) {
+ supertype = 'IsolateSpawnException';
+ }
+
return {
'message': message,
'type': error.runtimeType.toString(),
- 'isTestFailure': error is TestFailure,
+ 'supertype': supertype,
'toString': error.toString(),
'stackChain': new Chain.forTrace(stackTrace).toString()
};
@@ -57,20 +68,25 @@ class RemoteException implements Exception {
/// The returned [AsyncError] is guaranteed to have a [RemoteException] as its
/// error and a [Chain] as its stack trace.
static AsyncError deserialize(serialized) {
- var exception;
- if (serialized['isTestFailure']) {
- exception = new RemoteTestFailure._(
- serialized['message'],
- serialized['type'],
- serialized['toString']);
- } else {
- exception = new RemoteException._(
- serialized['message'],
- serialized['type'],
- serialized['toString']);
- }
+ return new AsyncError(
+ _deserializeException(serialized),
+ new Chain.parse(serialized['stackChain']));
+ }
- return new AsyncError(exception, new Chain.parse(serialized['stackChain']));
+ /// Deserializes the exception portion of [serialized].
+ static RemoteException _deserializeException(serialized) {
+ var message = serialized['message'];
+ var type = serialized['type'];
+ var toString = serialized['toString'];
+
+ switch (serialized['supertype']) {
+ case 'TestFailure':
+ return new _RemoteTestFailure(message, type, toString);
+ case 'IsolateSpawnException':
+ return new _RemoteIsolateSpawnException(message, type, toString);
+ default:
+ return new RemoteException._(message, type, toString);
+ }
}
RemoteException._(this.message, this.type, this._toString);
@@ -82,7 +98,14 @@ class RemoteException implements Exception {
///
/// It's important to preserve [TestFailure]-ness, because tests have different
/// results depending on whether an exception was a failure or an error.
-class RemoteTestFailure extends RemoteException implements TestFailure {
- RemoteTestFailure._(String message, String type, String toString)
+class _RemoteTestFailure extends RemoteException implements TestFailure {
+ _RemoteTestFailure(String message, String type, String toString)
+ : super._(message, type, toString);
+}
+
+/// A subclass of [RemoteException] that implements [IsolateSpawnException].
+class _RemoteIsolateSpawnException extends RemoteException
+ implements IsolateSpawnException {
+ _RemoteIsolateSpawnException(String message, String type, String toString)
: super._(message, type, toString);
}
diff --git a/lib/src/utils.dart b/lib/src/utils.dart
index da219276..5c12593e 100644
--- a/lib/src/utils.dart
+++ b/lib/src/utils.dart
@@ -13,6 +13,17 @@ import 'package:stack_trace/stack_trace.dart';
/// The return type should only ever by [Future] or void.
typedef AsyncFunction();
+/// A regular expression to match the exception prefix that some exceptions'
+/// [Object.toString] values contain.
+final _exceptionPrefix = new RegExp(r'^([A-Z][a-zA-Z]*)?(Exception|Error): ');
+
+/// Get a string description of an exception.
+///
+/// Many exceptions include the exception class name at the beginning of their
+/// [toString], so we remove that if it exists.
+String getErrorMessage(error) =>
+ error.toString().replaceFirst(_exceptionPrefix, '');
+
/// Indent each line in [str] by two spaces.
String indent(String str) =>
str.replaceAll(new RegExp("^", multiLine: true), " ");
@@ -34,6 +45,25 @@ class Pair<E, F> {
int get hashCode => first.hashCode ^ last.hashCode;
}
+/// A regular expression matching the path to a temporary file used to start an
+/// isolate.
+///
+/// These paths aren't relevant and are removed from stack traces.
+final _isolatePath =
+ new RegExp(r"/unittest_[A-Za-z0-9]{6}/runInIsolate\.dart$");
+
+/// Returns [stackTrace] converted to a [Chain] with all irrelevant frames
+/// folded together.
+Chain terseChain(StackTrace stackTrace) {
+ return new Chain.forTrace(stackTrace).foldFrames((frame) {
+ if (frame.package == 'unittest') return true;
+
+ // Filter out frames from our isolate bootstrap as well.
+ if (frame.uri.scheme != 'file') return false;
+ return frame.uri.path.contains(_isolatePath);
+ }, terse: true);
+}
+
/// Returns a Trace object from a StackTrace object or a String, or the
/// unchanged input if formatStacks is false;
Trace getTrace(stack, bool formatStacks, bool filterStacks) {
diff --git a/lib/src/vm_listener.dart b/lib/src/vm_listener.dart
index 7deb0b42..2a35f221 100644
--- a/lib/src/vm_listener.dart
+++ b/lib/src/vm_listener.dart
@@ -11,6 +11,7 @@ import 'declarer.dart';
import 'remote_exception.dart';
import 'suite.dart';
import 'test.dart';
+import 'utils.dart';
/// A class that runs tests in a separate isolate and communicates the results
/// back to the main isolate.
@@ -18,18 +19,54 @@ class VmListener {
/// The test suite to run.
final Suite _suite;
- /// Extracts metadata about all the tests in [main] and sends information
- /// about them over [sendPort].
+ /// Extracts metadata about all the tests in the function returned by
+ /// [getMain] and sends information about them over [sendPort].
+ ///
+ /// The main function is wrapped in a closure so that we can handle it being
+ /// undefined here rather than in the generated code.
///
/// Once that's done, this starts listening for commands about which tests to
/// run.
- static void start(SendPort sendPort, main()) {
+ static void start(SendPort sendPort, Function getMain()) {
+ var main;
+ try {
+ main = getMain();
+ } on NoSuchMethodError catch (_) {
+ _sendLoadException(sendPort, "No top-level main() function defined.");
+ return;
+ }
+
+ if (main is! Function) {
+ _sendLoadException(sendPort, "Top-level main getter is not a function.");
+ return;
+ } else if (main is! AsyncFunction) {
+ _sendLoadException(
+ sendPort, "Top-level main() function takes arguments.");
+ return;
+ }
+
var declarer = new Declarer();
- runZoned(main, zoneValues: {#unittest.declarer: declarer});
+ try {
+ runZoned(main, zoneValues: {#unittest.declarer: declarer});
+ } catch (error, stackTrace) {
+ sendPort.send({
+ "type": "error",
+ "error": RemoteException.serialize(error, stackTrace)
+ });
+ return;
+ }
+
new VmListener._(new Suite("VmListener", declarer.tests))
._listen(sendPort);
}
+ /// Sends a message over [sendPort] indicating that the tests failed to load.
+ ///
+ /// [message] should describe the failure.
+ static void _sendLoadException(SendPort sendPort, String message) {
+ sendPort.send({"type": "loadException", "message": message});
+ }
+
VmListener._(this._suite);
/// Send information about [_suite] across [sendPort] and start listening for
@@ -47,7 +84,10 @@ class VmListener {
});
}
- sendPort.send(tests);
+ sendPort.send({
+ "type": "success",
+ "tests": tests
+ });
}
/// Runs [test] and send the results across [sendPort].
diff --git a/pubspec.yaml b/pubspec.yaml
index b134cce7..049509e8 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -6,7 +6,8 @@ homepage: https://github.com/dart-lang/unittest
environment:
sdk: '>=1.0.0 <2.0.0'
dependencies:
- stack_trace: '>=0.9.0 <2.0.0'
+ args: '>=0.12.1 <0.13.0'
+ stack_trace: '>=1.2.0 <2.0.0'
# Using the pre-release version of matcher. When published we will go back
# using a tight version constraint to ensure that a constraint on unittest
diff --git a/test/console_reporter_test.dart b/test/console_reporter_test.dart
new file mode 100644
index 00000000..81661b7d
--- /dev/null
+++ b/test/console_reporter_test.dart
@@ -0,0 +1,179 @@
+// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+import 'package:unittest/src/io.dart';
+import 'package:unittest/unittest.dart';
+
+import 'io.dart';
+
+String _sandbox;
+
+void main() {
+ test("reports when no tests are run", () {
+ return withTempDir((path) {
+ new File(p.join(path, "test.dart")).writeAsStringSync("void main() {}");
+ var result = runUnittest(["test.dart"], workingDirectory: path);
+ expect(result.stdout, equals("No tests ran.\n"));
+ });
+ });
+
+ test("runs several successful tests and reports when each completes", () {
+ _expectReport("""
+ declarer.test('success 1', () {});
+ declarer.test('success 2', () {});
+ declarer.test('success 3', () {});""",
+ """
+ +0: success 1
+ +1: success 1
+ +1: success 2
+ +2: success 2
+ +2: success 3
+ +3: success 3
+ +3: All tests passed!""");
+ });
+
+ test("runs several failing tests and reports when each fails", () {
+ _expectReport("""
+ declarer.test('failure 1', () => throw new TestFailure('oh no'));
+ declarer.test('failure 2', () => throw new TestFailure('oh no'));
+ declarer.test('failure 3', () => throw new TestFailure('oh no'));""",
+ """
+ +0: failure 1
+ +0 -1: failure 1
+ oh no
+ test.dart 7:42 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ +0 -1: failure 2
+ +0 -2: failure 2
+ oh no
+ test.dart 8:42 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ +0 -2: failure 3
+ +0 -3: failure 3
+ oh no
+ test.dart 9:42 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ +0 -3: Some tests failed.""");
+ });
+
+ test("runs failing tests along with successful tests", () {
+ _expectReport("""
+ declarer.test('failure 1', () => throw new TestFailure('oh no'));
+ declarer.test('success 1', () {});
+ declarer.test('failure 2', () => throw new TestFailure('oh no'));
+ declarer.test('success 2', () {});""",
+ """
+ +0: failure 1
+ +0 -1: failure 1
+ oh no
+ test.dart 7:42 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ +0 -1: success 1
+ +1 -1: success 1
+ +1 -1: failure 2
+ +1 -2: failure 2
+ oh no
+ test.dart 9:42 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ +1 -2: success 2
+ +2 -2: success 2
+ +2 -2: Some tests failed.""");
+ });
+
+ test("gracefully handles multiple test failures in a row", () {
+ _expectReport("""
+ // This completer ensures that the test isolate isn't killed until all
+ // errors have been thrown.
+ var completer = new Completer();
+ declarer.test('failures', () {
+ new Future.microtask(() => throw 'first error');
+ new Future.microtask(() => throw 'second error');
+ new Future.microtask(() => throw 'third error');
+ new Future.microtask(completer.complete);
+ });
+ declarer.test('wait', () => completer.future);""",
+ """
+ +0: failures
+ +0 -1: failures
+ first error
+ test.dart 11:38 main.<fn>.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+ ===== asynchronous gap ===========================
+ dart:async Future.Future.microtask
+ test.dart 11:15 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ second error
+ test.dart 12:38 main.<fn>.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+ ===== asynchronous gap ===========================
+ dart:async Future.Future.microtask
+ test.dart 12:15 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ third error
+ test.dart 13:38 main.<fn>.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+ ===== asynchronous gap ===========================
+ dart:async Future.Future.microtask
+ test.dart 13:15 main.<fn>
+ dart:isolate _RawReceivePortImpl._handleMessage
+
+
+ +0 -1: wait
+ +1 -1: wait
+ +1 -1: Some tests failed.""");
+ });
+}
+
+final _prefixLength = "XX:XX ".length;
+
+void _expectReport(String tests, String expected) {
+ var dart = """
+import 'dart:async';
+
+import 'package:unittest/unittest.dart';
+
+void main() {
+ var declarer = Zone.current[#unittest.declarer];
+$tests
+}
+""";
+
+ expect(withTempDir((path) {
+ new File(p.join(path, "test.dart")).writeAsStringSync(dart);
+ var result = runUnittest(["test.dart"], workingDirectory: path);
+
+ // Convert CRs into newlines, remove excess trailing whitespace, and trim
+ // off timestamps.
+ var actual = result.stdout.trim().split(new RegExp(r"[\r\n]")).map((line) {
+ if (line.startsWith(" ") || line.isEmpty) return line.trimRight();
+ return line.trim().substring(_prefixLength);
+ }).join("\n");
+
+ // Un-indent the expected string.
+ var indentation = expected.indexOf(new RegExp("[^ ]"));
+ expected = expected.split("\n").map((line) {
+ if (line.isEmpty) return line;
+ return line.substring(indentation);
+ }).join("\n");
+
+ expect(actual, equals(expected));
+ }), completes);
+}
diff --git a/test/io.dart b/test/io.dart
index 663d454c..1dbc377e 100644
--- a/test/io.dart
+++ b/test/io.dart
@@ -8,7 +8,6 @@ import 'dart:io';
import 'package:path/path.dart' as p;
import 'package:stack_trace/stack_trace.dart';
-import 'package:unittest/unittest.dart';
/// The root directory of the `unittest` package.
final String packageDir = _computePackageDir();
@@ -17,9 +16,14 @@ String _computePackageDir() {
return p.dirname(p.dirname(p.fromUri(trace.frames.first.uri)));
}
-/// Returns a matcher that matches a [FileSystemException] with the given
-/// [message].
-Matcher isFileSystemException(String message) => predicate(
- (error) => error is FileSystemException && error.message == message,
- 'is a FileSystemException with message "$message"');
+/// Runs the unittest executable with the package root set properly.
+ProcessResult runUnittest(List<String> args, {String workingDirectory}) {
+ var allArgs = Platform.executableArguments.toList()
+ ..add(p.join(packageDir, 'bin/unittest.dart'))
+ ..add("--package-root=${p.join(packageDir, 'packages')}")
+ ..addAll(args);
+ // TODO(nweiz): Use ScheduledProcess once it's compatible.
+ return Process.runSync(Platform.executable, allArgs,
+ workingDirectory: workingDirectory);
+}
diff --git a/test/loader_test.dart b/test/loader_test.dart
index 95a4a8b1..47197fbf 100644
--- a/test/loader_test.dart
+++ b/test/loader_test.dart
@@ -83,7 +83,7 @@ void main() {
test("throws a nice error if the package root doesn't exist", () {
var loader = new Loader();
expect(() => loader.loadFile(p.join(_sandbox, 'a_test.dart')),
- throwsA(isFileSystemException(
+ throwsA(isLoadException(
"Directory ${p.join(_sandbox, 'packages')} does not exist.")));
});
});
diff --git a/test/runner_test.dart b/test/runner_test.dart
new file mode 100644
index 00000000..2b22667f
--- /dev/null
+++ b/test/runner_test.dart
@@ -0,0 +1,203 @@
+// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+import 'package:unittest/src/exit_codes.dart' as exit_codes;
+import 'package:unittest/unittest.dart';
+
+import 'io.dart';
+
+String _sandbox;
+
+final _success = """
+import 'dart:async';
+
+void main() {
+ var declarer = Zone.current[#unittest.declarer];
+ declarer.test("success", () {});
+}
+""";
+
+final _failure = """
+import 'dart:async';
+
+import 'package:unittest/unittest.dart';
+
+void main() {
+ var declarer = Zone.current[#unittest.declarer];
+ declarer.test("failure", () => throw new TestFailure("oh no"));
+}
+""";
+
+void main() {
+ setUp(() {
+ _sandbox = Directory.systemTemp.createTempSync('unittest_').path;
+ });
+
+ tearDown(() {
+ new Directory(_sandbox).deleteSync(recursive: true);
+ });
+
+ test("prints help information", () {
+ var result = _runUnittest(["--help"]);
+ expect(result.stdout, equals("""
+Runs tests in this package.
+
+Usage: pub run unittest:unittest [files or directories...]
+
+-h, --help Shows this usage information.
+"""));
+ expect(result.exitCode, equals(exit_codes.success));
+ });
+
+ group("fails gracefully if", () {
+ test("an invalid option is passed", () {
+ var result = _runUnittest(["--asdf"]);
+ expect(result.stderr, equals("""
+Could not find an option named "asdf".
+
+Usage: pub run unittest:unittest [files or directories...]
+
+-h, --help Shows this usage information.
+"""));
+ expect(result.exitCode, equals(exit_codes.usage));
+ });
+
+ test("a non-existent file is passed", () {
+ var result = _runUnittest(["file"]);
+ expect(result.stderr, equals('Failed to load "file": Does not exist.\n'));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ test("the default directory doesn't exist", () {
+ var result = _runUnittest([]);
+ expect(result.stderr, equals(
+ 'Failed to load "test": No test files were passed and the default '
+ 'directory doesn\'t exist.\n'));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ test("a test file fails to load", () {
+ var testPath = p.join(_sandbox, "test.dart");
+ new File(testPath).writeAsStringSync("invalid Dart file");
+ var result = _runUnittest(["test.dart"]);
+
+ expect(result.stderr, equals(
+ 'Failed to load "${p.relative(testPath, from: _sandbox)}":\n'
+ "line 1 pos 1: unexpected token 'invalid'\n"
+ "invalid Dart file\n"
+ "^\n"));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ test("a test file throws", () {
+ var testPath = p.join(_sandbox, "test.dart");
+ new File(testPath).writeAsStringSync("void main() => throw 'oh no';");
+
+ var result = _runUnittest(["test.dart"]);
+ expect(result.stderr, startsWith(
+ 'Failed to load "${p.relative(testPath, from: _sandbox)}": oh no\n'));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ test("a test file doesn't have a main defined", () {
+ var testPath = p.join(_sandbox, "test.dart");
+ new File(testPath).writeAsStringSync("void foo() {}");
+
+ var result = _runUnittest(["test.dart"]);
+ expect(result.stderr, startsWith(
+ 'Failed to load "${p.relative(testPath, from: _sandbox)}": No '
+ 'top-level main() function defined.\n'));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ test("a test file has a non-function main", () {
+ var testPath = p.join(_sandbox, "test.dart");
+ new File(testPath).writeAsStringSync("int main;");
+
+ var result = _runUnittest(["test.dart"]);
+ expect(result.stderr, startsWith(
+ 'Failed to load "${p.relative(testPath, from: _sandbox)}": Top-level '
+ 'main getter is not a function.\n'));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ test("a test file has a main with arguments", () {
+ var testPath = p.join(_sandbox, "test.dart");
+ new File(testPath).writeAsStringSync("void main(arg) {}");
+
+ var result = _runUnittest(["test.dart"]);
+ expect(result.stderr, startsWith(
+ 'Failed to load "${p.relative(testPath, from: _sandbox)}": Top-level '
+ 'main() function takes arguments.\n'));
+ expect(result.exitCode, equals(exit_codes.data));
+ });
+
+ // TODO(nweiz): test what happens when a test file is unreadable once issue
+ // 15078 is fixed.
+ });
+
+ group("runs successful tests", () {
+ test("defined in a single file", () {
+ new File(p.join(_sandbox, "test.dart")).writeAsStringSync(_success);
+ var result = _runUnittest(["test.dart"]);
+ expect(result.exitCode, equals(0));
+ });
+
+ test("defined in a directory", () {
+ for (var i = 0; i < 3; i++) {
+ new File(p.join(_sandbox, "${i}_test.dart"))
+ .writeAsStringSync(_success);
+ }
+
+ var result = _runUnittest(["."]);
+ expect(result.exitCode, equals(0));
+ });
+
+ test("defaulting to the test directory", () {
+ new Directory(p.join(_sandbox, "test")).createSync();
+ for (var i = 0; i < 3; i++) {
+ new File(p.join(_sandbox, "test", "${i}_test.dart"))
+ .writeAsStringSync(_success);
+ }
+
+ var result = _runUnittest([]);
+ expect(result.exitCode, equals(0));
+ });
+ });
+
+ group("runs failing tests", () {
+ test("defined in a single file", () {
+ new File(p.join(_sandbox, "test.dart")).writeAsStringSync(_failure);
+ var result = _runUnittest(["test.dart"]);
+ expect(result.exitCode, equals(1));
+ });
+
+ test("defined in a directory", () {
+ for (var i = 0; i < 3; i++) {
+ new File(p.join(_sandbox, "${i}_test.dart"))
+ .writeAsStringSync(_failure);
+ }
+
+ var result = _runUnittest(["."]);
+ expect(result.exitCode, equals(1));
+ });
+
+ test("defaulting to the test directory", () {
+ new Directory(p.join(_sandbox, "test")).createSync();
+ for (var i = 0; i < 3; i++) {
+ new File(p.join(_sandbox, "test", "${i}_test.dart"))
+ .writeAsStringSync(_failure);
+ }
+
+ var result = _runUnittest([]);
+ expect(result.exitCode, equals(1));
+ });
+ });
+}
+
+ProcessResult _runUnittest(List<String> args) =>
+ runUnittest(args, workingDirectory: _sandbox);
diff --git a/test/utils.dart b/test/utils.dart
index f9b4075d..64a3da71 100644
--- a/test/utils.dart
+++ b/test/utils.dart
@@ -8,6 +8,7 @@ import 'dart:async';
import 'dart:collection';
import 'package:unittest/src/live_test.dart';
+import 'package:unittest/src/load_exception.dart';
import 'package:unittest/src/remote_exception.dart';
import 'package:unittest/src/state.dart';
import 'package:unittest/unittest.dart';
@@ -71,6 +72,11 @@ Matcher isRemoteException(String message) => predicate(
(error) => error is RemoteException && error.message == message,
'is a RemoteException with message "$message"');
+/// Returns a matcher that matches a [LoadException] with the given [message].
+Matcher isLoadException(String message) => predicate(
+ (error) => error is LoadException && error.innerError == message,
+ 'is a LoadException with message "$message"');
+
/// Returns a [Future] that completes after pumping the event queue [times]
/// times.
///
diff --git a/test/vm_listener_test.dart b/test/vm_listener_test.dart
index b626ad15..47ee330a 100644
--- a/test/vm_listener_test.dart
+++ b/test/vm_listener_test.dart
@@ -9,6 +9,7 @@ import 'package:unittest/src/declarer.dart';
import 'package:unittest/src/invoker.dart';
import 'package:unittest/src/isolate_test.dart';
import 'package:unittest/src/live_test.dart';
+import 'package:unittest/src/remote_exception.dart';
import 'package:unittest/src/state.dart';
import 'package:unittest/src/suite.dart';
import 'package:unittest/src/vm_listener.dart';
@@ -41,7 +42,11 @@ void main() {
test("sends a list of available tests on startup", () {
return _spawnIsolate(_successfulTests).then((receivePort) {
return receivePort.first;
- }).then((tests) {
+ }).then((response) {
+ expect(response, containsPair("type", "success"));
+ expect(response, contains("tests"));
+
+ var tests = response["tests"];
expect(tests, hasLength(3));
expect(tests[0], containsPair("name", "successful 1"));
expect(tests[1], containsPair("name", "successful 2"));
@@ -49,6 +54,52 @@ void main() {
});
});
+ test("sends an error response if loading fails", () {
+ return _spawnIsolate(_loadError).then((receivePort) {
+ return receivePort.first;
+ }).then((response) {
+ expect(response, containsPair("type", "error"));
+ expect(response, contains("error"));
+
+ var error = RemoteException.deserialize(response["error"]).error;
+ expect(error.message, equals("oh no"));
+ expect(error.type, equals("String"));
+ });
+ });
+
+ test("sends an error response on a NoSuchMethodError", () {
+ return _spawnIsolate(_noSuchMethodError).then((receivePort) {
+ return receivePort.first;
+ }).then((response) {
+ expect(response, containsPair("type", "loadException"));
+ expect(response,
+ containsPair("message", "No top-level main() function defined."));
+ });
+ });
+
+ test("sends an error response on non-function main", () {
+ return _spawnIsolate(_nonFunction).then((receivePort) {
+ return receivePort.first;
+ }).then((response) {
+ expect(response, containsPair("type", "loadException"));
+ expect(response,
+ containsPair("message", "Top-level main getter is not a function."));
+ });
+ });
+
+ test("sends an error response on wrong-arity main", () {
+ return _spawnIsolate(_wrongArity).then((receivePort) {
+ return receivePort.first;
+ }).then((response) {
+ expect(response, containsPair("type", "loadException"));
+ expect(
+ response,
+ containsPair(
+ "message",
+ "Top-level main() function takes arguments."));
+ });
+ });
+
group("in a successful test", () {
test("the state changes from pending to running to complete", () {
return _isolateTest(_successfulTests).then((liveTest) {
@@ -218,7 +269,9 @@ Future<LiveTest> _isolateTest(void entryPoint(SendPort sendPort)) {
return _spawnIsolate(entryPoint).then((receivePort) {
return receivePort.first;
}).then((response) {
- var testMap = response.first;
+ expect(response, containsPair("type", "success"));
+
+ var testMap = response["tests"].first;
var test = new IsolateTest(testMap["name"], testMap["sendPort"]);
var suite = new Suite("suite", [test]);
_liveTest = test.load(suite);
@@ -238,9 +291,27 @@ Future<ReceivePort> _spawnIsolate(void entryPoint(SendPort sendPort)) {
});
}
+/// An isolate entrypoint that throws immediately.
+void _loadError(SendPort sendPort) =>
+ VmListener.start(sendPort, () => () => throw 'oh no');
+
+/// An isolate entrypoint that throws a NoSuchMethodError.
+void _noSuchMethodError(SendPort sendPort) {
+ return VmListener.start(sendPort, () =>
+ throw new NoSuchMethodError(null, #main, [], {}));
+}
+
+/// An isolate entrypoint that returns a non-function.
+void _nonFunction(SendPort sendPort) =>
+ VmListener.start(sendPort, () => null);
+
+/// An isolate entrypoint that returns a function with the wrong arity.
+void _wrongArity(SendPort sendPort) =>
+ VmListener.start(sendPort, () => (_) {});
+
/// An isolate entrypoint that defines three tests that succeed.
void _successfulTests(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("successful 1", () {});
_declarer.test("successful 2", () {});
_declarer.test("successful 3", () {});
@@ -249,14 +320,14 @@ void _successfulTests(SendPort sendPort) {
/// An isolate entrypoint that defines a test that fails.
void _failingTest(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("failure", () => throw new TestFailure('oh no'));
});
}
/// An isolate entrypoint that defines a test that fails after succeeding.
void _failAfterSucceedTest(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("fail after succeed", () {
pumpEventQueue().then((_) {
throw new TestFailure('oh no');
@@ -267,7 +338,7 @@ void _failAfterSucceedTest(SendPort sendPort) {
/// An isolate entrypoint that defines a test that fails multiple times.
void _multiFailTest(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("multiple failures", () {
Invoker.current.addOutstandingCallback();
new Future(() => throw new TestFailure("one"));
@@ -280,14 +351,14 @@ void _multiFailTest(SendPort sendPort) {
/// An isolate entrypoint that defines a test that errors.
void _errorTest(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("error", () => throw 'oh no');
});
}
/// An isolate entrypoint that defines a test that errors after succeeding.
void _errorAfterSucceedTest(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("error after succeed", () {
pumpEventQueue().then((_) => throw 'oh no');
});
@@ -296,7 +367,7 @@ void _errorAfterSucceedTest(SendPort sendPort) {
/// An isolate entrypoint that defines a test that errors multiple times.
void _multiErrorTest(SendPort sendPort) {
- VmListener.start(sendPort, () {
+ VmListener.start(sendPort, () => () {
_declarer.test("multiple errors", () {
Invoker.current.addOutstandingCallback();
new Future(() => throw "one");
--
GitLab