From 93c1ad0f0114f7e828886d56511af1058895dccc Mon Sep 17 00:00:00 2001
From: Gary Roumanis <groumanis@gmail.com>
Date: Wed, 22 Jan 2020 17:00:36 -0800
Subject: [PATCH] Flexible Coverage API (#1151)
* Flexible Coverage API
---
pkgs/test/pubspec.yaml | 2 +-
pkgs/test_core/CHANGELOG.md | 3 +-
pkgs/test_core/lib/src/runner/coverage.dart | 25 +++++---------
.../lib/src/runner/coverage_stub.dart | 3 +-
pkgs/test_core/lib/src/runner/engine.dart | 2 +-
pkgs/test_core/lib/src/runner/load_suite.dart | 14 ++++----
.../src/runner/plugin/platform_helpers.dart | 16 +++++----
.../lib/src/runner/runner_suite.dart | 26 ++++++++++----
.../test_core/lib/src/runner/vm/platform.dart | 34 ++++++++++++-------
pkgs/test_core/pubspec.yaml | 2 +-
10 files changed, 74 insertions(+), 53 deletions(-)
diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml
index 418f3128..b1abea3f 100644
--- a/pkgs/test/pubspec.yaml
+++ b/pkgs/test/pubspec.yaml
@@ -31,7 +31,7 @@ dependencies:
yaml: ^2.0.0
# Use an exact version until the test_api and test_core package are stable.
test_api: 0.2.14
- test_core: 0.2.19
+ test_core: 0.3.0
dev_dependencies:
fake_async: ^1.0.0
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index f28a3727..f16d824e 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,10 +1,11 @@
-## 0.2.19-dev
+## 0.3.0-dev
* Bump minimum SDK to `2.4.0` for safer usage of for-loop elements.
* Deprecate `PhantomJS` and provide warning when used. Support for `PhantomJS`
will be removed in version `2.0.0`.
* Differentiate between test-randomize-ordering-seed not set and 0 being chosen
as the random seed.
+* `deserializeSuite` now takes an optional `gatherCoverage` callback.
## 0.2.18
diff --git a/pkgs/test_core/lib/src/runner/coverage.dart b/pkgs/test_core/lib/src/runner/coverage.dart
index 200019b3..841bdd72 100644
--- a/pkgs/test_core/lib/src/runner/coverage.dart
+++ b/pkgs/test_core/lib/src/runner/coverage.dart
@@ -5,29 +5,20 @@
import 'dart:convert';
import 'dart:io';
-import 'package:coverage/coverage.dart';
import 'package:path/path.dart' as p;
import 'live_suite_controller.dart';
-/// Collects coverage and outputs to the [coverage] path.
-Future<void> gatherCoverage(
- String coverage, LiveSuiteController controller) async {
- final suite = controller.liveSuite.suite;
-
- if (!suite.platform.runtime.isDartVM) return;
-
- final isolateId = Uri.parse(suite.environment.observatoryUrl.fragment)
- .queryParameters['isolateId'];
-
- final cov = await collect(
- suite.environment.observatoryUrl, false, false, false, {},
- isolateIds: {isolateId});
-
- final outfile = File(p.join('$coverage', '${suite.path}.vm.json'))
+/// Collects coverage and outputs to the [coveragePath] path.
+Future<void> writeCoverage(
+ String coveragePath, LiveSuiteController controller) async {
+ var suite = controller.liveSuite.suite;
+ var coverage = await controller.liveSuite.suite.gatherCoverage();
+ final outfile = File(p.join(coveragePath,
+ '${suite.path}.${suite.platform.runtime.name.toLowerCase()}.json'))
..createSync(recursive: true);
final out = outfile.openWrite();
- out.write(json.encode(cov));
+ out.write(json.encode(coverage));
await out.flush();
await out.close();
}
diff --git a/pkgs/test_core/lib/src/runner/coverage_stub.dart b/pkgs/test_core/lib/src/runner/coverage_stub.dart
index 5a2285f0..64f69c75 100644
--- a/pkgs/test_core/lib/src/runner/coverage_stub.dart
+++ b/pkgs/test_core/lib/src/runner/coverage_stub.dart
@@ -4,6 +4,7 @@
import 'live_suite_controller.dart';
-Future<void> gatherCoverage(String coverage, LiveSuiteController controller) =>
+Future<void> writeCoverage(
+ String coveragePath, LiveSuiteController controller) =>
throw UnsupportedError(
'Coverage is only supported through the test runner.');
diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart
index e4a7fa43..d944e506 100644
--- a/pkgs/test_core/lib/src/runner/engine.dart
+++ b/pkgs/test_core/lib/src/runner/engine.dart
@@ -285,7 +285,7 @@ class Engine {
if (_closed) return;
await _runGroup(controller, controller.liveSuite.suite.group, []);
controller.noMoreLiveTests();
- if (_coverage != null) await gatherCoverage(_coverage, controller);
+ if (_coverage != null) await writeCoverage(_coverage, controller);
loadResource.allowRelease(() => controller.close());
});
}());
diff --git a/pkgs/test_core/lib/src/runner/load_suite.dart b/pkgs/test_core/lib/src/runner/load_suite.dart
index a8d2825a..fd6dfb09 100644
--- a/pkgs/test_core/lib/src/runner/load_suite.dart
+++ b/pkgs/test_core/lib/src/runner/load_suite.dart
@@ -6,27 +6,23 @@ import 'dart:async';
import 'package:stack_trace/stack_trace.dart';
import 'package:stream_channel/stream_channel.dart';
-
import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports
+import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
-import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
import 'package:test_api/src/utils.dart'; // ignore: implementation_imports
-import 'runner_suite.dart';
-import 'suite.dart';
-
import '../../test_core.dart';
-
-// ignore: uri_does_not_exist
import '../util/io_stub.dart'
// ignore: uri_does_not_exist
if (dart.library.io) '../util/io.dart';
import 'load_exception.dart';
import 'plugin/environment.dart';
+import 'runner_suite.dart';
+import 'suite.dart';
/// The timeout for loading a test suite.
///
@@ -214,4 +210,8 @@ class LoadSuite extends Suite implements RunnerSuite {
@override
Future close() async {}
+
+ @override
+ Future<Map<String, dynamic>> gatherCoverage() =>
+ throw UnsupportedError('Coverage is not supported for LoadSuite tests.');
}
diff --git a/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart b/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
index c63967c5..bd11058a 100644
--- a/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
+++ b/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
@@ -7,19 +7,18 @@ import 'dart:io';
import 'package:stack_trace/stack_trace.dart';
import 'package:stream_channel/stream_channel.dart';
-
import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
import 'package:test_api/src/util/remote_exception.dart'; // ignore: implementation_imports
-import '../runner_suite.dart';
-import '../environment.dart';
-import '../suite.dart';
import '../configuration.dart';
+import '../environment.dart';
import '../load_exception.dart';
+import '../runner_suite.dart';
import '../runner_test.dart';
+import '../suite.dart';
/// A helper method for creating a [RunnerSuiteController] containing tests
/// that communicate over [channel].
@@ -35,13 +34,17 @@ import '../runner_test.dart';
///
/// If [mapper] is passed, it will be used to adjust stack traces for any errors
/// emitted by tests.
+///
+/// [gatherCoverage] is a callback which returns a hit-map containing merged
+/// coverage report suitable for use with `package:coverage`.
RunnerSuiteController deserializeSuite(
String path,
SuitePlatform platform,
SuiteConfiguration suiteConfig,
Environment environment,
StreamChannel channel,
- Object message) {
+ Object message,
+ {Future<Map<String, dynamic>> Function() gatherCoverage}) {
var disconnector = Disconnector();
var suiteChannel = MultiChannel(channel.transform(disconnector));
@@ -110,7 +113,8 @@ RunnerSuiteController deserializeSuite(
return RunnerSuiteController(
environment, suiteConfig, suiteChannel, completer.future, platform,
path: path,
- onClose: () => disconnector.disconnect().catchError(handleError));
+ onClose: () => disconnector.disconnect().catchError(handleError),
+ gatherCoverage: gatherCoverage);
}
/// A utility class for storing state while deserializing tests.
diff --git a/pkgs/test_core/lib/src/runner/runner_suite.dart b/pkgs/test_core/lib/src/runner/runner_suite.dart
index 8470f62a..1b166d30 100644
--- a/pkgs/test_core/lib/src/runner/runner_suite.dart
+++ b/pkgs/test_core/lib/src/runner/runner_suite.dart
@@ -6,14 +6,13 @@ import 'dart:async';
import 'package:async/async.dart';
import 'package:stream_channel/stream_channel.dart';
-
import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
-import 'suite.dart';
import 'environment.dart';
+import 'suite.dart';
/// A suite produced and consumed by the test runner that has runner-specific
/// logic and lifecycle management.
@@ -77,6 +76,13 @@ class RunnerSuite extends Suite {
/// Closes the suite and releases any resources associated with it.
Future close() => _controller._close();
+
+ /// Collects a hit-map containing merged coverage.
+ ///
+ /// Result is suitable for input to the coverage formatters provided by
+ /// `package:coverage`.
+ Future<Map<String, dynamic>> gatherCoverage() async =>
+ (await _controller._gatherCoverage?.call()) ?? {};
}
/// A class that exposes and controls a [RunnerSuite].
@@ -106,10 +112,16 @@ class RunnerSuiteController {
/// The channel names that have already been used.
final _channelNames = <String>{};
+ /// Collects a hit-map containing merged coverage.
+ final Future<Map<String, dynamic>> Function() _gatherCoverage;
+
RunnerSuiteController(this._environment, this._config, this._suiteChannel,
Future<Group> groupFuture, SuitePlatform platform,
- {String path, Function() onClose})
- : _onClose = onClose {
+ {String path,
+ Function() onClose,
+ Future<Map<String, dynamic>> Function() gatherCoverage})
+ : _onClose = onClose,
+ _gatherCoverage = gatherCoverage {
_suite =
groupFuture.then((group) => RunnerSuite._(this, group, path, platform));
}
@@ -117,9 +129,11 @@ class RunnerSuiteController {
/// Used by [new RunnerSuite] to create a runner suite that's not loaded from
/// an external source.
RunnerSuiteController._local(this._environment, this._config,
- {Function() onClose})
+ {Function() onClose,
+ Future<Map<String, dynamic>> Function() gatherCoverage})
: _suiteChannel = null,
- _onClose = onClose;
+ _onClose = onClose,
+ _gatherCoverage = gatherCoverage;
/// Sets whether the suite is paused for debugging.
///
diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart
index 34853eac..12bf36c0 100644
--- a/pkgs/test_core/lib/src/runner/vm/platform.dart
+++ b/pkgs/test_core/lib/src/runner/vm/platform.dart
@@ -7,23 +7,23 @@ import 'dart:developer';
import 'dart:io';
import 'dart:isolate';
+import 'package:coverage/coverage.dart';
import 'package:path/path.dart' as p;
import 'package:stream_channel/isolate_channel.dart';
import 'package:stream_channel/stream_channel.dart';
import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/configuration.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/load_exception.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/plugin/environment.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports
-import 'package:test_core/src/util/dart.dart' // ignore: implementation_imports
- as dart;
import 'package:vm_service/vm_service.dart' hide Isolate;
import 'package:vm_service/vm_service_io.dart';
+import '../../runner/configuration.dart';
+import '../../runner/environment.dart';
+import '../../runner/load_exception.dart';
+import '../../runner/platform.dart';
+import '../../runner/plugin/platform_helpers.dart';
+import '../../runner/runner_suite.dart';
+import '../../runner/suite.dart';
+import '../../util/dart.dart' as dart;
import 'environment.dart';
/// A platform that loads tests in isolates spawned within this Dart process.
@@ -61,7 +61,7 @@ class VMPlatform extends PlatformPlugin {
sink.close();
}));
- VMEnvironment environment;
+ Environment environment;
IsolateRef isolateRef;
if (_config.debug) {
// Print an empty line because the VM prints an "Observatory listening on"
@@ -87,8 +87,11 @@ class VMPlatform extends PlatformPlugin {
environment = VMEnvironment(url, isolateRef, client);
}
- var controller = deserializeSuite(path, platform, suiteConfig,
- environment ?? PluginEnvironment(), channel, message);
+ environment ??= PluginEnvironment();
+
+ var controller = deserializeSuite(
+ path, platform, suiteConfig, environment, channel, message,
+ gatherCoverage: () => _gatherCoverage(environment));
if (isolateRef != null) {
await client.streamListen('Debug');
@@ -152,6 +155,13 @@ Future<Isolate> _spawnPrecompiledIsolate(
checked: true);
}
+Future<Map<String, dynamic>> _gatherCoverage(Environment environment) async {
+ final isolateId = Uri.parse(environment.observatoryUrl.fragment)
+ .queryParameters['isolateId'];
+ return await collect(environment.observatoryUrl, false, false, false, {},
+ isolateIds: {isolateId});
+}
+
Future<Isolate> _spawnPubServeIsolate(
String testPath, SendPort message, Uri pubServeUrl) async {
var url = pubServeUrl.resolveUri(
diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml
index b2455f6f..73c227ec 100644
--- a/pkgs/test_core/pubspec.yaml
+++ b/pkgs/test_core/pubspec.yaml
@@ -1,5 +1,5 @@
name: test_core
-version: 0.2.19-dev
+version: 0.3.0-dev
description: A basic library for writing tests and running them on the VM.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_core
--
GitLab