git.dart

// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';

import 'package:path/path.dart' as p;
import 'package:pub_semver/pub_semver.dart';

import '../git.dart' as git;
import '../io.dart';
import '../log.dart' as log;
import '../package.dart';
import '../package_name.dart';
import '../pubspec.dart';
import '../source.dart';
import '../system_cache.dart';
import '../utils.dart';
import 'cached.dart';

/// A package source that gets packages from Git repos.
class GitSource extends Source {
  final name = "git";

  BoundGitSource bind(SystemCache systemCache) =>
      new BoundGitSource(this, systemCache);

  /// Returns a reference to a git package with the given [name] and [url].
  ///
  /// If passed, [reference] is the Git reference. It defaults to `"HEAD"`.
  PackageRef refFor(String name, String url, {String reference, String path}) {
    if (path != null) assert(p.url.isRelative(path));
    return new PackageRef(name, this,
        {'url': url, 'ref': reference ?? 'HEAD', 'path': path ?? '.'});
  }

  /// Given a valid git package description, returns the URL of the repository
  /// it pulls from.
  String urlFromDescription(description) => description["url"];

  PackageRef parseRef(String name, description, {String containingPath}) {
    // TODO(rnystrom): Handle git URLs that are relative file paths (#8570).
    if (description is String) description = {'url': description};

    if (description is! Map) {
      throw new FormatException("The description must be a Git URL or a map "
          "with a 'url' key.");
    }

    if (description["url"] is! String) {
      throw new FormatException("The 'url' field of the description must be a "
          "string.");
    }

    _validateUrl(description["url"]);

    var ref = description["ref"];
    if (ref != null && ref is! String) {
      throw new FormatException("The 'ref' field of the description must be a "
          "string.");
    }

    var path = description["path"];
    if (path != null) {
      if (path is! String) {
        throw new FormatException(
            "The 'path' field of the description must be a string.");
      } else if (!p.url.isRelative(path)) {
        throw new FormatException(
            "The 'path' field of the description must be relative.");
      } else if (!p.url.isWithin('.', path)) {
        throw new FormatException(
            "The 'path' field of the description must not reach outside the "
            "repository.");
      }

      _validateUrl(path);
    }

    return new PackageRef(name, this,
        {"url": description["url"], "ref": ref ?? "HEAD", "path": path ?? "."});
  }

  PackageId parseId(String name, Version version, description) {
    if (description is! Map) {
      throw new FormatException("The description must be a map with a 'url' "
          "key.");
    }

    if (description["url"] is! String) {
      throw new FormatException("The 'url' field of the description must be a "
          "string.");
    }

    _validateUrl(description["url"]);

    var ref = description["ref"];
    if (ref != null && ref is! String) {
      throw new FormatException("The 'ref' field of the description must be a "
          "string.");
    }

    if (description["resolved-ref"] is! String) {
      throw new FormatException("The 'resolved-ref' field of the description "
          "must be a string.");
    }

    var path = description["path"];
    if (path != null) {
      if (path is! String) {
        throw new FormatException(
            "The 'path' field of the description must be a string.");
      } else if (!p.url.isRelative(path)) {
        throw new FormatException(
            "The 'path' field of the description must be relative.");
      }

      _validateUrl(path);
    }

    return new PackageId(name, this, version, {
      "url": description["url"],
      "ref": ref ?? "HEAD",
      "resolved-ref": description["resolved-ref"],
      "path": path ?? "."
    });
  }

  /// Throws a [FormatException] if [url] isn't a valid Git URL.
  void _validateUrl(String url) {
    // If the URL contains an @, it's probably an SSH hostname, which we don't
    // know how to validate.
    if (url.contains("@")) return;

    // Otherwise, we use Dart's URL parser to validate the URL.
    Uri.parse(url);
  }

  /// If [description] has a resolved ref, print it out in short-form.
  ///
  /// This helps distinguish different git commits with the same pubspec
  /// version.
  String formatDescription(String containingPath, description) {
    if (description is Map && description.containsKey('resolved-ref')) {
      var result = "${description['url']} at "
          "${description['resolved-ref'].substring(0, 6)}";
      if (description["path"] != ".") result += " in ${description["path"]}";
      return result;
    } else {
      return super.formatDescription(containingPath, description);
    }
  }

  /// Two Git descriptions are equal if both their URLs and their refs are
  /// equal.
  bool descriptionsEqual(description1, description2) {
    // TODO(nweiz): Do we really want to throw an error if you have two
    // dependencies on some repo, one of which specifies a ref and one of which
    // doesn't? If not, how do we handle that case in the version solver?
    if (description1['url'] != description2['url']) return false;
    if (description1['ref'] != description2['ref']) return false;
    if (description1['path'] != description2['path']) return false;

    if (description1.containsKey('resolved-ref') &&
        description2.containsKey('resolved-ref')) {
      return description1['resolved-ref'] == description2['resolved-ref'];
    }

    return true;
  }

  int hashDescription(description) {
    // Don't include the resolved ref in the hash code because we ignore it in
    // [descriptionsEqual] if only one description defines it.
    return description['url'].hashCode ^
        description['ref'].hashCode ^
        description['path'].hashCode;
  }
}

/// The [BoundSource] for [GitSource].
class BoundGitSource extends CachedSource {
  final GitSource source;

  final SystemCache systemCache;

  /// A map from revision cache locations to futures that will complete once
  /// they're finished being cloned.
  ///
  /// This lets us avoid race conditions when getting multiple different
  /// packages from the same repository.
  final _revisionCacheClones = <String, Future>{};

  /// The paths to the canonical clones of repositories for which "git fetch"
  /// has already been run during this run of pub.
  final _updatedRepos = new Set<String>();

  BoundGitSource(this.source, this.systemCache);

  /// Given a Git repo that contains a pub package, gets the name of the pub
  /// package.
  Future<String> getPackageNameFromRepo(String repo) {
    // Clone the repo to a temp directory.
    return withTempDir((tempDir) async {
      await _clone(repo, tempDir, shallow: true);
      var pubspec = new Pubspec.load(tempDir, systemCache.sources);
      return pubspec.name;
    });
  }

  Future<List<PackageId>> doGetVersions(PackageRef ref) async {
    await _ensureRepoCache(ref);
    var path = _repoCachePath(ref);
    var revision = await _firstRevision(path, ref.description['ref']);
    var pubspec =
        await _describeUncached(ref, revision, ref.description['path']);

    return [
      new PackageId(ref.name, source, pubspec.version, {
        'url': ref.description['url'],
        'ref': ref.description['ref'],
        'resolved-ref': revision,
        'path': ref.description['path']
      })
    ];
  }

  /// Since we don't have an easy way to read from a remote Git repo, this
  /// just installs [id] into the system cache, then describes it from there.
  Future<Pubspec> describeUncached(PackageId id) => _describeUncached(
      id.toRef(), id.description['resolved-ref'], id.description['path']);

  /// Like [describeUncached], but takes a separate [ref] and Git [revision]
  /// rather than a single ID.
  Future<Pubspec> _describeUncached(
      PackageRef ref, String revision, String path) async {
    await _ensureRevision(ref, revision);
    var repoPath = _repoCachePath(ref);

    // Normalize the path because Git treats "./" at the beginning of a path
    // specially.
    var pubspecPath = p.normalize(p.join(p.fromUri(path), 'pubspec.yaml'));

    var lines;
    try {
      lines = await git
          .run(["show", "$revision:$pubspecPath"], workingDir: repoPath);
    } on git.GitException catch (_) {
      fail('Could not find a file named "$pubspecPath" in '
          '${ref.description['url']} $revision.');
    }

    return new Pubspec.parse(lines.join("\n"), systemCache.sources,
        expectedName: ref.name);
  }

  /// Clones a Git repo to the local filesystem.
  ///
  /// The Git cache directory is a little idiosyncratic. At the top level, it
  /// contains a directory for each commit of each repository, named `<package
  /// name>-<commit hash>`. These are the canonical package directories that are
  /// linked to from the `packages/` directory.
  ///
  /// In addition, the Git system cache contains a subdirectory named `cache/`
  /// which contains a directory for each separate repository URL, named
  /// `<package name>-<url hash>`. These are used to check out the repository
  /// itself; each of the commit-specific directories are clones of a directory
  /// in `cache/`.
  Future<Package> downloadToSystemCache(PackageId id) async {
    var ref = id.toRef();
    if (!git.isInstalled) {
      fail("Cannot get ${id.name} from Git (${ref.description['url']}).\n"
          "Please ensure Git is correctly installed.");
    }

    ensureDir(p.join(systemCacheRoot, 'cache'));
    await _ensureRevision(ref, id.description['resolved-ref']);

    var revisionCachePath = _revisionCachePath(id);
    await _revisionCacheClones.putIfAbsent(revisionCachePath, () async {
      if (!entryExists(revisionCachePath)) {
        await _clone(_repoCachePath(ref), revisionCachePath);
        await _checkOut(revisionCachePath, id.description['resolved-ref']);
        _writePackageList(revisionCachePath, [id.description['path']]);
      } else {
        _updatePackageList(revisionCachePath, id.description['path']);
      }
    });

    return new Package.load(id.name,
        p.join(revisionCachePath, id.description['path']), systemCache.sources);
  }

  /// Returns the path to the revision-specific cache of [id].
  String getDirectory(PackageId id) =>
      p.join(_revisionCachePath(id), id.description['path']);

  List<Package> getCachedPackages() {
    // TODO(keertip): Implement getCachedPackages().
    throw new UnimplementedError(
        "The git source doesn't support listing its cached packages yet.");
  }

  /// Resets all cached packages back to the pristine state of the Git
  /// repository at the revision they are pinned to.
  Future<Pair<List<PackageId>, List<PackageId>>> repairCachedPackages() async {
    if (!dirExists(systemCacheRoot)) return new Pair([], []);

    var successes = <PackageId>[];
    var failures = <PackageId>[];

    var packages = listDir(systemCacheRoot)
        .where((entry) => dirExists(p.join(entry, ".git")))
        .expand((revisionCachePath) {
          return _readPackageList(revisionCachePath).map((relative) {
            // If we've already failed to load another package from this
            // repository, ignore it.
            if (!dirExists(revisionCachePath)) return null;

            var packageDir = p.join(revisionCachePath, relative);
            try {
              return new Package.load(null, packageDir, systemCache.sources);
            } catch (error, stackTrace) {
              log.error("Failed to load package", error, stackTrace);
              var name = p.basename(revisionCachePath).split('-').first;
              failures.add(new PackageId(name, source, Version.none, '???'));
              tryDeleteEntry(revisionCachePath);
              return null;
            }
          });
        })
        .where((package) => package != null)
        .toList();

    // Note that there may be multiple packages with the same name and version
    // (pinned to different commits). The sort order of those is unspecified.
    packages.sort(Package.orderByNameAndVersion);

    for (var package in packages) {
      // If we've already failed to repair another package in this repository,
      // ignore it.
      if (!dirExists(package.dir)) continue;

      var id = new PackageId(package.name, source, package.version, null);

      log.message("Resetting Git repository for "
          "${log.bold(package.name)} ${package.version}...");

      try {
        // Remove all untracked files.
        await git
            .run(["clean", "-d", "--force", "-x"], workingDir: package.dir);

        // Discard all changes to tracked files.
        await git.run(["reset", "--hard", "HEAD"], workingDir: package.dir);

        successes.add(id);
      } on git.GitException catch (error, stackTrace) {
        log.error("Failed to reset ${log.bold(package.name)} "
            "${package.version}. Error:\n$error");
        log.fine(stackTrace);
        failures.add(id);

        // Delete the revision cache path, not the subdirectory that contains the package.
        tryDeleteEntry(getDirectory(id));
      }
    }

    return new Pair(successes, failures);
  }

  /// Ensures that the canonical clone of the repository referred to by [ref]
  /// contains the given Git [revision].
  Future _ensureRevision(PackageRef ref, String revision) async {
    var path = _repoCachePath(ref);
    if (_updatedRepos.contains(path)) return;

    if (!entryExists(path)) await _createRepoCache(ref);

    // Try to list the revision. If it doesn't exist, git will fail and we'll
    // know we have to update the repository.
    try {
      await _firstRevision(path, revision);
    } on git.GitException catch (_) {
      await _updateRepoCache(ref);
    }
  }

  /// Ensures that the canonical clone of the repository referred to by [ref]
  /// exists and is up-to-date.
  Future _ensureRepoCache(PackageRef ref) async {
    var path = _repoCachePath(ref);
    if (_updatedRepos.contains(path)) return;

    if (!entryExists(path)) {
      await _createRepoCache(ref);
    } else {
      await _updateRepoCache(ref);
    }
  }

  /// Creates the canonical clone of the repository referred to by [ref].
  ///
  /// This assumes that the canonical clone doesn't yet exist.
  Future _createRepoCache(PackageRef ref) async {
    var path = _repoCachePath(ref);
    assert(!_updatedRepos.contains(path));

    await _clone(ref.description['url'], path, mirror: true);
    _updatedRepos.add(path);
  }

  /// Runs "git fetch" in the canonical clone of the repository referred to by
  /// [ref].
  ///
  /// This assumes that the canonical clone already exists.
  Future _updateRepoCache(PackageRef ref) async {
    var path = _repoCachePath(ref);
    if (_updatedRepos.contains(path)) return new Future.value();
    await git.run(["fetch"], workingDir: path);
    _updatedRepos.add(path);
  }

  /// Updates the package list file in [revisionCachePath] to include [path], if
  /// necessary.
  void _updatePackageList(String revisionCachePath, String path) {
    var packages = _readPackageList(revisionCachePath);
    if (packages.contains(path)) return;

    _writePackageList(revisionCachePath, packages..add(path));
  }

  /// Returns the list of packages in [revisionCachePath].
  List<String> _readPackageList(String revisionCachePath) {
    var path = _packageListPath(revisionCachePath);

    // If there's no package list file, this cache was created by an older
    // version of pub where pubspecs were only allowed at the root of the
    // repository.
    if (!fileExists(path)) return ['.'];
    return readTextFile(path).split("\n");
  }

  /// Writes a package list indicating that [packages] exist in
  /// [revisionCachePath].
  void _writePackageList(String revisionCachePath, List<String> packages) {
    writeTextFile(_packageListPath(revisionCachePath), packages.join('\n'));
  }

  /// The path in a revision cache repository in which we keep a list of the
  /// packages in the repository.
  String _packageListPath(String revisionCachePath) =>
      p.join(revisionCachePath, '.git/pub-packages');

  /// Runs "git rev-list" on [reference] in [path] and returns the first result.
  ///
  /// This assumes that the canonical clone already exists.
  Future<String> _firstRevision(String path, String reference) async {
    var lines = await git
        .run(["rev-list", "--max-count=1", reference], workingDir: path);
    return lines.first;
  }

  /// Clones the repo at the URI [from] to the path [to] on the local
  /// filesystem.
  ///
  /// If [mirror] is true, creates a bare, mirrored clone. This doesn't check
  /// out the working tree, but instead makes the repository a local mirror of
  /// the remote repository. See the manpage for `git clone` for more
  /// information.
  ///
  /// If [shallow] is true, creates a shallow clone that contains no history
  /// for the repository.
  Future _clone(String from, String to,
      {bool mirror: false, bool shallow: false}) {
    return new Future.sync(() {
      // Git on Windows does not seem to automatically create the destination
      // directory.
      ensureDir(to);
      var args = ["clone", from, to];

      if (mirror) args.insert(1, "--mirror");
      if (shallow) args.insertAll(1, ["--depth", "1"]);

      return git.run(args);
    }).then((result) => null);
  }

  /// Checks out the reference [ref] in [repoPath].
  Future _checkOut(String repoPath, String ref) {
    return git
        .run(["checkout", ref], workingDir: repoPath).then((result) => null);
  }

  String _revisionCachePath(PackageId id) => p.join(
      systemCacheRoot, "${_repoName(id)}-${id.description['resolved-ref']}");

  /// Returns the path to the canonical clone of the repository referred to by
  /// [id] (the one in `<system cache>/git/cache`).
  String _repoCachePath(PackageRef ref) {
    var repoCacheName = '${_repoName(ref)}-${sha1(ref.description['url'])}';
    return p.join(systemCacheRoot, 'cache', repoCacheName);
  }

  /// Returns a short, human-readable name for the repository URL in [packageName].
  ///
  /// This name is not guaranteed to be unique.
  String _repoName(PackageName packageName) {
    var name = p.url.basename(packageName.description['url']);
    if (name.endsWith('.git')) {
      name = name.substring(0, name.length - '.git'.length);
    }
    return name;
  }
}