Support lfs and submodules

doc/cache_layout.md

@@ -159,13 +159,26 @@ $PUB_CACHE/git/
 └── pub-c4e9ddc888c3aa89ef4462f0c4298929191e32b9/
 ```
 
-The `$PUB_CACHE/git/cache/` folder contains a "bare" checkout of each git-url (just the ). The
-folders are `$PUB_CACHE/git/cache/$name-$hash/` where `$name` is derived from base-name of the
-git url (without `.git`). and `$hash` is the sha1 of the git-url. This makes
-them recognizable and unique.
+The `$PUB_CACHE/git/cache/` folder contains a "bare" checkout of each git-url
+(just the .git folder without a working tree).
 
-The other sub-folders are the actual checkouts. They are clones of respective the `$PUB_CACHE/git/cache/$name-$hash/`
-folders checked out at a specific `ref`. The name is `$PUB_CACHE/git/$name-$resolvedRef/` where
+The folders are `$PUB_CACHE/git/cache/$name-$hash/` where `$name` is derived
+from base-name of the git url (without `.git`). and `$hash` is the sha1 of the
+git-url. This makes them recognizable and unique.
+
+The other sub-folders of `$PUB_CACHE/git` are the actual checkouts.
+
+Until Dart 3.1 These where clones of the respective
+`$PUB_CACHE/git/cache/$name-$hash/` folders checked out at a specific `ref`.
+
+After Dart 3.1 these are [git worktrees](https://git-scm.com/docs/git-worktree)
+instead of clones. This avoids repeating the entire .git folder for each checked
+out ref.
+
+In Dart 3.1 and later a clone will be recognized and replaced by a worktree when
+seen.
+
+The name of the checkout is `$PUB_CACHE/git/$name-$resolvedRef/` where
 `resolvedRef` is the commit-id that `ref` resolves to.
 
 ## Global packages

lib/src/source/git.dart

@@ -392,12 +392,11 @@ class GitSource extends CachedSource {
       final path = description.path;
       await _revisionCacheClones.putIfAbsent(revisionCachePath, () async {
         if (!entryExists(revisionCachePath)) {
-          await _cloneViaTemp(
+          await _createWorktree(
             _repoCachePath(description, cache),
             revisionCachePath,
-            cache,
+            resolvedRef,
           );
-          await _checkOut(revisionCachePath, resolvedRef);
           _writePackageList(revisionCachePath, [path]);
           didUpdate = true;
         } else {
@@ -436,7 +435,7 @@ class GitSource extends CachedSource {
     final result = <RepairResult>[];
 
     final packages = listDir(rootDir)
-        .where((entry) => dirExists(p.join(entry, '.git')))
+        .where((entry) => entryExists(p.join(entry, '.git')))
         .expand((revisionCachePath) {
           return _readPackageList(revisionCachePath).map((relative) {
             // If we've already failed to load another package from this
@@ -561,7 +560,7 @@ class GitSource extends CachedSource {
     final path = _repoCachePath(description, cache);
     assert(!_updatedRepos.contains(path));
     try {
-      await _cloneViaTemp(description.url, path, cache, mirror: true);
+      await _clone(description.url, path, cache);
     } catch (_) {
       await _deleteGitRepoIfInvalid(path);
       rethrow;
@@ -581,7 +580,7 @@ class GitSource extends CachedSource {
   ) async {
     final path = _repoCachePath(description, cache);
     if (_updatedRepos.contains(path)) return false;
-    await git.run([_gitDirArg(path), 'fetch'], workingDir: path);
+    await git.run([_gitDirArg(path), 'fetch', 'origin'], workingDir: path);
     _updatedRepos.add(path);
     return true;
   }
@@ -613,15 +612,15 @@ class GitSource extends CachedSource {
     }
   }
 
-  /// Updates the package list file in [revisionCachePath] to include [path], if
-  /// necessary.
+  /// Updates the package list file in [revisionCachePath] to include [package],
+  /// if necessary.
   ///
   /// Returns `true` if it had to update anything.
-  bool _updatePackageList(String revisionCachePath, String path) {
+  bool _updatePackageList(String revisionCachePath, String package) {
     final packages = _readPackageList(revisionCachePath);
-    if (packages.contains(path)) return false;
+    if (packages.contains(package)) return false;
 
-    _writePackageList(revisionCachePath, packages..add(path));
+    _writePackageList(revisionCachePath, packages..add(package));
     return true;
   }
 
@@ -645,7 +644,7 @@ class GitSource extends CachedSource {
   /// The path in a revision cache repository in which we keep a list of the
   /// packages in the repository.
   String _packageListPath(String revisionCachePath) =>
-      p.join(revisionCachePath, '.git/pub-packages');
+      p.join(revisionCachePath, '.pub-packages');
 
   /// Runs "git rev-list" on [reference] in [path] and returns the first result.
   ///
@@ -669,37 +668,38 @@ class GitSource extends CachedSource {
     return output;
   }
 
-  /// Clones the repo at the URI [from] to the path [to] on the local
-  /// filesystem.
+  /// Makes a working tree of the repo at the path [from] at ref [ref] to the
+  /// path [to] on the local filesystem.
   ///
-  /// If [mirror] is true, creates a bare, mirrored clone. This doesn't check
-  /// out the working tree, but instead makes the repository a local mirror of
-  /// the remote repository. See the manpage for `git clone` for more
-  /// information.
-  Future<void> _clone(
-    String from,
-    String to, {
-    bool mirror = false,
-  }) async {
+  /// Also checks out any submodules.
+  Future<void> _createWorktree(String from, String to, String ref) async {
     // Git on Windows does not seem to automatically create the destination
     // directory.
     ensureDir(to);
-    final args = ['clone', if (mirror) '--mirror', from, to];
-
-    await git.run(args);
+    await git.run(
+      [
+        _gitDirArg(from),
+        'worktree', 'add',
+        // Checkout <branch> even if already checked out in other worktree.
+        // Should not be necessary, but cannot hurt either.
+        '--force',
+        to,
+        ref,
+      ],
+      workingDir: from,
+    );
   }
 
-  /// Like [_clone], but clones to a temporary directory (inside the [cache])
-  /// and moves
-  Future<void> _cloneViaTemp(
-    String from,
-    String to,
-    SystemCache cache, {
-    bool mirror = false,
-  }) async {
+  /// Clones git repository at url [from] to the path [to]
+  /// clones to a temporary directory (inside the [cache])
+  /// and moves it to [to] as an atomic operation.
+  Future<void> _clone(String from, String to, SystemCache cache) async {
     final tempDir = cache.createTempDir();
     try {
-      await _clone(from, tempDir, mirror: mirror);
+      // Git on Windows does not seem to automatically create the destination
+      // directory.
+      ensureDir(tempDir);
+      await git.run(['clone', '--mirror', from, tempDir]);
     } catch (_) {
       deleteEntry(tempDir);
       rethrow;
@@ -713,12 +713,6 @@ class GitSource extends CachedSource {
     tryRenameDir(tempDir, to);
   }
 
-  /// Checks out the reference [ref] in [repoPath].
-  Future<void> _checkOut(String repoPath, String ref) {
-    return git
-        .run(['checkout', ref], workingDir: repoPath).then((result) => null);
-  }
-
   String _revisionCachePath(PackageId id, SystemCache cache) => p.join(
         cache.rootDirForSource(this),
         '${_repoName(id.description.description as GitDescription)}-'

test/descriptor.dart

@@ -310,8 +310,15 @@ Descriptor tokensFile([Map<String, dynamic> contents = const {}]) {
 
 /// Describes the application directory, containing only a pubspec specifying
 /// the given [dependencies].
-DirectoryDescriptor appDir({Map? dependencies, Map<String, Object>? pubspec}) =>
-    dir(appPath, [appPubspec(dependencies: dependencies, extras: pubspec)]);
+DirectoryDescriptor appDir({
+  Map? dependencies,
+  Map<String, Object>? pubspec,
+  Iterable<Descriptor>? contents,
+}) =>
+    dir(
+      appPath,
+      [appPubspec(dependencies: dependencies, extras: pubspec), ...?contents],
+    );
 
 /// Describes a `.dart_tools/package_config.json` file.
 ///

test/descriptor/git.dart

@@ -3,9 +3,11 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:io';
 
 import 'package:path/path.dart' as p;
 import 'package:pub/src/git.dart' as git;
+import 'package:test/test.dart';
 import 'package:test_descriptor/test_descriptor.dart';
 
 /// Describes a Git repository and its contents.
@@ -80,4 +82,37 @@ class GitRepoDescriptor extends DirectoryDescriptor {
       await _runGit(command, parent);
     }
   }
+
+  /// Serves this git directory on localhost a fresh port
+  /// returns the port.
+  Future<int> serve() async {
+    // Use this to invent a fresh host.
+    final s = await ServerSocket.bind('localhost', 0);
+    final port = s.port;
+    await s.close();
+    final process = await Process.start(
+      'git',
+      [
+        'daemon',
+        '--verbose',
+        '--export-all',
+        '--base-path=.git',
+        '--reuseaddr',
+        '--strict-paths',
+        '--port=$port',
+        '.git/',
+      ],
+      workingDirectory: p.join(sandbox, name),
+    );
+    final c = Completer<void>();
+    process.stderr.listen((x) {
+      if (!c.isCompleted) c.complete();
+    });
+    await c.future;
+    addTearDown(() async {
+      process.kill();
+      await process.exitCode;
+    });
+    return port;
+  }
 }

test/get/git/git_lfs_test.dart

@@ -0,0 +1,48 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+import 'package:test/test.dart';
+
+import '../../descriptor.dart' as d;
+import '../../test_pub.dart';
+
+void main() {
+  test('Can use LFS', () async {
+    ensureGit();
+
+    final foo = d.git('foo.git', [d.libPubspec('foo', '1.0.0')]);
+    await foo.create();
+    await foo.runGit(['lfs', 'install']);
+
+    await d.dir('foo.git', [
+      d.dir('lib', [d.file('foo.dart', 'main() => print("hi");')]),
+    ]).create();
+    await foo.runGit(['lfs', 'track', 'lib/foo.dart']);
+    await foo.runGit(['add', '.gitattributes']);
+    await foo.commit();
+
+    await d.appDir(
+      dependencies: {
+        'foo': {
+          'git': {'url': '../foo.git'},
+        },
+      },
+      contents: [
+        d.dir('bin', [d.file('main.dart', 'export "package:foo/foo.dart";')]),
+      ],
+    ).create();
+    await pubGet();
+
+    await runPub(args: ['run', 'myapp:main'], output: contains('hi'));
+
+    await d.git(
+      'foo.git',
+      [
+        d.dir('lib', [d.file('foo.dart', 'main() => print("bye");')]),
+      ],
+    ).commit();
+
+    await pubUpgrade();
+    await runPub(args: ['run', 'myapp:main'], output: contains('bye'));
+  });
+}