Implement a lease service to support remote cache eviction

src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java

@@ -42,10 +42,12 @@
 import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
 import com.google.devtools.build.lib.skyframe.TreeArtifactValue.ArchivedRepresentation;
 import com.google.devtools.build.lib.vfs.OutputPermissions;
+import com.google.devtools.build.lib.vfs.LeaseService;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -199,11 +201,18 @@ private static boolean validateArtifacts(
       NestedSet<Artifact> actionInputs,
       MetadataHandler metadataHandler,
       boolean checkOutput,
-      @Nullable CachedOutputMetadata cachedOutputMetadata) {
+      @Nullable CachedOutputMetadata cachedOutputMetadata,
+      @Nullable LeaseService leaseService) {
     Map<String, FileArtifactValue> mdMap = new HashMap<>();
     if (checkOutput) {
       for (Artifact artifact : action.getOutputs()) {
         FileArtifactValue metadata = getCachedMetadata(cachedOutputMetadata, artifact);
+        if (leaseService != null && metadata != null && metadata.isRemote()) {
+          if (!leaseService.isAlive(
+              metadata.getDigest(), metadata.getSize(), metadata.getLocationIndex())) {
+            metadata = null;
+          }
+        }
         if (metadata == null) {
           metadata = getMetadataMaybe(metadataHandler, artifact);
         }
@@ -434,7 +443,8 @@ public Token getTokenIfNeedToExecute(
       MetadataHandler metadataHandler,
       ArtifactExpander artifactExpander,
       Map<String, String> remoteDefaultPlatformProperties,
-      boolean loadCachedOutputMetadata)
+      boolean loadCachedOutputMetadata,
+      @Nullable LeaseService leaseService)
       throws InterruptedException {
     // TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that
     // produce only symlinks we should not check whether inputs are valid at all - all that matters
@@ -494,7 +504,8 @@ public Token getTokenIfNeedToExecute(
         clientEnv,
         outputPermissions,
         remoteDefaultPlatformProperties,
-        cachedOutputMetadata)) {
+        cachedOutputMetadata,
+        leaseService)) {
       if (entry != null) {
         removeCacheEntry(action);
       }
@@ -524,7 +535,8 @@ private boolean mustExecute(
       Map<String, String> clientEnv,
       OutputPermissions outputPermissions,
       Map<String, String> remoteDefaultPlatformProperties,
-      @Nullable CachedOutputMetadata cachedOutputMetadata)
+      @Nullable CachedOutputMetadata cachedOutputMetadata,
+      @Nullable LeaseService leaseService)
       throws InterruptedException {
     // Unconditional execution can be applied only for actions that are allowed to be executed.
     if (unconditionalExecution(action)) {
@@ -544,7 +556,7 @@ private boolean mustExecute(
       actionCache.accountMiss(MissReason.CORRUPTED_CACHE_ENTRY);
       return true;
     } else if (validateArtifacts(
-        entry, action, actionInputs, metadataHandler, true, cachedOutputMetadata)) {
+        entry, action, actionInputs, metadataHandler, true, cachedOutputMetadata, leaseService)) {
       reportChanged(handler, action);
       actionCache.accountMiss(MissReason.DIFFERENT_FILES);
       return true;
@@ -747,7 +759,8 @@ private void checkMiddlemanAction(
           action.getInputs(),
           metadataHandler,
           false,
-          /*cachedOutputMetadata=*/ null)) {
+          /*cachedOutputMetadata=*/ null,
+          /*leaseService=*/ null)) {
         reportChanged(handler, action);
         actionCache.accountMiss(MissReason.DIFFERENT_FILES);
         changed = true;
@@ -788,7 +801,8 @@ public Token getTokenUnconditionallyAfterFailureToRecordActionCacheHit(
       MetadataHandler metadataHandler,
       ArtifactExpander artifactExpander,
       Map<String, String> remoteDefaultPlatformProperties,
-      boolean loadCachedOutputMetadata)
+      boolean loadCachedOutputMetadata,
+      @Nullable LeaseService leaseService)
       throws InterruptedException {
     if (action != null) {
       removeCacheEntry(action);
@@ -802,7 +816,8 @@ public Token getTokenUnconditionallyAfterFailureToRecordActionCacheHit(
         metadataHandler,
         artifactExpander,
         remoteDefaultPlatformProperties,
-        loadCachedOutputMetadata);
+        loadCachedOutputMetadata,
+        leaseService);
   }
 
   /** Returns an action key. It is always set to the first output exec path string. */

src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java

@@ -503,6 +503,8 @@ private static RemoteFileArtifactValue decodeRemoteMetadata(
           PathFragment.create(getStringForIndex(indexer, VarInt.getVarInt(source)));
     }
 
+    // TODO(chiwang): setting expiredAtEpochMilli to 0 for now. Save/Load it once the implementation
+    //  details in other part are clear.
     return RemoteFileArtifactValue.create(
         digest, size, locationIndex, actionId, materializationExecPath);
   }

src/main/java/com/google/devtools/build/lib/remote/BUILD

@@ -112,6 +112,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/protobuf:failure_details_java_proto",
+        "//src/main/protobuf:remote_lease_java_proto",
         "//src/main/protobuf:spawn_java_proto",
         "//third_party:auth",
         "//third_party:caffeine",

src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java

@@ -32,6 +32,7 @@
 import com.google.devtools.build.lib.buildtool.buildevent.ProfilerStartedEvent;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.remote.common.MissingDigestsFinder.Intention;
 import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
 import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext.CachePolicy;
 import com.google.devtools.build.lib.remote.options.RemoteBuildEventUploadMode;
@@ -253,7 +254,9 @@ private Single<List<PathMetadata>> queryRemoteCache(
     if (digestsToQuery.isEmpty()) {
       return Single.just(knownRemotePaths);
     }
-    return toSingle(() -> remoteCache.findMissingDigests(context, digestsToQuery), executor)
+    return toSingle(
+            () -> remoteCache.findMissingDigests(context, Intention.WRITE, digestsToQuery),
+            executor)
         .onErrorResumeNext(
             error -> {
               reporterUploadError(error);

src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java

@@ -178,7 +178,7 @@ public static boolean isRemoteCacheOptions(RemoteOptions options) {
 
   @Override
   public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(
-      RemoteActionExecutionContext context, Iterable<Digest> digests) {
+      RemoteActionExecutionContext context, Intention intention, Iterable<Digest> digests) {
     if (Iterables.isEmpty(digests)) {
       return Futures.immediateFuture(ImmutableSet.of());
     }

src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java

@@ -51,6 +51,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.channels.ReadableByteChannel;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -74,6 +75,8 @@ public class RemoteActionFileSystem extends DelegateFileSystem {
   private final ActionInputMap inputArtifactData;
   private final ImmutableMap<PathFragment, Artifact> outputMapping;
   private final RemoteActionInputFetcher inputFetcher;
+  private final RemoteLeaseService remoteLeaseService;
+
   private final RemoteInMemoryFileSystem remoteOutputTree;
 
   @Nullable private MetadataInjector metadataInjector = null;
@@ -84,14 +87,17 @@ public class RemoteActionFileSystem extends DelegateFileSystem {
       String relativeOutputPath,
       ActionInputMap inputArtifactData,
       Iterable<Artifact> outputArtifacts,
-      RemoteActionInputFetcher inputFetcher) {
+      RemoteActionInputFetcher inputFetcher,
+      RemoteLeaseService remoteLeaseService) {
     super(localDelegate);
     this.execRoot = checkNotNull(execRootFragment, "execRootFragment");
     this.outputBase = execRoot.getRelative(checkNotNull(relativeOutputPath, "relativeOutputPath"));
     this.inputArtifactData = checkNotNull(inputArtifactData, "inputArtifactData");
     this.outputMapping =
         stream(outputArtifacts).collect(toImmutableMap(Artifact::getExecPath, a -> a));
     this.inputFetcher = checkNotNull(inputFetcher, "inputFetcher");
+    this.remoteLeaseService = remoteLeaseService;
+
     this.remoteOutputTree = new RemoteInMemoryFileSystem(getDigestFunction());
   }
 

src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java

@@ -34,6 +34,7 @@
 import com.google.devtools.build.lib.concurrent.ThreadSafety;
 import com.google.devtools.build.lib.exec.SpawnProgressEvent;
 import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
+import com.google.devtools.build.lib.remote.common.MissingDigestsFinder.Intention;
 import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
 import com.google.devtools.build.lib.remote.common.ProgressStatusListener;
 import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
@@ -115,11 +116,11 @@ public CachedActionResult downloadActionResult(
    * guaranteed to be a subset of {@code digests}.
    */
   public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(
-      RemoteActionExecutionContext context, Iterable<Digest> digests) {
+      RemoteActionExecutionContext context, Intention intention, Iterable<Digest> digests) {
     if (Iterables.isEmpty(digests)) {
       return immediateFuture(ImmutableSet.of());
     }
-    return cacheProtocol.findMissingDigests(context, digests);
+    return cacheProtocol.findMissingDigests(context, intention, digests);
   }
 
   /** Returns whether the action cache supports updating action results. */

src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionCache.java

@@ -33,6 +33,7 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
+import com.google.devtools.build.lib.remote.common.MissingDigestsFinder.Intention;
 import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
 import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
 import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
@@ -252,7 +253,7 @@ private Single<List<UploadTask>> findMissingBlobs(
                                   if (digestsToQuery.isEmpty()) {
                                     return immediateFuture(ImmutableSet.of());
                                   }
-                                  return findMissingDigests(context, digestsToQuery);
+                                  return findMissingDigests(context, Intention.WRITE, digestsToQuery);
                                 },
                                 directExecutor())
                             .map(

src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java

@@ -788,8 +788,7 @@ private void createSymlinks(Iterable<SymlinkMetadata> symlinks) throws IOExcepti
     }
   }
 
-  private void injectRemoteArtifacts(RemoteAction action, ActionResultMetadata metadata)
-      throws IOException {
+  private void injectRemoteArtifacts(RemoteAction action, ActionResultMetadata metadata) throws IOException {
     FileSystem actionFileSystem = action.getSpawnExecutionContext().getActionFileSystem();
     checkState(actionFileSystem instanceof RemoteActionFileSystem);