Add managed_directories attribute to workspace() function.

This is only a part of the incrementally updated user-owned directory feature; that is why the parsed value is not yet used in computations.

- Under --experimental_allow_incremental_repository_updates flag.
- Parse results are put into WorkspaceFileValue map field.

PiperOrigin-RevId: 244819268

src/main/java/com/google/devtools/build/lib/packages/StarlarkSemanticsOptions.java

@@ -60,6 +60,21 @@ public class StarlarkSemanticsOptions extends OptionsBase implements Serializabl
 
   // <== Add new options here in alphabetic order ==>
 
+  @Option(
+      name = "experimental_allow_incremental_repository_updates",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS,
+      effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+      metadataTags = {OptionMetadataTag.EXPERIMENTAL},
+      help =
+          "If used, it is possible to define a mapping between external repositories"
+              + " and some (mostly likely ignored by .bazelignore) directories."
+              + " The repository rule can read and update files in those directories,"
+              + " and the changes will be visible in the same build."
+              + " Use attribute 'managed_directories' of the global workspace()"
+              + " function in WORKSPACE file to define the mapping.")
+  public boolean experimentalAllowIncrementalRepositoryUpdates;
+
   @Option(
       name = "experimental_build_setting_api",
       defaultValue = "false",
@@ -581,6 +596,8 @@ public class StarlarkSemanticsOptions extends OptionsBase implements Serializabl
   public StarlarkSemantics toSkylarkSemantics() {
     return StarlarkSemantics.builder()
         // <== Add new options here in alphabetic order ==>
+        .experimentalAllowIncrementalRepositoryUpdates(
+            experimentalAllowIncrementalRepositoryUpdates)
         .experimentalBuildSettingApi(experimentalBuildSettingApi)
         .experimentalCcSkylarkApiEnabledPackages(experimentalCcSkylarkApiEnabledPackages)
         .experimentalEnableAndroidMigrationApis(experimentalEnableAndroidMigrationApis)

src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java

@@ -24,6 +24,7 @@
 import com.google.devtools.build.lib.analysis.skylark.SymbolGenerator;
 import com.google.devtools.build.lib.cmdline.LabelConstants;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.NullEventHandler;
 import com.google.devtools.build.lib.events.StoredEventHandler;
@@ -47,6 +48,7 @@
 import com.google.devtools.build.lib.syntax.StarlarkSemantics;
 import com.google.devtools.build.lib.syntax.ValidationEnvironment;
 import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.lib.vfs.Root;
 import com.google.devtools.build.lib.vfs.RootedPath;
 import java.io.File;
@@ -76,7 +78,6 @@ public class WorkspaceFactory {
   private final Path defaultSystemJavabaseDir;
   private final Mutability mutability;
 
-  private final boolean allowOverride;
   private final RuleFactory ruleFactory;
 
   private final WorkspaceGlobals workspaceGlobals;
@@ -119,7 +120,6 @@ public WorkspaceFactory(
     this.installDir = installDir;
     this.workspaceDir = workspaceDir;
     this.defaultSystemJavabaseDir = defaultSystemJavabaseDir;
-    this.allowOverride = allowOverride;
     this.environmentExtensions = environmentExtensions;
     this.ruleFactory = new RuleFactory(ruleClassProvider, AttributeContainer::new);
     this.workspaceGlobals = new WorkspaceGlobals(allowOverride, ruleFactory);
@@ -421,4 +421,8 @@ public Map<String, Extension> getImportMap() {
   public Map<String, Object> getVariableBindings() {
     return variableBindings;
   }
+
+  public Map<PathFragment, RepositoryName> getManagedDirectories() {
+    return workspaceGlobals.getManagedDirectories();
+  }
 }

src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java

@@ -22,6 +22,7 @@
 import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
 import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
 import com.google.devtools.build.lib.syntax.Environment.Extension;
+import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.lib.vfs.RootedPath;
 import com.google.devtools.build.skyframe.SkyFunctionName;
 import com.google.devtools.build.skyframe.SkyKey;
@@ -105,6 +106,9 @@ public String toString() {
   private final ImmutableMap<String, Integer> importToChunkMap;
   private final ImmutableMap<RepositoryName, ImmutableMap<RepositoryName, RepositoryName>>
       repositoryMapping;
+  // Mapping of the relative paths of the incrementally updated managed directories
+  // to the managing external repositories
+  private final ImmutableMap<PathFragment, RepositoryName> managedDirectories;
 
   /**
    * Create a WorkspaceFileValue containing the various values necessary to compute the split
@@ -122,6 +126,8 @@ public String toString() {
    * @param idx The index of this part of the split WORKSPACE file (0 for the first one, 1 for the
    *     second one and so on).
    * @param hasNext Is there a next part in the WORKSPACE file or this part the last one?
+   * @param managedDirectories Mapping of the relative paths of the incrementally updated managed
+   *     directories to the managing external repositories.
    */
   public WorkspaceFileValue(
       Package pkg,
@@ -130,7 +136,8 @@ public WorkspaceFileValue(
       Map<String, Object> bindings,
       RootedPath path,
       int idx,
-      boolean hasNext) {
+      boolean hasNext,
+      ImmutableMap<PathFragment, RepositoryName> managedDirectories) {
     this.pkg = Preconditions.checkNotNull(pkg);
     this.idx = idx;
     this.path = path;
@@ -139,6 +146,7 @@ public WorkspaceFileValue(
     this.importMap = ImmutableMap.copyOf(importMap);
     this.importToChunkMap = ImmutableMap.copyOf(importToChunkMap);
     this.repositoryMapping = pkg.getExternalPackageRepositoryMappings();
+    this.managedDirectories = managedDirectories;
   }
 
   /**
@@ -220,4 +228,8 @@ public ImmutableMap<String, Integer> getImportToChunkMap() {
       getRepositoryMapping() {
     return repositoryMapping;
   }
+
+  public ImmutableMap<PathFragment, RepositoryName> getManagedDirectories() {
+    return managedDirectories;
+  }
 }

src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java

@@ -17,6 +17,8 @@
 import static com.google.devtools.build.lib.syntax.Runtime.NONE;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.LabelValidator;
@@ -29,8 +31,12 @@
 import com.google.devtools.build.lib.syntax.EvalException;
 import com.google.devtools.build.lib.syntax.FuncallExpression;
 import com.google.devtools.build.lib.syntax.Runtime.NoneType;
+import com.google.devtools.build.lib.syntax.SkylarkDict;
 import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -42,14 +48,22 @@ public class WorkspaceGlobals implements WorkspaceGlobalsApi {
 
   private final boolean allowOverride;
   private final RuleFactory ruleFactory;
+  // Mapping of the relative paths of the incrementally updated managed directories
+  // to the managing external repositories
+  private final TreeMap<PathFragment, RepositoryName> managedDirectoriesMap;
 
   public WorkspaceGlobals(boolean allowOverride, RuleFactory ruleFactory) {
     this.allowOverride = allowOverride;
     this.ruleFactory = ruleFactory;
+    this.managedDirectoriesMap = Maps.newTreeMap();
   }
 
   @Override
-  public NoneType workspace(String name, FuncallExpression ast, Environment env)
+  public NoneType workspace(
+      String name,
+      SkylarkDict<String, Object> managedDirectories,
+      FuncallExpression ast,
+      Environment env)
       throws EvalException, InterruptedException {
     if (allowOverride) {
       if (!isLegalWorkspaceName(name)) {
@@ -80,6 +94,7 @@ public NoneType workspace(String name, FuncallExpression ast, Environment env)
             RepositoryName.createFromValidStrippedName(name),
             RepositoryName.MAIN);
       }
+      parseManagedDirectories(managedDirectories, ast);
       return NONE;
     } else {
       throw new EvalException(
@@ -88,6 +103,98 @@ public NoneType workspace(String name, FuncallExpression ast, Environment env)
     }
   }
 
+  private void parseManagedDirectories(
+      SkylarkDict<String, Object> managedDirectories, FuncallExpression ast) throws EvalException {
+    Map<PathFragment, String> nonNormalizedPathsMap = Maps.newHashMap();
+    for (Map.Entry<String, Object> entry : managedDirectories.entrySet()) {
+      RepositoryName repositoryName = createRepositoryName(entry.getKey(), ast.getLocation());
+      List<PathFragment> paths =
+          getManagedDirectoriesPaths(entry.getValue(), ast.getLocation(), nonNormalizedPathsMap);
+      for (PathFragment dir : paths) {
+        PathFragment floorKey = managedDirectoriesMap.floorKey(dir);
+        if (dir.equals(floorKey)) {
+          throw new EvalException(
+              ast.getLocation(),
+              String.format(
+                  "managed_directories attribute should not contain multiple"
+                      + " (or duplicate) repository mappings for the same directory ('%s').",
+                  nonNormalizedPathsMap.get(dir)));
+        }
+        PathFragment ceilingKey = managedDirectoriesMap.ceilingKey(dir);
+        boolean isDescendant = floorKey != null && dir.startsWith(floorKey);
+        if (isDescendant || (ceilingKey != null && ceilingKey.startsWith(dir))) {
+          throw new EvalException(
+              ast.getLocation(),
+              String.format(
+                  "managed_directories attribute value can not contain nested mappings."
+                      + " '%s' is a descendant of '%s'.",
+                  nonNormalizedPathsMap.get(isDescendant ? dir : ceilingKey),
+                  nonNormalizedPathsMap.get(isDescendant ? floorKey : dir)));
+        }
+        managedDirectoriesMap.put(dir, repositoryName);
+      }
+    }
+  }
+
+  private RepositoryName createRepositoryName(String key, Location location) throws EvalException {
+    if (!key.startsWith("@")) {
+      throw new EvalException(
+          location,
+          String.format(
+              "Cannot parse repository name '%s'. Repository name should start with '@'.", key));
+    }
+    try {
+      return RepositoryName.create(key);
+    } catch (LabelSyntaxException e) {
+      throw new EvalException(location, e);
+    }
+  }
+
+  private List<PathFragment> getManagedDirectoriesPaths(
+      Object directoriesList, Location location, Map<PathFragment, String> nonNormalizedPathsMap)
+      throws EvalException {
+    if (!(directoriesList instanceof SkylarkList)) {
+      throw new EvalException(
+          location,
+          "managed_directories attribute value should be of the type attr.string_list_dict(),"
+              + " mapping repository name to the list of managed directories.");
+    }
+    List<PathFragment> result = Lists.newArrayList();
+    for (Object obj : (SkylarkList) directoriesList) {
+      if (!(obj instanceof String)) {
+        throw new EvalException(
+            location,
+            String.format("Expected managed directory path (as string), but got '%s'.", obj));
+      }
+      String path = ((String) obj).trim();
+      if (path.isEmpty()) {
+        throw new EvalException(
+            location, "Expected managed directory path to be non-empty string.");
+      }
+      PathFragment pathFragment = PathFragment.create(path);
+      if (pathFragment.isAbsolute()) {
+        throw new EvalException(
+            location,
+            String.format(
+                "Expected managed directory path ('%s') to be relative to the workspace root.",
+                path));
+      }
+      if (pathFragment.containsUplevelReferences()) {
+        throw new EvalException(
+            location,
+            String.format(
+                "Expected managed directory path ('%s') to be under the workspace root.", path));
+      }
+      nonNormalizedPathsMap.put(pathFragment, path);
+      result.add(pathFragment);
+    }
+    return result;
+  }
+
+  public Map<PathFragment, RepositoryName> getManagedDirectories() {
+    return managedDirectoriesMap;
+  }
+
   @Override
   public NoneType registerExecutionPlatforms(
       SkylarkList<?> platformLabels, Location location, Environment env)

src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java

@@ -92,7 +92,8 @@ public SkyValue compute(SkyKey skyKey, Environment env)
             /* bindings = */ ImmutableMap.<String, Object>of(),
             workspaceRoot,
             /* idx = */ 0, // first fragment
-            /* hasNext = */ false);
+            /* hasNext = */ false,
+            ImmutableMap.of());
       } catch (NoSuchPackageException e) {
         throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
       }
@@ -149,7 +150,8 @@ public SkyValue compute(SkyKey skyKey, Environment env)
           parser.getVariableBindings(),
           workspaceRoot,
           key.getIndex(),
-          key.getIndex() < workspaceASTValue.getASTs().size() - 1);
+          key.getIndex() < workspaceASTValue.getASTs().size() - 1,
+          ImmutableMap.copyOf(parser.getManagedDirectories()));
     } catch (NoSuchPackageException e) {
       throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
     }

src/main/java/com/google/devtools/build/lib/skylarkbuildapi/WorkspaceGlobalsApi.java

@@ -22,7 +22,9 @@
 import com.google.devtools.build.lib.syntax.EvalException;
 import com.google.devtools.build.lib.syntax.FuncallExpression;
 import com.google.devtools.build.lib.syntax.Runtime.NoneType;
+import com.google.devtools.build.lib.syntax.SkylarkDict;
 import com.google.devtools.build.lib.syntax.SkylarkList;
+import com.google.devtools.build.lib.syntax.StarlarkSemantics.FlagIdentifier;
 
 /** A collection of global skylark build API functions that apply to WORKSPACE files. */
 @SkylarkGlobalLibrary
@@ -42,11 +44,31 @@ public interface WorkspaceGlobalsApi {
             type = String.class,
             doc = "the name of the workspace.",
             named = true,
-            positional = false)
+            positional = false),
+        @Param(
+            name = "managed_directories",
+            type = SkylarkDict.class,
+            generic1 = String.class,
+            noneable = true,
+            named = true,
+            positional = false,
+            enableOnlyWithFlag = FlagIdentifier.EXPERIMENTAL_ALLOW_INCREMENTAL_REPOSITORY_UPDATES,
+            defaultValue = "{}",
+            valueWhenDisabled = "{}",
+            doc =
+                "Dict (strings to list of strings) for defining the mappings between external"
+                    + " repositories and relative (to the workspace root) paths to directories"
+                    + " they incrementally update."
+                    + "\nManaged directories must be excluded from the source tree by listing"
+                    + " them (or their parent directories) in the .bazelignore file."),
       },
       useAst = true,
       useEnvironment = true)
-  NoneType workspace(String name, FuncallExpression ast, Environment env)
+  NoneType workspace(
+      String name,
+      SkylarkDict<String, Object> managedDirectories,
+      FuncallExpression ast,
+      Environment env)
       throws EvalException, InterruptedException;
 
   @SkylarkCallable(

src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java

@@ -39,6 +39,8 @@ public abstract class StarlarkSemantics {
    * the exact name of the flag transformed to upper case (for error representation).
    */
   public enum FlagIdentifier {
+    EXPERIMENTAL_ALLOW_INCREMENTAL_REPOSITORY_UPDATES(
+        StarlarkSemantics::experimentalAllowIncrementalRepositoryUpdates),
     EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS(
         StarlarkSemantics::experimentalEnableAndroidMigrationApis),
     EXPERIMENTAL_BUILD_SETTING_API(StarlarkSemantics::experimentalBuildSettingApi),
@@ -114,6 +116,8 @@ public boolean flagValue(FlagIdentifier flagIdentifier) {
       AutoValue_StarlarkSemantics.class;
 
   // <== Add new options here in alphabetic order ==>
+  public abstract boolean experimentalAllowIncrementalRepositoryUpdates();
+
   public abstract boolean experimentalBuildSettingApi();
 
   public abstract ImmutableList<String> experimentalCcSkylarkApiEnabledPackages();
@@ -209,6 +213,7 @@ public static Builder builderWithDefaults() {
           // <== Add new options here in alphabetic order ==>
           .experimentalBuildSettingApi(false)
           .experimentalCcSkylarkApiEnabledPackages(ImmutableList.of())
+          .experimentalAllowIncrementalRepositoryUpdates(false)
           .experimentalEnableAndroidMigrationApis(false)
           .experimentalGoogleLegacyApi(false)
           .experimentalJavaCommonCreateProviderEnabledPackages(ImmutableList.of())
@@ -253,6 +258,8 @@ public static Builder builderWithDefaults() {
   public abstract static class Builder {
 
     // <== Add new options here in alphabetic order ==>
+    public abstract Builder experimentalAllowIncrementalRepositoryUpdates(boolean value);
+
     public abstract Builder experimentalBuildSettingApi(boolean value);
 
     public abstract Builder experimentalCcSkylarkApiEnabledPackages(List<String> value);

src/test/java/com/google/devtools/build/lib/packages/SkylarkSemanticsConsistencyTest.java

@@ -118,6 +118,7 @@ public void canGetBuilderFromInstance() {
   private static StarlarkSemanticsOptions buildRandomOptions(Random rand) throws Exception {
     return parseOptions(
         // <== Add new options here in alphabetic order ==>
+        "--experimental_allow_incremental_repository_updates=" + rand.nextBoolean(),
         "--experimental_build_setting_api=" + rand.nextBoolean(),
         "--experimental_cc_skylark_api_enabled_packages="
             + rand.nextDouble()
@@ -172,6 +173,7 @@ private static StarlarkSemanticsOptions buildRandomOptions(Random rand) throws E
   private static StarlarkSemantics buildRandomSemantics(Random rand) {
     return StarlarkSemantics.builder()
         // <== Add new options here in alphabetic order ==>
+        .experimentalAllowIncrementalRepositoryUpdates(rand.nextBoolean())
         .experimentalBuildSettingApi(rand.nextBoolean())
         .experimentalCcSkylarkApiEnabledPackages(
             ImmutableList.of(String.valueOf(rand.nextDouble()), String.valueOf(rand.nextDouble())))