Update bcr_validation.py to handle overlay files (#2249)

Allow using the new overlay format
(#1566)

Tested with
#2240

This is an alternative to
#2046 (which I
hadn't seen previously 🙈 )

tools/bcr_validation.py

@@ -203,9 +203,8 @@ def add_module_dot_bazel_patch(self, diff, module_name, version):
     patches = source.get("patches", {})
     patches["module_dot_bazel.patch"] = integrity(read(patch_file))
     source["patches"] = patches
-    with open(self.registry.get_source_path(module_name, version), "w") as f:
-      json.dump(source, f, indent=4)
-      f.write("\n")
+    source_json_content = json.dumps(source, indent=4) + "\n"
+    self.registry.get_source_json_path(module_name, version).write_text(source_json_content)
 
   def verify_module_dot_bazel(self, module_name, version):
     source = self.registry.get_source(module_name, version)
@@ -225,6 +224,26 @@ def verify_module_dot_bazel(self, module_name, version):
             if actual_integrity != expected_integrity:
               self.report(BcrValidationResult.FAILED, f"The patch file `{patch_file}` has expected integrity value `{expected_integrity}`, but the real integrity value is `{actual_integrity}`.")
             apply_patch(source_root, source["patch_strip"], str(patch_file.resolve()))
+    if "overlay" in source:
+      version_dir = self.registry.get_version_dir(module_name, version)
+      for overlay_file, expected_integrity in source["overlay"].items():
+        overlay_src = version_dir / overlay_file
+        overlay_dst = source_root / overlay_file
+        try:
+          overlay_dst.resolve().relative_to(source_root)
+        except ValueError:
+          self.report(BcrValidationResult.FAILED, f"The overlay file path `{overlay_file}` must point inside the source archive.")
+          continue
+        try:
+          actual_integrity = integrity(read(overlay_src))
+        except FileNotFoundError:
+          self.report(BcrValidationResult.FAILED, f"The overlay file `{overlay_file}` does not exist")
+          continue
+        if actual_integrity != expected_integrity:
+          self.report(BcrValidationResult.FAILED, f"The overlay file `{overlay_file}` has expected integrity value `{expected_integrity}`, but the real integrity value is `{actual_integrity}`.")
+          continue
+        overlay_dst.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(overlay_src, overlay_dst)
 
     source_module_dot_bazel = source_root.joinpath("MODULE.bazel")
     if source_module_dot_bazel.exists():

tools/registry.py

@@ -243,26 +243,26 @@ def get_metadata(self, module_name):
   def get_metadata_path(self, module_name):
     return self.root / "modules" / module_name / "metadata.json"
 
+  def get_module_dir(self, module_name):
+    return self.root / "modules" / module_name
+
+  def get_version_dir(self, module_name, version):
+    return self.get_module_dir(module_name) / version
+
   def get_source(self, module_name, version):
-    source_path = self.root.joinpath("modules", module_name, version,
-                                     "source.json")
-    return json.load(source_path.open())
+    return json.loads(self.get_source_json_path(module_name, version).read_text())
 
-  def get_source_path(self, module_name, version):
-    return self.root.joinpath("modules", module_name, version,
-                              "source.json")
+  def get_source_json_path(self, module_name, version):
+    return self.get_version_dir(module_name, version) / "source.json"
 
   def get_presubmit_yml_path(self, module_name, version):
-    return self.root.joinpath("modules", module_name, version,
-                              "presubmit.yml")
+    return self.get_version_dir(module_name, version) / "presubmit.yml"
 
   def get_patch_file_path(self, module_name, version, patch_name):
-    return self.root.joinpath("modules", module_name, version,
-                              "patches", patch_name)
+    return self.get_version_dir(module_name, version) / "patches" / patch_name
 
   def get_module_dot_bazel_path(self, module_name, version):
-    return self.root.joinpath("modules", module_name, version,
-                              "MODULE.bazel")
+    return self.get_version_dir(module_name, version) / "MODULE.bazel"
 
   def contains(self, module_name, version=None):
     """
@@ -449,7 +449,7 @@ def update_integrity(self, module_name, version):
     """Update the SRI hashes of the source.json file of module at version."""
     source = self.get_source(module_name, version)
     source["integrity"] = integrity(download(source["url"]))
-    source_path = self.get_source_path(module_name, version)
+    source_path = self.get_source_json_path(module_name, version)
 
     patch_dir = source_path.parent / "patches"
     if patch_dir.exists():