-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement resource merging for flavored source sets (#89)
* Implement resource merging for flavored source sets * Push missed file * Ensure outputs are present and handle cases like empty resource files
- Loading branch information
1 parent
0206168
commit f7277a3
Showing
18 changed files
with
496 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| """ | ||
| Rule to merge android variant specific resource folders and account for overrides. | ||
| """ | ||
|
|
||
| def _to_path(f): | ||
| return f.path | ||
|
|
||
| def _resource_merger_impl(ctx): | ||
| outputs = ctx.outputs.merged_resources | ||
| label = ctx.label.name | ||
|
|
||
| # Args for compiler | ||
| args = ctx.actions.args() | ||
| args.set_param_file_format("multiline") | ||
| args.use_param_file("--flagfile=%s", use_always = True) | ||
| args.add("RESOURCE_MERGER") | ||
| args.add("--target", ctx.label.package) | ||
| args.add_joined( | ||
| "--source-sets", | ||
| ctx.attr.source_sets, | ||
| join_with = ",", | ||
| ) | ||
| args.add_joined( | ||
| "--output", | ||
| outputs, | ||
| join_with = ",", | ||
| map_each = _to_path, | ||
| ) | ||
|
|
||
| mnemonic = "MergeSourceSets" | ||
| ctx.actions.run( | ||
| mnemonic = mnemonic, | ||
| inputs = depset(ctx.files.resources + ctx.files.manifests), | ||
| outputs = outputs, | ||
| executable = ctx.executable._compiler, | ||
| arguments = [args], | ||
| progress_message = "%s %s" % (mnemonic, ctx.label), | ||
| execution_requirements = { | ||
| "supports-workers": "0", | ||
| "supports-multiplex-workers": "0", | ||
| "requires-worker-protocol": "json", | ||
| "worker-key-mnemonic": "MergeSourceSets", | ||
| }, | ||
| ) | ||
|
|
||
| return [DefaultInfo(files = depset(outputs))] | ||
|
|
||
| resource_merger = rule( | ||
| implementation = _resource_merger_impl, | ||
| attrs = { | ||
| "source_sets": attr.string_list(), | ||
| "resources": attr.label_list(allow_files = True, mandatory = True), | ||
| "manifests": attr.label_list(allow_files = True, mandatory = True), | ||
| "merged_resources": attr.output_list(mandatory = True), | ||
| "_compiler": attr.label( | ||
| default = Label("@grab_bazel_common//tools/aapt_lite:aapt_lite"), | ||
| executable = True, | ||
| cfg = "exec", | ||
| ), | ||
| }, | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,85 @@ | ||
| load("@grab_bazel_common//tools/res_value:res_value.bzl", "res_value") | ||
| load("@grab_bazel_common//rules/android/private:resource_merger.bzl", "resource_merger") | ||
|
|
||
| def build_resources(name, resource_files, res_values): | ||
| def _calculate_output_files(name, all_resources): | ||
| """ | ||
| Returns list of source `resource_files` and generated `resource_files` from the `res_value` macro | ||
| Resource merger would merge source resource files and write to a merged directory. Bazel needs to know output files in advance, so this | ||
| method tries to predict the output files so we can register them as predeclared outputs. | ||
| Args: | ||
| all_resources: All resource files sorted based on priority with higher priority appearing first. | ||
| """ | ||
| outputs = [] | ||
|
|
||
| # Multiple res folders root can contain same file name of resource, prevent creating same outputs by storing normalized resource paths | ||
| # eg: `res/values/strings.xml` | ||
| normalized_res_paths = {} | ||
|
|
||
| for file in all_resources: | ||
| res_name_and_dir = file.split("/")[-2:] # ["values", "values.xml"] etc | ||
| res_dir = res_name_and_dir[0] | ||
| res_name = res_name_and_dir[1] | ||
| if "values" in res_dir: | ||
| # Resource merging merges all values files into single values.xml file. | ||
| normalized_res_path = "%s/out/res/%s/values.xml" % (name, res_dir) | ||
| else: | ||
| normalized_res_path = "%s/out/res/%s/%s" % (name, res_dir, res_name) | ||
|
|
||
| if normalized_res_path not in normalized_res_paths: | ||
| normalized_res_paths[normalized_res_path] = normalized_res_path | ||
| outputs.append(normalized_res_path) | ||
| return outputs | ||
|
|
||
| def build_resources( | ||
| name, | ||
| resource_files, | ||
| resources, | ||
| res_values): | ||
| """ | ||
| return resource_files + res_value( | ||
| Calculates and returns resource_files either generated, merged or just the source ones based on parameters given. When `resources` are | ||
| declared and it has multiple resource roots then all those roots are merged into single directory and contents of the directory are returned. | ||
| Conversely if resource_files are used then sources are returned as is. In both cases, generated resources passed via res_values are | ||
| accounted for. | ||
| """ | ||
| generated_resources = res_value( | ||
| name = name + "_res_value", | ||
| strings = res_values.get("strings", default = {}), | ||
| ) | ||
|
|
||
| if (len(resources) != 0 and len(resource_files) != 0): | ||
| fail("Either resources or resource_files should be specified but not both") | ||
|
|
||
| if (len(resources) != 0): | ||
| # Resources are passed with the new format | ||
| # Merge sources and return the merged result | ||
|
|
||
| if (len(resources) == 1): | ||
| resource_dir = resources.keys()[0] | ||
| return native.glob([resource_dir + "/**"]) + generated_resources | ||
|
|
||
| source_sets = [] # Source sets in the format res_dir::manifest | ||
| all_resources = [] # All resources | ||
| all_manifests = [] | ||
|
|
||
| for resource_dir in resources.keys(): | ||
| resource_dict = resources.get(resource_dir) | ||
| all_resources.extend(native.glob([resource_dir + "/**"])) | ||
|
|
||
| manifest = resource_dict.get("manifest", "") | ||
| if manifest != "": | ||
| all_manifests.append(manifest) | ||
|
|
||
| source_sets.append("%s::%s" % (resource_dir, manifest)) | ||
|
|
||
| merge_target_name = name + "_res" | ||
| merged_resources = _calculate_output_files(merge_target_name, all_resources) | ||
| resource_merger( | ||
| name = merge_target_name, | ||
| source_sets = source_sets, | ||
| resources = all_resources, | ||
| manifests = all_manifests, | ||
| merged_resources = merged_resources, | ||
| ) | ||
| return merged_resources + generated_resources | ||
| else: | ||
| return resource_files + generated_resources |
37 changes: 37 additions & 0 deletions
37
tools/aapt_lite/src/main/java/com/google/devtools/build/android/BUILD.bazel
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") | ||
|
|
||
| kt_jvm_library( | ||
| name = "source_set", | ||
| srcs = [ | ||
| "SourceSet.kt", | ||
| ], | ||
| ) | ||
|
|
||
| java_library( | ||
| name = "merger", | ||
| srcs = glob([ | ||
| "*.java", | ||
| ]), | ||
| deps = [ | ||
| ":source_set", | ||
| "//tools/android:android_tools", | ||
| ], | ||
| ) | ||
|
|
||
| kt_jvm_library( | ||
| name = "resource", | ||
| srcs = [ | ||
| "OutputFixer.kt", | ||
| "ResourceMergerCommand.kt", | ||
| ], | ||
| visibility = [ | ||
| "//visibility:public", | ||
| ], | ||
| deps = [ | ||
| ":merger", | ||
| ":source_set", | ||
| "//tools/aapt_lite/src/main/java/com/grab/aapt/databinding/util", | ||
| "@bazel_common_maven//:com_github_ajalt_clikt", | ||
| "@bazel_common_maven//:org_jetbrains_kotlinx_kotlinx_coroutines_core", | ||
| ], | ||
| ) |
35 changes: 35 additions & 0 deletions
35
tools/aapt_lite/src/main/java/com/google/devtools/build/android/OutputFixer.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.google.devtools.build.android | ||
|
|
||
| import java.io.File | ||
|
|
||
| object OutputFixer { | ||
| const val EMPTY_RES_CONTENT = """<?xml version="1.0" encoding="UTF-8" standalone="no"?><resources/>""" | ||
| fun process(outputDir: File, declaredOutputs: List<String>) { | ||
| val outputDirPath = outputDir.toPath() | ||
|
|
||
| // Merged directories will have qualifiers added by bazel which will not match the path specified in declaredOutputs, manually | ||
| // walk and remove the suffixes like v4, v13 etc from the resource bucket directories. | ||
| outputDir.walk() | ||
| .filter { it != outputDirPath } | ||
| .filter { it.parentFile?.parentFile?.toPath() == outputDirPath } | ||
| .filter { it.isDirectory && it.name.matches(Regex(".*-v\\d+$")) } | ||
| .forEach { resBucket -> | ||
| val newName = resBucket.name.split("-").dropLast(1).joinToString(separator = "-") | ||
| resBucket.renameTo(File(resBucket.parent, newName)) | ||
| } | ||
|
|
||
| // Empty resource files especially xmls are skipped in the merged directory, in order to satisfy bazel action output requirements | ||
| // manually add empty resource files here. | ||
| declaredOutputs | ||
| .asSequence() | ||
| .filter { it.endsWith(".xml") } | ||
| .map { File(it) } | ||
| .filter { !it.exists() } | ||
| .forEach { file -> | ||
| file.run { | ||
| parentFile?.mkdirs() | ||
| writeText(EMPTY_RES_CONTENT) | ||
| } | ||
| } | ||
| } | ||
| } |
60 changes: 60 additions & 0 deletions
60
tools/aapt_lite/src/main/java/com/google/devtools/build/android/ResourceMerger.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package com.google.devtools.build.android; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.google.common.collect.ImmutableMap; | ||
| import com.google.common.collect.ImmutableSet; | ||
| import com.google.common.util.concurrent.MoreExecutors; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| public class ResourceMerger { | ||
|
|
||
| public static ParsedAndroidData emptyAndroidData() { | ||
| return ParsedAndroidData.of( | ||
| ImmutableSet.of(), | ||
| ImmutableMap.of(), | ||
| ImmutableMap.of(), | ||
| ImmutableMap.of()); | ||
| } | ||
|
|
||
| public static void merge(final List<SourceSet> sourceSets, final File outputDir) throws IOException { | ||
| final Path target = Paths.get(outputDir.getAbsolutePath()); | ||
| Collections.reverse(sourceSets); | ||
| final ImmutableList<DependencyAndroidData> deps = ImmutableList.copyOf(sourceSets | ||
| .stream() | ||
| .map(sourceSet -> new DependencyAndroidData( | ||
| /*resourceDirs*/ ImmutableList.copyOf(sourceSet.getResourceDirs()), | ||
| /*assetDirs*/ ImmutableList.copyOf(sourceSet.getAssetDirs()), | ||
| /*manifest*/ sourceSet.getManifest().toPath(), | ||
| /*rTxt*/ null, | ||
| /*symbols*/ null, | ||
| /*compiledSymbols*/ null | ||
| )).collect(Collectors.toList())); | ||
|
|
||
| final ParsedAndroidData androidData = ParsedAndroidData.from(deps); | ||
| final AndroidDataMerger merger = AndroidDataMerger.createWithDefaults(); | ||
|
|
||
| final UnwrittenMergedAndroidData unwrittenMergedAndroidData = merger.doMerge( | ||
| /*transitive*/ emptyAndroidData(), | ||
| /*direct*/ emptyAndroidData(), | ||
| /*parsedPrimary*/ androidData, | ||
| /*primaryManifest*/ null, | ||
| /*primaryOverrideAll*/ true, | ||
| /*throwOnResourceConflict*/ false | ||
| ); | ||
| final MergedAndroidData result = unwrittenMergedAndroidData.write( | ||
| AndroidDataWriter.createWith( | ||
| /*manifestDirectory*/ target, | ||
| /*resourceDirectory*/ target.resolve("res"), | ||
| /*assertsDirectory*/ target.resolve("assets"), | ||
| /*executorService*/ MoreExecutors.newDirectExecutorService()) | ||
| ); | ||
| } | ||
| } |
Oops, something went wrong.