From eb38453f1375388220ad56ca0b3e5a416de79c9f Mon Sep 17 00:00:00 2001 From: Andrew Lindesay Date: Sun, 25 Aug 2024 22:37:35 +1200 Subject: [PATCH 1/2] Implement py code generation handling It is possible to write Bazel rules that generate Python code and act as a `py_library`. The plugin is augmented with this change to have a means of detecting these sort of rules and be able to work with them. --- aspect/intellij_info_impl.bzl | 56 ++++++ base/BUILD | 4 +- base/src/META-INF/blaze-base.xml | 2 + .../BlazeQueryTargetTagFilter.java | 133 +++++++++++++ .../base/dependencies/TargetTagFilter.java | 74 ++++++++ .../google/idea/blaze/base/ideinfo/Tags.java | 7 +- .../base/model/primitives/LanguageClass.java | 41 ++-- .../base/sync/SyncProjectTargetsHelper.java | 52 ++++- .../BlazeQueryTargetTagFilterTest.java | 69 +++++++ docs/python/code-generators.md | 20 ++ docs/python/images/code-generators.svg | 179 ++++++++++++++++++ .../python/simple_code_generator/.gitignore | 3 + .../python/simple_code_generator/MODULE.bazel | 12 ++ .../python/simple_code_generator/README.md | 39 ++++ .../simple_code_generator/example/BUILD.bazel | 33 ++++ .../simple_code_generator/example/main.py | 21 ++ .../simple_code_generator/example/rules.bzl | 147 ++++++++++++++ .../example/things/places/rivers.py | 20 ++ .../idea/blaze/python/PythonBlazeRules.java | 24 ++- .../AbstractPyImportResolverStrategy.java | 132 ++++++++++++- .../blaze/python/PythonBlazeRulesTest.java | 64 +++++++ .../AbstractPyImportResolverStrategyTest.java | 126 ++++++++++++ 22 files changed, 1230 insertions(+), 28 deletions(-) create mode 100644 base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java create mode 100644 base/src/com/google/idea/blaze/base/dependencies/TargetTagFilter.java create mode 100644 base/tests/unittests/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilterTest.java create mode 100644 docs/python/code-generators.md create mode 100644 docs/python/images/code-generators.svg create mode 100644 examples/python/simple_code_generator/.gitignore create mode 100644 examples/python/simple_code_generator/MODULE.bazel create mode 100644 examples/python/simple_code_generator/README.md create mode 100644 examples/python/simple_code_generator/example/BUILD.bazel create mode 100644 examples/python/simple_code_generator/example/main.py create mode 100644 examples/python/simple_code_generator/example/rules.bzl create mode 100644 examples/python/simple_code_generator/example/things/places/rivers.py create mode 100644 python/tests/unittests/com/google/idea/blaze/python/PythonBlazeRulesTest.java diff --git a/aspect/intellij_info_impl.bzl b/aspect/intellij_info_impl.bzl index 161d9929c7a..c6989df164c 100644 --- a/aspect/intellij_info_impl.bzl +++ b/aspect/intellij_info_impl.bzl @@ -79,6 +79,8 @@ PY2 = 1 PY3 = 2 +TARGET_TAG_PY_CODE_GENERATOR = "intellij-py-code-generator" + # PythonCompatVersion enum; must match PyIdeInfo.PythonSrcsVersion SRC_PY2 = 1 @@ -338,6 +340,60 @@ def collect_py_info(target, ctx, semantics, ide_info, ide_info_file, output_grou args = _do_starlark_string_expansion(ctx, "args", args, data_deps) imports = getattr(ctx.rule.attr, "imports", []) + # If there are apparently no sources found from `srcs` and the target has the tag + # for code-generation then use the run-files as the sources and take the imports + # from the PyInfo. + + if 0 == len(sources) and TARGET_TAG_PY_CODE_GENERATOR in getattr(ctx.rule.attr, "tags", []): + def provider_import_to_attr_import(provider_import): + """\ + Remaps the imports from PyInfo + + The imports that are supplied on the `PyInfo` are relative to the runfiles and so are + not the same as those which might be supplied on an attribute of `py_library`. This + function will remap those back so they look as if they were `imports` attributes on + the rule. The form of the runfiles import is `//`. + The actual `workspace_name` is not interesting such that the first part can be simply + stripped. Next the package to the Label is stripped leaving a path that would have been + supplied on an `imports` attribute to a Rule. + """ + + # Other code in this file appears to assume *NIX path component separators? + + provider_import_parts = [p for p in provider_import.split("/")] + package_parts = [p for p in ctx.label.package.split("/")] + + if 0 == len(provider_import_parts): + return None + + scratch_parts = provider_import_parts[1:] # remove the workspace name or _main + + for p in package_parts: + if 0 != len(provider_import_parts) and scratch_parts[0] == p: + scratch_parts = scratch_parts[1:] + else: + return None + + return "/".join(scratch_parts) + + def provider_imports_to_attr_imports(): + result = [] + + for provider_import in target[PyInfo].imports.to_list(): + attr_import = provider_import_to_attr_import(provider_import) + if attr_import: + result.append(attr_import) + + return result + + if target[PyInfo].imports: + imports.extend(provider_imports_to_attr_imports()) + + runfiles = target[DefaultInfo].default_runfiles + + if runfiles and runfiles.files: + sources.extend([artifact_location(f) for f in runfiles.files.to_list()]) + ide_info["py_ide_info"] = struct_omit_none( launcher = py_launcher, python_version = _get_python_version(ctx), diff --git a/base/BUILD b/base/BUILD index 6e8e5202bcc..1c7482e41f6 100644 --- a/base/BUILD +++ b/base/BUILD @@ -46,7 +46,7 @@ java_library( "//shared:vcs", "//third_party/auto_value", "@error_prone_annotations//jar", - "@gson//jar" + "@gson//jar", ], ) @@ -125,6 +125,7 @@ java_library( name = "label_api", srcs = [ "src/com/google/idea/blaze/base/ideinfo/ArtifactLocation.java", + "src/com/google/idea/blaze/base/ideinfo/Tags.java", "src/com/google/idea/blaze/base/model/primitives/InvalidTargetException.java", "src/com/google/idea/blaze/base/model/primitives/Kind.java", "src/com/google/idea/blaze/base/model/primitives/Label.java", @@ -246,6 +247,7 @@ java_library( java_library( name = "workspace_language_checker_api", srcs = [ + "src/com/google/idea/blaze/base/ideinfo/Tags.java", "src/com/google/idea/blaze/base/model/primitives/LanguageClass.java", "src/com/google/idea/blaze/base/sync/projectview/WorkspaceLanguageChecker.java", ], diff --git a/base/src/META-INF/blaze-base.xml b/base/src/META-INF/blaze-base.xml index 34ad207f953..d8b887338e2 100644 --- a/base/src/META-INF/blaze-base.xml +++ b/base/src/META-INF/blaze-base.xml @@ -563,6 +563,7 @@ + + diff --git a/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java b/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java new file mode 100644 index 00000000000..40100cf0471 --- /dev/null +++ b/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java @@ -0,0 +1,133 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.base.dependencies; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.base.bazel.BuildSystem; +import com.google.idea.blaze.base.bazel.BuildSystem.BuildInvoker; +import com.google.idea.blaze.base.command.BlazeCommand; +import com.google.idea.blaze.base.command.BlazeCommandName; +import com.google.idea.blaze.base.command.BlazeFlags; +import com.google.idea.blaze.base.command.buildresult.BuildResultHelper; +import com.google.idea.blaze.base.model.primitives.Label; +import com.google.idea.blaze.base.model.primitives.TargetExpression; +import com.google.idea.blaze.base.scope.BlazeContext; +import com.google.idea.blaze.base.settings.Blaze; +import com.google.idea.blaze.exception.BuildException; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.SystemInfo; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.jetbrains.annotations.Nullable; + +/** + * This class is able to filter a list of targets selecting those that are + * tagged with one or more of a number of tags. + */ + +public class BlazeQueryTargetTagFilter implements TargetTagFilter { + + private static final Logger logger = + Logger.getInstance(BlazeQueryTargetTagFilter.class); + + /** + * Tags are checked against this {@link Pattern} to ensure they can be used with + * the Bazel queries run in this logic. See + * {@link com.google.idea.blaze.base.ideinfo.Tags} for example Tags that would + * be used inside the Plugin; all of which conform to this {@link Pattern}. + */ + private static final Pattern PATTERN_TAG = Pattern.compile("^[a-zA-Z0-9_-]+$"); + + @Nullable + @Override + public List doFilterCodeGen( + Project project, + BlazeContext context, + List targets, + Set tags) { + + if (targets.isEmpty() || tags.isEmpty()) { + return ImmutableList.of(); + } + + return runQuery(project, getQueryString(targets, tags), context); + } + + @VisibleForTesting + public static String getQueryString(List targets, Set tags) { + Preconditions.checkArgument(null != targets && !targets.isEmpty(), "the targets must be supplied"); + Preconditions.checkArgument(null != tags && !tags.isEmpty(), "the tags must be supplied"); + + for (String tag : tags) { + if (!PATTERN_TAG.matcher(tag).matches()) { + throw new IllegalStateException("the tag [" + tag + "] is not able to be used for filtering"); + } + } + + String targetsExpression = targets.stream().map(Object::toString).collect(Collectors.joining(" + ")); + String matchExpression = String.format("[\\[ ](%s)[,\\]]", String.join("|", ImmutableList.sortedCopyOf(tags))); + + if (SystemInfo.isWindows) { + return String.format("attr('tags', '%s', %s)", matchExpression, targetsExpression); + } + + return String.format("attr(\"tags\", \"%s\", %s)", matchExpression, targetsExpression); + } + + @javax.annotation.Nullable + private static List runQuery( + Project project, + String query, + BlazeContext context) { + + BuildSystem buildSystem = Blaze.getBuildSystemProvider(project).getBuildSystem(); + + BlazeCommand.Builder command = + BlazeCommand.builder( + buildSystem.getDefaultInvoker(project, context), BlazeCommandName.QUERY) + .addBlazeFlags("--output=label") + .addBlazeFlags(BlazeFlags.KEEP_GOING) + .addBlazeFlags(query); + + BuildInvoker invoker = buildSystem.getDefaultInvoker(project, context); + + try (BuildResultHelper helper = invoker.createBuildResultHelper(); + InputStream queryResultStream = invoker.getCommandRunner() + .runQuery(project, command, helper, context)) { + + return new BufferedReader(new InputStreamReader(queryResultStream, UTF_8)) + .lines() + .map(Label::createIfValid) + .collect(Collectors.toList()); + + } catch (IOException | BuildException e) { + logger.error(e.getMessage(), e); + return null; + } + } + +} diff --git a/base/src/com/google/idea/blaze/base/dependencies/TargetTagFilter.java b/base/src/com/google/idea/blaze/base/dependencies/TargetTagFilter.java new file mode 100644 index 00000000000..e82ea587f6d --- /dev/null +++ b/base/src/com/google/idea/blaze/base/dependencies/TargetTagFilter.java @@ -0,0 +1,74 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.base.dependencies; + +import com.google.common.collect.ImmutableList; +import com.google.idea.blaze.base.model.primitives.TargetExpression; +import com.google.idea.blaze.base.scope.BlazeContext; +import com.intellij.openapi.extensions.ExtensionPointName; +import com.intellij.openapi.project.Project; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import javax.annotation.Nullable; + +/** + * Implementations of this interface are able to filter targets for having + * one of a set of tags. + */ + +public interface TargetTagFilter { + + ExtensionPointName EP_NAME = + ExtensionPointName.create("com.google.idea.blaze.TargetTagFilter"); + + static boolean hasProvider() { + return EP_NAME.getExtensions().length != 0; + } + + /** + * This method will run a Bazel query to select for those targets having a tag + * which matches one of the supplied {@code tags}. + * + * @param tags is a list of tags that targets are expected to have configured in + * order to be filtered in. + * @param targets is a list of Bazel targets to filter. + * @return a subset of the supplied targets that include one of the supplied {code tags}. + */ + static List filterCodeGen( + Project project, + BlazeContext context, + List targets, + Set tags) { + return Arrays.stream(EP_NAME.getExtensions()) + .map(p -> p.doFilterCodeGen(project, context, targets, tags)) + .filter(Objects::nonNull) + .findFirst() + .orElse(ImmutableList.of()); + } + + /** + * {@see #filterCodeGen} + */ + @Nullable + List doFilterCodeGen( + Project project, + BlazeContext context, + List targets, + Set tags); + +} diff --git a/base/src/com/google/idea/blaze/base/ideinfo/Tags.java b/base/src/com/google/idea/blaze/base/ideinfo/Tags.java index c8513b74c4c..f4f9fc64932 100644 --- a/base/src/com/google/idea/blaze/base/ideinfo/Tags.java +++ b/base/src/com/google/idea/blaze/base/ideinfo/Tags.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 The Bazel Authors. All rights reserved. + * Copyright 2016-2024 The Bazel Authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,4 +30,9 @@ public final class Tags { /** Ignores the target. */ public static final String TARGET_TAG_EXCLUDE_TARGET = "intellij-exclude-target"; + + /** + * Signals to the IDE that a rule produces Python code rather than has code as input. + */ + public static final String TARGET_TAG_PY_CODE_GENERATOR = "intellij-py-code-generator"; } diff --git a/base/src/com/google/idea/blaze/base/model/primitives/LanguageClass.java b/base/src/com/google/idea/blaze/base/model/primitives/LanguageClass.java index e60322a5301..11ee4515852 100644 --- a/base/src/com/google/idea/blaze/base/model/primitives/LanguageClass.java +++ b/base/src/com/google/idea/blaze/base/model/primitives/LanguageClass.java @@ -18,21 +18,22 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.idea.blaze.base.ideinfo.ProtoWrapper; +import com.google.idea.blaze.base.ideinfo.Tags; import javax.annotation.Nullable; /** Language classes. */ public enum LanguageClass implements ProtoWrapper { - GENERIC("generic", ImmutableSet.of()), - C("c", ImmutableSet.of("c", "cc", "cpp", "h", "hh", "hpp")), - JAVA("java", ImmutableSet.of("java")), - ANDROID("android", ImmutableSet.of("aidl")), - JAVASCRIPT("javascript", ImmutableSet.of("js", "applejs")), - TYPESCRIPT("typescript", ImmutableSet.of("ts", "ats")), - DART("dart", ImmutableSet.of("dart")), - GO("go", ImmutableSet.of("go")), - PYTHON("python", ImmutableSet.of("py", "pyw")), - SCALA("scala", ImmutableSet.of("scala")), - KOTLIN("kotlin", ImmutableSet.of("kt")), + GENERIC("generic", ImmutableSet.of(), null), + C("c", ImmutableSet.of("c", "cc", "cpp", "h", "hh", "hpp"), null), + JAVA("java", ImmutableSet.of("java"), null), + ANDROID("android", ImmutableSet.of("aidl"), null), + JAVASCRIPT("javascript", ImmutableSet.of("js", "applejs"), null), + TYPESCRIPT("typescript", ImmutableSet.of("ts", "ats"), null), + DART("dart", ImmutableSet.of("dart"), null), + GO("go", ImmutableSet.of("go"), null), + PYTHON("python", ImmutableSet.of("py", "pyw"), Tags.TARGET_TAG_PY_CODE_GENERATOR), + SCALA("scala", ImmutableSet.of("scala"), null), + KOTLIN("kotlin", ImmutableSet.of("kt"), null), ; private static final ImmutableMap RECOGNIZED_EXTENSIONS = @@ -51,15 +52,31 @@ private static ImmutableMap extensionToClassMap() { private final String name; private final ImmutableSet recognizedFilenameExtensions; - LanguageClass(String name, ImmutableSet recognizedFilenameExtensions) { + /** + * The {@code codeGeneratorTag} is a tag that may be applied to a Bazel Rule's {@code tag} + * attribute to signal to the IDE that the Rule's Actions will generate source code. Each + * language has its own tag for this purpose. + * @see com.google.idea.blaze.base.sync.SyncProjectTargetsHelper + */ + private final String codeGeneratorTag; + + LanguageClass( + String name, + ImmutableSet recognizedFilenameExtensions, + String codeGeneratorTag) { this.name = name; this.recognizedFilenameExtensions = recognizedFilenameExtensions; + this.codeGeneratorTag = codeGeneratorTag; } public String getName() { return name; } + public String getCodeGeneratorTag() { + return codeGeneratorTag; + } + public static LanguageClass fromString(String name) { for (LanguageClass ruleClass : LanguageClass.values()) { if (ruleClass.name.equals(name)) { diff --git a/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java b/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java index 8825b8528f8..311a8990569 100644 --- a/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java +++ b/base/src/com/google/idea/blaze/base/sync/SyncProjectTargetsHelper.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 The Bazel Authors. All rights reserved. + * Copyright 2019-2024 The Bazel Authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ import com.google.idea.blaze.base.dependencies.DirectoryToTargetProvider; import com.google.idea.blaze.base.dependencies.SourceToTargetFilteringStrategy; import com.google.idea.blaze.base.dependencies.TargetInfo; +import com.google.idea.blaze.base.dependencies.TargetTagFilter; +import com.google.idea.blaze.base.model.primitives.LanguageClass; import com.google.idea.blaze.base.model.primitives.TargetExpression; import com.google.idea.blaze.base.model.primitives.WorkspaceRoot; import com.google.idea.blaze.base.projectview.ProjectViewSet; @@ -48,7 +50,10 @@ import com.google.idea.blaze.common.PrintOutput; import com.intellij.openapi.project.Project; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; /** Derives sync targets from the project directories. */ public final class SyncProjectTargetsHelper { @@ -146,15 +151,21 @@ private static ImmutableList deriveTargetsFromDirectories( return DirectoryToTargetProvider.expandDirectoryTargets( project, shouldSyncManualTargets(projectViewSet), importRoots, pathResolver, childContext); })).get(); // We still call no-timeout waitFor in ExternalTask.run() + if (context.isCancelled()) { throw new SyncCanceledException(); } + if (targets == null) { IssueOutput.error("Deriving targets from project directories failed." + fileBugSuggestion) .submit(context); throw new SyncFailedException(); } - ImmutableList retained = + + // retainedByKind will contain the targets which are to be kept because their Kind matches + // one of the languages actively in use in the IDE. + + ImmutableList retainedByKind = SourceToTargetFilteringStrategy.filterTargets(targets).stream() .filter( t -> @@ -163,11 +174,46 @@ private static ImmutableList deriveTargetsFromDirectories( .anyMatch(languageSettings::isLanguageActive)) .map(t -> t.label) .collect(toImmutableList()); + + // Gather together those targets that are rejected. Run the rejected targets through another + // Bazel query to see if they are code-generation (code-gen) ones. If any of them are then we + // should include those as well. In such cases the rule name might be something like + // `my_code_gen` which will not be detected as a library for example. + + List rejectedByKind = targets.stream() + .map(TargetInfo::getLabel) + .filter(label -> !retainedByKind.contains(label)) + .collect(Collectors.toUnmodifiableList()); + + List retainedByCodeGen = ImmutableList.of(); + + if (!rejectedByKind.isEmpty() && TargetTagFilter.hasProvider()) { + Set activeLanguageCodeGeneratorTags = languageSettings.getActiveLanguages() + .stream() + .map(LanguageClass::getCodeGeneratorTag) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + if (!activeLanguageCodeGeneratorTags.isEmpty()) { + retainedByCodeGen = TargetTagFilter.filterCodeGen( + project, + context, + rejectedByKind, + activeLanguageCodeGeneratorTags); + } + } + + ImmutableList retained = ImmutableList.builder() + .addAll(retainedByKind) + .addAll(retainedByCodeGen) + .build(); + context.output( PrintOutput.log( String.format( - "%d targets found under project directories; syncing %d of them.", + "%d targets found under project directories; syncing %d of them", targets.size(), retained.size()))); + return retained; } } diff --git a/base/tests/unittests/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilterTest.java b/base/tests/unittests/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilterTest.java new file mode 100644 index 00000000000..6eb0309a5cc --- /dev/null +++ b/base/tests/unittests/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilterTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.idea.blaze.base.dependencies; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.idea.blaze.base.model.primitives.InvalidTargetException; +import com.google.idea.blaze.base.model.primitives.TargetExpression; +import java.util.List; +import java.util.Set; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BlazeQueryTargetTagFilterTest { + + /** + * This is the "happy days" test showing the inputs yielding a sensible query. + */ + @Test + public void testAssembleQuery() throws InvalidTargetException { + List targetExpressions = ImmutableList.of( + TargetExpression.fromString("//flamingos:lib"), + TargetExpression.fromString("//flamingos:bin") + ); + + Set tags = ImmutableSet.of("sea-weed", "lake-weed"); + + // code under test + String actualQueryString = BlazeQueryTargetTagFilter.getQueryString(targetExpressions, tags); + + String expectedQueryString = "attr(\"tags\", \"[\\[ ](lake-weed|sea-weed)[,\\]]\", //flamingos:lib + //flamingos:bin)"; + assertThat(actualQueryString).isEqualTo(expectedQueryString); + } + + /** + * This test shows what happens if the logic attempts to create a query for a tag that might break + * the regular expression assembly. + */ + @Test(expected = IllegalStateException.class) + public void testAssembleQueryWithBadTag() throws InvalidTargetException { + List targetExpressions = ImmutableList.of(TargetExpression.fromString("//:bin")); + + Set tags = ImmutableSet.of("far.out"); + // ^^ note the dot is a special character in regex so is disallowed + + // code under test + BlazeQueryTargetTagFilter.getQueryString(targetExpressions, tags); + + // The expected behaviour is that this will exception because of the malformed tag. + } + +} diff --git a/docs/python/code-generators.md b/docs/python/code-generators.md new file mode 100644 index 00000000000..7e4b8cb3b0c --- /dev/null +++ b/docs/python/code-generators.md @@ -0,0 +1,20 @@ +# Python code generators + +## Background + +It is possible to create a _code generator_ Bazel Rule that acts as a `py_library`. The Bazel Action setup by the code generator Rule would execute a tool that outputs Python source code. A `py_binary` is then configured with the code generator Rule instance as a dependency and uses the generated source code. + +The following diagram shows how a code generator might look like in a typical scenario where code is generated from the structures present in a schema. + +![Code generators graph](images/code-generators.svg) + +## Example + +See the Python example [simple_code_generator](/examples/python/simple_code_generator). + +## Intelli-J + +The Bazel Intelli-J plugin is able to work with generated Python code in this situation. You need to ensure that your code generator has no `srcs` attribute and that it has a tag `intellij-py-code-generator`. This tag signals to the plugin that the Rule's output is generated code. + + + diff --git a/docs/python/images/code-generators.svg b/docs/python/images/code-generators.svg new file mode 100644 index 00000000000..9a5b97ecc31 --- /dev/null +++ b/docs/python/images/code-generators.svg @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SchemaFile + + + + + + + + + + + + + + + GeneratedSource + + + + + + + + + + + + + + + + + + + + + + + + + + Project Source + + + + + + + + Code Generator Rule + + + + + + + + BuildRule + + + + + + + + + + + + + Build Product + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/python/simple_code_generator/.gitignore b/examples/python/simple_code_generator/.gitignore new file mode 100644 index 00000000000..3dfd3023996 --- /dev/null +++ b/examples/python/simple_code_generator/.gitignore @@ -0,0 +1,3 @@ +MODULE.bazel.lock +.ijwb +bazel-* \ No newline at end of file diff --git a/examples/python/simple_code_generator/MODULE.bazel b/examples/python/simple_code_generator/MODULE.bazel new file mode 100644 index 00000000000..b4377ae5522 --- /dev/null +++ b/examples/python/simple_code_generator/MODULE.bazel @@ -0,0 +1,12 @@ +module( + name = "simple_code_generator", + version = "1.0.0", +) + +bazel_dep(name = "rules_python", version = "0.35.0") + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + configure_coverage_tool = True, + python_version = "3.12", +) diff --git a/examples/python/simple_code_generator/README.md b/examples/python/simple_code_generator/README.md new file mode 100644 index 00000000000..e80b241a869 --- /dev/null +++ b/examples/python/simple_code_generator/README.md @@ -0,0 +1,39 @@ +# simple_code_generator Python Example + +This simple project is intended to facilitate demonstration of Intelli-J working with a Bazel Python project that uses code-generator Bazel Rules. + +## Run the example + +``` +bazel run "//example:bin" +``` + +The expected output from the example is; + +``` +- Auckland City +- Regensburg City +- Darwin City +- Toulouse City +* Whanganui River +* Danube River +* Yarra River +* Volga River +# Bakerlite Plastic +# Polyethylene Plastic +# Nylon Plastic +``` + +## Explanation + +The example is a `py_binary` with three `deps` and which, via `main.py`, uses a function from each of the `deps` to print the list of cities, rivers and plastics above. + +- `:static_lib` provides the list of rivers and is sourced from checked-in source files. +- `:generated_files_lib` provides the list of cities and is sourced from a code-generator that outputs individual source files. +- `:generated_directory_lib` provides the list of plastics and is sourced from a code-generator that outputs a directory containing source files. + +In each case, `imports` are used. + +## Key observations + +Open the project in Intelli-J ensuring that Python is enabled as a language option. Perform a sync on the project. Open the `main.py` file. Observe that the imports and symbols for each of the three `deps` is recognized by the IDE. The targets `//example:generated_directory_lib` and `//example:generated_files_lib` have a tag `intellij-py-code-generator` which signals to the IDE that the target is a code generator. \ No newline at end of file diff --git a/examples/python/simple_code_generator/example/BUILD.bazel b/examples/python/simple_code_generator/example/BUILD.bazel new file mode 100644 index 00000000000..f9206b11d3f --- /dev/null +++ b/examples/python/simple_code_generator/example/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("//example:rules.bzl", "test_codegen_directory_py", "test_codegen_files_py") + +test_codegen_directory_py( + name = "generated_directory_lib", + tags = ["intellij-py-code-generator"], + visibility = ["//visibility:private"], +) + +test_codegen_files_py( + name = "generated_files_lib", + tags = ["intellij-py-code-generator"], + visibility = ["//visibility:private"], +) + +py_library( + name = "static_lib", + srcs = ["things/places/rivers.py"], + imports = ["things"], + visibility = ["//visibility:private"], +) + +py_binary( + name = "bin", + srcs = ["main.py"], + main = "main.py", + visibility = ["//visibility:private"], + deps = [ + "//example:generated_directory_lib", + "//example:generated_files_lib", + "//example:static_lib", + ], +) diff --git a/examples/python/simple_code_generator/example/main.py b/examples/python/simple_code_generator/example/main.py new file mode 100644 index 00000000000..1b4766c2337 --- /dev/null +++ b/examples/python/simple_code_generator/example/main.py @@ -0,0 +1,21 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from urban.cities import print_cities +from places.rivers import print_rivers +from artificial.plastics import print_plastics + +print_cities() +print_rivers() +print_plastics() diff --git a/examples/python/simple_code_generator/example/rules.bzl b/examples/python/simple_code_generator/example/rules.bzl new file mode 100644 index 00000000000..603acd43d51 --- /dev/null +++ b/examples/python/simple_code_generator/example/rules.bzl @@ -0,0 +1,147 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_python//python:defs.bzl", RulesPythonPyInfo = "PyInfo") + +_GENERATED_CONTENT_FILES = """\ +CITIES = ["Auckland", "Regensburg", "Darwin", "Toulouse"] + + +def print_cities(): + for city in CITIES: + print(f"- {city} City") +""" + +_SCRIPT_GENERATE_DIRECTORY = """\ +#!/usr/bin/env bash + +set -eu -o pipefail + +main() { + if [[ "$#" -lt 1 ]]; then + echo "expected a single argument of the base directory" + exit 1 + fi + + local base_dir + + base_dir="$1" + + mkdir -p "${base_dir}/artificial" + + touch "${base_dir}/artificial/__init__.py" + touch "${base_dir}/__init__.py" + + cat > "${base_dir}/artificial/plastics.py" << EOF +PLASTICS = ["Bakerlite", "Polyethylene", "Nylon"] + + +def print_plastics(): + for plastic in PLASTICS: + print(f"# {plastic} Plastic") +EOF +} + +main "$@" +""" + +def _test_codegen_directory_py_impl(ctx): + output_directory = ctx.actions.declare_directory("materials") + script_file = ctx.actions.declare_file("make_py.sh") + + ctx.actions.write(output = script_file, content = _SCRIPT_GENERATE_DIRECTORY) + + ctx.actions.run( + mnemonic = "TestCodeGenDirectoryPy", + executable = script_file, + arguments = [ctx.actions.args().add(output_directory.path)], + outputs = [output_directory], + ) + + # This would ideally use some normalized path handling here, but it is done here + # manually assuming a *NIX style system to reduce the complexity of the example. + imports_path = "/".join([ + ctx.label.repo_name or ctx.workspace_name, + "example", + "materials", + ]) + + return [ + DefaultInfo( + runfiles = ctx.runfiles([output_directory]), + files = depset([output_directory]), + ), + PyInfo( + transitive_sources = depset([output_directory]), + imports = depset([imports_path]), + ), + RulesPythonPyInfo( + transitive_sources = depset([output_directory]), + imports = depset([imports_path]), + ), + ] + +def _test_codegen_files_py_impl(ctx): + cities_output_file = ctx.actions.declare_file("infrastructure/urban/cities.py") + output_files = [cities_output_file] + ctx.actions.write(output = cities_output_file, content = _GENERATED_CONTENT_FILES) + + def setup_init_file(module_dir_path): + # This would ideally use some normalized path handling here, but it is done here + # manually assuming a *NIX style system to reduce the complexity of the example. + init_file = ctx.actions.declare_file(module_dir_path + "/__init__.py") + output_files.append(init_file) + ctx.actions.write(output = init_file, content = "") + + setup_init_file("infrastructure") + setup_init_file("infrastructure/urban") + + # This would ideally use some normalized path handling here, but it is done here + # manually assuming a *NIX style system to reduce the complexity of the example. + imports_path = "/".join([ + ctx.label.repo_name or ctx.workspace_name, + "example", + "infrastructure", + ]) + + return [ + DefaultInfo( + runfiles = ctx.runfiles(files = output_files), + files = depset(output_files), + ), + PyInfo( + transitive_sources = depset(output_files), + imports = depset([imports_path]), + ), + RulesPythonPyInfo( + transitive_sources = depset(output_files), + imports = depset([imports_path]), + ), + ] + +test_codegen_directory_py = rule( + implementation = _test_codegen_directory_py_impl, + provides = [DefaultInfo, PyInfo, RulesPythonPyInfo], + doc = """\ + Produces a Python code-generation library to demonstrate production of a directory of files. + """, +) + +test_codegen_files_py = rule( + implementation = _test_codegen_files_py_impl, + provides = [DefaultInfo, PyInfo, RulesPythonPyInfo], + doc = """\ + Produces a Python code-generation library to demonstrate production of files. + """, +) diff --git a/examples/python/simple_code_generator/example/things/places/rivers.py b/examples/python/simple_code_generator/example/things/places/rivers.py new file mode 100644 index 00000000000..e4f824f9e0b --- /dev/null +++ b/examples/python/simple_code_generator/example/things/places/rivers.py @@ -0,0 +1,20 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RIVERS = ["Whanganui", "Danube", "Yarra", "Volga"] + + +def print_rivers(): + for river in RIVERS: + print(f"* {river} River") diff --git a/python/src/com/google/idea/blaze/python/PythonBlazeRules.java b/python/src/com/google/idea/blaze/python/PythonBlazeRules.java index b587d64cb55..abd7db7603e 100644 --- a/python/src/com/google/idea/blaze/python/PythonBlazeRules.java +++ b/python/src/com/google/idea/blaze/python/PythonBlazeRules.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Bazel Authors. All rights reserved. + * Copyright 2018-2024 The Bazel Authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,20 +16,40 @@ package com.google.idea.blaze.python; import com.google.common.collect.ImmutableSet; +import com.google.devtools.intellij.ideinfo.IntellijIdeInfo.TargetIdeInfo; +import com.google.idea.blaze.base.ideinfo.Tags; import com.google.idea.blaze.base.model.primitives.Kind; import com.google.idea.blaze.base.model.primitives.LanguageClass; import com.google.idea.blaze.base.model.primitives.RuleType; +import java.util.function.Function; /** Contributes python rules to {@link Kind}. */ public final class PythonBlazeRules implements Kind.Provider { + private final static Kind PY_LIBRARY = Kind.Provider.create("py_library", LanguageClass.PYTHON, RuleType.LIBRARY); + @Override public ImmutableSet getTargetKinds() { return ImmutableSet.of( - Kind.Provider.create("py_library", LanguageClass.PYTHON, RuleType.LIBRARY), + PY_LIBRARY, Kind.Provider.create("py_binary", LanguageClass.PYTHON, RuleType.BINARY), Kind.Provider.create("py_test", LanguageClass.PYTHON, RuleType.TEST), Kind.Provider.create("py_appengine_binary", LanguageClass.PYTHON, RuleType.BINARY), Kind.Provider.create("py_web_test", LanguageClass.PYTHON, RuleType.TEST)); } + + @Override + public Function getTargetKindHeuristics() { + return (tii) -> { + + // If the target has tagged itself for code-generation then we can consider that it would be + // treated as if it were a `py_library`. + + if (tii.getTagsList().contains(Tags.TARGET_TAG_PY_CODE_GENERATOR)) { + return PY_LIBRARY; + } + + return null; + }; + } } diff --git a/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java b/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java index 79715ddfe10..73178643bf4 100644 --- a/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java +++ b/python/src/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategy.java @@ -15,8 +15,10 @@ */ package com.google.idea.blaze.python.resolve.provider; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.idea.blaze.base.command.buildresult.OutputArtifactResolver; import com.google.idea.blaze.base.ideinfo.ArtifactLocation; @@ -28,6 +30,7 @@ import com.google.idea.blaze.base.settings.BlazeImportSettings.ProjectType; import com.google.idea.blaze.base.sync.SyncCache; import com.google.idea.blaze.base.sync.workspace.ArtifactLocationDecoder; +import com.google.idea.blaze.base.sync.workspace.WorkspacePathResolver; import com.google.idea.blaze.python.resolve.BlazePyResolverUtils; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.text.StringUtil; @@ -47,6 +50,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Nullable; /** @@ -55,6 +59,19 @@ */ public abstract class AbstractPyImportResolverStrategy implements PyImportResolverStrategy { + /** + * This is a list of files, where the presence of one of these files represents either a + * Bazel Project or Bazel Repo. + */ + private final static Set BOUNDARY_MARKER_FILES = ImmutableSet.of( + "BUILD.bazel", + "BUILD", + "REPO.bazel", + "WORKSPACE.bazel", + "WORKSPACE", + "MODULE.bazel" + ); + private final static Path PATH_CURRENT_DIR = Path.of("."); private final ArtifactSupplierToPsiElementProviderMapper artifactSupplierToPsiElementProviderMapper; @@ -125,7 +142,7 @@ PySourcesIndex buildSourcesIndex(Project project, BlazeProjectData projectData) ArtifactLocationDecoder decoder = projectData.getArtifactLocationDecoder(); for (TargetIdeInfo target : projectData.getTargetMap().targets()) { List importRoots = assembleImportRoots(target); - for (ArtifactLocation source : getPySources(target)) { + for (ArtifactLocation source : getPySources(projectData.getWorkspacePathResolver(), target)) { List sourceImports = assembleSourceImportsFromImportRoots(importRoots, toImportString(source)); for (QualifiedName sourceImport : sourceImports) { @@ -144,16 +161,115 @@ PySourcesIndex buildSourcesIndex(Project project, BlazeProjectData projectData) return new PySourcesIndex(shortNames.build(), ImmutableMap.copyOf(map)); } - private static Collection getPySources(TargetIdeInfo target) { + /** + * This method will extract sources from the supplied target. If any of the sources + * are a directory rather than a file then it will descend through the directory + * transitively looking for any Python files. It is sometimes the case that + * generated code will supply source in a directory rather than as individual files. + */ + + private static Collection getPySources( + WorkspacePathResolver workspacePathResolver, + TargetIdeInfo target) { + Preconditions.checkArgument(null != workspacePathResolver); + Preconditions.checkArgument(null != target); + if (target.getPyIdeInfo() != null) { - return target.getPyIdeInfo().getSources(); + return getPySources(workspacePathResolver, target.getPyIdeInfo().getSources()); } if (target.getKind().hasLanguage(LanguageClass.PYTHON)) { - return target.getSources(); + return getPySources(workspacePathResolver, target.getSources()); } return ImmutableList.of(); } + private static List getPySources( + WorkspacePathResolver workspacePathResolver, + Collection sources) { + ImmutableList.Builder assembly = ImmutableList.builder(); + marshallPySources(workspacePathResolver, sources, assembly); + return assembly.build(); + } + + private static void marshallPySources( + WorkspacePathResolver workspacePathResolver, + Collection sources, + ImmutableList.Builder assembly) { + for (ArtifactLocation source : sources) { + marshallPySources(workspacePathResolver, source, assembly); + } + } + + /** + * Inspects the supplied {@code source}. If it is a Python file then it is added to the + * {@code assembly} If not then it will be then further processed as a {@link File}; + * likely a directory that may then potentially contain Python source files. + */ + + private static void marshallPySources( + WorkspacePathResolver workspacePathResolver, + ArtifactLocation source, + ImmutableList.Builder assembly) { + if (source.getRelativePath().endsWith(".py")) { + assembly.add(source); + } else { + if (!source.isSource()) { + marshallPySources( + source, + workspacePathResolver.resolveToFile(source.getExecutionRootRelativePath()), + 0, + assembly); + } + } + } + + /** + *

Assembles Python source files as instances of {@code ArtifactLocation} by inspecting + * the supplied {@code sourceFileOrDirectory}. The outputs are written to the + * {@code assembly}. This method is recursive.

+ * + *

If the logic should encounter a Bazel boundary file such as {@code BUILD.bazel} then + * it will stop walking the directory tree.

+ * + * @param depth indicates how far down the directory tree the traversal is. + */ + + private static void marshallPySources( + ArtifactLocation source, + File sourceFileOrDirectory, + int depth, + ImmutableList.Builder assembly) { + + if (sourceFileOrDirectory.isFile()) { + if (sourceFileOrDirectory.getName().endsWith(".py")) { + assembly.add(source); + } + } + + if (sourceFileOrDirectory.isDirectory() + && (0 == depth || !containsBoundaryMarkerFile(sourceFileOrDirectory))) { + String[] subFilenames = sourceFileOrDirectory.list(); + + if (null != subFilenames) { + for (String subFilename : subFilenames) { + Path subSourcePath = Path.of(source.getRelativePath(), subFilename); + marshallPySources( + ArtifactLocation.Builder.copy(source).setRelativePath(subSourcePath.toString()) + .build(), + new File(sourceFileOrDirectory, subFilename), + depth + 1, + assembly); + } + } + } + } + + private static boolean containsBoundaryMarkerFile(File directory) { + return BOUNDARY_MARKER_FILES.stream() + .map(filename -> new File(directory, filename)) + .anyMatch(File::exists); + } + /** * Maps a blaze artifact to the import string used to reference it. */ @@ -221,11 +337,9 @@ private static List assembleImportRoots(TargetIdeInfo target) { Path buildPath = Path.of(target.getBuildFile().getExecutionRootRelativePath()); Path buildParentPath = buildPath.getParent(); - // In the case of an external repo the build path could be `/BUILD` - // which is problematic because the basedir is `/` which makes sense - // in the context of a Bazel repo but not in the context of the overall - // file-system. For this reason, the special case of `/BUILD` is taken - // to have a basedir of `.`. + // In the case of an external repo the build path could be `/BUILD.bazel` + // which has a basedir of `/`. In this case we translate this to `.` so + // that it works in the sub file-system. if (null == buildParentPath || 0 == buildParentPath.getNameCount()) { buildParentPath = PATH_CURRENT_DIR; diff --git a/python/tests/unittests/com/google/idea/blaze/python/PythonBlazeRulesTest.java b/python/tests/unittests/com/google/idea/blaze/python/PythonBlazeRulesTest.java new file mode 100644 index 00000000000..471c71c9b8c --- /dev/null +++ b/python/tests/unittests/com/google/idea/blaze/python/PythonBlazeRulesTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2024 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.idea.blaze.python; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.intellij.ideinfo.IntellijIdeInfo.TargetIdeInfo; +import com.google.idea.blaze.base.model.primitives.Kind; +import com.google.idea.blaze.base.model.primitives.LanguageClass; +import com.google.idea.blaze.base.model.primitives.RuleType; +import java.util.function.Function; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import static com.google.common.truth.Truth.assertThat; + +@RunWith(JUnit4.class) +public class PythonBlazeRulesTest { + + private final static Kind.Provider rules = new PythonBlazeRules(); + + @Test + public void testTargetKindHeuristicsForCodeGenerator() { + Function heuristics = rules.getTargetKindHeuristics(); + TargetIdeInfo targetInfo = TargetIdeInfo.newBuilder() + .addAllTags(ImmutableSet.of("intellij-py-code-generator")) + .build(); + + // code under test + Kind kind = heuristics.apply(targetInfo); + + assertThat(kind).isNotNull(); + assertThat(kind.getRuleType()).isEqualTo(RuleType.LIBRARY); + assertThat(kind.getLanguageClasses()).containsExactly(LanguageClass.PYTHON); + } + + @Test + public void testTargetKindHeuristicsForNonCodeGenerator() { + Function heuristics = rules.getTargetKindHeuristics(); + TargetIdeInfo targetInfo = TargetIdeInfo.newBuilder() + .addAllTags(ImmutableSet.of("manual")) + .build(); + + // code under test + Kind kind = heuristics.apply(targetInfo); + + assertThat(kind).isNull(); + } + +} diff --git a/python/tests/unittests/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategyTest.java b/python/tests/unittests/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategyTest.java index 9654c506aaf..daad69ef6dd 100644 --- a/python/tests/unittests/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategyTest.java +++ b/python/tests/unittests/com/google/idea/blaze/python/resolve/provider/AbstractPyImportResolverStrategyTest.java @@ -19,16 +19,27 @@ import com.google.idea.blaze.base.BlazeTestCase; import com.google.idea.blaze.base.ideinfo.*; import com.google.idea.blaze.base.model.BlazeProjectData; +import com.google.idea.blaze.base.model.primitives.WorkspaceRoot; import com.google.idea.blaze.base.settings.BuildSystemName; import com.google.idea.blaze.base.sync.workspace.ArtifactLocationDecoder; import com.google.idea.blaze.base.sync.workspace.MockArtifactLocationDecoder; +import com.google.idea.blaze.base.sync.workspace.WorkspacePathResolver; +import com.google.idea.blaze.base.sync.workspace.WorkspacePathResolverImpl; import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiManager; import com.intellij.psi.util.QualifiedName; import com.jetbrains.python.psi.resolve.PyQualifiedNameResolveContext; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.LinkedHashMap; import org.jetbrains.annotations.Nullable; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; import java.io.File; @@ -38,6 +49,9 @@ public class AbstractPyImportResolverStrategyTest extends BlazeTestCase { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + /** * It is possible to specify import paths for a py_... target so that the modules in * the Python side can be specified. This test checks what happens in this situation. @@ -50,6 +64,7 @@ public void testBuildSourcesIndexWithAnImportRoot() { Project project = Mockito.mock(Project.class); BlazeProjectData projectData = Mockito.mock(BlazeProjectData.class); + Mockito.when(projectData.getWorkspacePathResolver()).thenReturn(Mockito.mock(WorkspacePathResolver.class)); Mockito.when(projectData.getTargetMap()).thenReturn(targetMap); Mockito.when(projectData.getArtifactLocationDecoder()).thenReturn( new MockArtifactLocationDecoder(new File("/workspaceroot"), false)); @@ -113,6 +128,7 @@ public void testBuildSourcesIndexWithNoImportRoot() { Project project = Mockito.mock(Project.class); BlazeProjectData projectData = Mockito.mock(BlazeProjectData.class); + Mockito.when(projectData.getWorkspacePathResolver()).thenReturn(Mockito.mock(WorkspacePathResolver.class)); Mockito.when(projectData.getTargetMap()).thenReturn(targetMap); Mockito.when(projectData.getArtifactLocationDecoder()).thenReturn( new MockArtifactLocationDecoder(new File("/workspaceroot"), false)); @@ -172,6 +188,7 @@ public void testBuildSourcesIndexWithDotImportRoot() { Project project = Mockito.mock(Project.class); BlazeProjectData projectData = Mockito.mock(BlazeProjectData.class); + Mockito.when(projectData.getWorkspacePathResolver()).thenReturn(Mockito.mock(WorkspacePathResolver.class)); Mockito.when(projectData.getTargetMap()).thenReturn(targetMap); Mockito.when(projectData.getArtifactLocationDecoder()).thenReturn( new MockArtifactLocationDecoder(new File("/workspaceroot"), false)); @@ -234,6 +251,7 @@ public void testBuildSourcesIndexWithBuildAtRootAndImport() { Project project = Mockito.mock(Project.class); BlazeProjectData projectData = Mockito.mock(BlazeProjectData.class); + Mockito.when(projectData.getWorkspacePathResolver()).thenReturn(Mockito.mock(WorkspacePathResolver.class)); Mockito.when(projectData.getTargetMap()).thenReturn(targetMap); Mockito.when(projectData.getArtifactLocationDecoder()).thenReturn( new MockArtifactLocationDecoder(new File("/workspaceroot"), false)); @@ -271,6 +289,114 @@ public void testBuildSourcesIndexWithBuildAtRootAndImport() { } } + /** + * If the {@link TargetIdeInfo} has generated code then the sources might be a directory. In this + * case, the logic should walk the directory tree and pull {@code .py} source files from it. There + * are plenty of other test cases in this test class that cover the situation where the sources + * are not coming from a directory. + */ + + @Test + public void testGetPySourcesWithDirectory() throws IOException { + AbstractPyImportResolverStrategy strategy = new TestPyImportResolverStrategy(); + + PyIdeInfo.Builder pyIdeInfoBuilder = PyIdeInfo.builder() + .addSources(ImmutableSet.of( + ArtifactLocation.builder() + .setRootExecutionPathFragment("bazel-out/anyos_arm64-fastbuild/bin") + .setRelativePath("example/materials") + //^ This would be a typical source path for a code generation scenario. + .setIsSource(false) + // ^ When working with code-gen, the source is marked as false. + .build() + )) + .addImports(ImmutableSet.of()); + + TargetMap targetMap = TargetMapBuilder.builder() + .addTarget( + TargetIdeInfo.builder() + .setLabel("//example:generated_directory_lib") + .setBuildFile(source("example/BUILD")) + .setPyInfo(pyIdeInfoBuilder) + ) + .build(); + + // Configure a directory tree with the files in it that can be searched. + + File rootDirectory = temporaryFolder.newFolder("workspaceroot"); + File pythonModuleDirectory = new File(rootDirectory, "bazel-out/anyos_arm64-fastbuild/bin/example/materials/artificial"); + assertThat(pythonModuleDirectory.mkdirs()).isTrue(); + touchFile(new File(pythonModuleDirectory, "plastics.py")); + + // This will not happen in this situation, but just in case the directory is not within + // `bazel-out` then the logic should exclude anything below a barrier file such as + // `BUILD.bazel` + + File outOfBoundsDirectory = new File(rootDirectory, "bazel-out/anyos_arm64-fastbuild/bin/example/anotherworkspace"); + assertThat(outOfBoundsDirectory.mkdirs()).isTrue(); + touchFile(new File(outOfBoundsDirectory, "BUILD.bazel")); + touchFile(new File(outOfBoundsDirectory, "metals.py")); + + Project project = Mockito.mock(Project.class); + BlazeProjectData projectData = Mockito.mock(BlazeProjectData.class); + WorkspacePathResolver workspacePathResolver = new WorkspacePathResolverImpl(new WorkspaceRoot(rootDirectory)); + + Mockito.when(projectData.getWorkspacePathResolver()).thenReturn(workspacePathResolver); + Mockito.when(projectData.getTargetMap()).thenReturn(targetMap); + Mockito.when(projectData.getArtifactLocationDecoder()) + .thenReturn(new MockArtifactLocationDecoder(rootDirectory, false)); + + // code under test + PySourcesIndex actualSourcesIndex = strategy.buildSourcesIndex(project, projectData); + + // Expect the logic under test has found the source code `plastics.py` but has not found the + // sources that lie under the `anotherworkspace` directory. + + // Verify the short names capture the right mapping to possible modules. The path from which + // the modules are calculated is from the .../bin directory. + Set shortNamesImports = actualSourcesIndex.shortNames.get("plastics"); + assertThat(shortNamesImports).containsExactly( + QualifiedName.fromComponents("example", "materials", "artificial", "plastics") + ); + + // Verify that the mappings from possible modules to actual files works. Because the strategy + // class was initialized with a special class to provide the PsiElement, we know that the + // `toString()` method will return specific information that we can relay on in this test. + // The Python source is found in the directory. The Python source that is behind a "boundary" + // (in this case a `BUILD.bazel` file) is not found. + + assertThat(actualSourcesIndex.sourceMap).hasSize(2); + // ^ Only one file should have been found. The file and its parent directory are included. + + PsiManager manager = Mockito.mock(PsiManager.class); + + HashMap expectedBarModuleToSourceMap = new LinkedHashMap<>(); + expectedBarModuleToSourceMap.put( + QualifiedName.fromComponents("example", "materials", "artificial", "plastics"), + "bazel-out/anyos_arm64-fastbuild/bin/example/materials/artificial/plastics.py"); + expectedBarModuleToSourceMap.put( + QualifiedName.fromComponents("example", "materials", "artificial"), + "bazel-out/anyos_arm64-fastbuild/bin/example/materials/artificial"); + + expectedBarModuleToSourceMap.forEach((expectedModule, expectedPath) -> { + PsiElement fullElement = actualSourcesIndex.sourceMap.get(expectedModule).get(manager); + assertThat(fullElement).isNotNull(); + assertThat(fullElement.toString()).isEqualTo(expectedPath); + }); + + } + + /** + * Writes a couple of bytes to the supplied {@link File}. + */ + private void touchFile(File file) { + try (OutputStream outputStream = new FileOutputStream(file)) { + outputStream.write("TEST".getBytes()); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + private TargetMap assembleTestTargetMap(Set importPaths) { PyIdeInfo.Builder pyIdeInfoBuilder = PyIdeInfo.builder() .addSources(ImmutableSet.of(source("whistle/foo/river/lib/bar.py"))) From 94345b5ffefb96321b3122e0b5c8fd2fbc34f54c Mon Sep 17 00:00:00 2001 From: Andrew Lindesay Date: Thu, 10 Oct 2024 17:41:20 +1300 Subject: [PATCH 2/2] Implement py code generation handling - fix method signature problem It is possible to write Bazel rules that generate Python code and act as a `py_library`. The plugin is augmented with this change to have a means of detecting these sort of rules and be able to work with them. --- .../idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java b/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java index 40100cf0471..f22c100bd57 100644 --- a/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java +++ b/base/src/com/google/idea/blaze/base/dependencies/BlazeQueryTargetTagFilter.java @@ -108,7 +108,7 @@ private static List runQuery( BlazeCommand.Builder command = BlazeCommand.builder( - buildSystem.getDefaultInvoker(project, context), BlazeCommandName.QUERY) + buildSystem.getDefaultInvoker(project, context), BlazeCommandName.QUERY, project) .addBlazeFlags("--output=label") .addBlazeFlags(BlazeFlags.KEEP_GOING) .addBlazeFlags(query);