Resolve names inside BUILD files to take into account available targe…

…t types (#454)

Previously, all BUILD files would have functions like scala_library or java_library underlined red despite them being valid.

Now, we additionally query while exporting for available target types and save them to the PropertiesComponent.

This will work with newest pants version, so we might need to wait until there is nightly release. Not sure what is the procedure for changes both to pants and this plugin.

Fixes #355

common/com/twitter/intellij/pants/util/PantsConstants.java

@@ -9,7 +9,6 @@
 import com.twitter.intellij.pants.model.JdkRef;
 import org.jetbrains.annotations.NotNull;
 
-
 public class PantsConstants {
   public static final String PANTS = "pants";
   public static final String PANTS_CONSOLE_NAME = "Pants Console";
@@ -43,6 +42,7 @@ public class PantsConstants {
   // Used to initialize project sdk therefore use project processing weight, i.e, the highest.
   public static final Key<JdkRef> SDK_KEY = Key.create(JdkRef.class, ProjectKeys.PROJECT.getProcessingWeight());
 
+  public static final String PANTS_AVAILABLE_TARGETS_KEY = "available_targets";
   public static final String PANTS_CLI_OPTION_EXPORT_OUTPUT_FILE = "--export-output-file";
   public static final String PANTS_CLI_OPTION_LIST_OUTPUT_FILE = "--list-output-file";
   public static final String PANTS_CLI_OPTION_EXPORT_CLASSPATH_MANIFEST_JAR = "--export-classpath-manifest-jar-only";

common/com/twitter/intellij/pants/util/PantsUtil.java

@@ -105,9 +105,12 @@
 
 public class PantsUtil {
   public static final Gson gson = new Gson();
-  public static final Type TYPE_LIST_STRING = new TypeToken<List<String>>() {}.getType();
-  public static final Type TYPE_SET_STRING = new TypeToken<Set<String>>() {}.getType();
-  public static final Type TYPE_MAP_STRING_INTEGER = new TypeToken<Map<String, Integer>>() {}.getType();
+  public static final Type TYPE_LIST_STRING = new TypeToken<List<String>>() {
+  }.getType();
+  public static final Type TYPE_SET_STRING = new TypeToken<Set<String>>() {
+  }.getType();
+  public static final Type TYPE_MAP_STRING_INTEGER = new TypeToken<Map<String, Integer>>() {
+  }.getType();
   public static final ScheduledExecutorService scheduledThreadPool = Executors.newSingleThreadScheduledExecutor(
     new ThreadFactory() {
       @Override
@@ -211,6 +214,18 @@ public static Optional<VirtualFile> findPantsIniFile(Optional<VirtualFile> worki
     return workingDir.map(file -> file.findChild(PantsConstants.PANTS_INI));
   }
 
+  public static boolean isCompatiblePantsVersion(String projectPath, String minVersion) {
+    return PantsUtil.findPantsExecutable(projectPath)
+      .flatMap(exec -> PantsOptions.getPantsOptions(exec.getPath()).get("pants_version"))
+      .map(version -> PantsUtil.isCompatibleVersion(version, minVersion))
+      .orElse(false);
+  }
+
+  public static boolean isCompatibleVersion(String current, String minimum) {
+    String currentVersion = current.replaceAll("rc.+", "").trim();
+    return versionCompare(currentVersion, minimum) >= 0;
+  }
+
   private static Optional<String> findVersionInFile(@NotNull VirtualFile file) {
     try {
       final String fileContent = VfsUtilCore.loadText(file);
@@ -376,23 +391,28 @@ public static Collection<String> listAllTargets(@NotNull String projectPath) thr
       }
       else {
         List<String> errorLogs = Lists.newArrayList(
-          String.format("Could not list targets: Pants exited with status %d",
-                        processOutput.getExitCode()),
+          String.format(
+            "Could not list targets: Pants exited with status %d",
+            processOutput.getExitCode()
+          ),
           String.format("argv: '%s'", cmd.getCommandLineString()),
           "stdout:",
           processOutput.getStdout(),
           "stderr:",
-          processOutput.getStderr());
+          processOutput.getStderr()
+        );
         final String errorMessage = String.join("\n", errorLogs);
         LOG.warn(errorMessage);
         throw new PantsException(errorMessage);
       }
     }
     catch (IOException | ExecutionException e) {
       final String processCreationFailureMessage =
-        String.format("Could not execute command: '%s' due to error: '%s'",
-                      cmd.getCommandLineString(),
-                      e.getMessage());
+        String.format(
+          "Could not execute command: '%s' due to error: '%s'",
+          cmd.getCommandLineString(),
+          e.getMessage()
+        );
       LOG.warn(processCreationFailureMessage, e);
       throw new PantsException(processCreationFailureMessage);
     }
@@ -506,7 +526,7 @@ class SeedPantsProjectKeys {
     }
     if (versionCompare(version, PANTS_IDEA_PLUGIN_VERESION_MIN) < 0 ||
         versionCompare(version, PANTS_IDEA_PLUGIN_VERESION_MAX) > 0
-      ) {
+    ) {
       Messages.showInfoMessage(project, PantsBundle.message("pants.idea.plugin.goal.version.unsupported"), "Version Error");
       return false;
     }
@@ -829,15 +849,15 @@ public static Set<String> filterGenTargets(@NotNull Collection<String> addresses
   }
 
   /**
-   * @param pantsExecutable path to the pants executable file for the
-   * project. This function will return erroneous output if you use a directory path. The
-   * pants executable can be found from a project path with {@link #findPantsExecutable(String)}.
+   * @param pantsExecutable  path to the pants executable file for the
+   *                         project. This function will return erroneous output if you use a directory path. The
+   *                         pants executable can be found from a project path with {@link #findPantsExecutable(String)}.
    * @param parentDisposable Disposable object to use if a new JDK is added to
-   * the project jdk table (otherwise null). Integration tests should use getTestRootDisposable() for
-   * this argument to avoid exceptions during teardown.
+   *                         the project jdk table (otherwise null). Integration tests should use getTestRootDisposable() for
+   *                         this argument to avoid exceptions during teardown.
    * @return The default Sdk object to use for the project at the given pants
    * executable path.
-   *
+   * <p>
    * This method will add a JDK to the project JDK table if it needs to create
    * one, which mutates global state (protected by a read/write lock).
    */

pants.ini

@@ -5,7 +5,7 @@ local_artifact_cache = %(buildroot)s/.cache
 jvm_options: ["-Xmx1g", "-XX:MaxPermSize=256m"]
 
 [GLOBAL]
-pants_version: 1.20.0rc2
+pants_version: 1.22.0
 print_exception_stacktrace: True
 pants_ignore: +[
     'out/',

src/com/twitter/intellij/pants/psi/resolve/PantsReferenceResolveProvider.java

@@ -3,6 +3,7 @@
 
 package com.twitter.intellij.pants.psi.resolve;
 
+import com.intellij.ide.util.PropertiesComponent;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
 import com.intellij.util.Function;
@@ -12,20 +13,36 @@
 import com.jetbrains.python.psi.resolve.RatedResolveResult;
 import com.jetbrains.python.psi.types.TypeEvalContext;
 import com.twitter.intellij.pants.index.PantsTargetIndex;
+import com.twitter.intellij.pants.util.PantsConstants;
 import com.twitter.intellij.pants.util.PantsUtil;
 import org.jetbrains.annotations.NotNull;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 
 public class PantsReferenceResolveProvider implements PyReferenceResolveProvider {
+
   @NotNull
   @Override
   public List<RatedResolveResult> resolveName(@NotNull PyQualifiedExpression expression, @NotNull TypeEvalContext context) {
     PsiFile containingFile = expression.getContainingFile();
-    return PantsUtil.isBUILDFileName(containingFile.getName()) ?
-           resolvePantsName(expression) :
-           Collections.<RatedResolveResult>emptyList();
+    if (isOneOfAvailableTargetTypes(expression)) {
+      List<RatedResolveResult> resolved = new LinkedList<>();
+      resolved.add(new RatedResolveResult(RatedResolveResult.RATE_NORMAL, expression));
+      return resolved;
+    }
+    else {
+      return PantsUtil.isBUILDFileName(containingFile.getName()) ?
+             resolvePantsName(expression) :
+             Collections.<RatedResolveResult>emptyList();
+    }
+  }
+
+  private boolean isOneOfAvailableTargetTypes(@NotNull PyQualifiedExpression expression) {
+    String[] allBuildTypes = PropertiesComponent.getInstance().getValues(PantsConstants.PANTS_AVAILABLE_TARGETS_KEY);
+    return allBuildTypes != null && Arrays.asList(allBuildTypes).contains(expression.getReferencedName());
   }
 
   private List<RatedResolveResult> resolvePantsName(@NotNull PyQualifiedExpression element) {

src/com/twitter/intellij/pants/service/PantsCompileOptionsExecutor.java

@@ -20,6 +20,7 @@
 import com.twitter.intellij.pants.model.IJRC;
 import com.twitter.intellij.pants.model.PantsCompileOptions;
 import com.twitter.intellij.pants.model.PantsExecutionOptions;
+import com.twitter.intellij.pants.model.PantsOptions;
 import com.twitter.intellij.pants.settings.PantsExecutionSettings;
 import com.twitter.intellij.pants.util.PantsUtil;
 import org.jetbrains.annotations.Nls;
@@ -197,7 +198,8 @@ private ProcessOutput getProcessOutput(
   }
 
   @NotNull
-  private GeneralCommandLine getPantsExportCommand(final File outputFile, @NotNull Consumer<String> statusConsumer) throws IOException {
+  private GeneralCommandLine getPantsExportCommand(final File outputFile, @NotNull Consumer<String> statusConsumer)
+    throws IOException {
     final GeneralCommandLine commandLine = PantsUtil.defaultCommandLine(getProjectPath());
 
     // Grab the import stage pants rc file for IntelliJ.
@@ -213,6 +215,11 @@ private GeneralCommandLine getPantsExportCommand(final File outputFile, @NotNull
     }
     commandLine.addParameter("--target-spec-file=" + targetSpecsFile.getPath());
     commandLine.addParameter("--no-quiet");
+
+    if (PantsUtil.isCompatiblePantsVersion(getProjectPath(), "1.24.0")) {
+      commandLine.addParameter("--export-available-target-types");
+    }
+
     if (getOptions().isImportSourceDepsAsJars()) {
       commandLine.addParameter("export-dep-as-jar");
     }

src/com/twitter/intellij/pants/service/project/PantsResolver.java

@@ -6,6 +6,7 @@
 import com.google.gson.JsonSyntaxException;
 import com.intellij.execution.ExecutionException;
 import com.intellij.execution.process.ProcessAdapter;
+import com.intellij.ide.util.PropertiesComponent;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.externalSystem.model.DataNode;
 import com.intellij.openapi.externalSystem.model.ExternalSystemException;
@@ -20,6 +21,7 @@
 import com.twitter.intellij.pants.service.PantsCompileOptionsExecutor;
 import com.twitter.intellij.pants.service.project.model.graph.BuildGraph;
 import com.twitter.intellij.pants.service.project.model.ProjectInfo;
+import com.twitter.intellij.pants.util.PantsConstants;
 import com.twitter.intellij.pants.util.PantsUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -100,6 +102,7 @@ public void addInfoTo(@NotNull DataNode<ProjectData> projectInfoDataNode) {
 
     Optional<BuildGraph> buildGraph = constructBuildGraph(projectInfoDataNode);
 
+    PropertiesComponent.getInstance().setValues(PantsConstants.PANTS_AVAILABLE_TARGETS_KEY, myProjectInfo.getAvailableTargetTypes());
     final Map<String, DataNode<ModuleData>> modules = new HashMap<>();
     for (PantsResolverExtension resolver : PantsResolverExtension.EP_NAME.getExtensions()) {
       resolver.resolve(myProjectInfo, myExecutor, projectInfoDataNode, modules, buildGraph);

src/com/twitter/intellij/pants/service/project/model/ProjectInfo.java

@@ -4,6 +4,7 @@
 package com.twitter.intellij.pants.service.project.model;
 
 import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.SerializedName;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.containers.ContainerUtil;
@@ -12,6 +13,8 @@
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.TestOnly;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -40,6 +43,14 @@ public ProjectInfo() {
   // name to info
   protected Map<String, TargetInfo> targets;
 
+  /* This might need to be expanded to show all properties that
+   * a target type can contain like:
+   *
+   * {"java_library" : [{ "dependencies" : "list(str)" }]}
+   */
+  @SerializedName("available_target_types")
+  protected String[] availableTargetTypes = {};
+
   protected String version;
 
   @NotNull
@@ -53,7 +64,7 @@ public String getVersion() {
   private static <T> List<Map.Entry<String, T>> getSortedEntries(Map<String, T> map) {
     return ContainerUtil.sorted(
       map.entrySet(),
-      new Comparator<Map.Entry<String,T>>() {
+      new Comparator<Map.Entry<String, T>>() {
         @Override
         public int compare(Map.Entry<String, T> o1, Map.Entry<String, T> o2) {
           return StringUtil.naturalCompare(o1.getKey(), o2.getKey());
@@ -86,6 +97,11 @@ public void setTargets(Map<String, TargetInfo> targets) {
     this.targets = targets;
   }
 
+  @NotNull
+  public String[] getAvailableTargetTypes() {
+    return availableTargetTypes;
+  }
+
   @Nullable
   public PythonSetup getPythonSetup() {
     return python_setup;

testFramework/com/twitter/intellij/pants/testFramework/OSSPantsIntegrationTest.java

@@ -63,7 +63,7 @@ protected void assertEmptyBeforeRunTask(RunConfiguration configuration) {
    * Assert Project has the right JDK and language level (JVM project only).
    */
   protected void assertProjectJdkAndLanguageLevel() {
-    final String pantsExecutablePath = PantsUtil.findPantsExecutable(getParentPath()).get().getPath();
+    final String pantsExecutablePath = PantsUtil.findPantsExecutable(getProjectPath()).get().getPath();
     assertEquals(
       ProjectRootManager.getInstance(myProject).getProjectSdk().getHomePath(),
       getDefaultJavaSdk(pantsExecutablePath).get().getHomePath()

tests/com/twitter/intellij/pants/highlighting/PantsHighlightingIntegrationTest.java

@@ -3,31 +3,24 @@
 
 package com.twitter.intellij.pants.highlighting;
 
-import com.google.common.collect.Lists;
 import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer;
-import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerEx;
-import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl;
 import com.intellij.codeInsight.daemon.impl.HighlightInfo;
 import com.intellij.codeInsight.intention.IntentionAction;
 import com.intellij.openapi.editor.Editor;
 import com.intellij.openapi.editor.RangeMarker;
 import com.intellij.openapi.fileEditor.FileEditorManager;
 import com.intellij.openapi.fileEditor.OpenFileDescriptor;
-import com.intellij.openapi.fileEditor.TextEditor;
-import com.intellij.openapi.fileEditor.impl.text.TextEditorProvider;
 import com.intellij.openapi.util.Condition;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiDocumentManager;
-import com.intellij.psi.PsiFile;
 import com.intellij.util.containers.ContainerUtil;
 import com.twitter.intellij.pants.testFramework.OSSPantsIntegrationTest;
 import org.jetbrains.annotations.Nls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Collections;
 import java.util.List;
 
 abstract public class PantsHighlightingIntegrationTest extends OSSPantsIntegrationTest {

tests/com/twitter/intellij/pants/integration/OSSProjectInfoResolveTest.java

@@ -16,8 +16,10 @@
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 
 public class OSSProjectInfoResolveTest extends OSSPantsIntegrationTest {
   private static Consumer<String> STRING_CONSUMER = new Consumer<String>() {
@@ -63,6 +65,17 @@ public void testTargetType() {
     assertFalse(greetTarget.isScalaTarget());
   }
 
+  public void testAvailableTargetTypes() {
+    final ProjectInfo info = resolveProjectInfo("examples/src/scala/org/pantsbuild/example/hello/");
+
+    // this should be only tested after export version 1.0.13
+    if (PantsUtil.isCompatibleVersion(info.getVersion(), "1.0.13")) {
+      final List<String> availableTargetTypes = Arrays.asList(info.getAvailableTargetTypes());
+      assertNotEmpty(availableTargetTypes);
+      assertContain(availableTargetTypes, "scala_library", "java_library");
+    }
+  }
+
   public void testTargetJars() {
     final ProjectInfo info = resolveProjectInfo("intellij-integration/3rdparty/hadoop/::");
 

tests/com/twitter/intellij/pants/integration/TargetFileResolutionIntegrationTest.java

@@ -0,0 +1,36 @@
+// Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+package com.twitter.intellij.pants.integration;
+
+import com.intellij.codeInsight.TargetElementUtil;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.PsiElement;
+import com.intellij.psi.PsiFile;
+import com.intellij.psi.PsiManager;
+import com.intellij.psi.PsiReference;
+import com.twitter.intellij.pants.testFramework.OSSPantsIntegrationTest;
+import com.twitter.intellij.pants.util.PantsUtil;
+
+import java.io.IOException;
+import java.util.Collection;
+
+public class TargetFileResolutionIntegrationTest extends OSSPantsIntegrationTest {
+
+  public void testAvailableTargetTypes() throws IOException {
+    String helloProjectPath = "examples/src/scala/org/pantsbuild/example/hello/";
+    doImport(helloProjectPath);
+    // should be only tested with pants versions above 1.24.0
+    if (PantsUtil.isCompatiblePantsVersion(myProjectRoot.getPath(), "1.24.0")) {
+      VirtualFile vfile = myProjectRoot.findFileByRelativePath(helloProjectPath + "BUILD");
+      assertNotNull(vfile);
+      String input = new String(vfile.contentsToByteArray());
+      PsiFile build = PsiManager.getInstance(myProject).findFile(vfile);
+      final PsiReference reference = build.findReferenceAt(input.indexOf("target(") + 1);
+      assertNotNull("no reference", reference);
+      final Collection<PsiElement> elements = TargetElementUtil.getInstance().getTargetCandidates(reference);
+      assertNotNull(elements);
+      assertEquals(1, elements.size());
+    }
+  }
+}