feat: Support for textDocument/semanticTokens

Fixes redhat-developer#238

Signed-off-by: azerr <[email protected]>

docs/LSPSupport.md

@@ -58,7 +58,7 @@ Current state of [Language Features]( https://microsoft.github.io/language-serve
  * ✅ [textDocument/foldingRange](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_foldingRange)  (see [implementation details](#folding-range))
  * ❌ [textDocument/selectionRange](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_selectionRange).
  * ❌ [textDocument/documentSymbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_documentSymbol).
- * ❌ [textDocument/semanticTokens](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens).
+ * ✅ [textDocument/semanticTokens](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens) (see [implementation details](#semantic-tokens))
  * ❌ [textDocument/inlineValue](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_inlineValue).
  * ❌ [workspace/inlineValue/refresh](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#workspace_inlineValue_refresh).
  * ❌ [textDocument/moniker](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_moniker).
@@ -454,4 +454,43 @@ is rendered as a `Sticky balloon` notification:
 
 You can change the notification behavior of `LSP/window/showMessageRequest` by using the standard UI `Notifications` preferences :
 
-![window/showMessageRequest Notification](./images/lsp-support/window_showMessageRequest_Notification.png)
+![window/showMessageRequest Notification](./images/lsp-support/window_showMessageRequest_Notification.png)
+
+#### Semantic Tokens
+
+ * Use only textDocument/semanticTokens/full
+ * Use an implementation of RainbowVisitor. You need to enable semantic tokens (for the moment I don't know how to select by default this checkbox)
+
+![](images/lsp-support/textDocument_semanticTokens_enable.png)
+ * Use SemanticTokensColorsProvider API to get TextAttributesKey from tokenType, tokenModifiers:
+
+```java
+   public @Nullable TextAttributesKey getTextAttributesKey(@NotNull String tokenType,
+   @NotNull List<String> tokenModifiers,
+   @NotNull PsiFile file);
+```
+ * By default uses DefaultSemanticTokensColorsProvider
+ * TODO: provide an extension point to use a custom SemanticTokensColorsProvider
+ * Provide a semantic tokens inspector view ![](images/lsp-support/textDocument_semanticTokens_inspector.png)
+
+##### DefaultSemanticTokensColorsProvider
+
+The following table lists the currently predefined mappings:
+
+ * the `Semantic token types` column shows the standard LSP Semantic token types. 
+ * the `Semantic modifiers types` column shows the standard LSP Semantic modifier types. 
+ * the `SemanticTokensHighlightingColors` column defines the `TextAttributesKey` constants declared in the LSP4IJ `SemanticTokensHighlightingColors` class.
+ * the `DefaultLanguageHighlighterColors` column defines the default `TextAttributesKey` used by IntelliJ and show the mapping between `SemanticTokensHighlightingColors`
+and `DefaultLanguageHighlighterColors`.
+
+| Semantic token types | Semantic modifier types | SemanticTokensHighlightingColors | DefaultLanguageHighlighterColors |
+|----------------------|-------------------------|----------------------------------|----------------------------------|
+| namespace            | definition              | NAMESPACE_DECLARATION            | CLASS_REFERENCE                  |
+| namespace            | declaration             | NAMESPACE_DECLARATION            | CLASS_REFERENCE                  |
+| namespace            |                         | NAMESPACE                        | CLASS_NAME                       |
+| type (TODO)          |                         |                                  |                                  |
+
+If you need other mapping:
+
+ * if you think it is a generic mapping, please create a contribution to define a new `SemanticTokensHighlightingColors` constants
+ * if the mapping is specific to your language, use the 'semanticTokensColorsProvider' extension point to defines your new mapping.

src/main/java/com/redhat/devtools/lsp4ij/LSPFileSupport.java

@@ -29,6 +29,7 @@
 import com.redhat.devtools.lsp4ij.features.references.LSPReferenceSupport;
 import com.redhat.devtools.lsp4ij.features.rename.LSPPrepareRenameSupport;
 import com.redhat.devtools.lsp4ij.features.rename.LSPRenameSupport;
+import com.redhat.devtools.lsp4ij.features.semanticTokens.LSPSemanticTokensSupport;
 import com.redhat.devtools.lsp4ij.features.signatureHelp.LSPSignatureHelpSupport;
 import com.redhat.devtools.lsp4ij.features.typeDefinition.LSPTypeDefinitionSupport;
 import org.jetbrains.annotations.ApiStatus;
@@ -78,6 +79,8 @@ public class LSPFileSupport extends UserDataHolderBase implements Disposable {
 
     private final LSPTypeDefinitionSupport typeDefinitionSupport;
 
+    private final LSPSemanticTokensSupport semanticTokensSupport;
+
     private LSPFileSupport(@NotNull PsiFile file) {
         this.file = file;
         this.codeLensSupport = new LSPCodeLensSupport(file);
@@ -97,6 +100,7 @@ private LSPFileSupport(@NotNull PsiFile file) {
         this.referenceSupport = new LSPReferenceSupport(file);
         this.declarationSupport = new LSPDeclarationSupport(file);
         this.typeDefinitionSupport = new LSPTypeDefinitionSupport(file);
+        this.semanticTokensSupport = new LSPSemanticTokensSupport(file);
         file.putUserData(LSP_FILE_SUPPORT_KEY, this);
     }
 
@@ -121,8 +125,9 @@ public void dispose() {
         getReferenceSupport().cancel();
         getDeclarationSupport().cancel();
         getTypeDefinitionSupport().cancel();
+        getSemanticTokensSupport().cancel();
         var map = getUserMap();
-        for(var key : map.getKeys()) {
+        for (var key : map.getKeys()) {
             var value = map.get(key);
             if (value instanceof Disposable disposable) {
                 disposable.dispose();
@@ -283,6 +288,15 @@ public LSPTypeDefinitionSupport getTypeDefinitionSupport() {
         return typeDefinitionSupport;
     }
 
+    /**
+     * Returns the LSP semantic tokens support.
+     *
+     * @return the LSP semantic tokens support.
+     */
+    public LSPSemanticTokensSupport getSemanticTokensSupport() {
+        return semanticTokensSupport;
+    }
+
     /**
      * Return the existing LSP file support for the given Psi file, or create a new one if necessary.
      *

src/main/java/com/redhat/devtools/lsp4ij/LSPIJUtils.java

@@ -484,10 +484,13 @@ public static Module getModule(@Nullable VirtualFile file, @NotNull Project proj
      * @return a valid offset from the given position in the given document.
      */
     public static int toOffset(@NotNull Position position, @NotNull Document document) {
+        return toOffset(position.getLine(), position.getCharacter(), document);
+    }
+
+    public static int toOffset(int line, int character, @NotNull Document document) {
         // See https://github.com/microsoft/vscode-languageserver-node/blob/8e625564b531da607859b8cb982abb7cdb2fbe2e/textDocument/src/main.ts#L304
 
         // Adjust position line/character according to this comment https://github.com/microsoft/vscode-languageserver-node/blob/ed3cd0f78c1495913bda7318ace2be7f968008af/textDocument/src/main.ts#L26
-        int line = position.getLine();
         if (line >= document.getLineCount()) {
             // The line number is greater than the number of lines in a document, it defaults back to the number of lines in the document.
             return document.getTextLength();
@@ -498,7 +501,7 @@ public static int toOffset(@NotNull Position position, @NotNull Document documen
         int lineOffset = document.getLineStartOffset(line);
         int nextLineOffset = document.getLineEndOffset(line);
         // If the character value is greater than the line length it defaults back to the line length
-        return Math.max(Math.min(lineOffset + position.getCharacter(), nextLineOffset), lineOffset);
+        return Math.max(Math.min(lineOffset + character, nextLineOffset), lineOffset);
     }
 
     /**
@@ -953,7 +956,7 @@ private static void doApplyEdits(@Nullable Editor editor,
      * @param caretOffset the current caret offset and -1 if caret must be not moved.
      * @return the increment (positive or negative) used to update caret offset.
      */
-    private static int applyEdit(int start,
+    public static int applyEdit(int start,
                                  int end,
                                  @Nullable String newText,
                                  @NotNull Document document,

src/main/java/com/redhat/devtools/lsp4ij/LSPRequestConstants.java

@@ -27,6 +27,7 @@ public class LSPRequestConstants {
     public static final String TEXT_DOCUMENT_DEFINITION = "textDocument/definition";
     public static final String TEXT_DOCUMENT_DOCUMENT_LINK = "textDocument/documentLink";
     public static final String TEXT_DOCUMENT_FOLDING_RANGE = "textDocument/foldingRange";
+    public static final String TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL = "textDocument/semanticTokensFull";
     public static final String TEXT_DOCUMENT_TYPE_DEFINITION = "textDocument/typeDefinition";
     public static final String TEXT_DOCUMENT_CODE_ACTION = "textDocument/codeAction";
     public static final String TEXT_DOCUMENT_CODE_LENS = "textDocument/codeLens";

src/main/java/com/redhat/devtools/lsp4ij/LanguageServerItem.java

@@ -11,6 +11,7 @@
 package com.redhat.devtools.lsp4ij;
 
 import com.intellij.psi.PsiFile;
+import com.redhat.devtools.lsp4ij.features.semanticTokens.SemanticTokensColorsProvider;
 import org.eclipse.lsp4j.*;
 import org.eclipse.lsp4j.jsonrpc.messages.Either;
 import org.eclipse.lsp4j.services.LanguageServer;
@@ -405,6 +406,16 @@ public static boolean isCodeActionResolveSupported(@Nullable ServerCapabilities
         return false;
     }
 
+    /**
+     * Returns true if the language server can support semantic tokens and false otherwise.
+     *
+     * @param serverCapabilities the server capabilities.
+     * @return true if the language server can support semantic tokens and false otherwise.
+     */
+    public static boolean isSemanticTokensSupported(@Nullable ServerCapabilities serverCapabilities) {
+        return serverCapabilities != null &&
+                serverCapabilities.getSemanticTokensProvider() != null;
+    }
 
     /**
      * Returns true if the language server can support rename and false otherwise.
@@ -498,4 +509,7 @@ private static boolean hasCapability(Boolean capability) {
         return capability != null && capability;
     }
 
+    public SemanticTokensColorsProvider getSemanticTokensColorsProvider() {
+        return getServerWrapper().getServerDefinition().getSemanticTokensColorsProvider();
+    }
 }

src/main/java/com/redhat/devtools/lsp4ij/client/LanguageClientImpl.java

@@ -11,6 +11,7 @@
 package com.redhat.devtools.lsp4ij.client;
 
 import com.google.gson.JsonObject;
+import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.command.WriteCommandAction;
 import com.intellij.openapi.editor.Editor;
@@ -150,6 +151,27 @@ private void refreshInlayHintsForAllOpenedFiles() {
         }
     }
 
+    @Override
+    public CompletableFuture<Void> refreshSemanticTokens() {
+        return CompletableFuture.runAsync(() -> {
+            if (wrapper == null) {
+                return;
+            }
+            refreshSemanticTokensForAllOpenedFiles();
+        });
+    }
+
+    private void refreshSemanticTokensForAllOpenedFiles() {
+        for (var fileData : wrapper.getConnectedFiles()) {
+            VirtualFile file = fileData.getFile();
+            final PsiFile psiFile = LSPIJUtils.getPsiFile(file, project);
+            if (psiFile != null) {
+                // Should be enough?
+                DaemonCodeAnalyzer.getInstance(psiFile.getProject()).restart(psiFile);
+            }
+        }
+    }
+
     @Override
     public CompletableFuture<Void> createProgress(WorkDoneProgressCreateParams params) {
         return progressManager.createProgress(params);

src/main/java/com/redhat/devtools/lsp4ij/console/explorer/LanguageServerExplorerLifecycleListener.java

@@ -13,6 +13,7 @@
  *******************************************************************************/
 package com.redhat.devtools.lsp4ij.console.explorer;
 
+import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.project.Project;
 import com.redhat.devtools.lsp4ij.LanguageServerWrapper;
 import com.redhat.devtools.lsp4ij.ServerStatus;
@@ -62,9 +63,12 @@ public void handleLSPMessage(Message message, MessageConsumer messageConsumer, L
             return;
         }
 
-        TracingMessageConsumer tracing = getLSPRequestCacheFor(languageServer);
-        String log = tracing.log(message, messageConsumer, serverTrace);
-        invokeLaterIfNeeded(() -> showTrace(processTreeNode, log));
+        ApplicationManager.getApplication()
+                .executeOnPooledThread(() -> {
+                    TracingMessageConsumer tracing = getLSPRequestCacheFor(languageServer);
+                    String log = tracing.log(message, messageConsumer, serverTrace);
+                    invokeLaterIfNeeded(() -> showTrace(processTreeNode, log));
+                });
     }
 
     @Override

src/main/java/com/redhat/devtools/lsp4ij/features/documentation/LSPDocumentationTargetProvider.java

@@ -11,7 +11,6 @@
 package com.redhat.devtools.lsp4ij.features.documentation;
 
 import com.intellij.openapi.editor.Document;
-import com.intellij.openapi.editor.Editor;
 import com.intellij.openapi.progress.ProcessCanceledException;
 import com.intellij.openapi.project.DumbService;
 import com.intellij.openapi.project.Project;
@@ -83,7 +82,6 @@ public class LSPDocumentationTargetProvider implements DocumentationTargetProvid
             // textDocument/hover has been collected correctly
             List<Hover> hovers = hoverFuture.getNow(null);
             if (hovers != null) {
-                Editor editor = LSPIJUtils.editorForElement(psiFile);
                 return hovers
                         .stream()
                         .map(hover -> toDocumentTarget(hover, document, psiFile))