Skip to content

Commit

Permalink
Adjust SemanticTokensService to LSP specification
Browse files Browse the repository at this point in the history 
TokenType is 0 based and represents the index in the list of supported
token types.

0 is valid for TokenType and TokenModifiers so we do not need to check
for 0 here.

Signed-off-by: Jonathan Pollert <[email protected]>
  • Loading branch information
@jnt0r
jnt0r committed Dec 6, 2023
1 parent c23f8b4 commit bbbf392
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 15 deletions.
9 changes: 6 additions & 3 deletions org.eclipse.xtext.ide.tests/src/org/eclipse/xtext/ide/tests/server/SemanticTokensTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

/**
* @author Rubén Porras Campo - Initial test
* @author Jonathan Pollert - Adjust to LSP specification
*/
public class SemanticTokensTest extends AbstractTestLangLanguageServerTest {
@Test
Expand All @@ -34,9 +35,11 @@ public void testSemanticTokensFull() {
it.setModel(model);

List<List<Integer>> expectedTokens = new ArrayList<>();
expectedTokens.add(ImmutableList.of(0,0,4,1,0));
expectedTokens.add(ImmutableList.of(3,0,4,1,0));
expectedTokens.add(ImmutableList.of(0,9,7,1,0));
expectedTokens.add(ImmutableList.of(0,0,4,15,0));
expectedTokens.add(ImmutableList.of(0,5,3,1,16));
expectedTokens.add(ImmutableList.of(3,0,4,15,0));
expectedTokens.add(ImmutableList.of(0,5,3,1,16));
expectedTokens.add(ImmutableList.of(0,4,7,15,0));

it.setExpected(expectedTokens.stream().flatMap(List::stream).collect(Collectors.toList()));
});
Expand Down
9 changes: 8 additions & 1 deletion ...g/eclipse/xtext/ide/tests/testlanguage/syntaxcoloring/SemanticHighlightingCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
package org.eclipse.xtext.ide.tests.testlanguage.syntaxcoloring;

import org.eclipse.emf.ecore.EObject;
import org.eclipse.lsp4j.SemanticTokenModifiers;
import org.eclipse.lsp4j.SemanticTokenTypes;
import org.eclipse.xtext.Keyword;
import org.eclipse.xtext.TerminalRule;
import org.eclipse.xtext.ide.editor.syntaxcoloring.DefaultSemanticHighlightingCalculator;
import org.eclipse.xtext.ide.editor.syntaxcoloring.HighlightingStyles;
import org.eclipse.xtext.ide.editor.syntaxcoloring.IHighlightedPositionAcceptor;
import org.eclipse.xtext.ide.tests.testlanguage.testLanguage.TestLanguagePackage;
import org.eclipse.xtext.ide.tests.testlanguage.testLanguage.TypeDeclaration;
import org.eclipse.xtext.nodemodel.ICompositeNode;
import org.eclipse.xtext.nodemodel.INode;
import org.eclipse.xtext.nodemodel.util.NodeModelUtils;
Expand All @@ -26,7 +30,7 @@ public class SemanticHighlightingCalculator extends DefaultSemanticHighlightingC
@Override
protected boolean highlightElement(EObject object, IHighlightedPositionAcceptor acceptor,
CancelIndicator cancelIndicator) {
if (object.eClass() == TestLanguagePackage.Literals.TYPE_DECLARATION) {
if (object instanceof TypeDeclaration) {
ICompositeNode node = NodeModelUtils.getNode(object);
for (INode childNode: node.getChildren()) {
if (childNode.getGrammarElement() instanceof Keyword) {
Expand All @@ -35,6 +39,9 @@ protected boolean highlightElement(EObject object, IHighlightedPositionAcceptor
acceptor.addPosition(childNode.getOffset(), childNode.getLength(), HighlightingStyles.KEYWORD_ID);
}
}
NodeModelUtils.findNodesForFeature(object, childNode.getGrammarElement().eClass().getEStructuralFeature("name")).forEach(n -> {
acceptor.addPosition(n.getOffset(), n.getLength(), HighlightingStyles.TYPE_ID, HighlightingStyles.DEFINITION_ID);
});
}
}

Expand Down
4 changes: 4 additions & 0 deletions ...eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/syntaxcoloring/HighlightingStyles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,9 @@ public interface HighlightingStyles {
String DEFAULT_ID = "default";
String INVALID_TOKEN_ID = "error";
String TASK_ID = "task";
String TYPE_ID = "type";

// LSP modifier
String DEFINITION_ID = "definition";

}
59 changes: 48 additions & 11 deletions ...ipse.xtext.ide/src/org/eclipse/xtext/ide/server/semantictokens/SemanticTokensService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
import org.eclipse.emf.common.util.WrappedException;
import org.eclipse.lsp4j.Position;
import org.eclipse.lsp4j.SemanticTokenModifiers;
import org.eclipse.lsp4j.SemanticTokenTypes;
import org.eclipse.lsp4j.SemanticTokens;
import org.eclipse.lsp4j.SemanticTokensParams;
import org.eclipse.xtext.ide.editor.syntaxcoloring.HighlightingStyles;
import org.eclipse.xtext.ide.editor.syntaxcoloring.ISemanticHighlightingCalculator;
import org.eclipse.xtext.ide.editor.syntaxcoloring.LightweightPosition;
import org.eclipse.xtext.ide.editor.syntaxcoloring.MergingHighlightedPositionAcceptor;
Expand All @@ -37,6 +37,7 @@
* A service for Semantic Tokens.
*
* @author Rubén Porras Campo - Initial contribution and API
* @author Jonathan Pollert - Adjust to LSP specification
*
* @since 2.29
*/
Expand All @@ -45,7 +46,7 @@ public class SemanticTokensService {
@Inject
private IResourceServiceProvider.Registry languagesRegistry;

private final List<String> tokenTypes = new ArrayList<>();
private final List<String> tokenTypes = new ArrayList<>();
private final List<String> tokenModifiers = new ArrayList<>();

protected SemanticTokensService() {
Expand All @@ -54,13 +55,42 @@ protected SemanticTokensService() {
}

protected void addTokenTypes() {
tokenTypes.add(HighlightingStyles.KEYWORD_ID);
tokenTypes.add(HighlightingStyles.DEFAULT_ID);
tokenTypes.add(HighlightingStyles.NUMBER_ID);
tokenTypes.add(SemanticTokenTypes.Namespace);
tokenTypes.add(SemanticTokenTypes.Type);
tokenTypes.add(SemanticTokenTypes.Class);
tokenTypes.add(SemanticTokenTypes.Enum);
tokenTypes.add(SemanticTokenTypes.Interface);
tokenTypes.add(SemanticTokenTypes.Struct);
tokenTypes.add(SemanticTokenTypes.TypeParameter);
tokenTypes.add(SemanticTokenTypes.Parameter);
tokenTypes.add(SemanticTokenTypes.Variable);
tokenTypes.add(SemanticTokenTypes.Property);
tokenTypes.add(SemanticTokenTypes.EnumMember);
tokenTypes.add(SemanticTokenTypes.Event);
tokenTypes.add(SemanticTokenTypes.Function);
tokenTypes.add(SemanticTokenTypes.Method);
tokenTypes.add(SemanticTokenTypes.Macro);
tokenTypes.add(SemanticTokenTypes.Keyword);
tokenTypes.add(SemanticTokenTypes.Modifier);
tokenTypes.add(SemanticTokenTypes.Comment);
tokenTypes.add(SemanticTokenTypes.String);
tokenTypes.add(SemanticTokenTypes.Number);
tokenTypes.add(SemanticTokenTypes.Regexp);
tokenTypes.add(SemanticTokenTypes.Operator);
tokenTypes.add(SemanticTokenTypes.Decorator);
}

protected void addTokenModifiers() {
tokenModifiers.add(SemanticTokenModifiers.Abstract);
tokenModifiers.add(SemanticTokenModifiers.Async);
tokenModifiers.add(SemanticTokenModifiers.Declaration);
tokenModifiers.add(SemanticTokenModifiers.DefaultLibrary);
tokenModifiers.add(SemanticTokenModifiers.Definition);
tokenModifiers.add(SemanticTokenModifiers.Deprecated);
tokenModifiers.add(SemanticTokenModifiers.Documentation);
tokenModifiers.add(SemanticTokenModifiers.Modification);
tokenModifiers.add(SemanticTokenModifiers.Readonly);
tokenModifiers.add(SemanticTokenModifiers.Static);
}

private List<LightweightPosition> getPositions(final XtextResource resource,
Expand All @@ -85,7 +115,7 @@ private List<LightweightPosition> getPositions(final XtextResource resource,
* @param resource
* the resource
* @param params
* the params
* the parameters
* @param cancelIndicator
* the cancel indicator
* @return the semantic tokens
Expand All @@ -105,16 +135,18 @@ public SemanticTokens semanticTokensFull(final Document document, final XtextRes
Integer positionTokenType = getTokenType(lightweightPosition.getIds());
Integer positionTokenModifiers = getTokenModifiers(lightweightPosition.getIds());

if (positionTokenType != 0 || positionTokenModifiers != 0) {
if (positionTokenType != -1) {
int deltaOffset = lightweightPosition.getOffset() - lastOffset;
reader.skip(deltaOffset);
Position position = reader.getPosition();

int deltaLine = position.getLine() - lastLine;
data.add(deltaLine); // delta line
lastLine = position.getLine();
lastOffset = lightweightPosition.getOffset();

data.add(deltaLine); // delta line
// delta start relative to previous token if on the same line or to 0
data.add(deltaLine == 0 ? deltaOffset : position.getCharacter());
lastOffset = lightweightPosition.getOffset();
data.add(lightweightPosition.getLength()); // length
data.add(positionTokenType); // token type
data.add(positionTokenModifiers); // token modifiers
Expand All @@ -134,14 +166,19 @@ public List<String> getTokenModifiers() {
return tokenModifiers;
}

/**
* @param ids List of tokenType ids to search for in the list of supported tokenTypes
* @return index of the first tokenType found by id in the list of supported tokenTypes or -1 if no supported tokenType has been found
*/
private int getTokenType(final String[] ids) {
for (String id : ids) {
Integer index = tokenTypes.indexOf(id);
if (index != -1) {
return index + 1;
return index;
}
}
return 0;
// return -1 to indicate an invalid tokenType.
return -1;
}

private Integer getTokenModifiers(final String[] ids) {
Expand Down

0 comments on commit bbbf392

Please sign in to comment.