Skip to content

Commit

Permalink
sus-lsp now takes care of bulk of token colouring. LSP only does sema…
Browse files Browse the repository at this point in the history 
…ntic tokens now
  • Loading branch information
@VonTum
VonTum committed Feb 24, 2024
1 parent d4fa45f commit 0e6b633
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 185 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ repository = "https://github.com/pc2/sus-compiler"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
console = "0.15.7" # for terminal colors
# console = "0.15.7" # for terminal colors
# chumsky = "0.9.2"
ariadne = {path = "ariadne"} # for nice errors
num = "*"
Expand Down
53 changes: 20 additions & 33 deletions src/dev_aid/lsp.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ use lsp_server::{Connection, Message, Response};
use lsp_types::notification::Notification;

use crate::{
arena_alloc::ArenaVector, ast::{IdentifierType, Module}, dev_aid::syntax_highlighting::create_token_ide_info, errors::{CompileError, ErrorCollector, ErrorLevel}, file_position::{CharLine, FileText, Span}, flattening::FlatID, instantiation::{SubModuleOrWire, CALCULATE_LATENCY_LATER}, linker::{FileData, FileUUID, FileUUIDMarker, Linker, LocationInfo}, parser::perform_full_semantic_parse
arena_alloc::ArenaVector, ast::{IdentifierType, Module}, errors::{CompileError, ErrorCollector, ErrorLevel}, file_position::{CharLine, FileText, Span}, flattening::FlatID, instantiation::{SubModuleOrWire, CALCULATE_LATENCY_LATER}, linker::{FileData, FileUUID, FileUUIDMarker, Linker, LocationInfo}, parser::perform_full_semantic_parse, walk_name_color
};

use super::syntax_highlighting::{IDETokenType, IDEIdentifierType, IDEToken};
use super::syntax_highlighting::IDEIdentifierType;

struct LoadedFileCache {
linker : Linker,
Expand Down Expand Up @@ -124,33 +124,24 @@ pub fn lsp_main(port : u16, debug : bool) -> Result<(), Box<dyn Error + Sync + S
Ok(())
}

fn get_semantic_token_type_from_ide_token(tok : &IDEToken) -> u32 {
match &tok.typ {
IDETokenType::Comment => 0,
IDETokenType::Keyword => 1,
IDETokenType::Operator => 2,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Input)) => 4,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Output)) => 4,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::State)) => 3,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Local)) => 3,
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Generative)) => 3,
IDETokenType::Identifier(IDEIdentifierType::Constant) => 9, // make it 'OPERATOR'?
IDETokenType::Identifier(IDEIdentifierType::Unknown) => 2, // make it 'OPERATOR'?
IDETokenType::Identifier(IDEIdentifierType::Interface) => 7, // FUNCTION
IDETokenType::Identifier(IDEIdentifierType::Type) => 5, // All others are 'TYPE'
IDETokenType::Number => 6,
IDETokenType::Invalid => 2, // make it 'OPERATOR'?
IDETokenType::InvalidBracket => 2, // make it 'OPERATOR'?
IDETokenType::OpenBracket(_) => 2,
IDETokenType::CloseBracket(_) => 2,
fn get_semantic_token_type_from_ide_token(tok : IDEIdentifierType) -> u32 {
match tok {
IDEIdentifierType::Value(IdentifierType::Input) => 4,
IDEIdentifierType::Value(IdentifierType::Output) => 4,
IDEIdentifierType::Value(IdentifierType::State) => 3,
IDEIdentifierType::Value(IdentifierType::Local) => 3,
IDEIdentifierType::Value(IdentifierType::Generative) => 3,
IDEIdentifierType::Constant => 9, // make it 'OPERATOR'?
IDEIdentifierType::Interface => 7, // FUNCTION
IDEIdentifierType::Type => 5, // All others are 'TYPE'
}
}

// Produces a bitset with 'modifier bits'
fn get_modifiers_for_token(tok : &IDEToken) -> u32 {
match &tok.typ {
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::State)) => 1, // repurpose ASYNC for "State"
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Generative)) => 8, // repurpose READONLY
fn get_modifiers_for_token(tok : IDEIdentifierType) -> u32 {
match tok {
IDEIdentifierType::Value(IdentifierType::State) => 1, // repurpose ASYNC for "State"
IDEIdentifierType::Value(IdentifierType::Generative) => 8, // repurpose READONLY
_other => 0
}
}
Expand All @@ -168,15 +159,15 @@ fn to_position_range(range : std::ops::Range<CharLine>) -> lsp_types::Range {
lsp_types::Range{start : to_position(range.start), end : to_position(range.end)}
}

fn convert_to_semantic_tokens(file_data : &FileData, ide_tokens : &mut[(IDEToken, Span)]) -> Vec<SemanticToken> {
fn convert_to_semantic_tokens(file_data : &FileData, ide_tokens : &mut[(IDEIdentifierType, Span)]) -> Vec<SemanticToken> {
ide_tokens.sort_by(|a, b| a.1.cmp(&b.1));

let mut cursor = Position {line : 0, character : 0};
let mut semantic_tokens = Vec::with_capacity(file_data.tokens.len());

for (ide_kind, span) in ide_tokens.iter() {
let typ = get_semantic_token_type_from_ide_token(ide_kind);
let mod_bits = get_modifiers_for_token(ide_kind);
let typ = get_semantic_token_type_from_ide_token(*ide_kind);
let mod_bits = get_modifiers_for_token(*ide_kind);

let tok_range = file_data.file_text.get_span_linechar_range(*span);
let start_pos = to_position(tok_range.start);
Expand Down Expand Up @@ -206,11 +197,7 @@ fn convert_to_semantic_tokens(file_data : &FileData, ide_tokens : &mut[(IDEToken
}

fn do_syntax_highlight(file_data : &FileData, linker : &Linker) -> Vec<SemanticToken> {
let ide_tokens = create_token_ide_info(&file_data, linker);

let mut ide_tokens : Vec<(IDEToken, Span)> = ide_tokens.iter().enumerate().map(|(idx, tok_typ)| (*tok_typ, Span::new_single_token(idx))).collect();


let mut ide_tokens = walk_name_color(&file_data.associated_values, linker);

convert_to_semantic_tokens(file_data, &mut ide_tokens)
}
Expand Down
155 changes: 11 additions & 144 deletions src/dev_aid/syntax_highlighting.rs
View file
Original file line number Diff line number Diff line change
@@ -1,116 +1,25 @@

use std::{ops::Range, path::PathBuf};

use crate::{arena_alloc::ArenaVector, ast::*, errors::{CompileError, ErrorLevel}, file_position::{FileText, Span}, flattening::{Instruction, WireSource}, linker::{FileData, FileUUID, FileUUIDMarker, Linker, NameElem}, parser::*, tokenizer::*};
use crate::{arena_alloc::ArenaVector, ast::*, errors::{CompileError, ErrorLevel}, file_position::{FileText, Span}, flattening::{Instruction, WireSource}, linker::{FileUUID, FileUUIDMarker, Linker, NameElem}, parser::*};

use ariadne::*;
use console::Style;


#[derive(Debug,Clone,Copy,PartialEq,Eq)]
pub enum IDEIdentifierType {
Value(IdentifierType),
Type,
Interface,
Constant,
Unknown
}

#[derive(Debug,Clone,Copy,PartialEq,Eq)]
pub enum IDETokenType {
Comment,
Keyword,
Operator,
Identifier(IDEIdentifierType),
Number,
Invalid,
InvalidBracket,
OpenBracket(usize), // Bracket depth
CloseBracket(usize) // Bracket depth
}

#[derive(Debug,Clone,Copy)]
pub struct IDEToken {
pub typ : IDETokenType
Constant
}

pub struct SyntaxHighlightSettings {
pub show_tokens : bool
}

fn pretty_print_chunk_with_whitespace(whitespace_start : usize, file_text : &str, text_span : Range<usize>, st : Style) {
let whitespace_text = &file_text[whitespace_start..text_span.start];

print!("{}{}", whitespace_text, st.apply_to(&file_text[text_span]));
}

fn print_tokens(file_text : &FileText) {
let mut whitespace_start : usize = 0;
for tok_idx in 0..file_text.num_tokens() {
let styles = [Style::new().magenta(), Style::new().yellow(), Style::new().blue()];
let st = styles[tok_idx % styles.len()].clone().underlined();

let token_range = file_text.get_token_range(tok_idx);
pretty_print_chunk_with_whitespace(whitespace_start, &file_text.file_text, token_range.clone(), st);
whitespace_start = token_range.end;
}

print!("{}\n", &file_text.file_text[whitespace_start..]);
}

fn pretty_print(file_text : &FileText, ide_infos : &[IDEToken]) {
let mut whitespace_start : usize = 0;

for (tok_idx, token) in ide_infos.iter().enumerate() {
let bracket_styles = [Style::new().magenta(), Style::new().yellow(), Style::new().blue()];
let st = match token.typ {
IDETokenType::Comment => Style::new().green().dim(),
IDETokenType::Keyword => Style::new().blue(),
IDETokenType::Operator => Style::new().white().bright(),
IDETokenType::Identifier(IDEIdentifierType::Unknown) => Style::new().red().underlined(),
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Local)) => Style::new().blue().bright(),
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::State)) => Style::new().blue().bright().underlined(),
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Input)) => Style::new().blue().bright(),
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Output)) => Style::new().blue().dim(),
IDETokenType::Identifier(IDEIdentifierType::Value(IdentifierType::Generative)) => Style::new().blue().bright().bold(),
IDETokenType::Identifier(IDEIdentifierType::Constant) => Style::new().blue().bold(),
IDETokenType::Identifier(IDEIdentifierType::Type) => Style::new().magenta().bright(),
IDETokenType::Identifier(IDEIdentifierType::Interface) => Style::new().yellow(),
IDETokenType::Number => Style::new().green().bright(),
IDETokenType::Invalid | IDETokenType::InvalidBracket => Style::new().red().underlined(),
IDETokenType::OpenBracket(depth) | IDETokenType::CloseBracket(depth) => {
bracket_styles[depth % bracket_styles.len()].clone()
}
};

let tok_span = file_text.get_token_range(tok_idx);
pretty_print_chunk_with_whitespace(whitespace_start, &file_text.file_text, tok_span.clone(), st);
whitespace_start = tok_span.end;
}

print!("{}\n", &file_text.file_text[whitespace_start..]);
}

fn add_ide_bracket_depths_recursive<'a>(result : &mut [IDEToken], current_depth : usize, token_hierarchy : &[TokenTreeNode]) {
for tok in token_hierarchy {
if let TokenTreeNode::Block(_, sub_block, span) = tok {
let inner_span = span.inner_span();
let outer_span = span.outer_span();
let left_bracket_span = Span::difference_left(outer_span, inner_span);
let right_bracket_span = Span::difference_right(outer_span, inner_span);
result[left_bracket_span.assert_is_single_token()].typ = IDETokenType::OpenBracket(current_depth);
add_ide_bracket_depths_recursive(result, current_depth+1, sub_block);
result[right_bracket_span.assert_is_single_token()].typ = IDETokenType::CloseBracket(current_depth);
}
}
}

fn set_span_name_color(span : Span, typ : IDEIdentifierType, result : &mut [IDEToken]) {
for tok_idx in span {
result[tok_idx].typ = IDETokenType::Identifier(typ);
}
}
fn walk_name_color(all_objects : &[NameElem], linker : &Linker, result : &mut [IDEToken]) {
pub fn walk_name_color(all_objects : &[NameElem], linker : &Linker) -> Vec<(IDEIdentifierType, Span)> {
let mut result : Vec<(IDEIdentifierType, Span)> = Vec::new();
for obj_uuid in all_objects {
let (ide_typ, link_info) = match obj_uuid {
NameElem::Module(id) => {
Expand All @@ -122,31 +31,31 @@ fn walk_name_color(all_objects : &[NameElem], linker : &Linker, result : &mut [I
&WireSource::WireRead(from_wire) => {
let decl = module.flattened.instructions[from_wire].extract_wire_declaration();
if !decl.is_declared_in_this_module {continue;} // Virtual wires don't appear in this program text
result[w.span.assert_is_single_token()].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
result.push((IDEIdentifierType::Value(decl.identifier_type), w.span));
}
WireSource::UnaryOp { op:_, right:_ } => {}
WireSource::BinaryOp { op:_, left:_, right:_ } => {}
WireSource::ArrayAccess { arr:_, arr_idx:_ } => {}
WireSource::Constant(_) => {}
WireSource::NamedConstant(_name) => {
set_span_name_color(w.span, IDEIdentifierType::Constant, result);
result.push((IDEIdentifierType::Constant, w.span));
}
}
}
Instruction::Declaration(decl) => {
if !decl.is_declared_in_this_module {continue;} // Virtual wires don't appear in this program text
result[decl.name_token].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
decl.typ_expr.for_each_located_type(&mut |_, span| {
set_span_name_color(span, IDEIdentifierType::Type, result);
result.push((IDEIdentifierType::Type, span));
});
result.push((IDEIdentifierType::Value(decl.identifier_type), Span::new_single_token(decl.name_token)));
}
Instruction::Write(conn) => {
let decl = module.flattened.instructions[conn.to.root].extract_wire_declaration();
if !decl.is_declared_in_this_module {continue;} // Virtual wires don't appear in this program text
result[conn.to.root_span.assert_is_single_token()].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
result.push((IDEIdentifierType::Value(decl.identifier_type), conn.to.root_span));
}
Instruction::SubModule(sm) => {
set_span_name_color(sm.module_name_span, IDEIdentifierType::Interface, result);
result.push((IDEIdentifierType::Interface, sm.module_name_span));
}
Instruction::IfStatement(_) | Instruction::ForStatement(_) => {}
}
Expand All @@ -156,39 +65,8 @@ fn walk_name_color(all_objects : &[NameElem], linker : &Linker, result : &mut [I
_other => {todo!("Name Color for non-modules not implemented")}
};

set_span_name_color(link_info.name_span, ide_typ, result);
result.push((ide_typ, link_info.name_span));
}
}

pub fn create_token_ide_info<'a>(parsed: &FileData, linker : &Linker) -> Vec<IDEToken> {
let mut result : Vec<IDEToken> = Vec::new();
result.reserve(parsed.tokens.len());

for &tok_typ in &parsed.tokens {
let initial_typ = if is_keyword(tok_typ) {
IDETokenType::Keyword
} else if is_bracket(tok_typ) != IsBracket::NotABracket {
IDETokenType::InvalidBracket // Brackets are initially invalid. They should be overwritten by the token_hierarchy step. The ones that don't get overwritten are invalid
} else if is_symbol(tok_typ) {
IDETokenType::Operator
} else if tok_typ == TOKEN_IDENTIFIER {
IDETokenType::Identifier(IDEIdentifierType::Unknown)
} else if tok_typ == TOKEN_NUMBER {
IDETokenType::Number
} else if tok_typ == TOKEN_COMMENT {
IDETokenType::Comment
} else {
assert!(tok_typ == TOKEN_INVALID);
IDETokenType::Invalid
};

result.push(IDEToken{typ : initial_typ})
}

add_ide_bracket_depths_recursive(&mut result, 0, &parsed.token_hierarchy);

walk_name_color(&parsed.associated_values, linker, &mut result);

result
}

Expand Down Expand Up @@ -313,14 +191,3 @@ pub fn print_all_errors(linker : &Linker, paths_arena : &mut ArenaVector<(PathBu
}
}
}

pub fn syntax_highlight_file(linker : &Linker, file_uuid : FileUUID, settings : &SyntaxHighlightSettings) {
let f = &linker.files[file_uuid];

if settings.show_tokens {
print_tokens(&f.file_text);
}

let ide_tokens = create_token_ide_info(f, linker);
pretty_print(&f.file_text, &ide_tokens);
}
7 changes: 2 additions & 5 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#![doc = include_str!("../README.md")]
//#![allow(dead_code)]

mod util;
Expand Down Expand Up @@ -141,11 +142,7 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {

let (linker, mut paths_arena) = compile_all(file_paths);
print_all_errors(&linker, &mut paths_arena);
for (id, (path, _)) in &paths_arena {
println!("\n\n[{}]: ", path.to_string_lossy());
syntax_highlight_file(&linker, id, &settings);
}


// #[cfg(feature = "codegen")]
if let Some(module_name) = codegen {
//let gen_ctx = codegen::GenerationContext::new();
Expand Down
3 changes: 1 addition & 2 deletions src/tokenizer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@ use crate::util::const_str_position_in_tuples;

pub type TokenTypeIdx = u8;

pub const ALL_KEYWORDS : [(&'static str, u8); 18] = [
pub const ALL_KEYWORDS : [(&'static str, u8); 17] = [
("template", 0),
("module", 0),
("pipeline", 0),
("interface", 0),
("timeline", 0),
("assume", 0),
Expand Down

0 comments on commit 0e6b633

Please sign in to comment.