Refactor linker, to make it more self-contained

src/compiler_top.rs

@@ -0,0 +1,83 @@
+use std::rc::Rc;
+
+use tree_sitter::Parser;
+
+use crate::{
+    errors::ErrorCollector,
+    file_position::FileText,
+    flattening::{initialization::gather_initial_file_data, FlattenedModule},
+    instantiation::InstantiatedModule,
+    linker::{FileData, FileUUID, Linker, ModuleUUID},
+    parser::report_all_tree_errors
+};
+
+pub fn add_file(text : String, linker : &mut Linker) -> FileUUID {
+    let mut parser = Parser::new();
+    parser.set_language(&tree_sitter_sus::language()).unwrap();
+    let tree = parser.parse(&text, None).unwrap();
+
+    let file_id = linker.files.reserve();
+    linker.files.alloc_reservation(file_id, FileData{
+        parsing_errors : ErrorCollector::new(file_id, text.len()),
+        file_text : FileText::new(text),
+        tree,
+        associated_values : Vec::new()
+    });
+
+    let file_data = &linker.files[file_id];
+
+    report_all_tree_errors(&file_data.file_text, &file_data.tree, &file_data.parsing_errors);
+
+    let mut builder = linker.get_file_builder(file_id);
+    gather_initial_file_data(&mut builder);
+
+    file_id
+}
+
+pub fn update_file(text : String, file_id : FileUUID, linker : &mut Linker) {
+    let file_data = linker.remove_everything_in_file(file_id);
+
+    let mut parser = Parser::new();
+    parser.set_language(&tree_sitter_sus::language()).unwrap();
+    let tree = parser.parse(&text, None).unwrap();
+
+    file_data.parsing_errors.reset(text.len());
+    file_data.file_text = FileText::new(text);
+    file_data.tree = tree;
+
+    report_all_tree_errors(&file_data.file_text, &file_data.tree, &file_data.parsing_errors);
+
+    let mut builder = linker.get_file_builder(file_id);
+    gather_initial_file_data(&mut builder);
+}
+
+pub fn recompile_all(linker : &mut Linker) {
+    // Flatten all modules
+    let id_vec : Vec<ModuleUUID> = linker.modules.iter().map(|(id, _)| id).collect();
+    for id in id_vec {
+        let md = &linker.modules[id];// Have to get them like this, so we don't have a mutable borrow on self.modules across the loop
+        println!("Flattening {}", md.link_info.name);
+
+        let flattened = FlattenedModule::flatten(&linker, md);
+        println!("Typechecking {}", &md.link_info.name);
+
+        let md = &mut linker.modules[id]; // Convert to mutable ptr
+        md.flattened = flattened;
+        md.instantiations.clear_instances();
+    }
+
+    // Can't merge these loops, because instantiation can only be done once all modules have been type checked
+    for (id, _md) in &linker.modules {
+        //md.print_flattened_module();
+        // Already instantiate any modules without parameters
+        // Currently this is all modules
+        let _inst = instantiate(&linker, id);
+    }
+}
+
+pub fn instantiate(linker : &Linker, module_id : ModuleUUID) -> Option<Rc<InstantiatedModule>> {
+    let md = &linker.modules[module_id];
+    println!("Instantiating {}", md.link_info.name);
+
+    md.instantiations.instantiate(&md.link_info.name, &md.flattened, linker)
+}

src/dev_aid/lsp.rs

@@ -7,7 +7,15 @@ use lsp_server::{Connection, Message, Response};
 use lsp_types::notification::Notification;
 
 use crate::{
-    arena_alloc::ArenaVector, errors::{CompileError, ErrorCollector, ErrorLevel}, file_position::{FileText, LineCol, Span}, flattening::{FlatID, Instruction, IdentifierType, Module}, instantiation::{SubModuleOrWire, CALCULATE_LATENCY_LATER}, linker::{FileData, FileUUID, FileUUIDMarker, Linker, LocationInfo}, parser::perform_full_semantic_parse, walk_name_color
+    arena_alloc::ArenaVector,
+    compiler_top::{add_file, recompile_all, update_file},
+    errors::{CompileError, ErrorCollector, ErrorLevel},
+    file_position::{FileText, LineCol, Span},
+    flattening::{FlatID, IdentifierType, Instruction, Module, WireInstance, WireSource},
+    instantiation::{SubModuleOrWire, CALCULATE_LATENCY_LATER},
+    linker::{FileData, FileUUID, FileUUIDMarker, Linker, NameElem},
+    typing::WrittenType,
+    walk_name_color
 };
 
 use super::syntax_highlighting::IDEIdentifierType;
@@ -27,29 +35,22 @@ impl LoadedFileCache {
             .map(|(uuid, _uri_found)| uuid)
     }
     fn update_text(&mut self, uri : Url, new_file_text : String) {
-        let found_opt = self.find_uri(&uri);
-        let found_opt_was_none = found_opt.is_none();
-        let file_uuid : FileUUID = found_opt.unwrap_or_else(|| self.linker.reserve_file());
-        let full_parse = perform_full_semantic_parse(new_file_text, file_uuid);
-
-        if found_opt_was_none {
-            self.linker.add_reserved_file(file_uuid, full_parse);
-            self.uris.insert(file_uuid, uri.clone());
+        if let Some(found_file_uuid) = self.find_uri(&uri) {
+            update_file(new_file_text, found_file_uuid, &mut self.linker);
         } else {
-            self.linker.relink(file_uuid, full_parse);
+            let file_uuid = add_file(new_file_text, &mut self.linker);
+            self.uris.insert(file_uuid, uri.clone());
         }
-        self.linker.recompile_all();
+
+        recompile_all(&mut self.linker);
     }
     fn ensure_contains_file(&mut self, uri : &Url) -> FileUUID {
         if let Some(found) = self.find_uri(uri) {
             found
         } else {
-            let file_uuid = self.linker.reserve_file();
             let file_text = std::fs::read_to_string(uri.to_file_path().unwrap()).unwrap();
-            let full_parse = perform_full_semantic_parse(file_text, file_uuid);
-            self.linker.add_reserved_file(file_uuid, full_parse);
-            self.uris.insert(file_uuid, uri.clone());
-            self.linker.recompile_all();
+            let file_uuid = add_file(file_text, &mut self.linker);
+            recompile_all(&mut self.linker);
             file_uuid
         }
     }
@@ -223,8 +224,10 @@ fn convert_diagnostic(err : CompileError, main_file_text : &FileText, linker : &
     let mut related_info = Vec::new();
     for info in err.infos {
         let info_file_text = &linker.files[info.file].file_text;
-        assert!(info_file_text.is_span_valid(info.position), "bad info: {}; in err: {}", info.info, err.reason);
-        let info_pos = cvt_span_to_lsp_range(info.position, info_file_text);
+        let file_name = uris[info.file].to_string();
+        let info_span = info.position;
+        assert!(info_file_text.is_span_valid(info_span), "bad info in {file_name}:\n{}; in err: {}.\nSpan is {info_span}, but file length is {}", info.info, err.reason, info_file_text.len());
+        let info_pos = cvt_span_to_lsp_range(info_span, info_file_text);
         let location = Location{uri : uris[info.file].clone(), range : info_pos};
         related_info.push(DiagnosticRelatedInformation { location, message: info.info });
     }
@@ -254,14 +257,107 @@ fn send_errors_warnings(connection: &Connection, errors : ErrorCollector, main_f
     Ok(())
 }
 
+
+#[derive(Clone, Copy, Debug)]
+enum LocationInfo<'linker> {
+    WireRef(&'linker Module, FlatID),
+    Temporary(&'linker Module, FlatID, &'linker WireInstance),
+    Type(&'linker WrittenType),
+    Global(NameElem)
+}
+
+struct LocationInfoBuilder<'linker> {
+    best_instruction : Option<LocationInfo<'linker>>,
+    best_span : Span,
+    position : usize
+}
+
+impl<'linker> LocationInfoBuilder<'linker> {
+    fn new(token_idx : usize) -> Self {
+        Self{
+            best_instruction : None,
+            best_span : Span::MAX_POSSIBLE_SPAN,
+            position: token_idx
+        }
+    }
+    fn update(&mut self, span : Span, info : LocationInfo<'linker>) {
+        if span.contains_pos(self.position) && span.size() <= self.best_span.size() {
+            //assert!(span.size() < self.best_span.size());
+            // May not be the case. Do prioritize later ones, as they tend to be nested
+            self.best_span = span;
+            self.best_instruction = Some(info);
+        }
+    }
+}
+
+
+fn get_info_about_source_location<'linker>(linker : &'linker Linker, position : usize, file : FileUUID) -> Option<(LocationInfo<'linker>, Span)> {
+    let mut location_builder = LocationInfoBuilder::new(position);
+
+    for global in &linker.files[file].associated_values {
+        match *global {
+            NameElem::Module(md_id) => {
+                let md = &linker.modules[md_id];
+                if md.link_info.span.contains_pos(position) {
+                    location_builder.update(md.link_info.name_span, LocationInfo::Global(NameElem::Module(md_id)));
+                    for (id, inst) in &md.flattened.instructions {
+                        match inst {
+                            Instruction::SubModule(sm) => {
+                                location_builder.update(sm.module_name_span, LocationInfo::Global(NameElem::Module(sm.module_uuid)));
+                            }
+                            Instruction::Declaration(decl) => {
+                                match decl.typ_expr.get_deepest_selected(position) {
+                                    Some(WrittenType::Named(span, name_id)) => {
+                                        location_builder.update(*span, LocationInfo::Global(NameElem::Type(*name_id)));
+                                    }
+                                    Some(typ) => {
+                                        location_builder.update(typ.get_span(), LocationInfo::Type(typ));
+                                    }
+                                    None => {}
+                                }
+                                if decl.declaration_itself_is_not_written_to && decl.name_span.contains_pos(position) {
+                                    location_builder.update(decl.name_span, LocationInfo::WireRef(md, id));
+                                }
+                            }
+                            Instruction::Wire(wire) => {
+                                let loc_info = if let WireSource::WireRead(decl_id) = &wire.source {
+                                    LocationInfo::WireRef(md, *decl_id)
+                                } else {
+                                    LocationInfo::Temporary(md, id, wire)
+                                };
+                                location_builder.update(wire.span, loc_info);
+                            }
+                            Instruction::Write(write) => {
+                                location_builder.update(write.to.root_span, LocationInfo::WireRef(md, write.to.root));
+                            }
+                            Instruction::IfStatement(_) | Instruction::ForStatement(_) => {}
+                        };
+                    }
+                }
+            }
+            NameElem::Type(_) => {
+                todo!()
+            }
+            NameElem::Constant(_) => {
+                todo!()
+            }
+        }
+    }
+    if let Some(instr) = location_builder.best_instruction {
+        Some((instr, location_builder.best_span))
+    } else {
+        None
+    }
+}
+
 fn get_hover_info<'l>(file_cache : &'l LoadedFileCache, text_pos : &lsp_types::TextDocumentPositionParams) -> Option<(&'l FileData, LocationInfo<'l>, lsp_types::Range)> {
     let uuid = file_cache.find_uri(&text_pos.text_document.uri).unwrap();
 
     let file_data = &file_cache.linker.files[uuid];
 
     let byte_pos = file_data.file_text.linecol_to_byte_clamp(from_position(text_pos.position));
 
-    let (info, span) = file_cache.linker.get_info_about_source_location(byte_pos, uuid)?;
+    let (info, span) = get_info_about_source_location(&file_cache.linker, byte_pos, uuid)?;
     //let span = Span::new_single_token(token_idx);
 
     let char_line_range = file_data.file_text.get_span_linecol_range(span);
@@ -289,17 +385,15 @@ fn initialize_all_files(init_params : &InitializeParams) -> LoadedFileCache {
             for file in std::fs::read_dir(path).unwrap() {
                 let file_path = file.unwrap().path();
                 if file_path.is_file() && file_path.extension() == Some(OsStr::new("sus")) {
-                    let file_uuid = linker.reserve_file();
                     let file_text = std::fs::read_to_string(&file_path).unwrap();
-                    let full_parse = perform_full_semantic_parse(file_text, file_uuid);
-                    linker.add_reserved_file(file_uuid, full_parse);
+                    let file_uuid = add_file(file_text, &mut linker);
                     uris.insert(file_uuid, Url::from_file_path(&file_path).unwrap());
                 }
             }
         }
     }
     let mut result = LoadedFileCache::new(linker, uris);
-    result.linker.recompile_all();
+    recompile_all(&mut result.linker);
     result
 }
 
@@ -477,7 +571,12 @@ fn handle_notification(connection: &Connection, notification : lsp_server::Notif
         notification::DidChangeTextDocument::METHOD => {
             println!("DidChangeTextDocument");
             let params : DidChangeTextDocumentParams = serde_json::from_value(notification.params).expect("JSON Encoding Error while parsing params");
-            file_cache.update_text(params.text_document.uri, params.content_changes.into_iter().next().unwrap().text);
+
+            let mut content_change_iter = params.content_changes.into_iter();
+            let only_change = content_change_iter.next().unwrap();
+            assert!(content_change_iter.next().is_none());
+            assert!(only_change.range.is_none());
+            file_cache.update_text(params.text_document.uri, only_change.text);
 
             push_all_errors(connection, &file_cache)?;
         }

src/dev_aid/syntax_highlighting.rs

@@ -1,11 +1,17 @@
 
 use std::{ops::Range, path::PathBuf};
 
-use crate::{arena_alloc::ArenaVector, errors::{CompileError, ErrorLevel}, file_position::Span, flattening::{IdentifierType, Instruction, WireSource}, linker::{FileUUID, FileUUIDMarker, Linker, NameElem}, parser::*};
+use crate::{
+    arena_alloc::ArenaVector,
+    compiler_top::{add_file, recompile_all},
+    errors::{CompileError, ErrorLevel},
+    file_position::Span,
+    flattening::{IdentifierType, Instruction, WireSource},
+    linker::{FileUUID, FileUUIDMarker, Linker, NameElem}
+};
 
 use ariadne::*;
 
-
 #[derive(Debug,Clone,Copy,PartialEq,Eq)]
 pub enum IDEIdentifierType {
     Value(IdentifierType),
@@ -74,7 +80,6 @@ pub fn compile_all(file_paths : Vec<PathBuf>) -> (Linker, ArenaVector<(PathBuf,
     let mut linker = Linker::new();
     let mut paths_arena : ArenaVector<(PathBuf, Source), FileUUIDMarker> = ArenaVector::new();
     for file_path in file_paths {
-        let uuid = linker.reserve_file();
         let file_text = match std::fs::read_to_string(&file_path) {
             Ok(file_text) => file_text,
             Err(reason) => {
@@ -83,13 +88,13 @@ pub fn compile_all(file_paths : Vec<PathBuf>) -> (Linker, ArenaVector<(PathBuf,
             }
         };
 
-        let full_parse = perform_full_semantic_parse(file_text, uuid);
+        let source = Source::from(file_text.clone());
+        let uuid = add_file(file_text, &mut linker);
 
-        paths_arena.insert(uuid, (file_path, Source::from(full_parse.file_text.file_text.clone())));
-        linker.add_reserved_file(uuid, full_parse);
+        paths_arena.insert(uuid, (file_path, source));
     }
 
-    linker.recompile_all();
+    recompile_all(&mut linker);
 
     (linker, paths_arena)
 }

src/errors.rs

@@ -45,6 +45,10 @@ impl ErrorCollector {
     pub fn new(file : FileUUID, file_len : usize) -> Self {
         Self{errors : RefCell::new(Vec::new()), file, file_len, did_error : Cell::new(false)}
     }
+    pub fn reset(&mut self, file_len : usize) {
+        self.file_len = file_len;
+        self.errors.get_mut().clear();
+    }
 
     pub fn new_for_same_file(&self) -> Self {
         Self{errors : RefCell::new(Vec::new()), file : self.file, file_len : self.file_len, did_error : self.did_error.clone()}

src/flattening/initialization.rs

@@ -0,0 +1,32 @@
+use sus_proc_macro::{field, kind};
+
+use crate::{
+    errors::ErrorCollector,
+    flattening::{FlattenedModule, Module},
+    instantiation::InstantiationList,
+    linker::{FileBuilder, LinkInfo},
+    parser::Cursor
+};
+
+
+pub fn gather_initial_file_data(builder : &mut FileBuilder) {
+    let mut walker = Cursor::new_at_root(builder.tree, builder.file_text);
+    walker.list(kind!("source_file"), |cursor| {
+        let (kind, span) = cursor.kind_span();
+        assert!(kind == kind!("module"));
+        let name_span = cursor.go_down_no_check(|cursor| cursor.field_span(field!("name"), kind!("identifier")));
+        let md = Module{
+            link_info: LinkInfo {
+                documentation: cursor.extract_gathered_comments(),
+                file: builder.file_id,
+                name: builder.file_text[name_span].to_owned(),
+                name_span,
+                span
+            },
+            flattened: FlattenedModule::empty(ErrorCollector::new(builder.file_id, builder.file_text.len())),
+            instantiations: InstantiationList::new()
+        };
+
+        builder.add_module(md);
+    });
+}