From bd75161e862b6a7267fae7316bc15e45c470df9b Mon Sep 17 00:00:00 2001 From: ef3d0c3e Date: Fri, 18 Oct 2024 12:43:51 +0200 Subject: [PATCH] Cleanup --- src/elements/section.rs | 9 +- src/lsp/mod.rs | 1 - src/lsp/parser.rs | 85 ------------ src/lsp/semantic.rs | 275 ++++++++++++++++++++++----------------- src/main.rs | 1 + src/parser/langparser.rs | 2 +- src/parser/mod.rs | 1 - src/parser/parser.rs | 2 +- src/parser/semantics.rs | 65 --------- src/parser/source.rs | 2 +- src/server.rs | 21 ++- 11 files changed, 174 insertions(+), 290 deletions(-) delete mode 100644 src/lsp/parser.rs delete mode 100644 src/parser/semantics.rs diff --git a/src/elements/section.rs b/src/elements/section.rs index c4b26e6..ee5d2e1 100644 --- a/src/elements/section.rs +++ b/src/elements/section.rs @@ -20,7 +20,6 @@ use mlua::Lua; use regex::Regex; use section_style::SectionLinkPos; use section_style::SectionStyle; -use std::cell::RefCell; use std::cell::RefMut; use std::ops::Range; use std::rc::Rc; @@ -333,16 +332,16 @@ impl RegexRule for SectionRule { .ok() .unwrap() }) { - sems.add(token.source(), matches.get(1).unwrap().range(), 0, 0); + sems.add(token.source(), matches.get(1).unwrap().range(), sems.token.section_heading); if let Some(reference) = matches.get(2) { - sems.add(token.source(), reference.start()-1..reference.end()+1, 1, 0); + sems.add(token.source(), reference.start()-1..reference.end()+1, sems.token.section_reference); } if let Some(kind) = matches.get(3) { - sems.add(token.source(), kind.range(), 3, 0); + sems.add(token.source(), kind.range(), sems.token.section_kind); } - //sems.add(token.source(), matches.get(5).unwrap().range(), 2, 0); + sems.add(token.source(), matches.get(5).unwrap().range(), sems.token.section_name); } result diff --git a/src/lsp/mod.rs b/src/lsp/mod.rs index 3d10d7d..515416e 100644 --- a/src/lsp/mod.rs +++ b/src/lsp/mod.rs @@ -1,2 +1 @@ pub mod semantic; -pub mod parser; diff --git a/src/lsp/parser.rs b/src/lsp/parser.rs deleted file mode 100644 index e47dabe..0000000 --- a/src/lsp/parser.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::rc::Rc; - -use crate::parser::source::Cursor; -use crate::parser::source::Source; - -#[derive(Debug, Clone)] -pub struct LineCursor { - pub pos: usize, - pub line: usize, - pub line_pos: usize, - pub source: Rc, -} - -impl LineCursor { - /// Creates [`LineCursor`] at position - /// - /// # Error - /// This function will panic if [`pos`] is not utf8 aligned - /// - /// Note: this is a convenience function, it should be used - /// with parsimony as it is expensive - pub fn at(&mut self, pos: usize) { - if pos > self.pos { - let start = self.pos; - eprintln!("slice{{{}}}, want={pos}", &self.source.content().as_str()[start..pos]); - let mut it = self.source.content().as_str()[start..] // pos+1 - .chars() - .peekable(); - - let mut prev = self.source.content().as_str()[..start + 1] - .chars() - .rev() - .next(); - eprintln!("prev={prev:#?}"); - while self.pos < pos { - let c = it.next().unwrap(); - let len = c.len_utf8(); - - if self.pos != 0 && prev == Some('\n') { - self.line += 1; - self.line_pos = 0; - } else { - self.line_pos += len; - } - self.pos += len; - - eprintln!("({}, {c:#?}, {} {}, {prev:#?})", self.pos, self.line, self.line_pos); - prev = Some(c); - } - if self.pos != 0 && prev == Some('\n') { - self.line += 1; - self.line_pos = 0; - } - } else if pos < self.pos { - todo!(""); - self.source.content().as_str()[pos..self.pos] - .char_indices() - .rev() - .for_each(|(len, c)| { - self.pos -= len; - if c == '\n' { - self.line -= 1; - } - }); - self.line_pos = self.source.content().as_str()[..self.pos] - .char_indices() - .rev() - .find(|(_, c)| *c == '\n') - .map(|(line_start, _)| self.pos - line_start) - .unwrap_or(0); - } - - // May fail if pos is not utf8-aligned - assert_eq!(pos, self.pos); - } -} - -impl From<&LineCursor> for Cursor { - fn from(value: &LineCursor) -> Self { - Self { - pos: value.pos, - source: value.source.clone(), - } - } -} diff --git a/src/lsp/semantic.rs b/src/lsp/semantic.rs index 013bcfe..273426c 100644 --- a/src/lsp/semantic.rs +++ b/src/lsp/semantic.rs @@ -1,130 +1,171 @@ use std::any::Any; +use std::cell::RefCell; +use std::ops::Range; +use std::rc::Rc; use tower_lsp::lsp_types::SemanticToken; +use tower_lsp::lsp_types::SemanticTokenModifier; use tower_lsp::lsp_types::SemanticTokenType; -use crate::document::document::Document; -use crate::document::element::Element; -use crate::elements::comment::Comment; -use crate::elements::paragraph::Paragraph; -use crate::elements::section::Section; -use crate::parser::rule::Rule; +use crate::parser::source::LineCursor; +use crate::parser::source::Source; -use super::parser::LineCursor; -pub trait SemanticProvider: Rule { - fn get_semantic_tokens( - &self, - cursor: &LineCursor, - match_data: Box, - ) -> Vec; -} - -pub mod nml_semantic { - use tower_lsp::lsp_types::SemanticTokenType; - - pub const SECTION_HEADING: SemanticTokenType = SemanticTokenType::new("type"); - pub const SECTION_NAME: SemanticTokenType = SemanticTokenType::new("string"); - pub const REFERENCE: SemanticTokenType = SemanticTokenType::new("event"); -} - -pub const LEGEND_TYPE: &[SemanticTokenType] = &[ - SemanticTokenType::COMMENT, - SemanticTokenType::VARIABLE, - SemanticTokenType::STRING, - SemanticTokenType::PARAMETER, +pub const TOKEN_TYPE: &[SemanticTokenType] = &[ + SemanticTokenType::NAMESPACE, + SemanticTokenType::TYPE, + SemanticTokenType::CLASS, + SemanticTokenType::ENUM, + SemanticTokenType::INTERFACE, + SemanticTokenType::STRUCT, + SemanticTokenType::TYPE_PARAMETER, + SemanticTokenType::PARAMETER, + SemanticTokenType::VARIABLE, + SemanticTokenType::PROPERTY, + SemanticTokenType::ENUM_MEMBER, + SemanticTokenType::EVENT, + SemanticTokenType::FUNCTION, + SemanticTokenType::METHOD, + SemanticTokenType::MACRO, + SemanticTokenType::KEYWORD, + SemanticTokenType::MODIFIER, + SemanticTokenType::COMMENT, + SemanticTokenType::STRING, + SemanticTokenType::NUMBER, + SemanticTokenType::REGEXP, + SemanticTokenType::OPERATOR, + SemanticTokenType::DECORATOR, ]; -// TODO... -pub fn provide( - semantic_tokens: &mut Vec, - cursor: &mut LineCursor, - elem: &Box, -) { - if cursor.source != elem.location().source() { - return; - } +pub const TOKEN_MODIFIERS: &[SemanticTokenModifier] = &[ + SemanticTokenModifier::DECLARATION, + SemanticTokenModifier::DEFINITION, + SemanticTokenModifier::READONLY, + SemanticTokenModifier::STATIC, + SemanticTokenModifier::DEPRECATED, + SemanticTokenModifier::ABSTRACT, + SemanticTokenModifier::ASYNC, + SemanticTokenModifier::MODIFICATION, + SemanticTokenModifier::DOCUMENTATION, + SemanticTokenModifier::DEFAULT_LIBRARY, +]; - let prev = cursor.clone(); - - /*if let Some(comm) = elem.downcast_ref::() { - cursor.at(elem.location().start()); - let delta_start = if cursor.line == prev.line { - cursor.line_pos - prev.line_pos - } else if cursor.line == 0 { - cursor.line_pos - } else { - cursor.line_pos + 1 - }; - semantic_tokens.push(SemanticToken { - delta_line: (cursor.line - prev.line) as u32, - delta_start: delta_start as u32, - length: (elem.location().end() - elem.location().start()) as u32, - token_type: 0, - token_modifiers_bitset: 0, - }); - } else */if let Some(sect) = elem.downcast_ref::
() { - cursor.at(elem.location().start() + 1); - eprintln!("section {cursor:#?}"); - let delta_start = if cursor.line == prev.line { - cursor.line_pos - prev.line_pos - } else if cursor.line == 0 { - cursor.line_pos - } else { - 0 - }; - semantic_tokens.push(SemanticToken { - delta_line: (cursor.line - prev.line) as u32, - delta_start: delta_start as u32, - length: (elem.location().end() - elem.location().start()) as u32, - token_type: 0, - token_modifiers_bitset: 0, - }); - } +fn token_index(name: &str) -> u32 +{ + TOKEN_TYPE.iter() + .enumerate() + .find(|(_, token)| token.as_str() == name) + .map(|(index, _)| index as u32) + .unwrap_or(0) } - -pub fn semantic_token_from_document(document: &dyn Document) -> Vec { - let mut semantic_tokens = vec![]; - - let source = document.source(); - let mut cursor = LineCursor { - pos: 0, - line: 0, - line_pos: 0, - source: source.clone(), +fn modifier_index(name: &str) -> u32 +{ + TOKEN_MODIFIERS.iter() + .enumerate() + .find(|(_, token)| token.as_str() == name) + .map(|(index, _)| index as u32) + .unwrap_or(0) +} +macro_rules! token { + ($key:expr) => { + { + (token_index($key), 0) + } + }; + ($key:expr, $($mods:tt)*) => { + { + let mut bitset : u32 = 0; + $( + bitset |= 1 << modifier_index($mods); + )* + (token_index($key), bitset) + } }; -/* - semantic_tokens.push(SemanticToken { - delta_line: 2, - delta_start: 1, - length: 5, - token_type: 0, - token_modifiers_bitset: 0, - }); - - semantic_tokens.push(SemanticToken { - delta_line: 1, - delta_start: 1, - length: 5, - token_type: 1, - token_modifiers_bitset: 0, - });*/ - - document.content().borrow() - .iter() - .for_each(|elem| { - if let Some(container) = elem.as_container() - { - container - .contained() - .iter() - .for_each(|elem| provide(&mut semantic_tokens, &mut cursor, elem)); - } - else - { - provide(&mut semantic_tokens, &mut cursor, elem); - } - }); - - semantic_tokens +} + +#[derive(Debug)] +pub struct Tokens +{ + pub section_heading: (u32, u32), + pub section_reference: (u32, u32), + pub section_kind: (u32, u32), + pub section_name: (u32, u32), +} + +impl Tokens +{ + pub fn new() -> Self + { + Self { + section_heading : token!("number"), + section_reference : token!("enum", "async"), + section_kind : token!("enum"), + section_name : token!("string"), + } + } +} + +/// Semantics for a buffer +#[derive(Debug)] +pub struct Semantics { + /// The tokens + pub token: Tokens, + + /// The current cursor + cursor: RefCell, + + /// Semantic tokens + pub tokens: RefCell>, +} + +impl Semantics { + pub fn new(source: Rc) -> Semantics { + Self { + token: Tokens::new(), + cursor: RefCell::new(LineCursor::new(source)), + tokens: RefCell::new(vec![]), + } + } + + pub fn add( + &self, + source: Rc, + range: Range, + token: (u32, u32) + ) { + let mut tokens = self.tokens.borrow_mut(); + let mut cursor = self.cursor.borrow_mut(); + let mut current = cursor.clone(); + cursor.move_to(range.start); + + while cursor.pos != range.end { + let end = source.content()[cursor.pos..] + .find('\n') + .unwrap_or(source.content().len() - cursor.pos); + let len = usize::min(range.end - cursor.pos, end); + + let delta_line = cursor.line - current.line; + let delta_start = if delta_line == 0 { + if let Some(last) = tokens.last() { + cursor.line_pos - current.line_pos + last.length as usize + } else { + cursor.line_pos - current.line_pos + } + } else { + cursor.line_pos + }; + + //eprintln!("CURRENT={:#?}, CURS={:#?}", current, cursor); + tokens.push(SemanticToken { + delta_line: delta_line as u32, + delta_start: delta_start as u32, + length: len as u32, + token_type: token.0, + token_modifiers_bitset: token.1 + }); + current = cursor.clone(); + let pos = cursor.pos; + cursor.move_to(pos + len); + } + } } diff --git a/src/main.rs b/src/main.rs index b86bb33..cc7b18f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ mod document; mod elements; mod lua; mod parser; +mod lsp; use std::env; use std::io::BufWriter; diff --git a/src/parser/langparser.rs b/src/parser/langparser.rs index 686f9b1..dde7396 100644 --- a/src/parser/langparser.rs +++ b/src/parser/langparser.rs @@ -5,12 +5,12 @@ use crate::document::document::Document; use crate::document::element::DocumentEnd; use crate::document::langdocument::LangDocument; use crate::elements::text::Text; +use crate::lsp::semantic::Semantics; use super::parser::Parser; use super::parser::ParserState; use super::parser::ReportColors; use super::rule::Rule; -use super::semantics::Semantics; use super::source::Cursor; use super::source::Source; use super::source::Token; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index d7d499e..433cbec 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -7,4 +7,3 @@ pub mod util; pub mod style; pub mod layout; pub mod customstyle; -pub mod semantics; diff --git a/src/parser/parser.rs b/src/parser/parser.rs index be56635..84cd9ed 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -11,7 +11,6 @@ use unicode_segmentation::UnicodeSegmentation; use super::customstyle::CustomStyleHolder; use super::layout::LayoutHolder; use super::rule::Rule; -use super::semantics::Semantics; use super::source::Cursor; use super::source::Source; use super::state::RuleStateHolder; @@ -22,6 +21,7 @@ use crate::document::element::ContainerElement; use crate::document::element::ElemKind; use crate::document::element::Element; use crate::elements::paragraph::Paragraph; +use crate::lsp::semantic::Semantics; use crate::lua::kernel::Kernel; use crate::lua::kernel::KernelHolder; use crate::parser::source::SourceFile; diff --git a/src/parser/semantics.rs b/src/parser/semantics.rs deleted file mode 100644 index b54ed3a..0000000 --- a/src/parser/semantics.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::{ops::Range, rc::Rc}; - -use tower_lsp::lsp_types::SemanticToken; - -use super::source::{LineCursor, Source}; - - -/// Semantics for a buffer -#[derive(Debug)] -pub struct Semantics -{ - /// The current cursor - cursor: LineCursor, - - /// Semantic tokens - pub tokens: Vec, -} - -impl Semantics -{ - pub fn new(source: Rc) -> Semantics - { - Self { - cursor: LineCursor::new(source), - tokens: vec![] - } - } - - pub fn add(&mut self, source: Rc, range: Range, token_type: u32, token_modifier: u32) - { - let mut current = self.cursor.clone(); - self.cursor.move_to(range.start); - - while self.cursor.pos != range.end - { - let end = source.content()[self.cursor.pos..].find('\n') - .unwrap_or(source.content().len() - self.cursor.pos); - let len = usize::min(range.end - self.cursor.pos, end); - - let delta_line = self.cursor.line - current.line; - let delta_start = if delta_line == 0 - { - if let Some(last) = self.tokens.last() - { - self.cursor.line_pos - current.line_pos + last.length as usize - } - else - { - self.cursor.line_pos - current.line_pos - } - } else { self.cursor.line_pos }; - - eprintln!("CURRENT={:#?}, CURS={:#?}", current, self.cursor); - self.tokens.push(SemanticToken{ - delta_line: delta_line as u32, - delta_start: delta_start as u32, - length: len as u32, - token_type, - token_modifiers_bitset: token_modifier, - }); - current = self.cursor.clone(); - self.cursor.move_to(self.cursor.pos + len); - } - } -} diff --git a/src/parser/source.rs b/src/parser/source.rs index 750388b..f568cf7 100644 --- a/src/parser/source.rs +++ b/src/parser/source.rs @@ -199,7 +199,7 @@ impl LineCursor { self.line_pos += c.width().unwrap_or(1); self.pos += len; - eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' ')); + //eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' ')); prev = Some(c); } if self.pos != 0 && prev == Some('\n') { diff --git a/src/server.rs b/src/server.rs index 06473f0..8726965 100644 --- a/src/server.rs +++ b/src/server.rs @@ -6,18 +6,13 @@ mod lsp; mod lua; mod parser; -use std::collections::HashMap; use std::rc::Rc; -use std::sync::Arc; use dashmap::DashMap; -use document::document::Document; -use document::element::Element; -use lsp::semantic::semantic_token_from_document; +use lsp::semantic::Tokens; use parser::langparser::LangParser; use parser::parser::Parser; use parser::parser::ParserState; -use parser::semantics::Semantics; use parser::source::SourceFile; use tower_lsp::jsonrpc::Result; use tower_lsp::lsp_types::*; @@ -58,14 +53,14 @@ impl Backend { .unwrap() }) { self.semantic_token_map - .insert(params.uri.to_string(), sems.tokens.to_owned()); + .insert(params.uri.to_string(), sems.tokens.borrow().to_owned()); }; } } #[tower_lsp::async_trait] impl LanguageServer for Backend { - async fn initialize(&self, _: InitializeParams) -> Result { + async fn initialize(&self, params: InitializeParams) -> Result { Ok(InitializeResult { server_info: None, capabilities: ServerCapabilities { @@ -85,17 +80,17 @@ impl LanguageServer for Backend { text_document_registration_options: { TextDocumentRegistrationOptions { document_selector: Some(vec![DocumentFilter { - language: Some("nml".to_string()), - scheme: Some("file".to_string()), - pattern: None, + language: Some("nml".into()), + scheme: Some("file".into()), + pattern: Some("*.nml".into()), }]), } }, semantic_tokens_options: SemanticTokensOptions { work_done_progress_options: WorkDoneProgressOptions::default(), legend: SemanticTokensLegend { - token_types: lsp::semantic::LEGEND_TYPE.into(), - token_modifiers: vec![], + token_types: lsp::semantic::TOKEN_TYPE.into(), + token_modifiers: lsp::semantic::TOKEN_MODIFIERS.into(), }, range: None, //Some(true), full: Some(SemanticTokensFullOptions::Bool(true)),