diff --git a/src/elements/section.rs b/src/elements/section.rs
index c4b26e6..ee5d2e1 100644
--- a/src/elements/section.rs
+++ b/src/elements/section.rs
@@ -20,7 +20,6 @@ use mlua::Lua;
use regex::Regex;
use section_style::SectionLinkPos;
use section_style::SectionStyle;
-use std::cell::RefCell;
use std::cell::RefMut;
use std::ops::Range;
use std::rc::Rc;
@@ -333,16 +332,16 @@ impl RegexRule for SectionRule {
.ok()
.unwrap()
}) {
- sems.add(token.source(), matches.get(1).unwrap().range(), 0, 0);
+ sems.add(token.source(), matches.get(1).unwrap().range(), sems.token.section_heading);
if let Some(reference) = matches.get(2)
{
- sems.add(token.source(), reference.start()-1..reference.end()+1, 1, 0);
+ sems.add(token.source(), reference.start()-1..reference.end()+1, sems.token.section_reference);
}
if let Some(kind) = matches.get(3)
{
- sems.add(token.source(), kind.range(), 3, 0);
+ sems.add(token.source(), kind.range(), sems.token.section_kind);
}
- //sems.add(token.source(), matches.get(5).unwrap().range(), 2, 0);
+ sems.add(token.source(), matches.get(5).unwrap().range(), sems.token.section_name);
}
result
diff --git a/src/lsp/mod.rs b/src/lsp/mod.rs
index 3d10d7d..515416e 100644
--- a/src/lsp/mod.rs
+++ b/src/lsp/mod.rs
@@ -1,2 +1 @@
pub mod semantic;
-pub mod parser;
diff --git a/src/lsp/parser.rs b/src/lsp/parser.rs
deleted file mode 100644
index e47dabe..0000000
--- a/src/lsp/parser.rs
+++ /dev/null
@@ -1,85 +0,0 @@
-use std::rc::Rc;
-
-use crate::parser::source::Cursor;
-use crate::parser::source::Source;
-
-#[derive(Debug, Clone)]
-pub struct LineCursor {
- pub pos: usize,
- pub line: usize,
- pub line_pos: usize,
- pub source: Rc<dyn Source>,
-}
-
-impl LineCursor {
- /// Creates [`LineCursor`] at position
- ///
- /// # Error
- /// This function will panic if [`pos`] is not utf8 aligned
- ///
- /// Note: this is a convenience function, it should be used
- /// with parsimony as it is expensive
- pub fn at(&mut self, pos: usize) {
- if pos > self.pos {
- let start = self.pos;
- eprintln!("slice{{{}}}, want={pos}", &self.source.content().as_str()[start..pos]);
- let mut it = self.source.content().as_str()[start..] // pos+1
- .chars()
- .peekable();
-
- let mut prev = self.source.content().as_str()[..start + 1]
- .chars()
- .rev()
- .next();
- eprintln!("prev={prev:#?}");
- while self.pos < pos {
- let c = it.next().unwrap();
- let len = c.len_utf8();
-
- if self.pos != 0 && prev == Some('\n') {
- self.line += 1;
- self.line_pos = 0;
- } else {
- self.line_pos += len;
- }
- self.pos += len;
-
- eprintln!("({}, {c:#?}, {} {}, {prev:#?})", self.pos, self.line, self.line_pos);
- prev = Some(c);
- }
- if self.pos != 0 && prev == Some('\n') {
- self.line += 1;
- self.line_pos = 0;
- }
- } else if pos < self.pos {
- todo!("");
- self.source.content().as_str()[pos..self.pos]
- .char_indices()
- .rev()
- .for_each(|(len, c)| {
- self.pos -= len;
- if c == '\n' {
- self.line -= 1;
- }
- });
- self.line_pos = self.source.content().as_str()[..self.pos]
- .char_indices()
- .rev()
- .find(|(_, c)| *c == '\n')
- .map(|(line_start, _)| self.pos - line_start)
- .unwrap_or(0);
- }
-
- // May fail if pos is not utf8-aligned
- assert_eq!(pos, self.pos);
- }
-}
-
-impl From<&LineCursor> for Cursor {
- fn from(value: &LineCursor) -> Self {
- Self {
- pos: value.pos,
- source: value.source.clone(),
- }
- }
-}
diff --git a/src/lsp/semantic.rs b/src/lsp/semantic.rs
index 013bcfe..273426c 100644
--- a/src/lsp/semantic.rs
+++ b/src/lsp/semantic.rs
@@ -1,130 +1,171 @@
use std::any::Any;
+use std::cell::RefCell;
+use std::ops::Range;
+use std::rc::Rc;
use tower_lsp::lsp_types::SemanticToken;
+use tower_lsp::lsp_types::SemanticTokenModifier;
use tower_lsp::lsp_types::SemanticTokenType;
-use crate::document::document::Document;
-use crate::document::element::Element;
-use crate::elements::comment::Comment;
-use crate::elements::paragraph::Paragraph;
-use crate::elements::section::Section;
-use crate::parser::rule::Rule;
+use crate::parser::source::LineCursor;
+use crate::parser::source::Source;
-use super::parser::LineCursor;
-pub trait SemanticProvider: Rule {
- fn get_semantic_tokens(
- &self,
- cursor: &LineCursor,
- match_data: Box<dyn Any>,
- ) -> Vec<SemanticToken>;
-}
-
-pub mod nml_semantic {
- use tower_lsp::lsp_types::SemanticTokenType;
-
- pub const SECTION_HEADING: SemanticTokenType = SemanticTokenType::new("type");
- pub const SECTION_NAME: SemanticTokenType = SemanticTokenType::new("string");
- pub const REFERENCE: SemanticTokenType = SemanticTokenType::new("event");
-}
-
-pub const LEGEND_TYPE: &[SemanticTokenType] = &[
- SemanticTokenType::COMMENT,
- SemanticTokenType::VARIABLE,
- SemanticTokenType::STRING,
- SemanticTokenType::PARAMETER,
+pub const TOKEN_TYPE: &[SemanticTokenType] = &[
+ SemanticTokenType::NAMESPACE,
+ SemanticTokenType::TYPE,
+ SemanticTokenType::CLASS,
+ SemanticTokenType::ENUM,
+ SemanticTokenType::INTERFACE,
+ SemanticTokenType::STRUCT,
+ SemanticTokenType::TYPE_PARAMETER,
+ SemanticTokenType::PARAMETER,
+ SemanticTokenType::VARIABLE,
+ SemanticTokenType::PROPERTY,
+ SemanticTokenType::ENUM_MEMBER,
+ SemanticTokenType::EVENT,
+ SemanticTokenType::FUNCTION,
+ SemanticTokenType::METHOD,
+ SemanticTokenType::MACRO,
+ SemanticTokenType::KEYWORD,
+ SemanticTokenType::MODIFIER,
+ SemanticTokenType::COMMENT,
+ SemanticTokenType::STRING,
+ SemanticTokenType::NUMBER,
+ SemanticTokenType::REGEXP,
+ SemanticTokenType::OPERATOR,
+ SemanticTokenType::DECORATOR,
];
-// TODO...
-pub fn provide(
- semantic_tokens: &mut Vec<SemanticToken>,
- cursor: &mut LineCursor,
- elem: &Box<dyn Element>,
-) {
- if cursor.source != elem.location().source() {
- return;
- }
+pub const TOKEN_MODIFIERS: &[SemanticTokenModifier] = &[
+ SemanticTokenModifier::DECLARATION,
+ SemanticTokenModifier::DEFINITION,
+ SemanticTokenModifier::READONLY,
+ SemanticTokenModifier::STATIC,
+ SemanticTokenModifier::DEPRECATED,
+ SemanticTokenModifier::ABSTRACT,
+ SemanticTokenModifier::ASYNC,
+ SemanticTokenModifier::MODIFICATION,
+ SemanticTokenModifier::DOCUMENTATION,
+ SemanticTokenModifier::DEFAULT_LIBRARY,
+];
- let prev = cursor.clone();
-
- /*if let Some(comm) = elem.downcast_ref::<Comment>() {
- cursor.at(elem.location().start());
- let delta_start = if cursor.line == prev.line {
- cursor.line_pos - prev.line_pos
- } else if cursor.line == 0 {
- cursor.line_pos
- } else {
- cursor.line_pos + 1
- };
- semantic_tokens.push(SemanticToken {
- delta_line: (cursor.line - prev.line) as u32,
- delta_start: delta_start as u32,
- length: (elem.location().end() - elem.location().start()) as u32,
- token_type: 0,
- token_modifiers_bitset: 0,
- });
- } else */if let Some(sect) = elem.downcast_ref::<Section>() {
- cursor.at(elem.location().start() + 1);
- eprintln!("section {cursor:#?}");
- let delta_start = if cursor.line == prev.line {
- cursor.line_pos - prev.line_pos
- } else if cursor.line == 0 {
- cursor.line_pos
- } else {
- 0
- };
- semantic_tokens.push(SemanticToken {
- delta_line: (cursor.line - prev.line) as u32,
- delta_start: delta_start as u32,
- length: (elem.location().end() - elem.location().start()) as u32,
- token_type: 0,
- token_modifiers_bitset: 0,
- });
- }
+fn token_index(name: &str) -> u32
+{
+ TOKEN_TYPE.iter()
+ .enumerate()
+ .find(|(_, token)| token.as_str() == name)
+ .map(|(index, _)| index as u32)
+ .unwrap_or(0)
}
-
-pub fn semantic_token_from_document(document: &dyn Document) -> Vec<SemanticToken> {
- let mut semantic_tokens = vec![];
-
- let source = document.source();
- let mut cursor = LineCursor {
- pos: 0,
- line: 0,
- line_pos: 0,
- source: source.clone(),
+fn modifier_index(name: &str) -> u32
+{
+ TOKEN_MODIFIERS.iter()
+ .enumerate()
+ .find(|(_, token)| token.as_str() == name)
+ .map(|(index, _)| index as u32)
+ .unwrap_or(0)
+}
+macro_rules! token {
+ ($key:expr) => {
+ {
+ (token_index($key), 0)
+ }
+ };
+ ($key:expr, $($mods:tt)*) => {
+ {
+ let mut bitset : u32 = 0;
+ $(
+ bitset |= 1 << modifier_index($mods);
+ )*
+ (token_index($key), bitset)
+ }
};
-/*
- semantic_tokens.push(SemanticToken {
- delta_line: 2,
- delta_start: 1,
- length: 5,
- token_type: 0,
- token_modifiers_bitset: 0,
- });
-
- semantic_tokens.push(SemanticToken {
- delta_line: 1,
- delta_start: 1,
- length: 5,
- token_type: 1,
- token_modifiers_bitset: 0,
- });*/
-
- document.content().borrow()
- .iter()
- .for_each(|elem| {
- if let Some(container) = elem.as_container()
- {
- container
- .contained()
- .iter()
- .for_each(|elem| provide(&mut semantic_tokens, &mut cursor, elem));
- }
- else
- {
- provide(&mut semantic_tokens, &mut cursor, elem);
- }
- });
-
- semantic_tokens
+}
+
+#[derive(Debug)]
+pub struct Tokens
+{
+ pub section_heading: (u32, u32),
+ pub section_reference: (u32, u32),
+ pub section_kind: (u32, u32),
+ pub section_name: (u32, u32),
+}
+
+impl Tokens
+{
+ pub fn new() -> Self
+ {
+ Self {
+ section_heading : token!("number"),
+ section_reference : token!("enum", "async"),
+ section_kind : token!("enum"),
+ section_name : token!("string"),
+ }
+ }
+}
+
+/// Semantics for a buffer
+#[derive(Debug)]
+pub struct Semantics {
+ /// The tokens
+ pub token: Tokens,
+
+ /// The current cursor
+ cursor: RefCell<LineCursor>,
+
+ /// Semantic tokens
+ pub tokens: RefCell<Vec<SemanticToken>>,
+}
+
+impl Semantics {
+ pub fn new(source: Rc<dyn Source>) -> Semantics {
+ Self {
+ token: Tokens::new(),
+ cursor: RefCell::new(LineCursor::new(source)),
+ tokens: RefCell::new(vec![]),
+ }
+ }
+
+ pub fn add(
+ &self,
+ source: Rc<dyn Source>,
+ range: Range<usize>,
+ token: (u32, u32)
+ ) {
+ let mut tokens = self.tokens.borrow_mut();
+ let mut cursor = self.cursor.borrow_mut();
+ let mut current = cursor.clone();
+ cursor.move_to(range.start);
+
+ while cursor.pos != range.end {
+ let end = source.content()[cursor.pos..]
+ .find('\n')
+ .unwrap_or(source.content().len() - cursor.pos);
+ let len = usize::min(range.end - cursor.pos, end);
+
+ let delta_line = cursor.line - current.line;
+ let delta_start = if delta_line == 0 {
+ if let Some(last) = tokens.last() {
+ cursor.line_pos - current.line_pos + last.length as usize
+ } else {
+ cursor.line_pos - current.line_pos
+ }
+ } else {
+ cursor.line_pos
+ };
+
+ //eprintln!("CURRENT={:#?}, CURS={:#?}", current, cursor);
+ tokens.push(SemanticToken {
+ delta_line: delta_line as u32,
+ delta_start: delta_start as u32,
+ length: len as u32,
+ token_type: token.0,
+ token_modifiers_bitset: token.1
+ });
+ current = cursor.clone();
+ let pos = cursor.pos;
+ cursor.move_to(pos + len);
+ }
+ }
}
diff --git a/src/main.rs b/src/main.rs
index b86bb33..cc7b18f 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,6 +4,7 @@ mod document;
mod elements;
mod lua;
mod parser;
+mod lsp;
use std::env;
use std::io::BufWriter;
diff --git a/src/parser/langparser.rs b/src/parser/langparser.rs
index 686f9b1..dde7396 100644
--- a/src/parser/langparser.rs
+++ b/src/parser/langparser.rs
@@ -5,12 +5,12 @@ use crate::document::document::Document;
use crate::document::element::DocumentEnd;
use crate::document::langdocument::LangDocument;
use crate::elements::text::Text;
+use crate::lsp::semantic::Semantics;
use super::parser::Parser;
use super::parser::ParserState;
use super::parser::ReportColors;
use super::rule::Rule;
-use super::semantics::Semantics;
use super::source::Cursor;
use super::source::Source;
use super::source::Token;
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index d7d499e..433cbec 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -7,4 +7,3 @@ pub mod util;
pub mod style;
pub mod layout;
pub mod customstyle;
-pub mod semantics;
diff --git a/src/parser/parser.rs b/src/parser/parser.rs
index be56635..84cd9ed 100644
--- a/src/parser/parser.rs
+++ b/src/parser/parser.rs
@@ -11,7 +11,6 @@ use unicode_segmentation::UnicodeSegmentation;
use super::customstyle::CustomStyleHolder;
use super::layout::LayoutHolder;
use super::rule::Rule;
-use super::semantics::Semantics;
use super::source::Cursor;
use super::source::Source;
use super::state::RuleStateHolder;
@@ -22,6 +21,7 @@ use crate::document::element::ContainerElement;
use crate::document::element::ElemKind;
use crate::document::element::Element;
use crate::elements::paragraph::Paragraph;
+use crate::lsp::semantic::Semantics;
use crate::lua::kernel::Kernel;
use crate::lua::kernel::KernelHolder;
use crate::parser::source::SourceFile;
diff --git a/src/parser/semantics.rs b/src/parser/semantics.rs
deleted file mode 100644
index b54ed3a..0000000
--- a/src/parser/semantics.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-use std::{ops::Range, rc::Rc};
-
-use tower_lsp::lsp_types::SemanticToken;
-
-use super::source::{LineCursor, Source};
-
-
-/// Semantics for a buffer
-#[derive(Debug)]
-pub struct Semantics
-{
- /// The current cursor
- cursor: LineCursor,
-
- /// Semantic tokens
- pub tokens: Vec<SemanticToken>,
-}
-
-impl Semantics
-{
- pub fn new(source: Rc<dyn Source>) -> Semantics
- {
- Self {
- cursor: LineCursor::new(source),
- tokens: vec![]
- }
- }
-
- pub fn add(&mut self, source: Rc<dyn Source>, range: Range<usize>, token_type: u32, token_modifier: u32)
- {
- let mut current = self.cursor.clone();
- self.cursor.move_to(range.start);
-
- while self.cursor.pos != range.end
- {
- let end = source.content()[self.cursor.pos..].find('\n')
- .unwrap_or(source.content().len() - self.cursor.pos);
- let len = usize::min(range.end - self.cursor.pos, end);
-
- let delta_line = self.cursor.line - current.line;
- let delta_start = if delta_line == 0
- {
- if let Some(last) = self.tokens.last()
- {
- self.cursor.line_pos - current.line_pos + last.length as usize
- }
- else
- {
- self.cursor.line_pos - current.line_pos
- }
- } else { self.cursor.line_pos };
-
- eprintln!("CURRENT={:#?}, CURS={:#?}", current, self.cursor);
- self.tokens.push(SemanticToken{
- delta_line: delta_line as u32,
- delta_start: delta_start as u32,
- length: len as u32,
- token_type,
- token_modifiers_bitset: token_modifier,
- });
- current = self.cursor.clone();
- self.cursor.move_to(self.cursor.pos + len);
- }
- }
-}
diff --git a/src/parser/source.rs b/src/parser/source.rs
index 750388b..f568cf7 100644
--- a/src/parser/source.rs
+++ b/src/parser/source.rs
@@ -199,7 +199,7 @@ impl LineCursor {
self.line_pos += c.width().unwrap_or(1);
self.pos += len;
- eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' '));
+ //eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' '));
prev = Some(c);
}
if self.pos != 0 && prev == Some('\n') {
diff --git a/src/server.rs b/src/server.rs
index 06473f0..8726965 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -6,18 +6,13 @@ mod lsp;
mod lua;
mod parser;
-use std::collections::HashMap;
use std::rc::Rc;
-use std::sync::Arc;
use dashmap::DashMap;
-use document::document::Document;
-use document::element::Element;
-use lsp::semantic::semantic_token_from_document;
+use lsp::semantic::Tokens;
use parser::langparser::LangParser;
use parser::parser::Parser;
use parser::parser::ParserState;
-use parser::semantics::Semantics;
use parser::source::SourceFile;
use tower_lsp::jsonrpc::Result;
use tower_lsp::lsp_types::*;
@@ -58,14 +53,14 @@ impl Backend {
.unwrap()
}) {
self.semantic_token_map
- .insert(params.uri.to_string(), sems.tokens.to_owned());
+ .insert(params.uri.to_string(), sems.tokens.borrow().to_owned());
};
}
}
#[tower_lsp::async_trait]
impl LanguageServer for Backend {
- async fn initialize(&self, _: InitializeParams) -> Result<InitializeResult> {
+ async fn initialize(&self, params: InitializeParams) -> Result<InitializeResult> {
Ok(InitializeResult {
server_info: None,
capabilities: ServerCapabilities {
@@ -85,17 +80,17 @@ impl LanguageServer for Backend {
text_document_registration_options: {
TextDocumentRegistrationOptions {
document_selector: Some(vec![DocumentFilter {
- language: Some("nml".to_string()),
- scheme: Some("file".to_string()),
- pattern: None,
+ language: Some("nml".into()),
+ scheme: Some("file".into()),
+ pattern: Some("*.nml".into()),
}]),
}
},
semantic_tokens_options: SemanticTokensOptions {
work_done_progress_options: WorkDoneProgressOptions::default(),
legend: SemanticTokensLegend {
- token_types: lsp::semantic::LEGEND_TYPE.into(),
- token_modifiers: vec![],
+ token_types: lsp::semantic::TOKEN_TYPE.into(),
+ token_modifiers: lsp::semantic::TOKEN_MODIFIERS.into(),
},
range: None, //Some(true),
full: Some(SemanticTokensFullOptions::Bool(true)),