Cleanup

src/elements/section.rs

@@ -20,7 +20,6 @@ use mlua::Lua;
 use regex::Regex;
 use section_style::SectionLinkPos;
 use section_style::SectionStyle;
-use std::cell::RefCell;
 use std::cell::RefMut;
 use std::ops::Range;
 use std::rc::Rc;
@@ -333,16 +332,16 @@ impl RegexRule for SectionRule {
 				.ok()
 				.unwrap()
 		}) {
-			sems.add(token.source(), matches.get(1).unwrap().range(), 0, 0);
+			sems.add(token.source(), matches.get(1).unwrap().range(), sems.token.section_heading);
 			if let Some(reference) = matches.get(2)
 			{
-				sems.add(token.source(), reference.start()-1..reference.end()+1, 1, 0);
+				sems.add(token.source(), reference.start()-1..reference.end()+1, sems.token.section_reference);
 			}
 			if let Some(kind) = matches.get(3)
 			{
-				sems.add(token.source(), kind.range(), 3, 0);
+				sems.add(token.source(), kind.range(), sems.token.section_kind);
 			}
-			//sems.add(token.source(), matches.get(5).unwrap().range(), 2, 0);
+			sems.add(token.source(), matches.get(5).unwrap().range(), sems.token.section_name);
 		}
 
 		result

src/lsp/mod.rs

@@ -1,2 +1 @@
 pub mod semantic;
-pub mod parser;

src/lsp/parser.rs

@@ -1,85 +0,0 @@
-use std::rc::Rc;
-
-use crate::parser::source::Cursor;
-use crate::parser::source::Source;
-
-#[derive(Debug, Clone)]
-pub struct LineCursor {
-	pub pos: usize,
-	pub line: usize,
-	pub line_pos: usize,
-	pub source: Rc<dyn Source>,
-}
-
-impl LineCursor {
-	/// Creates [`LineCursor`] at position
-	///
-	/// # Error
-	/// This function will panic if [`pos`] is not utf8 aligned
-	///
-	/// Note: this is a convenience function, it should be used
-	/// with parsimony as it is expensive
-	pub fn at(&mut self, pos: usize) {
-		if pos > self.pos {
-			let start = self.pos;
-			eprintln!("slice{{{}}}, want={pos}", &self.source.content().as_str()[start..pos]);
-			let mut it = self.source.content().as_str()[start..] // pos+1
-				.chars()
-				.peekable();
-
-			let mut prev = self.source.content().as_str()[..start + 1]
-				.chars()
-				.rev()
-				.next();
-			eprintln!("prev={prev:#?}");
-			while self.pos < pos {
-				let c = it.next().unwrap();
-				let len = c.len_utf8();
-
-				if self.pos != 0 && prev == Some('\n') {
-					self.line += 1;
-					self.line_pos = 0;
-				} else {
-					self.line_pos += len;
-				}
-				self.pos += len;
-
-				eprintln!("({}, {c:#?}, {} {}, {prev:#?})", self.pos, self.line, self.line_pos);
-				prev = Some(c);
-			}
-			if self.pos != 0 && prev == Some('\n') {
-				self.line += 1;
-				self.line_pos = 0;
-			}
-		} else if pos < self.pos {
-			todo!("");
-			self.source.content().as_str()[pos..self.pos]
-				.char_indices()
-				.rev()
-				.for_each(|(len, c)| {
-					self.pos -= len;
-					if c == '\n' {
-						self.line -= 1;
-					}
-				});
-			self.line_pos = self.source.content().as_str()[..self.pos]
-				.char_indices()
-				.rev()
-				.find(|(_, c)| *c == '\n')
-				.map(|(line_start, _)| self.pos - line_start)
-				.unwrap_or(0);
-		}
-
-		// May fail if pos is not utf8-aligned
-		assert_eq!(pos, self.pos);
-	}
-}
-
-impl From<&LineCursor> for Cursor {
-	fn from(value: &LineCursor) -> Self {
-		Self {
-			pos: value.pos,
-			source: value.source.clone(),
-		}
-	}
-}

src/lsp/semantic.rs

@@ -1,130 +1,171 @@
 use std::any::Any;
+use std::cell::RefCell;
+use std::ops::Range;
+use std::rc::Rc;
 
 use tower_lsp::lsp_types::SemanticToken;
+use tower_lsp::lsp_types::SemanticTokenModifier;
 use tower_lsp::lsp_types::SemanticTokenType;
 
-use crate::document::document::Document;
+use crate::parser::source::LineCursor;
-use crate::document::element::Element;
+use crate::parser::source::Source;
-use crate::elements::comment::Comment;
-use crate::elements::paragraph::Paragraph;
-use crate::elements::section::Section;
-use crate::parser::rule::Rule;
 
-use super::parser::LineCursor;
 
-pub trait SemanticProvider: Rule {
+pub const TOKEN_TYPE: &[SemanticTokenType] = &[
-	fn get_semantic_tokens(
+    SemanticTokenType::NAMESPACE,
-		&self,
+    SemanticTokenType::TYPE,
-		cursor: &LineCursor,
+    SemanticTokenType::CLASS,
-		match_data: Box<dyn Any>,
+    SemanticTokenType::ENUM,
-	) -> Vec<SemanticToken>;
+    SemanticTokenType::INTERFACE,
-}
+    SemanticTokenType::STRUCT,
-
+    SemanticTokenType::TYPE_PARAMETER,
-pub mod nml_semantic {
+    SemanticTokenType::PARAMETER,
-    use tower_lsp::lsp_types::SemanticTokenType;
+    SemanticTokenType::VARIABLE,
-
+    SemanticTokenType::PROPERTY,
-    pub const SECTION_HEADING: SemanticTokenType = SemanticTokenType::new("type");
+    SemanticTokenType::ENUM_MEMBER,
-    pub const SECTION_NAME: SemanticTokenType = SemanticTokenType::new("string");
+    SemanticTokenType::EVENT,
-    pub const REFERENCE: SemanticTokenType = SemanticTokenType::new("event");
+    SemanticTokenType::FUNCTION,
-}
+    SemanticTokenType::METHOD,
-
+    SemanticTokenType::MACRO,
-pub const LEGEND_TYPE: &[SemanticTokenType] = &[
+    SemanticTokenType::KEYWORD,
-	SemanticTokenType::COMMENT,
+    SemanticTokenType::MODIFIER,
-	SemanticTokenType::VARIABLE,
+    SemanticTokenType::COMMENT,
-	SemanticTokenType::STRING,
+    SemanticTokenType::STRING,
-	SemanticTokenType::PARAMETER,
+    SemanticTokenType::NUMBER,
+    SemanticTokenType::REGEXP,
+    SemanticTokenType::OPERATOR,
+    SemanticTokenType::DECORATOR,
 ];
 
-// TODO...
+pub const TOKEN_MODIFIERS: &[SemanticTokenModifier] = &[
-pub fn provide(
+    SemanticTokenModifier::DECLARATION,
-	semantic_tokens: &mut Vec<SemanticToken>,
+    SemanticTokenModifier::DEFINITION,
-	cursor: &mut LineCursor,
+    SemanticTokenModifier::READONLY,
-	elem: &Box<dyn Element>,
+    SemanticTokenModifier::STATIC,
-) {
+    SemanticTokenModifier::DEPRECATED,
-	if cursor.source != elem.location().source() {
+    SemanticTokenModifier::ABSTRACT,
-		return;
+    SemanticTokenModifier::ASYNC,
-	}
+    SemanticTokenModifier::MODIFICATION,
+    SemanticTokenModifier::DOCUMENTATION,
+    SemanticTokenModifier::DEFAULT_LIBRARY,
+];
 
-	let prev = cursor.clone();
+fn token_index(name: &str) -> u32
-
+{
-	/*if let Some(comm) = elem.downcast_ref::<Comment>() {
+	TOKEN_TYPE.iter()
-		cursor.at(elem.location().start());
+		.enumerate()
-		let delta_start = if cursor.line == prev.line {
+		.find(|(_, token)| token.as_str() == name)
-			cursor.line_pos - prev.line_pos
+		.map(|(index, _)| index as u32)
-		} else if cursor.line == 0 {
+		.unwrap_or(0)
-			cursor.line_pos
-		} else {
-			cursor.line_pos + 1
-		};
-		semantic_tokens.push(SemanticToken {
-			delta_line: (cursor.line - prev.line) as u32,
-			delta_start: delta_start as u32,
-			length: (elem.location().end() - elem.location().start()) as u32,
-			token_type: 0,
-			token_modifiers_bitset: 0,
-		});
-	} else */if let Some(sect) = elem.downcast_ref::<Section>() {
-		cursor.at(elem.location().start() + 1);
-		eprintln!("section {cursor:#?}");
-		let delta_start = if cursor.line == prev.line {
-			cursor.line_pos - prev.line_pos
-		} else if cursor.line == 0 {
-			cursor.line_pos
-		} else {
-			0
-		};
-		semantic_tokens.push(SemanticToken {
-			delta_line: (cursor.line - prev.line) as u32,
-			delta_start: delta_start as u32,
-			length: (elem.location().end() - elem.location().start()) as u32,
-			token_type: 0,
-			token_modifiers_bitset: 0,
-		});
-	}
 }
-
+fn modifier_index(name: &str) -> u32
-pub fn semantic_token_from_document(document: &dyn Document) -> Vec<SemanticToken> {
+{
-	let mut semantic_tokens = vec![];
+	TOKEN_MODIFIERS.iter()
-
+		.enumerate()
-	let source = document.source();
+		.find(|(_, token)| token.as_str() == name)
-	let mut cursor = LineCursor {
+		.map(|(index, _)| index as u32)
-		pos: 0,
+		.unwrap_or(0)
-		line: 0,
+}
-		line_pos: 0,
+macro_rules! token {
-		source: source.clone(),
+	($key:expr) => {
+		{
+			(token_index($key), 0)
+		}
+	};
+	($key:expr, $($mods:tt)*) => {
+		{
+			let mut bitset : u32 = 0;
+			$(
+				bitset |= 1 << modifier_index($mods);
+			)*
+				(token_index($key), bitset)
+		}
 	};
-/*
+}
-	semantic_tokens.push(SemanticToken {
+
-		delta_line: 2,
+#[derive(Debug)]
-		delta_start: 1,
+pub struct Tokens
-		length: 5,
+{
-		token_type: 0,
+	pub section_heading: (u32, u32),
-		token_modifiers_bitset: 0,
+	pub section_reference: (u32, u32),
-	});
+	pub section_kind: (u32, u32),
-
+	pub section_name: (u32, u32),
-	semantic_tokens.push(SemanticToken {
+}
-		delta_line: 1,
+
-		delta_start: 1,
+impl Tokens
-		length: 5,
+{
-		token_type: 1,
+	pub fn new() -> Self
-		token_modifiers_bitset: 0,
+	{
-	});*/
+		Self {
-
+			section_heading : token!("number"),
-	document.content().borrow()
+			section_reference : token!("enum", "async"),
-		.iter()
+			section_kind : token!("enum"),
-		.for_each(|elem| {
+			section_name : token!("string"),
-			if let Some(container) = elem.as_container()
+		}
-			{
+	}
-				container
+}
-					.contained()
+
-					.iter()
+/// Semantics for a buffer
-					.for_each(|elem| provide(&mut semantic_tokens, &mut cursor, elem));
+#[derive(Debug)]
-			}
+pub struct Semantics {
-			else
+	/// The tokens
-			{
+	pub token: Tokens,
-				provide(&mut semantic_tokens, &mut cursor, elem);
+
-			}
+	/// The current cursor
-		});
+	cursor: RefCell<LineCursor>,
-
+
-	semantic_tokens
+	/// Semantic tokens
+	pub tokens: RefCell<Vec<SemanticToken>>,
+}
+
+impl Semantics {
+	pub fn new(source: Rc<dyn Source>) -> Semantics {
+		Self {
+			token: Tokens::new(),
+			cursor: RefCell::new(LineCursor::new(source)),
+			tokens: RefCell::new(vec![]),
+		}
+	}
+
+	pub fn add(
+		&self,
+		source: Rc<dyn Source>,
+		range: Range<usize>,
+		token: (u32, u32)
+	) {
+		let mut tokens = self.tokens.borrow_mut();
+		let mut cursor = self.cursor.borrow_mut();
+		let mut current = cursor.clone();
+		cursor.move_to(range.start);
+
+		while cursor.pos != range.end {
+			let end = source.content()[cursor.pos..]
+				.find('\n')
+				.unwrap_or(source.content().len() - cursor.pos);
+			let len = usize::min(range.end - cursor.pos, end);
+
+			let delta_line = cursor.line - current.line;
+			let delta_start = if delta_line == 0 {
+				if let Some(last) = tokens.last() {
+					cursor.line_pos - current.line_pos + last.length as usize
+				} else {
+					cursor.line_pos - current.line_pos
+				}
+			} else {
+				cursor.line_pos
+			};
+
+			//eprintln!("CURRENT={:#?}, CURS={:#?}", current, cursor);
+			tokens.push(SemanticToken {
+				delta_line: delta_line as u32,
+				delta_start: delta_start as u32,
+				length: len as u32,
+				token_type: token.0,
+				token_modifiers_bitset: token.1
+			});
+			current = cursor.clone();
+			let pos = cursor.pos;
+			cursor.move_to(pos + len);
+		}
+	}
 }

src/main.rs

@@ -4,6 +4,7 @@ mod document;
 mod elements;
 mod lua;
 mod parser;
+mod lsp;
 
 use std::env;
 use std::io::BufWriter;

src/parser/langparser.rs

@@ -5,12 +5,12 @@ use crate::document::document::Document;
 use crate::document::element::DocumentEnd;
 use crate::document::langdocument::LangDocument;
 use crate::elements::text::Text;
+use crate::lsp::semantic::Semantics;
 
 use super::parser::Parser;
 use super::parser::ParserState;
 use super::parser::ReportColors;
 use super::rule::Rule;
-use super::semantics::Semantics;
 use super::source::Cursor;
 use super::source::Source;
 use super::source::Token;

src/parser/mod.rs

@@ -7,4 +7,3 @@ pub mod util;
 pub mod style;
 pub mod layout;
 pub mod customstyle;
-pub mod semantics;

src/parser/parser.rs

@@ -11,7 +11,6 @@ use unicode_segmentation::UnicodeSegmentation;
 use super::customstyle::CustomStyleHolder;
 use super::layout::LayoutHolder;
 use super::rule::Rule;
-use super::semantics::Semantics;
 use super::source::Cursor;
 use super::source::Source;
 use super::state::RuleStateHolder;
@@ -22,6 +21,7 @@ use crate::document::element::ContainerElement;
 use crate::document::element::ElemKind;
 use crate::document::element::Element;
 use crate::elements::paragraph::Paragraph;
+use crate::lsp::semantic::Semantics;
 use crate::lua::kernel::Kernel;
 use crate::lua::kernel::KernelHolder;
 use crate::parser::source::SourceFile;

src/parser/semantics.rs

@@ -1,65 +0,0 @@
-use std::{ops::Range, rc::Rc};
-
-use tower_lsp::lsp_types::SemanticToken;
-
-use super::source::{LineCursor, Source};
-
-
-/// Semantics for a buffer
-#[derive(Debug)]
-pub struct Semantics
-{
-	/// The current cursor
-	cursor: LineCursor,
-
-	/// Semantic tokens
-	pub tokens: Vec<SemanticToken>,
-}
-
-impl Semantics
-{
-	pub fn new(source: Rc<dyn Source>) -> Semantics
-	{
-		Self {
-			cursor: LineCursor::new(source),
-			tokens: vec![]
-		}
-	}
-
-	pub fn add(&mut self, source: Rc<dyn Source>, range: Range<usize>, token_type: u32, token_modifier: u32)
-	{
-		let mut current = self.cursor.clone();
-		self.cursor.move_to(range.start);
-
-		while self.cursor.pos != range.end
-		{
-			let end = source.content()[self.cursor.pos..].find('\n')
-				.unwrap_or(source.content().len() - self.cursor.pos);
-			let len = usize::min(range.end - self.cursor.pos, end);
-
-			let delta_line = self.cursor.line - current.line;
-			let delta_start = if delta_line == 0
-			{
-				if let Some(last) = self.tokens.last()
-				{
-					self.cursor.line_pos - current.line_pos + last.length as usize
-				}
-				else
-				{
-					self.cursor.line_pos - current.line_pos
-				}
-			} else { self.cursor.line_pos };
-
-			eprintln!("CURRENT={:#?}, CURS={:#?}", current, self.cursor);
-			self.tokens.push(SemanticToken{
-				delta_line: delta_line as u32,
-				delta_start: delta_start as u32,
-				length: len as u32,
-				token_type,
-				token_modifiers_bitset: token_modifier,
-			});
-			current = self.cursor.clone();
-			self.cursor.move_to(self.cursor.pos + len);
-		}
-	}
-}

src/parser/source.rs

@@ -199,7 +199,7 @@ impl LineCursor {
 				self.line_pos += c.width().unwrap_or(1);
 				self.pos += len;
 
-				eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' '));
+				//eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' '));
 				prev = Some(c);
 			}
 			if self.pos != 0 && prev == Some('\n') {

src/server.rs

@@ -6,18 +6,13 @@ mod lsp;
 mod lua;
 mod parser;
 
-use std::collections::HashMap;
 use std::rc::Rc;
-use std::sync::Arc;
 
 use dashmap::DashMap;
-use document::document::Document;
+use lsp::semantic::Tokens;
-use document::element::Element;
-use lsp::semantic::semantic_token_from_document;
 use parser::langparser::LangParser;
 use parser::parser::Parser;
 use parser::parser::ParserState;
-use parser::semantics::Semantics;
 use parser::source::SourceFile;
 use tower_lsp::jsonrpc::Result;
 use tower_lsp::lsp_types::*;
@@ -58,14 +53,14 @@ impl Backend {
 				.unwrap()
 		}) {
 			self.semantic_token_map
-				.insert(params.uri.to_string(), sems.tokens.to_owned());
+				.insert(params.uri.to_string(), sems.tokens.borrow().to_owned());
 		};
 	}
 }
 
 #[tower_lsp::async_trait]
 impl LanguageServer for Backend {
-	async fn initialize(&self, _: InitializeParams) -> Result<InitializeResult> {
+	async fn initialize(&self, params: InitializeParams) -> Result<InitializeResult> {
 		Ok(InitializeResult {
 			server_info: None,
 			capabilities: ServerCapabilities {
@@ -85,17 +80,17 @@ impl LanguageServer for Backend {
 							text_document_registration_options: {
 								TextDocumentRegistrationOptions {
 									document_selector: Some(vec![DocumentFilter {
-										language: Some("nml".to_string()),
+										language: Some("nml".into()),
-										scheme: Some("file".to_string()),
+										scheme: Some("file".into()),
-										pattern: None,
+										pattern: Some("*.nml".into()),
 									}]),
 								}
 							},
 							semantic_tokens_options: SemanticTokensOptions {
 								work_done_progress_options: WorkDoneProgressOptions::default(),
 								legend: SemanticTokensLegend {
-									token_types: lsp::semantic::LEGEND_TYPE.into(),
+									token_types: lsp::semantic::TOKEN_TYPE.into(),
-									token_modifiers: vec![],
+									token_modifiers: lsp::semantic::TOKEN_MODIFIERS.into(),
 								},
 								range: None, //Some(true),
 								full: Some(SemanticTokensFullOptions::Bool(true)),