ef3d0c3e/nml
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cleanup

Browse source
This commit is contained in:
ef3d0c3e 2024-10-18 12:43:51 +02:00
parent 1965ff6006
commit bd75161e86
11 changed files with 174 additions and 290 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/elements/section.rs
View file

@ -20,7 +20,6 @@ use mlua::Lua;
use regex::Regex;
use section_style::SectionLinkPos;
use section_style::SectionStyle;
use std::cell::RefCell;
use std::cell::RefMut;
use std::ops::Range;
use std::rc::Rc;
@ -333,16 +332,16 @@ impl RegexRule for SectionRule {
.ok()
.unwrap()
}) {
sems.add(token.source(), matches.get(1).unwrap().range(), 0, 0);
sems.add(token.source(), matches.get(1).unwrap().range(), sems.token.section_heading);
if let Some(reference) = matches.get(2)
{
sems.add(token.source(), reference.start()-1..reference.end()+1, 1, 0);
sems.add(token.source(), reference.start()-1..reference.end()+1, sems.token.section_reference);
}
if let Some(kind) = matches.get(3)
{
sems.add(token.source(), kind.range(), 3, 0);
sems.add(token.source(), kind.range(), sems.token.section_kind);
}
//sems.add(token.source(), matches.get(5).unwrap().range(), 2, 0);
sems.add(token.source(), matches.get(5).unwrap().range(), sems.token.section_name);
}
result

1
src/lsp/mod.rs
View file

@ -1,2 +1 @@
pub mod semantic;
pub mod parser;

85
src/lsp/parser.rs
View file

@ -1,85 +0,0 @@
use std::rc::Rc;
use crate::parser::source::Cursor;
use crate::parser::source::Source;
#[derive(Debug, Clone)]
pub struct LineCursor {
pub pos: usize,
pub line: usize,
pub line_pos: usize,
pub source: Rc<dyn Source>,
}
impl LineCursor {
/// Creates [`LineCursor`] at position
///
/// # Error
/// This function will panic if [`pos`] is not utf8 aligned
///
/// Note: this is a convenience function, it should be used
/// with parsimony as it is expensive
pub fn at(&mut self, pos: usize) {
if pos > self.pos {
let start = self.pos;
eprintln!("slice{{{}}}, want={pos}", &self.source.content().as_str()[start..pos]);
let mut it = self.source.content().as_str()[start..] // pos+1
.chars()
.peekable();
let mut prev = self.source.content().as_str()[..start + 1]
.chars()
.rev()
.next();
eprintln!("prev={prev:#?}");
while self.pos < pos {
let c = it.next().unwrap();
let len = c.len_utf8();
if self.pos != 0 && prev == Some('\n') {
self.line += 1;
self.line_pos = 0;
} else {
self.line_pos += len;
}
self.pos += len;
eprintln!("({}, {c:#?}, {} {}, {prev:#?})", self.pos, self.line, self.line_pos);
prev = Some(c);
}
if self.pos != 0 && prev == Some('\n') {
self.line += 1;
self.line_pos = 0;
}
} else if pos < self.pos {
todo!("");
self.source.content().as_str()[pos..self.pos]
.char_indices()
.rev()
.for_each(|(len, c)| {
self.pos -= len;
if c == '\n' {
self.line -= 1;
}
});
self.line_pos = self.source.content().as_str()[..self.pos]
.char_indices()
.rev()
.find(|(_, c)| *c == '\n')
.map(|(line_start, _)| self.pos - line_start)
.unwrap_or(0);
}
// May fail if pos is not utf8-aligned
assert_eq!(pos, self.pos);
}
}
impl From<&LineCursor> for Cursor {
fn from(value: &LineCursor) -> Self {
Self {
pos: value.pos,
source: value.source.clone(),
}
}
}

259
src/lsp/semantic.rs
View file

@ -1,130 +1,171 @@
use std::any::Any;
use std::cell::RefCell;
use std::ops::Range;
use std::rc::Rc;
use tower_lsp::lsp_types::SemanticToken;
use tower_lsp::lsp_types::SemanticTokenModifier;
use tower_lsp::lsp_types::SemanticTokenType;
use crate::document::document::Document;
use crate::document::element::Element;
use crate::elements::comment::Comment;
use crate::elements::paragraph::Paragraph;
use crate::elements::section::Section;
use crate::parser::rule::Rule;
use crate::parser::source::LineCursor;
use crate::parser::source::Source;
use super::parser::LineCursor;
pub trait SemanticProvider: Rule {
fn get_semantic_tokens(
&self,
cursor: &LineCursor,
match_data: Box<dyn Any>,
) -> Vec<SemanticToken>;
}
pub mod nml_semantic {
use tower_lsp::lsp_types::SemanticTokenType;
pub const SECTION_HEADING: SemanticTokenType = SemanticTokenType::new("type");
pub const SECTION_NAME: SemanticTokenType = SemanticTokenType::new("string");
pub const REFERENCE: SemanticTokenType = SemanticTokenType::new("event");
}
pub const LEGEND_TYPE: &[SemanticTokenType] = &[
SemanticTokenType::COMMENT,
SemanticTokenType::VARIABLE,
SemanticTokenType::STRING,
pub const TOKEN_TYPE: &[SemanticTokenType] = &[
SemanticTokenType::NAMESPACE,
SemanticTokenType::TYPE,
SemanticTokenType::CLASS,
SemanticTokenType::ENUM,
SemanticTokenType::INTERFACE,
SemanticTokenType::STRUCT,
SemanticTokenType::TYPE_PARAMETER,
SemanticTokenType::PARAMETER,
SemanticTokenType::VARIABLE,
SemanticTokenType::PROPERTY,
SemanticTokenType::ENUM_MEMBER,
SemanticTokenType::EVENT,
SemanticTokenType::FUNCTION,
SemanticTokenType::METHOD,
SemanticTokenType::MACRO,
SemanticTokenType::KEYWORD,
SemanticTokenType::MODIFIER,
SemanticTokenType::COMMENT,
SemanticTokenType::STRING,
SemanticTokenType::NUMBER,
SemanticTokenType::REGEXP,
SemanticTokenType::OPERATOR,
SemanticTokenType::DECORATOR,
];
// TODO...
pub fn provide(
semantic_tokens: &mut Vec<SemanticToken>,
cursor: &mut LineCursor,
elem: &Box<dyn Element>,
pub const TOKEN_MODIFIERS: &[SemanticTokenModifier] = &[
SemanticTokenModifier::DECLARATION,
SemanticTokenModifier::DEFINITION,
SemanticTokenModifier::READONLY,
SemanticTokenModifier::STATIC,
SemanticTokenModifier::DEPRECATED,
SemanticTokenModifier::ABSTRACT,
SemanticTokenModifier::ASYNC,
SemanticTokenModifier::MODIFICATION,
SemanticTokenModifier::DOCUMENTATION,
SemanticTokenModifier::DEFAULT_LIBRARY,
];
fn token_index(name: &str) -> u32
{
TOKEN_TYPE.iter()
.enumerate()
.find(|(_, token)| token.as_str() == name)
.map(|(index, _)| index as u32)
.unwrap_or(0)
}
fn modifier_index(name: &str) -> u32
{
TOKEN_MODIFIERS.iter()
.enumerate()
.find(|(_, token)| token.as_str() == name)
.map(|(index, _)| index as u32)
.unwrap_or(0)
}
macro_rules! token {
($key:expr) => {
{
(token_index($key), 0)
}
};
($key:expr, $($mods:tt)*) => {
{
let mut bitset : u32 = 0;
$(
bitset |= 1 << modifier_index($mods);
)*
(token_index($key), bitset)
}
};
}
#[derive(Debug)]
pub struct Tokens
{
pub section_heading: (u32, u32),
pub section_reference: (u32, u32),
pub section_kind: (u32, u32),
pub section_name: (u32, u32),
}
impl Tokens
{
pub fn new() -> Self
{
Self {
section_heading : token!("number"),
section_reference : token!("enum", "async"),
section_kind : token!("enum"),
section_name : token!("string"),
}
}
}
/// Semantics for a buffer
#[derive(Debug)]
pub struct Semantics {
/// The tokens
pub token: Tokens,
/// The current cursor
cursor: RefCell<LineCursor>,
/// Semantic tokens
pub tokens: RefCell<Vec<SemanticToken>>,
}
impl Semantics {
pub fn new(source: Rc<dyn Source>) -> Semantics {
Self {
token: Tokens::new(),
cursor: RefCell::new(LineCursor::new(source)),
tokens: RefCell::new(vec![]),
}
}
pub fn add(
&self,
source: Rc<dyn Source>,
range: Range<usize>,
token: (u32, u32)
) {
if cursor.source != elem.location().source() {
return;
}
let mut tokens = self.tokens.borrow_mut();
let mut cursor = self.cursor.borrow_mut();
let mut current = cursor.clone();
cursor.move_to(range.start);
let prev = cursor.clone();
while cursor.pos != range.end {
let end = source.content()[cursor.pos..]
.find('\n')
.unwrap_or(source.content().len() - cursor.pos);
let len = usize::min(range.end - cursor.pos, end);
/*if let Some(comm) = elem.downcast_ref::<Comment>() {
cursor.at(elem.location().start());
let delta_start = if cursor.line == prev.line {
cursor.line_pos - prev.line_pos
} else if cursor.line == 0 {
cursor.line_pos
let delta_line = cursor.line - current.line;
let delta_start = if delta_line == 0 {
if let Some(last) = tokens.last() {
cursor.line_pos - current.line_pos + last.length as usize
} else {
cursor.line_pos + 1
};
semantic_tokens.push(SemanticToken {
delta_line: (cursor.line - prev.line) as u32,
delta_start: delta_start as u32,
length: (elem.location().end() - elem.location().start()) as u32,
token_type: 0,
token_modifiers_bitset: 0,
});
} else */if let Some(sect) = elem.downcast_ref::<Section>() {
cursor.at(elem.location().start() + 1);
eprintln!("section {cursor:#?}");
let delta_start = if cursor.line == prev.line {
cursor.line_pos - prev.line_pos
} else if cursor.line == 0 {
cursor.line_pos
cursor.line_pos - current.line_pos
}
} else {
0
cursor.line_pos
};
semantic_tokens.push(SemanticToken {
delta_line: (cursor.line - prev.line) as u32,
//eprintln!("CURRENT={:#?}, CURS={:#?}", current, cursor);
tokens.push(SemanticToken {
delta_line: delta_line as u32,
delta_start: delta_start as u32,
length: (elem.location().end() - elem.location().start()) as u32,
token_type: 0,
token_modifiers_bitset: 0,
length: len as u32,
token_type: token.0,
token_modifiers_bitset: token.1
});
current = cursor.clone();
let pos = cursor.pos;
cursor.move_to(pos + len);
}
}
pub fn semantic_token_from_document(document: &dyn Document) -> Vec<SemanticToken> {
let mut semantic_tokens = vec![];
let source = document.source();
let mut cursor = LineCursor {
pos: 0,
line: 0,
line_pos: 0,
source: source.clone(),
};
/*
semantic_tokens.push(SemanticToken {
delta_line: 2,
delta_start: 1,
length: 5,
token_type: 0,
token_modifiers_bitset: 0,
});
semantic_tokens.push(SemanticToken {
delta_line: 1,
delta_start: 1,
length: 5,
token_type: 1,
token_modifiers_bitset: 0,
});*/
document.content().borrow()
.iter()
.for_each(|elem| {
if let Some(container) = elem.as_container()
{
container
.contained()
.iter()
.for_each(|elem| provide(&mut semantic_tokens, &mut cursor, elem));
}
else
{
provide(&mut semantic_tokens, &mut cursor, elem);
}
});
semantic_tokens
}

1
src/main.rs
View file

@ -4,6 +4,7 @@ mod document;
mod elements;
mod lua;
mod parser;
mod lsp;
use std::env;
use std::io::BufWriter;

2
src/parser/langparser.rs
View file

@ -5,12 +5,12 @@ use crate::document::document::Document;
use crate::document::element::DocumentEnd;
use crate::document::langdocument::LangDocument;
use crate::elements::text::Text;
use crate::lsp::semantic::Semantics;
use super::parser::Parser;
use super::parser::ParserState;
use super::parser::ReportColors;
use super::rule::Rule;
use super::semantics::Semantics;
use super::source::Cursor;
use super::source::Source;
use super::source::Token;

1
src/parser/mod.rs
View file

@ -7,4 +7,3 @@ pub mod util;
pub mod style;
pub mod layout;
pub mod customstyle;
pub mod semantics;

2
src/parser/parser.rs
View file

@ -11,7 +11,6 @@ use unicode_segmentation::UnicodeSegmentation;
use super::customstyle::CustomStyleHolder;
use super::layout::LayoutHolder;
use super::rule::Rule;
use super::semantics::Semantics;
use super::source::Cursor;
use super::source::Source;
use super::state::RuleStateHolder;
@ -22,6 +21,7 @@ use crate::document::element::ContainerElement;
use crate::document::element::ElemKind;
use crate::document::element::Element;
use crate::elements::paragraph::Paragraph;
use crate::lsp::semantic::Semantics;
use crate::lua::kernel::Kernel;
use crate::lua::kernel::KernelHolder;
use crate::parser::source::SourceFile;

65
src/parser/semantics.rs
View file

@ -1,65 +0,0 @@
use std::{ops::Range, rc::Rc};
use tower_lsp::lsp_types::SemanticToken;
use super::source::{LineCursor, Source};
/// Semantics for a buffer
#[derive(Debug)]
pub struct Semantics
{
/// The current cursor
cursor: LineCursor,
/// Semantic tokens
pub tokens: Vec<SemanticToken>,
}
impl Semantics
{
pub fn new(source: Rc<dyn Source>) -> Semantics
{
Self {
cursor: LineCursor::new(source),
tokens: vec![]
}
}
pub fn add(&mut self, source: Rc<dyn Source>, range: Range<usize>, token_type: u32, token_modifier: u32)
{
let mut current = self.cursor.clone();
self.cursor.move_to(range.start);
while self.cursor.pos != range.end
{
let end = source.content()[self.cursor.pos..].find('\n')
.unwrap_or(source.content().len() - self.cursor.pos);
let len = usize::min(range.end - self.cursor.pos, end);
let delta_line = self.cursor.line - current.line;
let delta_start = if delta_line == 0
{
if let Some(last) = self.tokens.last()
{
self.cursor.line_pos - current.line_pos + last.length as usize
}
else
{
self.cursor.line_pos - current.line_pos
}
} else { self.cursor.line_pos };
eprintln!("CURRENT={:#?}, CURS={:#?}", current, self.cursor);
self.tokens.push(SemanticToken{
delta_line: delta_line as u32,
delta_start: delta_start as u32,
length: len as u32,
token_type,
token_modifiers_bitset: token_modifier,
});
current = self.cursor.clone();
self.cursor.move_to(self.cursor.pos + len);
}
}
}

2
src/parser/source.rs
View file

@ -199,7 +199,7 @@ impl LineCursor {
self.line_pos += c.width().unwrap_or(1);
self.pos += len;
eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' '));
//eprintln!("({}, {c:#?}, {} {}, {})", self.pos, self.line, self.line_pos, prev.unwrap_or(' '));
prev = Some(c);
}
if self.pos != 0 && prev == Some('\n') {

21
src/server.rs
View file

@ -6,18 +6,13 @@ mod lsp;
mod lua;
mod parser;
use std::collections::HashMap;
use std::rc::Rc;
use std::sync::Arc;
use dashmap::DashMap;
use document::document::Document;
use document::element::Element;
use lsp::semantic::semantic_token_from_document;
use lsp::semantic::Tokens;
use parser::langparser::LangParser;
use parser::parser::Parser;
use parser::parser::ParserState;
use parser::semantics::Semantics;
use parser::source::SourceFile;
use tower_lsp::jsonrpc::Result;
use tower_lsp::lsp_types::*;
@ -58,14 +53,14 @@ impl Backend {
.unwrap()
}) {
self.semantic_token_map
.insert(params.uri.to_string(), sems.tokens.to_owned());
.insert(params.uri.to_string(), sems.tokens.borrow().to_owned());
};
}
}
#[tower_lsp::async_trait]
impl LanguageServer for Backend {
async fn initialize(&self, _: InitializeParams) -> Result<InitializeResult> {
async fn initialize(&self, params: InitializeParams) -> Result<InitializeResult> {
Ok(InitializeResult {
server_info: None,
capabilities: ServerCapabilities {
@ -85,17 +80,17 @@ impl LanguageServer for Backend {
text_document_registration_options: {
TextDocumentRegistrationOptions {
document_selector: Some(vec![DocumentFilter {
language: Some("nml".to_string()),
scheme: Some("file".to_string()),
pattern: None,
language: Some("nml".into()),
scheme: Some("file".into()),
pattern: Some("*.nml".into()),
}]),
}
},
semantic_tokens_options: SemanticTokensOptions {
work_done_progress_options: WorkDoneProgressOptions::default(),
legend: SemanticTokensLegend {
token_types: lsp::semantic::LEGEND_TYPE.into(),
token_modifiers: vec![],
token_types: lsp::semantic::TOKEN_TYPE.into(),
token_modifiers: lsp::semantic::TOKEN_MODIFIERS.into(),
},
range: None, //Some(true),
full: Some(SemanticTokensFullOptions::Bool(true)),