From 461738dab93ed1587419dfd5c440a606644e36b1 Mon Sep 17 00:00:00 2001 From: ef3d0c3e Date: Sun, 21 Jul 2024 15:56:56 +0200 Subject: [PATCH] Lua & first bindings --- src/cache/cache.rs | 9 ++ src/document/element.rs | 30 ------- src/document/variable.rs | 4 +- src/elements/code.rs | 4 + src/elements/comment.rs | 3 + src/elements/import.rs | 3 + src/elements/link.rs | 5 ++ src/elements/list.rs | 4 + src/elements/mod.rs | 1 + src/elements/paragraph.rs | 4 + src/elements/raw.rs | 4 + src/elements/registrar.rs | 3 +- src/elements/script.rs | 168 +++++++++++++++++++++++--------------- src/elements/section.rs | 71 +++++++++++----- src/elements/style.rs | 3 + src/elements/tex.rs | 4 + src/elements/text.rs | 63 ++++++++++++++ src/elements/variable.rs | 10 ++- src/lsp/mod.rs | 1 + src/lsp/parser.rs | 124 +++++++++++++++++++++++----- src/lsp/semantic.rs | 90 ++++++++++++++++++++ src/lua/kernel.rs | 65 +++++++++++++-- src/parser/langparser.rs | 6 +- src/parser/parser.rs | 7 +- src/parser/rule.rs | 14 ++++ src/parser/source.rs | 10 +++ src/parser/state.rs | 10 ++- src/server.rs | 120 ++++++++------------------- 28 files changed, 601 insertions(+), 239 deletions(-) create mode 100644 src/elements/text.rs create mode 100644 src/lsp/semantic.rs diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 3c0494a..650422d 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -45,6 +45,15 @@ pub trait Cached .map(|_| ()) } + /// Attempts to retrieve a cached element from the compilation database + /// or create it (and insert it), if it doesn't exist + /// + /// # Error + /// + /// Will return an error if the database connection(s) fail, + /// or if not cached, an error from the generator [`f`] + /// + /// Note that on error, [`f`] may still have been called fn cached(&self, con: &mut Connection, f: F) -> Result<::Value, CachedError> where diff --git a/src/document/element.rs b/src/document/element.rs index 7befd80..e7e9abd 100644 --- a/src/document/element.rs +++ b/src/document/element.rs @@ -63,33 +63,3 @@ pub trait ReferenceableElement : Element { /// Reference name fn reference_name(&self) -> Option<&String>; } - -#[derive(Debug)] -pub struct Text -{ - location: Token, - content: String, -} - -impl Text -{ - pub fn new(location: Token, content: String) -> Text - { - Text { - location: location, - content: content - } - } -} - -impl Element for Text -{ - fn location(&self) -> &Token { &self.location } - fn kind(&self) -> ElemKind { ElemKind::Inline } - fn element_name(&self) -> &'static str { "Text" } - fn to_string(&self) -> String { format!("{self:#?}") } - - fn compile(&self, compiler: &Compiler, _document: &Document) -> Result { - Ok(compiler.sanitize(self.content.as_str())) - } -} diff --git a/src/document/variable.rs b/src/document/variable.rs index d2eb70c..5ce1af4 100644 --- a/src/document/variable.rs +++ b/src/document/variable.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, rc::Rc}; -use crate::parser::{parser::Parser, source::{Source, Token, VirtualSource}}; -use super::{document::Document, element::Text}; +use crate::{elements::text::Text, parser::{parser::Parser, source::{Source, Token, VirtualSource}}}; +use super::{document::Document}; // TODO enforce to_string(from_string(to_string())) == to_string() diff --git a/src/elements/code.rs b/src/elements/code.rs index a3ad380..e41a6cf 100644 --- a/src/elements/code.rs +++ b/src/elements/code.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, ops::Range, rc::Rc, sync::Once}; use ariadne::{Fmt, Label, Report, ReportKind}; use crypto::{digest::Digest, sha2::Sha512}; +use mlua::{Function, Lua}; use regex::{Captures, Regex}; use syntect::{easy::HighlightLines, highlighting::ThemeSet, parsing::SyntaxSet}; @@ -387,4 +388,7 @@ impl RegexRule for CodeRule reports } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/comment.rs b/src/elements/comment.rs index 15836f7..021f412 100644 --- a/src/elements/comment.rs +++ b/src/elements/comment.rs @@ -1,3 +1,4 @@ +use mlua::{Function, Lua}; use regex::{Captures, Regex}; use crate::parser::{parser::Parser, rule::RegexRule, source::{Source, Token}}; use ariadne::{Report, Label, ReportKind}; @@ -78,4 +79,6 @@ impl RegexRule for CommentRule { return reports; } + + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/import.rs b/src/elements/import.rs index 7db46f4..393e689 100644 --- a/src/elements/import.rs +++ b/src/elements/import.rs @@ -1,3 +1,4 @@ +use mlua::{Function, Lua}; use regex::Regex; use crate::parser::{parser::{Parser, ReportColors}, rule::RegexRule, source::{Source, SourceFile, Token}}; use ariadne::{Report, Fmt, Label, ReportKind}; @@ -152,4 +153,6 @@ impl RegexRule for ImportRule { return result; } + + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/link.rs b/src/elements/link.rs index 634dbab..8a5bf0a 100644 --- a/src/elements/link.rs +++ b/src/elements/link.rs @@ -1,4 +1,6 @@ +use mlua::{Function, Lua}; use regex::Regex; +use serde::{Deserialize, Serialize}; use crate::parser::{parser::Parser, rule::RegexRule, source::{Source, Token}, util}; use ariadne::{Report, Fmt, Label, ReportKind}; use crate::{compiler::compiler::{Compiler, Target}, document::{document::Document, element::{ElemKind, Element}}}; @@ -146,4 +148,7 @@ impl RegexRule for LinkRule { return result; } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/list.rs b/src/elements/list.rs index 0b44ddd..dd05ef8 100644 --- a/src/elements/list.rs +++ b/src/elements/list.rs @@ -2,6 +2,7 @@ use std::{any::Any, cell::Ref, ops::Range, rc::Rc}; use crate::{compiler::compiler::{Compiler, Target}, document::{document::Document, element::{ElemKind, Element}}, parser::{parser::Parser, rule::Rule, source::{Cursor, Source, Token, VirtualSource}}}; use ariadne::{Label, Report, ReportKind}; +use mlua::{Function, Lua}; use regex::Regex; use super::paragraph::Paragraph; @@ -332,4 +333,7 @@ impl Rule for ListRule (end_cursor, reports) } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/mod.rs b/src/elements/mod.rs index af33474..7561a42 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -1,4 +1,5 @@ pub mod registrar; +pub mod text; pub mod comment; pub mod paragraph; pub mod variable; diff --git a/src/elements/paragraph.rs b/src/elements/paragraph.rs index 449a452..c170160 100644 --- a/src/elements/paragraph.rs +++ b/src/elements/paragraph.rs @@ -1,6 +1,7 @@ use std::{any::Any, ops::Range, rc::Rc}; use ariadne::Report; +use mlua::{Function, Lua}; use regex::Regex; use crate::{compiler::compiler::{Compiler, Target}, document::{document::Document, element::{ElemKind, Element}}, parser::{parser::Parser, rule::Rule, source::{Cursor, Source, Token}}}; @@ -124,4 +125,7 @@ impl Rule for ParagraphRule (end_cursor, Vec::new()) } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/raw.rs b/src/elements/raw.rs index ed53894..c24df80 100644 --- a/src/elements/raw.rs +++ b/src/elements/raw.rs @@ -1,3 +1,4 @@ +use mlua::{Function, Lua}; use regex::{Captures, Regex}; use crate::{compiler::compiler::Compiler, document::element::{ElemKind, Element}, parser::{parser::Parser, rule::RegexRule, source::{Source, Token}, util::{self, Property, PropertyParser}}}; use ariadne::{Fmt, Label, Report, ReportKind}; @@ -161,4 +162,7 @@ impl RegexRule for RawRule reports } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/registrar.rs b/src/elements/registrar.rs index e3b7efc..5ea20c2 100644 --- a/src/elements/registrar.rs +++ b/src/elements/registrar.rs @@ -1,6 +1,6 @@ use crate::parser::parser::Parser; -use super::{code::CodeRule, comment::CommentRule, import::ImportRule, link::LinkRule, list::ListRule, paragraph::ParagraphRule, raw::RawRule, script::ScriptRule, section::SectionRule, style::StyleRule, tex::TexRule, variable::{VariableRule, VariableSubstitutionRule}}; +use super::{code::CodeRule, comment::CommentRule, import::ImportRule, link::LinkRule, list::ListRule, paragraph::ParagraphRule, raw::RawRule, script::ScriptRule, section::SectionRule, style::StyleRule, tex::TexRule, text::TextRule, variable::{VariableRule, VariableSubstitutionRule}}; pub fn register(parser: &mut P) @@ -19,4 +19,5 @@ pub fn register(parser: &mut P) parser.add_rule(Box::new(StyleRule::new()), None); parser.add_rule(Box::new(SectionRule::new()), None); parser.add_rule(Box::new(LinkRule::new()), None); + parser.add_rule(Box::new(TextRule::default()), None); } diff --git a/src/elements/script.rs b/src/elements/script.rs index 01cb39f..93bd96d 100644 --- a/src/elements/script.rs +++ b/src/elements/script.rs @@ -1,24 +1,28 @@ +use mlua::{Function, Lua}; use regex::{Captures, Regex}; -use crate::{document::element::Text, lua::kernel::{Kernel, KernelHolder}, parser::{parser::{Parser, ReportColors}, rule::RegexRule, source::{Source, Token, VirtualSource}, util}}; +use crate::{lua::kernel::{Kernel, KernelContext, KernelHolder}, parser::{parser::{Parser, ReportColors}, rule::RegexRule, source::{Source, Token, VirtualSource}, util}}; use ariadne::{Fmt, Label, Report, ReportKind}; use crate::document::document::Document; use std::{ops::Range, rc::Rc}; +use super::text::Text; + pub struct ScriptRule { re: [Regex; 2], - eval_kinds: [(&'static str, &'static str); 2] + eval_kinds: [(&'static str, &'static str); 3] } impl ScriptRule { pub fn new() -> Self { Self { re: [ - Regex::new(r"(?:^|\n)@<(?:(.*)\n?)((?:\\.|[^\[\]\\])*?)(?:\n?)>@").unwrap(), - Regex::new(r"%<([^\s[:alpha:]])?(?:\[(.*?)\])?((?:\\.|[^\[\]\\])*?)(?:\n?)>%").unwrap() + Regex::new(r"(?:^|\n)@<(?:(.*)\n?)((?:\\.|[^\\\\])*?)(?:\n?)>@").unwrap(), + Regex::new(r"%<([^\s[:alpha:]])?(?:\[(.*?)\])?((?:\\.|[^\\\\])*?)(?:\n?)>%").unwrap() ], eval_kinds: [ - ("", "Eval to text"), + ("", "Eval"), + ("\"", "Eval to text"), ("!", "Eval and parse"), ] } @@ -87,7 +91,7 @@ impl RegexRule for ScriptRule }) .unwrap_or("main"); let kernel = parser.get_kernel(kernel_name).unwrap_or_else(|| { - parser.insert_kernel(kernel_name.to_string(), Kernel::new()) + parser.insert_kernel(kernel_name.to_string(), Kernel::new(parser)) }); let kernel_data = matches.get(if index == 0 {2} else {3}) @@ -115,15 +119,16 @@ impl RegexRule for ScriptRule format!("{}#{}:lua_kernel@{kernel_name}", token.source().name(), matches.get(0).unwrap().start()), util::process_escaped('\\', ">@", kernel_content) )) as Rc; - - let chunk = kernel.lua.load(source.content()) - .set_name(kernel_name); - if index == 0 // @< ... >@ -> Exec + + let execute = |lua: &Lua| { - match chunk.exec() + let chunk = lua.load(source.content()) + .set_name(kernel_name); + + if index == 0 // Exec { - Ok(_) => {}, - Err(e) => { + if let Err(e) = chunk.exec() + { reports.push( Report::build(ReportKind::Error, source.clone(), 0) .with_message("Invalid kernel code") @@ -132,70 +137,99 @@ impl RegexRule for ScriptRule .with_message(format!("Kernel execution failed:\n{}", e.to_string())) .with_color(parser.colors().error)) .finish()); - } + return reports; + } } - } - else if index == 1 // %< ... >% -> Eval - { - let kind = match matches.get(1) { - None => 0, - Some(kind) => { - match self.validate_kind(parser.colors(), kind.as_str()) + else // Eval + { + // Validate kind + let kind = match matches.get(1) { + None => 0, + Some(kind) => { + match self.validate_kind(parser.colors(), kind.as_str()) + { + Ok(kind) => kind, + Err(msg) => { + reports.push( + Report::build(ReportKind::Error, token.source(), kind.start()) + .with_message("Invalid kernel code kind") + .with_label( + Label::new((token.source(), kind.range())) + .with_message(msg) + .with_color(parser.colors().error)) + .finish()); + return reports; + } + } + } + }; + + if kind == 0 // Eval + { + if let Err(e) = chunk.eval::<()>() { - Ok(kind) => kind, - Err(msg) => { + reports.push( + Report::build(ReportKind::Error, source.clone(), 0) + .with_message("Invalid kernel code") + .with_label( + Label::new((source.clone(), 0..source.content().len())) + .with_message(format!("Kernel evaluation failed:\n{}", e.to_string())) + .with_color(parser.colors().error)) + .finish()); + } + } + else // Eval to string + { + match chunk.eval::() + { + Ok(result) => { + if kind == 1 // Eval to text + { + if !result.is_empty() + { + parser.push(document, Box::new(Text::new( + Token::new(1..source.content().len(), source.clone()), + util::process_text(document, result.as_str()), + ))); + } + } + else if kind == 2 // Eval and Parse + { + let parse_source = Rc::new(VirtualSource::new( + Token::new(0..source.content().len(), source.clone()), + format!("parse({})", source.name()), + result + )) as Rc; + + parser.parse_into(parse_source, document); + } + }, + Err(e) => { reports.push( - Report::build(ReportKind::Error, token.source(), kind.start()) - .with_message("Invalid kernel code kind") + Report::build(ReportKind::Error, source.clone(), 0) + .with_message("Invalid kernel code") .with_label( - Label::new((token.source(), kind.range())) - .with_message(msg) + Label::new((source.clone(), 0..source.content().len())) + .with_message(format!("Kernel evaluation failed:\n{}", e.to_string())) .with_color(parser.colors().error)) .finish()); - return reports; } } } - }; - - match chunk.eval::() - { - Ok(result) => { - if kind == 0 // Eval to text - { - if !result.is_empty() - { - parser.push(document, Box::new(Text::new( - Token::new(1..source.content().len(), source.clone()), - util::process_text(document, result.as_str()), - ))); - } - } - else if kind == 1 // Eval and Parse - { - let parse_source = Rc::new(VirtualSource::new( - Token::new(0..source.content().len(), source.clone()), - format!("parse({})", source.name()), - result - )) as Rc; - //println!("SRC={parse_source:#?}, {}", parse_source.content()); - - parser.parse_into(parse_source, document); - } - }, - Err(e) => { - reports.push( - Report::build(ReportKind::Error, source.clone(), 0) - .with_message("Invalid kernel code") - .with_label( - Label::new((source.clone(), 0..source.content().len())) - .with_message(format!("Kernel evaluation failed:\n{}", e.to_string())) - .with_color(parser.colors().error)) - .finish()); - } } - } - reports + reports + }; + + let ctx = KernelContext { + location: Token::new(0..source.content().len(), source.clone()), + parser, + document + }; + + kernel.run_with_context(ctx, execute) } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/section.rs b/src/elements/section.rs index dfa794f..4ea478d 100644 --- a/src/elements/section.rs +++ b/src/elements/section.rs @@ -1,23 +1,17 @@ +use mlua::{Error::BadArgument, Function, Lua}; use regex::Regex; -use crate::{compiler::compiler::Target, parser::{parser::Parser, rule::RegexRule, source::{Source, Token}}}; +use crate::{compiler::compiler::Target, lua::kernel::CTX, parser::{parser::Parser, rule::RegexRule, source::{Source, Token}}}; use ariadne::{Report, Fmt, Label, ReportKind}; use crate::{compiler::compiler::Compiler, document::{document::Document, element::{ElemKind, Element, ReferenceableElement}}}; -use std::{ops::Range, rc::Rc}; +use std::{ops::Range, rc::Rc, sync::Arc}; #[derive(Debug)] pub struct Section { - location: Token, - title: String, // Section title - depth: usize, // Section depth - kind: u8, // Section kind, e.g numbered, unnumbred, ... - reference: Option, // Section reference name -} - -impl Section -{ - pub fn new(location: Token, title: String, depth: usize, kind: u8, reference: Option) -> Self { - Self { location: location, title, depth, kind, reference } - } + pub(self) location: Token, + pub(self) title: String, // Section title + pub(self) depth: usize, // Section depth + pub(self) kind: u8, // Section kind, e.g numbered, unnumbred, ... + pub(self) reference: Option, // Section reference name } impl Element for Section @@ -194,15 +188,50 @@ impl RegexRule for SectionRule { }; parser.push(document, Box::new( - Section::new( - token.clone(), - section_name, - section_depth, - section_kind, - section_refname - ) + Section { + location: token.clone(), + title: section_name, + depth: section_depth, + kind: section_kind, + reference: section_refname + } )); return result; } + + fn lua_bindings<'lua>(&self, lua: &'lua Lua) -> Vec<(String, Function<'lua>)> + { + let mut bindings = vec![]; + + bindings.push(("push".to_string(), lua.create_function( + |_, (title, depth, kind, reference) : (String, usize, String, Option)| { + let kind = match kind.as_str() { + "*+" | "+*" => section_kind::NO_NUMBER | section_kind::NO_TOC, + "*" => section_kind::NO_NUMBER, + "+" => section_kind::NO_TOC, + "" => section_kind::NONE, + _ => return Err(BadArgument { + to: Some("push".to_string()), + pos: 3, + name: Some("kind".to_string()), + cause: Arc::new(mlua::Error::external( + format!("Unknown section kind specified")))}) + }; + + CTX.with_borrow(|ctx| ctx.as_ref().map(|ctx| { + ctx.parser.push(ctx.document, Box::new(Section { + location: ctx.location.clone(), + title, + depth, + kind, + reference + })); + })); + + Ok(()) + }).unwrap())); + + bindings + } } diff --git a/src/elements/style.rs b/src/elements/style.rs index 631c147..6af05a1 100644 --- a/src/elements/style.rs +++ b/src/elements/style.rs @@ -1,3 +1,4 @@ +use mlua::{Function, Lua}; use regex::{Captures, Regex}; use crate::{compiler::compiler::{Compiler, Target}, document::element::{ElemKind, Element}, parser::{parser::Parser, rule::RegexRule, source::{Source, Token}, state::State}}; use ariadne::{Fmt, Label, Report, ReportKind}; @@ -182,4 +183,6 @@ impl RegexRule for StyleRule return result; } + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/tex.rs b/src/elements/tex.rs index 6e05370..9900233 100644 --- a/src/elements/tex.rs +++ b/src/elements/tex.rs @@ -2,6 +2,7 @@ use std::{io::{Read, Write}, ops::Range, process::{Command, Stdio}, rc::Rc, sync use ariadne::{Fmt, Label, Report, ReportKind}; use crypto::{digest::Digest, sha2::Sha512}; +use mlua::{Function, Lua}; use regex::{Captures, Regex}; use crate::{cache::cache::{Cached, CachedError}, compiler::compiler::{Compiler, Target}, document::{document::Document, element::{ElemKind, Element}}, parser::{parser::Parser, rule::RegexRule, source::{Source, Token}, util}}; @@ -260,4 +261,7 @@ impl RegexRule for TexRule reports } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/elements/text.rs b/src/elements/text.rs new file mode 100644 index 0000000..1eed8a5 --- /dev/null +++ b/src/elements/text.rs @@ -0,0 +1,63 @@ +use mlua::{Function, Lua}; + +use crate::{compiler::compiler::Compiler, document::{document::Document, element::{ElemKind, Element}}, lua::kernel::CTX, parser::{rule::Rule, source::Token}}; + +#[derive(Debug)] +pub struct Text +{ + pub(self) location: Token, + pub(self) content: String, +} + +impl Text +{ + pub fn new(location: Token, content: String) -> Text + { + Text { + location: location, + content: content + } + } +} + +impl Element for Text +{ + fn location(&self) -> &Token { &self.location } + fn kind(&self) -> ElemKind { ElemKind::Inline } + fn element_name(&self) -> &'static str { "Text" } + fn to_string(&self) -> String { format!("{self:#?}") } + + fn compile(&self, compiler: &Compiler, _document: &Document) -> Result { + Ok(compiler.sanitize(self.content.as_str())) + } +} + +#[derive(Default)] +pub struct TextRule; + +impl Rule for TextRule +{ + fn name(&self) -> &'static str { "Text" } + + fn next_match(&self, cursor: &crate::parser::source::Cursor) -> Option<(usize, Box)> { None } + + fn on_match(&self, parser: &dyn crate::parser::parser::Parser, document: &crate::document::document::Document, cursor: crate::parser::source::Cursor, match_data: Option>) -> (crate::parser::source::Cursor, Vec, std::ops::Range)>>) { panic!("Text canno match"); } + + fn lua_bindings<'lua>(&self, lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { + let mut bindings = vec![]; + + bindings.push(("push".to_string(), lua.create_function( + |_, content: String| { + CTX.with_borrow(|ctx| ctx.as_ref().map(|ctx| { + ctx.parser.push(ctx.document, Box::new(Text { + location: ctx.location.clone(), + content, + })); + })); + + Ok(()) + }).unwrap())); + + bindings + } +} diff --git a/src/elements/variable.rs b/src/elements/variable.rs index afb7f02..0a63daa 100644 --- a/src/elements/variable.rs +++ b/src/elements/variable.rs @@ -1,3 +1,4 @@ +use mlua::{Function, Lua}; use regex::Regex; use crate::parser::{parser::{Parser, ReportColors}, rule::RegexRule, source::{Source, Token}}; use ariadne::{Report, Fmt, Label, ReportKind}; @@ -20,7 +21,6 @@ impl VariableRule { } } - pub fn make_variable(&self, colors: &ReportColors, location: Token, kind: usize, name: String, value: String) -> Result, String> { match self.kinds[kind].0.as_str() @@ -89,6 +89,8 @@ impl RegexRule for VariableRule { fn regexes(&self) -> &[Regex] { &self.re } + + fn on_regex_match(&self, _: usize, parser: &dyn Parser, document: &Document, token: Token, matches: regex::Captures) -> Vec, Range)>> { let mut result = vec![]; @@ -197,6 +199,9 @@ impl RegexRule for VariableRule { return result; } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } pub struct VariableSubstitutionRule @@ -326,4 +331,7 @@ impl RegexRule for VariableSubstitutionRule return result; } + + // TODO + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { vec![] } } diff --git a/src/lsp/mod.rs b/src/lsp/mod.rs index 67c567f..3d10d7d 100644 --- a/src/lsp/mod.rs +++ b/src/lsp/mod.rs @@ -1 +1,2 @@ +pub mod semantic; pub mod parser; diff --git a/src/lsp/parser.rs b/src/lsp/parser.rs index 09a4146..320fd9b 100644 --- a/src/lsp/parser.rs +++ b/src/lsp/parser.rs @@ -1,30 +1,116 @@ -use std::{cell::RefCell, collections::HashMap}; +use std::rc::Rc; -use crate::{elements::registrar::register, lua::kernel::Kernel, parser::{rule::Rule, state::StateHolder}}; +use crate::parser::source::{Cursor, Source}; -struct LSParser +#[derive(Debug, Clone)] +pub struct LineCursor { - rules: Vec>, - - // Parser state - pub state: RefCell, - //pub kernels: RefCell>, + pub pos: usize, + pub line: usize, + pub line_pos: usize, + pub source: Rc, } -impl LSParser { - pub fn default() -> Self +impl LineCursor +{ + /// Creates [`LineCursor`] at position + /// + /// # Error + /// This function will panic if [`pos`] is not utf8 aligned + /// + /// Note: this is a convenience function, it should be used + /// with parsimony as it is expensive + pub fn at(&mut self, pos: usize) { - let mut parser = LSParser { - rules: vec![], - state: RefCell::new(StateHolder::new()), - //kernels: RefCell::new(HashMap::new()), - }; + if pos > self.pos + { + let start = self.pos; + //eprintln!("slice{{{}}}, want={pos}", &self.source.content().as_str()[start..pos]); + let mut it = self.source.content() + .as_str()[start..] // pos+1 + .chars() + .peekable(); - // TODO: Main kernel - //register(&mut parser); + let mut prev = self.source.content() + .as_str()[..start+1] + .chars() + .rev() + .next(); + //eprintln!("prev={prev:#?}"); + while self.pos < pos + { + let c = it.next().unwrap(); + let len = c.len_utf8(); - parser + self.pos += len; + if prev == Some('\n') + { + self.line += 1; + self.line_pos = 0; + } + else + { + self.line_pos += len; + } + + //eprintln!("({}, {c:#?}) ({} {})", self.pos, self.line, self.line_pos); + prev = Some(c); + } + + /* + self.source.content() + .as_str()[start..pos+1] + .char_indices() + .for_each(|(at, c)| { + self.pos = at+start; + + if c == '\n' + { + self.line += 1; + self.line_pos = 0; + } + else + { + self.line_pos += c.len_utf8(); + } + + }); + */ + } + else if pos < self.pos + { + todo!(""); + self.source.content() + .as_str()[pos..self.pos] + .char_indices() + .rev() + .for_each(|(len, c)| { + self.pos -= len; + if c == '\n' + { + self.line -= 1; + } + }); + self.line_pos = self.source.content() + .as_str()[..self.pos] + .char_indices() + .rev() + .find(|(_, c)| *c == '\n') + .map(|(line_start, _)| self.pos-line_start) + .unwrap_or(0); + } + + // May fail if pos is not utf8-aligned + assert_eq!(pos, self.pos); } } - +impl From<&LineCursor> for Cursor +{ + fn from(value: &LineCursor) -> Self { + Self { + pos: value.pos, + source: value.source.clone() + } + } +} diff --git a/src/lsp/semantic.rs b/src/lsp/semantic.rs new file mode 100644 index 0000000..49418ea --- /dev/null +++ b/src/lsp/semantic.rs @@ -0,0 +1,90 @@ +use std::any::Any; + +use tower_lsp::lsp_types::{SemanticToken, SemanticTokenType}; + +use crate::{document::{document::Document, element::Element}, elements::{comment::Comment, paragraph::Paragraph, section::Section}, parser::rule::Rule}; + +use super::parser::LineCursor; + +pub trait SemanticProvider: Rule +{ + fn get_semantic_tokens(&self, cursor: &LineCursor, match_data: Box) -> Vec; +} + +pub const LEGEND_TYPE : &[SemanticTokenType] = &[ + SemanticTokenType::COMMENT, + SemanticTokenType::VARIABLE, + SemanticTokenType::STRING, + SemanticTokenType::PARAMETER, +]; + +// TODO... +pub fn provide(semantic_tokens: &mut Vec, cursor: &mut LineCursor, elem: &Box) { + if cursor.source != elem.location().source() { return } + + let prev = cursor.clone(); + + if let Some(comm) = elem.downcast_ref::() + { + cursor.at(elem.location().start()); + let delta_start = if cursor.line == prev.line + { + cursor.line_pos - prev.line_pos + } else if cursor.line == 0 { cursor.line_pos } + else { cursor.line_pos+1 }; + semantic_tokens.push(SemanticToken { + delta_line: (cursor.line-prev.line) as u32, + delta_start: delta_start as u32, + length: (elem.location().end() - elem.location().start()) as u32, + token_type: 0, + token_modifiers_bitset: 0, + }); + } + else if let Some(sect) = elem.downcast_ref::
() + { + eprintln!("section"); + cursor.at(elem.location().start()); + let delta_start = if cursor.line == prev.line + { + cursor.line_pos - prev.line_pos + } else if cursor.line == 0 { cursor.line_pos } + else { cursor.line_pos+1 }; + semantic_tokens.push(SemanticToken { + delta_line: (cursor.line-prev.line) as u32, + delta_start: delta_start as u32, + length: (elem.location().end() - elem.location().start()) as u32, + token_type: 0, + token_modifiers_bitset: 0, + }); + } +} + +pub fn semantic_token_from_document(document: &Document) -> Vec +{ + let mut semantic_tokens = vec![]; + + let source = document.source(); + let mut cursor = LineCursor { + pos: 0, + line: 0, + line_pos: 0, + source: source.clone() + }; + + document.content.borrow() + .iter() + .for_each(|elem| { + if let Some(paragraph) = elem.downcast_ref::() + { + paragraph.content + .iter() + .for_each(|elem| provide(&mut semantic_tokens, &mut cursor, elem)); + } + else + { + provide(&mut semantic_tokens, &mut cursor, elem); + } + }); + + semantic_tokens +} diff --git a/src/lua/kernel.rs b/src/lua/kernel.rs index 18070bb..7ac6b7b 100644 --- a/src/lua/kernel.rs +++ b/src/lua/kernel.rs @@ -1,16 +1,71 @@ -use std::cell::RefMut; +use std::{cell::{RefCell, RefMut}, rc::Rc}; -use mlua::Lua; +use mlua::{Error, FromLua, Lua, UserData, UserDataMethods}; +use crate::{document::document::Document, parser::{parser::Parser, source::Token}}; + +pub struct KernelContext<'a> +{ + pub location: Token, + pub parser: &'a dyn Parser, + pub document: &'a Document<'a>, + //pub parser: &'a dyn Parser, +} + +thread_local! { + pub static CTX: RefCell>> = RefCell::new(None); +} + +#[derive(Debug)] pub struct Kernel { - pub lua: Lua, + lua: Lua, } impl Kernel { - pub fn new() -> Self { - Self { lua: Lua::new() } + + // TODO: Take parser as arg and + // iterate over the rules + // to find export the bindings (if some) + pub fn new(parser: &dyn Parser) -> Self { + let lua = Lua::new(); + + { + let nml_table = lua.create_table().unwrap(); + + for rule in parser.rules() + { + let table = lua.create_table().unwrap(); + let name = rule.name().to_lowercase(); + + for (fun_name, fun) in rule.lua_bindings(&lua) + { + table.set(fun_name, fun).unwrap(); + } + + nml_table.set(name, table).unwrap(); + } + lua.globals().set("nml", nml_table).unwrap(); + } + + Self { lua } } + + /// Runs a procedure with a context + /// + /// This is the only way lua code shoule be ran, because exported + /// functions may require the context in order to operate + pub fn run_with_context(&self, context: KernelContext, f: F) + -> T + where + F: FnOnce(&Lua) -> T + { + CTX.set(Some(unsafe { std::mem::transmute(context) })); + let ret = f(&self.lua); + CTX.set(None); + + ret + } } pub trait KernelHolder diff --git a/src/parser/langparser.rs b/src/parser/langparser.rs index 57f6e55..2789a39 100644 --- a/src/parser/langparser.rs +++ b/src/parser/langparser.rs @@ -2,11 +2,12 @@ use std::{cell::{RefCell, RefMut}, collections::{HashMap, HashSet}, ops::Range, use ariadne::{Label, Report}; -use crate::{document::{document::Document, element::{ElemKind, Element, Text}}, elements::{paragraph::Paragraph, registrar::register}, lua::kernel::{Kernel, KernelHolder}, parser::source::{SourceFile, VirtualSource}}; +use crate::{document::{document::Document, element::{ElemKind, Element}}, elements::{paragraph::Paragraph, registrar::register, text::Text}, lua::kernel::{Kernel, KernelHolder}, parser::source::{SourceFile, VirtualSource}}; use super::{parser::{Parser, ReportColors}, rule::Rule, source::{Cursor, Source, Token}, state::StateHolder, util}; /// Parser for the language +#[derive(Debug)] pub struct LangParser { rules: Vec>, @@ -30,8 +31,9 @@ impl LangParser kernels: RefCell::new(HashMap::new()), }; register(&mut s); + s.kernels.borrow_mut() - .insert("main".to_string(), Kernel::new()); + .insert("main".to_string(), Kernel::new(&s)); s } diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 05ac33f..aab0ba3 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -1,7 +1,5 @@ use std::any::Any; -use std::cell::{Ref, RefCell, RefMut}; -use std::collections::{HashMap, HashSet}; -use std::ops::Range; +use std::cell::{Ref, RefMut}; use std::rc::Rc; use unicode_segmentation::UnicodeSegmentation; @@ -11,8 +9,9 @@ use super::state::StateHolder; use crate::document::document::Document; use crate::document::element::Element; use ariadne::Color; -use crate::lua::kernel::{Kernel, KernelHolder}; +use crate::lua::kernel::KernelHolder; +#[derive(Debug)] pub struct ReportColors { pub error: Color, diff --git a/src/parser/rule.rs b/src/parser/rule.rs index 682f98f..c344800 100644 --- a/src/parser/rule.rs +++ b/src/parser/rule.rs @@ -1,6 +1,7 @@ use super::parser::Parser; use super::source::{Cursor, Source, Token}; use ariadne::Report; +use mlua::{Function, Lua}; use crate::document::document::Document; use std::any::Any; @@ -14,6 +15,15 @@ pub trait Rule { fn next_match(&self, cursor: &Cursor) -> Option<(usize, Box)>; /// Callback when rule matches fn on_match(&self, parser: &dyn Parser, document: &Document, cursor: Cursor, match_data: Option>) -> (Cursor, Vec, Range)>>); + /// Export bindings to lua + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)>; +} + +impl core::fmt::Debug for dyn Rule +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Rule{{{}}}", self.name()) + } } /* @@ -64,6 +74,8 @@ pub trait RegexRule /// Callback on regex rule match fn on_regex_match(&self, index: usize, parser: &dyn Parser, document: &Document, token: Token, matches: regex::Captures) -> Vec, Range)>>; + + fn lua_bindings<'lua>(&self, _lua: &'lua Lua) -> Vec<(String, Function<'lua>)>; } impl Rule for T { @@ -100,4 +112,6 @@ impl Rule for T { let token_end = token.end(); return (cursor.at(token_end), self.on_regex_match(*index, parser, document, token, captures)); } + + fn lua_bindings<'lua>(&self, lua: &'lua Lua) -> Vec<(String, Function<'lua>)> { self.lua_bindings(lua) } } diff --git a/src/parser/source.rs b/src/parser/source.rs index 5ed6104..5e12acb 100644 --- a/src/parser/source.rs +++ b/src/parser/source.rs @@ -2,6 +2,7 @@ use std::{fs, ops::Range, rc::Rc}; use core::fmt::Debug; use downcast_rs::{impl_downcast, Downcast}; +use serde::{Deserialize, Serialize}; /// Trait for source content pub trait Source: Downcast @@ -69,6 +70,15 @@ impl SourceFile }), } } + + pub fn with_content(path: String, content: String, location: Option) -> Self + { + Self { + location: location, + path: path, + content: content, + } + } } impl Source for SourceFile diff --git a/src/parser/state.rs b/src/parser/state.rs index 975500e..2ec01f9 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -8,7 +8,7 @@ use crate::document::document::Document; use super::{parser::Parser, source::Source}; /// Scope for state objects -#[derive(PartialEq, PartialOrd)] +#[derive(PartialEq, PartialOrd, Debug)] pub enum Scope { /// Global state @@ -31,7 +31,15 @@ pub trait State: Downcast } impl_downcast!(State); +impl core::fmt::Debug for dyn State +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "State{{Scope: {:#?}}}", self.scope()) + } +} + /// Object owning all the states +#[derive(Debug)] pub struct StateHolder { data: HashMap>> diff --git a/src/server.rs b/src/server.rs index abe0004..b73e404 100644 --- a/src/server.rs +++ b/src/server.rs @@ -12,7 +12,12 @@ use std::rc::Rc; use std::sync::Arc; use dashmap::DashMap; -use document::variable::Variable; +use document::document::Document; +use document::element::Element; +use lsp::semantic::{semantic_token_from_document, LEGEND_TYPE}; +use parser::langparser::LangParser; +use parser::parser::Parser; +use parser::source::SourceFile; use tower_lsp::jsonrpc::Result; use tower_lsp::lsp_types::*; use tower_lsp::{Client, LanguageServer, LspService, Server}; @@ -21,87 +26,35 @@ use tower_lsp::{Client, LanguageServer, LspService, Server}; struct Backend { client: Client, document_map: DashMap, + //ast_map: DashMap>>, //variables: DashMap>>, + semantic_token_map: DashMap>, } #[derive(Debug)] struct TextDocumentItem { uri: Url, text: String, - version: i32, } impl Backend { async fn on_change(&self, params: TextDocumentItem) { self.document_map .insert(params.uri.to_string(), params.text.clone()); - let ParserResult { - ast, - parse_errors, - semantic_tokens, - } = parse(¶ms.text); - let diagnostics = parse_errors - .into_iter() - .filter_map(|item| { - let (message, span) = match item.reason() { - chumsky::error::SimpleReason::Unclosed { span, delimiter } => { - (format!("Unclosed delimiter {}", delimiter), span.clone()) - } - chumsky::error::SimpleReason::Unexpected => ( - format!( - "{}, expected {}", - if item.found().is_some() { - "Unexpected token in input" - } else { - "Unexpected end of input" - }, - if item.expected().len() == 0 { - "something else".to_string() - } else { - item.expected() - .map(|expected| match expected { - Some(expected) => expected.to_string(), - None => "end of input".to_string(), - }) - .collect::>() - .join(", ") - } - ), - item.span(), - ), - chumsky::error::SimpleReason::Custom(msg) => (msg.to_string(), item.span()), - }; - || -> Option { - // let start_line = rope.try_char_to_line(span.start)?; - // let first_char = rope.try_line_to_char(start_line)?; - // let start_column = span.start - first_char; - let start_position = offset_to_position(span.start, &rope)?; - let end_position = offset_to_position(span.end, &rope)?; - // let end_line = rope.try_char_to_line(span.end)?; - // let first_char = rope.try_line_to_char(end_line)?; - // let end_column = span.end - first_char; - Some(Diagnostic::new_simple( - Range::new(start_position, end_position), - message, - )) - }() - }) - .collect::>(); - - self.client - .publish_diagnostics(params.uri.clone(), diagnostics, Some(params.version)) - .await; - - if let Some(ast) = ast { - self.ast_map.insert(params.uri.to_string(), ast); - } - // self.client - // .log_message(MessageType::INFO, &format!("{:?}", semantic_tokens)) - // .await; - self.semantic_token_map - .insert(params.uri.to_string(), semantic_tokens); - } + // TODO: Create a custom parser for the lsp + // Which will require a dyn Document to work + let source = SourceFile::with_content( + params.uri.to_string(), + params.text.clone(), + None); + let parser = LangParser::default(); + let doc = parser.parse(Rc::new(source), None); + + let semantic_tokens = semantic_token_from_document(&doc); + self.semantic_token_map + .insert(params.uri.to_string(), semantic_tokens); + } } #[tower_lsp::async_trait] @@ -135,7 +88,7 @@ impl LanguageServer for Backend { semantic_tokens_options: SemanticTokensOptions { work_done_progress_options: WorkDoneProgressOptions::default(), legend: SemanticTokensLegend { - token_types: vec![SemanticTokenType::COMMENT, SemanticTokenType::MACRO], + token_types: LEGEND_TYPE.into(), token_modifiers: vec![], }, range: None, //Some(true), @@ -167,7 +120,6 @@ impl LanguageServer for Backend { self.on_change(TextDocumentItem { uri: params.text_document.uri, text: params.text_document.text, - version: params.text_document.version, }) .await } @@ -176,7 +128,6 @@ impl LanguageServer for Backend { self.on_change(TextDocumentItem { uri: params.text_document.uri, text: std::mem::take(&mut params.content_changes[0].text), - version: params.text_document.version, }) .await } @@ -200,22 +151,17 @@ impl LanguageServer for Backend { self.client .log_message(MessageType::LOG, "semantic_token_full") .await; - let semantic_tokens = || -> Option> { - let semantic_tokens = vec![ - SemanticToken { - delta_line: 1, - delta_start: 2, - length: 5, - token_type: 1, - token_modifiers_bitset: 0, - } - ]; - Some(semantic_tokens) - }(); - if let Some(semantic_token) = semantic_tokens { + + if let Some(semantic_tokens) = self.semantic_token_map.get(&uri) { + let data = semantic_tokens.iter() + .filter_map(|token| { + Some(token.clone()) + }) + .collect::>(); + return Ok(Some(SemanticTokensResult::Tokens(SemanticTokens { result_id: None, - data: semantic_token, + data: data, }))); } Ok(None) @@ -230,7 +176,9 @@ async fn main() { let (service, socket) = LspService::new( |client| Backend { - client + client, + document_map: DashMap::new(), + semantic_token_map: DashMap::new(), }); Server::new(stdin, stdout, socket).serve(service).await; }