ef3d0c3e/nml
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix bugs

Browse source
This commit is contained in:
ef3d0c3e 2024-10-19 22:02:10 +02:00
parent 8e45b001a3
commit 02b34d5424
7 changed files with 149 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -40,6 +40,7 @@ tower-lsp = "0.20.0"
unicode-segmentation = "1.11.0"
walkdir = "2.5.0"
runtime-format = "0.1.3"
unicode-width = "0.2.0"
[dev-dependencies]
rand = "0.8.5"

32
src/elements/comment.rs
View file

@ -26,7 +26,12 @@ impl Element for Comment {
fn location(&self) -> &Token { &self.location }
fn kind(&self) -> ElemKind { ElemKind::Invisible }
fn element_name(&self) -> &'static str { "Comment" }
fn compile(&self, _compiler: &Compiler, _document: &dyn Document, _cursor: usize) -> Result<String, String> {
fn compile(
&self,
_compiler: &Compiler,
_document: &dyn Document,
_cursor: usize,
) -> Result<String, String> {
Ok("".to_string())
}
}
@ -90,10 +95,11 @@ impl RegexRule for CommentRule {
}),
);
if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics)
if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{
let comment = matches.get(1).unwrap().range();
sems.add(comment.start-2..comment.end, tokens.comment);
sems.add(comment.start - 2..comment.end, tokens.comment);
}
reports
@ -108,7 +114,8 @@ mod tests {
use crate::parser::langparser::LangParser;
use crate::parser::parser::Parser;
use crate::parser::source::SourceFile;
use crate::{validate_document, validate_semantics};
use crate::validate_document;
use crate::validate_semantics;
use super::*;
@ -137,20 +144,23 @@ COMMENT ::Test
}
#[test]
fn semantic()
{
fn semantic() {
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
"".to_string(),
r#"
::Test
::Another
:: Another
"#
.to_string(),
None,
.to_string(),
None,
));
let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
let (_, state) = parser.parse(
ParserState::new_with_semantics(&parser, None),
source.clone(),
None,
);
validate_semantics!(state, source.clone(), 0,
comment { delta_line == 1, delta_start == 0, length == 6 };

88
src/elements/link.rs
View file

@ -38,7 +38,12 @@ impl Element for Link {
fn location(&self) -> &Token { &self.location }
fn kind(&self) -> ElemKind { ElemKind::Inline }
fn element_name(&self) -> &'static str { "Link" }
fn compile(&self, compiler: &Compiler, document: &dyn Document, cursor: usize) -> Result<String, String> {
fn compile(
&self,
compiler: &Compiler,
document: &dyn Document,
cursor: usize,
) -> Result<String, String> {
match compiler.target() {
Target::HTML => {
let mut result = format!(
@ -47,7 +52,9 @@ impl Element for Link {
);
for elem in &self.display {
result += elem.compile(compiler, document, cursor+result.len())?.as_str();
result += elem
.compile(compiler, document, cursor + result.len())?
.as_str();
}
result += "</a>";
@ -135,9 +142,13 @@ impl RegexRule for LinkRule {
return reports;
}
if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics)
if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{
sems.add(display.range().start-1..display.range().start, tokens.link_display_sep);
sems.add(
display.range().start - 1..display.range().start,
tokens.link_display_sep,
);
}
let source = Rc::new(VirtualSource::new(
Token::new(display.range(), token.source()),
@ -212,20 +223,18 @@ impl RegexRule for LinkRule {
}),
);
//if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
// RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
// .ok()
// .unwrap()
//}) {
// let name = matches.get(1).unwrap().range();
// sems.add(token.source(), name.start-1..name.start, sems.token.link_name_sep);
// sems.add(token.source(), name.clone(), sems.token.link_name);
// sems.add(token.source(), name.end..name.end+1, sems.token.link_name_sep);
// let url = matches.get(2).unwrap().range();
// sems.add(token.source(), url.start-1..url.start, sems.token.link_url_sep);
// sems.add(token.source(), url.clone(), sems.token.link_url);
// sems.add(token.source(), url.end..url.end+1, sems.token.link_url_sep);
//}
if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{
sems.add(
matches.get(1).unwrap().end()..matches.get(1).unwrap().end() + 1,
tokens.link_display_sep,
);
let url = matches.get(2).unwrap().range();
sems.add(url.start - 1..url.start, tokens.link_url_sep);
sems.add(url.clone(), tokens.link_url);
sems.add(url.end..url.end + 1, tokens.link_url_sep);
}
reports
}
@ -257,9 +266,7 @@ impl RegexRule for LinkRule {
});
return;
}
Ok(mut paragraph) => {
std::mem::take(&mut paragraph.content)
}
Ok(mut paragraph) => std::mem::take(&mut paragraph.content),
};
ctx.state.push(
@ -290,7 +297,8 @@ mod tests {
use crate::parser::langparser::LangParser;
use crate::parser::parser::Parser;
use crate::parser::source::SourceFile;
use crate::{validate_document, validate_semantics};
use crate::validate_document;
use crate::validate_semantics;
use super::*;
@ -353,8 +361,7 @@ nml.link.push("**BOLD link**", "another url")
}
#[test]
fn semantics()
{
fn semantics() {
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
@ -364,29 +371,20 @@ nml.link.push("**BOLD link**", "another url")
None,
));
let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
println!("{:#?}", state.shared.semantics);
/*
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
[link](url)
"#
.to_string(),
let (_, state) = parser.parse(
ParserState::new_with_semantics(&parser, None),
source.clone(),
None,
));
let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
);
validate_semantics!(state, source.clone(), 0,
link_name_sep { delta_line == 1, delta_start == 0, length == 1 };
link_name { delta_line == 0, delta_start == 1, length == 4 };
link_name_sep { delta_line == 0, delta_start == 4, length == 1 };
link_url_sep { delta_line == 0, delta_start == 1, length == 1 };
link_url { delta_line == 0, delta_start == 1, length == 3 };
link_url_sep { delta_line == 0, delta_start == 3, length == 1 };
);
*/
link_display_sep { delta_line == 1, delta_start == 0, length == 1 };
style_marker { delta_line == 0, delta_start == 3, length == 2 };
style_marker { delta_line == 0, delta_start == 3, length == 2 };
link_display_sep { delta_line == 0, delta_start == 3, length == 1 };
link_url_sep { delta_line == 0, delta_start == 1, length == 1 };
link_url { delta_line == 0, delta_start == 1, length == 3 };
link_url_sep { delta_line == 0, delta_start == 3, length == 1 };
);
}
}

99
src/elements/reference.rs
View file

@ -21,6 +21,7 @@ use crate::document::document::Document;
use crate::document::element::ElemKind;
use crate::document::element::Element;
use crate::document::references::validate_refname;
use crate::lsp::semantic::Semantics;
use crate::parser::parser::ParserState;
use crate::parser::parser::ReportColors;
use crate::parser::rule::RegexRule;
@ -311,7 +312,7 @@ impl RegexRule for ReferenceRule {
.downcast_rc::<reference_style::ExternalReferenceStyle>()
.unwrap();
// §{#refname}
// &{#refname}
if refdoc.is_empty() {
state.push(
document,
@ -322,7 +323,7 @@ impl RegexRule for ReferenceRule {
style,
}),
);
// §{docname#refname}
// &{docname#refname}
} else {
state.push(
document,
@ -335,25 +336,26 @@ impl RegexRule for ReferenceRule {
);
}
/*
if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
.ok()
.unwrap()
}) {
if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{
let link = matches.get(1).unwrap().range();
sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator);
sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep);
sems.add(link.start - 2..link.start - 1, tokens.reference_operator);
sems.add(link.start - 1..link.start, tokens.reference_link_sep);
if !refdoc.is_empty()
{
sems.add(token.source(), link.start.. refdoc.len()+link.start, sems.token.reference_doc);
if !refdoc.is_empty() {
sems.add(link.start..refdoc.len() + link.start, tokens.reference_doc);
}
sems.add(token.source(), refdoc.len()+link.start.. refdoc.len()+link.start+1, sems.token.reference_doc_sep);
sems.add(token.source(), refdoc.len()+link.start+1..link.end, sems.token.reference_link);
sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep);
sems.add(
refdoc.len() + link.start..refdoc.len() + link.start + 1,
tokens.reference_doc_sep,
);
sems.add(
refdoc.len() + link.start + 1..link.end,
tokens.reference_link,
);
sems.add(link.end..link.end + 1, tokens.reference_link_sep);
}
*/
} else {
state.push(
document,
@ -363,35 +365,26 @@ impl RegexRule for ReferenceRule {
caption,
}),
);
/*
if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
.ok()
.unwrap()
}) {
if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{
let link = matches.get(1).unwrap().range();
sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator);
sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep);
sems.add(token.source(), link.clone(), sems.token.reference_link);
sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep);
sems.add(link.start - 2..link.start - 1, tokens.reference_operator);
sems.add(link.start - 1..link.start, tokens.reference_link_sep);
sems.add(link.clone(), tokens.reference_link);
sems.add(link.end..link.end + 1, tokens.reference_link_sep);
}
*/
}
/*
if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
.ok()
.unwrap()
}) {
if let Some(props) = matches.get(2).map(|m| m.range())
{
sems.add(token.source(), props.start-1..props.start, sems.token.reference_props_sep);
sems.add(token.source(), props.clone(), sems.token.reference_props);
sems.add(token.source(), props.end..props.end+1, sems.token.reference_props_sep);
}
if let (Some((sems, tokens)), Some(props)) = (
Semantics::from_source(token.source(), &state.shared.semantics),
matches.get(2).map(|m| m.range()),
) {
sems.add(props.start - 1..props.start, tokens.reference_props_sep);
sems.add(props.clone(), tokens.reference_props);
sems.add(props.end..props.end + 1, tokens.reference_props_sep);
}
*/
reports
}
@ -446,9 +439,9 @@ mod tests {
r#"
#{ref} Referenceable section
§{ref}[caption=Section]
§{ref}[caption=Another]
§{ref2}[caption=Before]
&{ref}[caption=Section]
&{ref}[caption=Another]
&{ref2}[caption=Before]
#{ref2} Another section
"#
@ -475,9 +468,9 @@ mod tests {
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
§{DocA#ref}[caption=Section]
§{DocB#ref}
§{#ref}[caption='ref' from any document]
&{DocA#ref}[caption=Section]
&{DocB#ref}
&{#ref}[caption='ref' from any document]
"#
.to_string(),
None,
@ -510,8 +503,8 @@ mod tests {
@html.page_title = 1
@compiler.output = b.html
§{#ref}
§{a#ref}
&{#ref}
&{a#ref}
#{ref2} Another Referenceable section
"#
.into(),
@ -523,10 +516,10 @@ mod tests {
"format_specific": "[SPECIFIC {refdoc}:{refname}]"
}
§{#ref}[caption=from 0]
§{#ref}
§{#ref2}[caption=from 1]
§{b#ref2}
&{#ref}[caption=from 0]
&{#ref}
&{#ref2}[caption=from 1]
&{b#ref2}
"#
.into(),
],

5
src/elements/section.rs
View file

@ -552,7 +552,9 @@ nml.section.push("6", 6, "", "refname")
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
#{📫} test
# First section
##{📫}+ test
#{refname}*+ Another section
"#
.to_string(),
None,
@ -560,7 +562,6 @@ nml.section.push("6", 6, "", "refname")
let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
println!("{:#?}", state.shared.semantics);
validate_semantics!(state, source.clone(), 0,
section_heading { delta_line == 1, delta_start == 0, length == 1 };
section_name { delta_line == 0, delta_start == 1 };

21
src/lsp/semantic.rs
View file

@ -134,7 +134,7 @@ impl Tokens {
link_display_sep: token!("macro"),
link_url_sep: token!("macro"),
link_url: token!("operator", "readonly", "abstract", "abstract"),
link_url: token!("function", "readonly", "abstract", "abstract"),
style_marker: token!("operator"),
@ -170,10 +170,8 @@ pub struct SemanticsData {
pub tokens: RefCell<Vec<SemanticToken>>,
}
impl SemanticsData
{
pub fn new(source: Rc<dyn Source>) -> Self
{
impl SemanticsData {
pub fn new(source: Rc<dyn Source>) -> Self {
Self {
cursor: RefCell::new(LineCursor::new(source)),
tokens: RefCell::new(vec![]),
@ -192,9 +190,8 @@ impl<'a> Semantics<'a> {
fn from_source_impl(
source: Rc<dyn Source>,
semantics: &'a Option<RefCell<SemanticsHolder>>,
range: Range<usize>)
-> Option<(Self, Ref<'a, Tokens>)>
{
range: Range<usize>,
) -> Option<(Self, Ref<'a, Tokens>)> {
if let Some(location) = source
.clone()
.downcast_rc::<VirtualSource>()
@ -238,12 +235,13 @@ impl<'a> Semantics<'a> {
}
let range = source.location().map_or_else(
|| 0..source.content().len(),
|location| location.range.clone());
|location| location.range.clone(),
);
return Self::from_source_impl(source, semantics, range);
}
pub fn add(&self, range: Range<usize>, token: (u32, u32)) {
let range = self.range.start+range.start..self.range.start+range.end;
let range = self.range.start + range.start..self.range.start + range.end;
let mut tokens = self.sems.tokens.borrow_mut();
let mut cursor = self.sems.cursor.borrow_mut();
let mut current = cursor.clone();
@ -270,8 +268,7 @@ impl<'a> Semantics<'a> {
token_type: token.0,
token_modifiers_bitset: token.1,
});
if cursor.pos + len == range.end
{
if cursor.pos + len == range.end {
break;
}
current = cursor.clone();

89
src/parser/source.rs
View file

@ -5,6 +5,7 @@ use std::rc::Rc;
use downcast_rs::impl_downcast;
use downcast_rs::Downcast;
use unicode_width::UnicodeWidthChar;
/// Trait for source content
pub trait Source: Downcast + Debug {
@ -24,17 +25,13 @@ impl core::fmt::Display for dyn Source {
}
impl std::cmp::PartialEq for dyn Source {
fn eq(&self, other: &Self) -> bool {
self.name() == other.name()
}
fn eq(&self, other: &Self) -> bool { self.name() == other.name() }
}
impl std::cmp::Eq for dyn Source {}
impl std::hash::Hash for dyn Source {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.name().hash(state)
}
fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.name().hash(state) }
}
#[derive(Debug)]
@ -48,12 +45,7 @@ impl SourceFile {
// TODO: Create a SourceFileRegistry holding already loaded files to avoid reloading them
pub fn new(path: String, location: Option<Token>) -> Result<Self, String> {
match fs::read_to_string(&path) {
Err(_) => {
Err(format!(
"Unable to read file content: `{}`",
path
))
}
Err(_) => Err(format!("Unable to read file content: `{}`", path)),
Ok(content) => Ok(Self {
location,
path,
@ -70,22 +62,13 @@ impl SourceFile {
}
}
pub fn path(&self) -> &String
{
&self.path
}
pub fn path(&self) -> &String { &self.path }
}
impl Source for SourceFile {
fn location(&self) -> Option<&Token> {
self.location.as_ref()
}
fn name(&self) -> &String {
&self.path
}
fn content(&self) -> &String {
&self.content
}
fn location(&self) -> Option<&Token> { self.location.as_ref() }
fn name(&self) -> &String { &self.path }
fn content(&self) -> &String { &self.content }
}
#[derive(Debug)]
@ -106,15 +89,9 @@ impl VirtualSource {
}
impl Source for VirtualSource {
fn location(&self) -> Option<&Token> {
Some(&self.location)
}
fn name(&self) -> &String {
&self.name
}
fn content(&self) -> &String {
&self.content
}
fn location(&self) -> Option<&Token> { Some(&self.location) }
fn name(&self) -> &String { &self.name }
fn content(&self) -> &String { &self.content }
}
#[derive(Debug)]
@ -124,9 +101,7 @@ pub struct Cursor {
}
impl Cursor {
pub fn new(pos: usize, source: Rc<dyn Source>) -> Self {
Self { pos, source }
}
pub fn new(pos: usize, source: Rc<dyn Source>) -> Self { Self { pos, source } }
/// Creates [`cursor`] at [`new_pos`] in the same [`file`]
pub fn at(&self, new_pos: usize) -> Self {
@ -145,9 +120,7 @@ impl Clone for Cursor {
}
}
fn clone_from(&mut self, source: &Self) {
*self = source.clone()
}
fn clone_from(&mut self, source: &Self) { *self = source.clone() }
}
/// Cursor type used for the language server
@ -165,8 +138,7 @@ pub struct LineCursor {
impl LineCursor {
/// Creates a [`LineCursor`] at the begining of the source
pub fn new(source: Rc<dyn Source>) -> LineCursor
{
pub fn new(source: Rc<dyn Source>) -> LineCursor {
Self {
pos: 0,
line: 0,
@ -182,14 +154,9 @@ impl LineCursor {
pub fn move_to(&mut self, pos: usize) {
if self.pos < pos {
let start = self.pos;
let mut it = self.source.content().as_str()[start..]
.chars()
.peekable();
let mut it = self.source.content().as_str()[start..].chars().peekable();
let mut prev = self.source.content().as_str()[..start]
.chars()
.rev()
.next();
let mut prev = self.source.content().as_str()[..start].chars().rev().next();
while self.pos < pos {
let c = it.next().unwrap();
let len = c.len_utf8();
@ -198,7 +165,7 @@ impl LineCursor {
self.line += 1;
self.line_pos = 0;
}
self.line_pos += 1;
self.line_pos += c.width().unwrap_or(1);
self.pos += len;
prev = Some(c);
}
@ -214,9 +181,7 @@ impl LineCursor {
.rev()
.peekable();
let mut prev = self.source.content().as_str()[start..]
.chars()
.next();
let mut prev = self.source.content().as_str()[start..].chars().next();
while self.pos > pos {
let c = it.next().unwrap();
let len = c.len_utf8();
@ -225,7 +190,7 @@ impl LineCursor {
self.line -= 1;
self.line_pos = 0;
}
self.line_pos -= 1;
self.line_pos -= c.width().unwrap_or(1);
self.pos -= len;
prev = Some(c);
}
@ -247,13 +212,9 @@ pub struct Token {
}
impl Token {
pub fn new(range: Range<usize>, source: Rc<dyn Source>) -> Self {
Self { range, source }
}
pub fn new(range: Range<usize>, source: Rc<dyn Source>) -> Self { Self { range, source } }
pub fn source(&self) -> Rc<dyn Source> {
self.source.clone()
}
pub fn source(&self) -> Rc<dyn Source> { self.source.clone() }
/// Construct Token from a range
pub fn from(start: &Cursor, end: &Cursor) -> Self {
@ -265,11 +226,7 @@ impl Token {
}
}
pub fn start(&self) -> usize {
self.range.start
}
pub fn start(&self) -> usize { self.range.start }
pub fn end(&self) -> usize {
self.range.end
}
pub fn end(&self) -> usize { self.range.end }
}