diff --git a/Cargo.toml b/Cargo.toml index e2f2d5f..4ad8676 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ tower-lsp = "0.20.0" unicode-segmentation = "1.11.0" walkdir = "2.5.0" runtime-format = "0.1.3" +unicode-width = "0.2.0" [dev-dependencies] rand = "0.8.5" diff --git a/src/elements/comment.rs b/src/elements/comment.rs index 0ff57e7..6aacd6c 100644 --- a/src/elements/comment.rs +++ b/src/elements/comment.rs @@ -26,7 +26,12 @@ impl Element for Comment { fn location(&self) -> &Token { &self.location } fn kind(&self) -> ElemKind { ElemKind::Invisible } fn element_name(&self) -> &'static str { "Comment" } - fn compile(&self, _compiler: &Compiler, _document: &dyn Document, _cursor: usize) -> Result { + fn compile( + &self, + _compiler: &Compiler, + _document: &dyn Document, + _cursor: usize, + ) -> Result { Ok("".to_string()) } } @@ -90,10 +95,11 @@ impl RegexRule for CommentRule { }), ); - if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics) + if let Some((sems, tokens)) = + Semantics::from_source(token.source(), &state.shared.semantics) { let comment = matches.get(1).unwrap().range(); - sems.add(comment.start-2..comment.end, tokens.comment); + sems.add(comment.start - 2..comment.end, tokens.comment); } reports @@ -108,7 +114,8 @@ mod tests { use crate::parser::langparser::LangParser; use crate::parser::parser::Parser; use crate::parser::source::SourceFile; - use crate::{validate_document, validate_semantics}; + use crate::validate_document; + use crate::validate_semantics; use super::*; @@ -137,20 +144,23 @@ COMMENT ::Test } #[test] - fn semantic() - { + fn semantic() { let source = Rc::new(SourceFile::with_content( - "".to_string(), - r#" + "".to_string(), + r#" ::Test ::Another :: Another "# - .to_string(), - None, + .to_string(), + None, )); let parser = LangParser::default(); - let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); + let (_, state) = parser.parse( + ParserState::new_with_semantics(&parser, None), + source.clone(), + None, + ); validate_semantics!(state, source.clone(), 0, comment { delta_line == 1, delta_start == 0, length == 6 }; diff --git a/src/elements/link.rs b/src/elements/link.rs index b2e4ae1..b739d8b 100644 --- a/src/elements/link.rs +++ b/src/elements/link.rs @@ -38,7 +38,12 @@ impl Element for Link { fn location(&self) -> &Token { &self.location } fn kind(&self) -> ElemKind { ElemKind::Inline } fn element_name(&self) -> &'static str { "Link" } - fn compile(&self, compiler: &Compiler, document: &dyn Document, cursor: usize) -> Result { + fn compile( + &self, + compiler: &Compiler, + document: &dyn Document, + cursor: usize, + ) -> Result { match compiler.target() { Target::HTML => { let mut result = format!( @@ -47,7 +52,9 @@ impl Element for Link { ); for elem in &self.display { - result += elem.compile(compiler, document, cursor+result.len())?.as_str(); + result += elem + .compile(compiler, document, cursor + result.len())? + .as_str(); } result += ""; @@ -135,9 +142,13 @@ impl RegexRule for LinkRule { return reports; } - if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics) + if let Some((sems, tokens)) = + Semantics::from_source(token.source(), &state.shared.semantics) { - sems.add(display.range().start-1..display.range().start, tokens.link_display_sep); + sems.add( + display.range().start - 1..display.range().start, + tokens.link_display_sep, + ); } let source = Rc::new(VirtualSource::new( Token::new(display.range(), token.source()), @@ -212,20 +223,18 @@ impl RegexRule for LinkRule { }), ); - //if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { - // RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) - // .ok() - // .unwrap() - //}) { - // let name = matches.get(1).unwrap().range(); - // sems.add(token.source(), name.start-1..name.start, sems.token.link_name_sep); - // sems.add(token.source(), name.clone(), sems.token.link_name); - // sems.add(token.source(), name.end..name.end+1, sems.token.link_name_sep); - // let url = matches.get(2).unwrap().range(); - // sems.add(token.source(), url.start-1..url.start, sems.token.link_url_sep); - // sems.add(token.source(), url.clone(), sems.token.link_url); - // sems.add(token.source(), url.end..url.end+1, sems.token.link_url_sep); - //} + if let Some((sems, tokens)) = + Semantics::from_source(token.source(), &state.shared.semantics) + { + sems.add( + matches.get(1).unwrap().end()..matches.get(1).unwrap().end() + 1, + tokens.link_display_sep, + ); + let url = matches.get(2).unwrap().range(); + sems.add(url.start - 1..url.start, tokens.link_url_sep); + sems.add(url.clone(), tokens.link_url); + sems.add(url.end..url.end + 1, tokens.link_url_sep); + } reports } @@ -257,9 +266,7 @@ impl RegexRule for LinkRule { }); return; } - Ok(mut paragraph) => { - std::mem::take(&mut paragraph.content) - } + Ok(mut paragraph) => std::mem::take(&mut paragraph.content), }; ctx.state.push( @@ -290,7 +297,8 @@ mod tests { use crate::parser::langparser::LangParser; use crate::parser::parser::Parser; use crate::parser::source::SourceFile; - use crate::{validate_document, validate_semantics}; + use crate::validate_document; + use crate::validate_semantics; use super::*; @@ -353,8 +361,7 @@ nml.link.push("**BOLD link**", "another url") } #[test] - fn semantics() - { + fn semantics() { let source = Rc::new(SourceFile::with_content( "".to_string(), r#" @@ -364,29 +371,20 @@ nml.link.push("**BOLD link**", "another url") None, )); let parser = LangParser::default(); - let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); - - println!("{:#?}", state.shared.semantics); - /* - let source = Rc::new(SourceFile::with_content( - "".to_string(), - r#" -[link](url) - "# - .to_string(), + let (_, state) = parser.parse( + ParserState::new_with_semantics(&parser, None), + source.clone(), None, - )); - let parser = LangParser::default(); - let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); + ); validate_semantics!(state, source.clone(), 0, - link_name_sep { delta_line == 1, delta_start == 0, length == 1 }; - link_name { delta_line == 0, delta_start == 1, length == 4 }; - link_name_sep { delta_line == 0, delta_start == 4, length == 1 }; - link_url_sep { delta_line == 0, delta_start == 1, length == 1 }; - link_url { delta_line == 0, delta_start == 1, length == 3 }; - link_url_sep { delta_line == 0, delta_start == 3, length == 1 }; - ); - */ + link_display_sep { delta_line == 1, delta_start == 0, length == 1 }; + style_marker { delta_line == 0, delta_start == 3, length == 2 }; + style_marker { delta_line == 0, delta_start == 3, length == 2 }; + link_display_sep { delta_line == 0, delta_start == 3, length == 1 }; + link_url_sep { delta_line == 0, delta_start == 1, length == 1 }; + link_url { delta_line == 0, delta_start == 1, length == 3 }; + link_url_sep { delta_line == 0, delta_start == 3, length == 1 }; + ); } } diff --git a/src/elements/reference.rs b/src/elements/reference.rs index b73cb3e..6cc1aaf 100644 --- a/src/elements/reference.rs +++ b/src/elements/reference.rs @@ -21,6 +21,7 @@ use crate::document::document::Document; use crate::document::element::ElemKind; use crate::document::element::Element; use crate::document::references::validate_refname; +use crate::lsp::semantic::Semantics; use crate::parser::parser::ParserState; use crate::parser::parser::ReportColors; use crate::parser::rule::RegexRule; @@ -311,7 +312,7 @@ impl RegexRule for ReferenceRule { .downcast_rc::() .unwrap(); - // §{#refname} + // &{#refname} if refdoc.is_empty() { state.push( document, @@ -322,7 +323,7 @@ impl RegexRule for ReferenceRule { style, }), ); - // §{docname#refname} + // &{docname#refname} } else { state.push( document, @@ -335,25 +336,26 @@ impl RegexRule for ReferenceRule { ); } - /* - if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { - RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) - .ok() - .unwrap() - }) { + if let Some((sems, tokens)) = + Semantics::from_source(token.source(), &state.shared.semantics) + { let link = matches.get(1).unwrap().range(); - sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator); - sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep); + sems.add(link.start - 2..link.start - 1, tokens.reference_operator); + sems.add(link.start - 1..link.start, tokens.reference_link_sep); - if !refdoc.is_empty() - { - sems.add(token.source(), link.start.. refdoc.len()+link.start, sems.token.reference_doc); + if !refdoc.is_empty() { + sems.add(link.start..refdoc.len() + link.start, tokens.reference_doc); } - sems.add(token.source(), refdoc.len()+link.start.. refdoc.len()+link.start+1, sems.token.reference_doc_sep); - sems.add(token.source(), refdoc.len()+link.start+1..link.end, sems.token.reference_link); - sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep); + sems.add( + refdoc.len() + link.start..refdoc.len() + link.start + 1, + tokens.reference_doc_sep, + ); + sems.add( + refdoc.len() + link.start + 1..link.end, + tokens.reference_link, + ); + sems.add(link.end..link.end + 1, tokens.reference_link_sep); } - */ } else { state.push( document, @@ -363,35 +365,26 @@ impl RegexRule for ReferenceRule { caption, }), ); - /* - if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { - RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) - .ok() - .unwrap() - }) { + + if let Some((sems, tokens)) = + Semantics::from_source(token.source(), &state.shared.semantics) + { let link = matches.get(1).unwrap().range(); - sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator); - sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep); - sems.add(token.source(), link.clone(), sems.token.reference_link); - sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep); + sems.add(link.start - 2..link.start - 1, tokens.reference_operator); + sems.add(link.start - 1..link.start, tokens.reference_link_sep); + sems.add(link.clone(), tokens.reference_link); + sems.add(link.end..link.end + 1, tokens.reference_link_sep); } - */ } - /* - if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { - RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) - .ok() - .unwrap() - }) { - if let Some(props) = matches.get(2).map(|m| m.range()) - { - sems.add(token.source(), props.start-1..props.start, sems.token.reference_props_sep); - sems.add(token.source(), props.clone(), sems.token.reference_props); - sems.add(token.source(), props.end..props.end+1, sems.token.reference_props_sep); - } + if let (Some((sems, tokens)), Some(props)) = ( + Semantics::from_source(token.source(), &state.shared.semantics), + matches.get(2).map(|m| m.range()), + ) { + sems.add(props.start - 1..props.start, tokens.reference_props_sep); + sems.add(props.clone(), tokens.reference_props); + sems.add(props.end..props.end + 1, tokens.reference_props_sep); } - */ reports } @@ -446,9 +439,9 @@ mod tests { r#" #{ref} Referenceable section -§{ref}[caption=Section] -§{ref}[caption=Another] -§{ref2}[caption=Before] +&{ref}[caption=Section] +&{ref}[caption=Another] +&{ref2}[caption=Before] #{ref2} Another section "# @@ -475,9 +468,9 @@ mod tests { let source = Rc::new(SourceFile::with_content( "".to_string(), r#" -§{DocA#ref}[caption=Section] -§{DocB#ref} -§{#ref}[caption='ref' from any document] +&{DocA#ref}[caption=Section] +&{DocB#ref} +&{#ref}[caption='ref' from any document] "# .to_string(), None, @@ -510,8 +503,8 @@ mod tests { @html.page_title = 1 @compiler.output = b.html -§{#ref} -§{a#ref} +&{#ref} +&{a#ref} #{ref2} Another Referenceable section "# .into(), @@ -523,10 +516,10 @@ mod tests { "format_specific": "[SPECIFIC {refdoc}:{refname}]" } -§{#ref}[caption=from 0] -§{#ref} -§{#ref2}[caption=from 1] -§{b#ref2} +&{#ref}[caption=from 0] +&{#ref} +&{#ref2}[caption=from 1] +&{b#ref2} "# .into(), ], diff --git a/src/elements/section.rs b/src/elements/section.rs index e15cd97..6ba21b8 100644 --- a/src/elements/section.rs +++ b/src/elements/section.rs @@ -552,7 +552,9 @@ nml.section.push("6", 6, "", "refname") let source = Rc::new(SourceFile::with_content( "".to_string(), r#" -#{📫} test +# First section +##{📫}+ test +#{refname}*+ Another section "# .to_string(), None, @@ -560,7 +562,6 @@ nml.section.push("6", 6, "", "refname") let parser = LangParser::default(); let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); - println!("{:#?}", state.shared.semantics); validate_semantics!(state, source.clone(), 0, section_heading { delta_line == 1, delta_start == 0, length == 1 }; section_name { delta_line == 0, delta_start == 1 }; diff --git a/src/lsp/semantic.rs b/src/lsp/semantic.rs index 745c855..d4f2457 100644 --- a/src/lsp/semantic.rs +++ b/src/lsp/semantic.rs @@ -134,7 +134,7 @@ impl Tokens { link_display_sep: token!("macro"), link_url_sep: token!("macro"), - link_url: token!("operator", "readonly", "abstract", "abstract"), + link_url: token!("function", "readonly", "abstract", "abstract"), style_marker: token!("operator"), @@ -170,10 +170,8 @@ pub struct SemanticsData { pub tokens: RefCell>, } -impl SemanticsData -{ - pub fn new(source: Rc) -> Self - { +impl SemanticsData { + pub fn new(source: Rc) -> Self { Self { cursor: RefCell::new(LineCursor::new(source)), tokens: RefCell::new(vec![]), @@ -192,9 +190,8 @@ impl<'a> Semantics<'a> { fn from_source_impl( source: Rc, semantics: &'a Option>, - range: Range) - -> Option<(Self, Ref<'a, Tokens>)> - { + range: Range, + ) -> Option<(Self, Ref<'a, Tokens>)> { if let Some(location) = source .clone() .downcast_rc::() @@ -228,7 +225,7 @@ impl<'a> Semantics<'a> { } return None; } - + pub fn from_source( source: Rc, semantics: &'a Option>, @@ -238,12 +235,13 @@ impl<'a> Semantics<'a> { } let range = source.location().map_or_else( || 0..source.content().len(), - |location| location.range.clone()); + |location| location.range.clone(), + ); return Self::from_source_impl(source, semantics, range); } pub fn add(&self, range: Range, token: (u32, u32)) { - let range = self.range.start+range.start..self.range.start+range.end; + let range = self.range.start + range.start..self.range.start + range.end; let mut tokens = self.sems.tokens.borrow_mut(); let mut cursor = self.sems.cursor.borrow_mut(); let mut current = cursor.clone(); @@ -270,8 +268,7 @@ impl<'a> Semantics<'a> { token_type: token.0, token_modifiers_bitset: token.1, }); - if cursor.pos + len == range.end - { + if cursor.pos + len == range.end { break; } current = cursor.clone(); diff --git a/src/parser/source.rs b/src/parser/source.rs index c2edcab..3e14ae3 100644 --- a/src/parser/source.rs +++ b/src/parser/source.rs @@ -5,6 +5,7 @@ use std::rc::Rc; use downcast_rs::impl_downcast; use downcast_rs::Downcast; +use unicode_width::UnicodeWidthChar; /// Trait for source content pub trait Source: Downcast + Debug { @@ -24,17 +25,13 @@ impl core::fmt::Display for dyn Source { } impl std::cmp::PartialEq for dyn Source { - fn eq(&self, other: &Self) -> bool { - self.name() == other.name() - } + fn eq(&self, other: &Self) -> bool { self.name() == other.name() } } impl std::cmp::Eq for dyn Source {} impl std::hash::Hash for dyn Source { - fn hash(&self, state: &mut H) { - self.name().hash(state) - } + fn hash(&self, state: &mut H) { self.name().hash(state) } } #[derive(Debug)] @@ -48,12 +45,7 @@ impl SourceFile { // TODO: Create a SourceFileRegistry holding already loaded files to avoid reloading them pub fn new(path: String, location: Option) -> Result { match fs::read_to_string(&path) { - Err(_) => { - Err(format!( - "Unable to read file content: `{}`", - path - )) - } + Err(_) => Err(format!("Unable to read file content: `{}`", path)), Ok(content) => Ok(Self { location, path, @@ -70,22 +62,13 @@ impl SourceFile { } } - pub fn path(&self) -> &String - { - &self.path - } + pub fn path(&self) -> &String { &self.path } } impl Source for SourceFile { - fn location(&self) -> Option<&Token> { - self.location.as_ref() - } - fn name(&self) -> &String { - &self.path - } - fn content(&self) -> &String { - &self.content - } + fn location(&self) -> Option<&Token> { self.location.as_ref() } + fn name(&self) -> &String { &self.path } + fn content(&self) -> &String { &self.content } } #[derive(Debug)] @@ -106,15 +89,9 @@ impl VirtualSource { } impl Source for VirtualSource { - fn location(&self) -> Option<&Token> { - Some(&self.location) - } - fn name(&self) -> &String { - &self.name - } - fn content(&self) -> &String { - &self.content - } + fn location(&self) -> Option<&Token> { Some(&self.location) } + fn name(&self) -> &String { &self.name } + fn content(&self) -> &String { &self.content } } #[derive(Debug)] @@ -124,9 +101,7 @@ pub struct Cursor { } impl Cursor { - pub fn new(pos: usize, source: Rc) -> Self { - Self { pos, source } - } + pub fn new(pos: usize, source: Rc) -> Self { Self { pos, source } } /// Creates [`cursor`] at [`new_pos`] in the same [`file`] pub fn at(&self, new_pos: usize) -> Self { @@ -145,9 +120,7 @@ impl Clone for Cursor { } } - fn clone_from(&mut self, source: &Self) { - *self = source.clone() - } + fn clone_from(&mut self, source: &Self) { *self = source.clone() } } /// Cursor type used for the language server @@ -165,8 +138,7 @@ pub struct LineCursor { impl LineCursor { /// Creates a [`LineCursor`] at the begining of the source - pub fn new(source: Rc) -> LineCursor - { + pub fn new(source: Rc) -> LineCursor { Self { pos: 0, line: 0, @@ -182,14 +154,9 @@ impl LineCursor { pub fn move_to(&mut self, pos: usize) { if self.pos < pos { let start = self.pos; - let mut it = self.source.content().as_str()[start..] - .chars() - .peekable(); + let mut it = self.source.content().as_str()[start..].chars().peekable(); - let mut prev = self.source.content().as_str()[..start] - .chars() - .rev() - .next(); + let mut prev = self.source.content().as_str()[..start].chars().rev().next(); while self.pos < pos { let c = it.next().unwrap(); let len = c.len_utf8(); @@ -197,8 +164,8 @@ impl LineCursor { if self.pos != start && prev == Some('\n') { self.line += 1; self.line_pos = 0; - } - self.line_pos += 1; + } + self.line_pos += c.width().unwrap_or(1); self.pos += len; prev = Some(c); } @@ -214,9 +181,7 @@ impl LineCursor { .rev() .peekable(); - let mut prev = self.source.content().as_str()[start..] - .chars() - .next(); + let mut prev = self.source.content().as_str()[start..].chars().next(); while self.pos > pos { let c = it.next().unwrap(); let len = c.len_utf8(); @@ -224,8 +189,8 @@ impl LineCursor { if self.pos != start && prev == Some('\n') { self.line -= 1; self.line_pos = 0; - } - self.line_pos -= 1; + } + self.line_pos -= c.width().unwrap_or(1); self.pos -= len; prev = Some(c); } @@ -247,13 +212,9 @@ pub struct Token { } impl Token { - pub fn new(range: Range, source: Rc) -> Self { - Self { range, source } - } + pub fn new(range: Range, source: Rc) -> Self { Self { range, source } } - pub fn source(&self) -> Rc { - self.source.clone() - } + pub fn source(&self) -> Rc { self.source.clone() } /// Construct Token from a range pub fn from(start: &Cursor, end: &Cursor) -> Self { @@ -265,11 +226,7 @@ impl Token { } } - pub fn start(&self) -> usize { - self.range.start - } + pub fn start(&self) -> usize { self.range.start } - pub fn end(&self) -> usize { - self.range.end - } + pub fn end(&self) -> usize { self.range.end } }