diff --git a/Cargo.toml b/Cargo.toml
index e2f2d5f..4ad8676 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,6 +40,7 @@ tower-lsp = "0.20.0"
unicode-segmentation = "1.11.0"
walkdir = "2.5.0"
runtime-format = "0.1.3"
+unicode-width = "0.2.0"
[dev-dependencies]
rand = "0.8.5"
diff --git a/src/elements/comment.rs b/src/elements/comment.rs
index 0ff57e7..6aacd6c 100644
--- a/src/elements/comment.rs
+++ b/src/elements/comment.rs
@@ -26,7 +26,12 @@ impl Element for Comment {
fn location(&self) -> &Token { &self.location }
fn kind(&self) -> ElemKind { ElemKind::Invisible }
fn element_name(&self) -> &'static str { "Comment" }
- fn compile(&self, _compiler: &Compiler, _document: &dyn Document, _cursor: usize) -> Result<String, String> {
+ fn compile(
+ &self,
+ _compiler: &Compiler,
+ _document: &dyn Document,
+ _cursor: usize,
+ ) -> Result<String, String> {
Ok("".to_string())
}
}
@@ -90,10 +95,11 @@ impl RegexRule for CommentRule {
}),
);
- if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics)
+ if let Some((sems, tokens)) =
+ Semantics::from_source(token.source(), &state.shared.semantics)
{
let comment = matches.get(1).unwrap().range();
- sems.add(comment.start-2..comment.end, tokens.comment);
+ sems.add(comment.start - 2..comment.end, tokens.comment);
}
reports
@@ -108,7 +114,8 @@ mod tests {
use crate::parser::langparser::LangParser;
use crate::parser::parser::Parser;
use crate::parser::source::SourceFile;
- use crate::{validate_document, validate_semantics};
+ use crate::validate_document;
+ use crate::validate_semantics;
use super::*;
@@ -137,20 +144,23 @@ COMMENT ::Test
}
#[test]
- fn semantic()
- {
+ fn semantic() {
let source = Rc::new(SourceFile::with_content(
- "".to_string(),
- r#"
+ "".to_string(),
+ r#"
::Test
::Another
:: Another
"#
- .to_string(),
- None,
+ .to_string(),
+ None,
));
let parser = LangParser::default();
- let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
+ let (_, state) = parser.parse(
+ ParserState::new_with_semantics(&parser, None),
+ source.clone(),
+ None,
+ );
validate_semantics!(state, source.clone(), 0,
comment { delta_line == 1, delta_start == 0, length == 6 };
diff --git a/src/elements/link.rs b/src/elements/link.rs
index b2e4ae1..b739d8b 100644
--- a/src/elements/link.rs
+++ b/src/elements/link.rs
@@ -38,7 +38,12 @@ impl Element for Link {
fn location(&self) -> &Token { &self.location }
fn kind(&self) -> ElemKind { ElemKind::Inline }
fn element_name(&self) -> &'static str { "Link" }
- fn compile(&self, compiler: &Compiler, document: &dyn Document, cursor: usize) -> Result<String, String> {
+ fn compile(
+ &self,
+ compiler: &Compiler,
+ document: &dyn Document,
+ cursor: usize,
+ ) -> Result<String, String> {
match compiler.target() {
Target::HTML => {
let mut result = format!(
@@ -47,7 +52,9 @@ impl Element for Link {
);
for elem in &self.display {
- result += elem.compile(compiler, document, cursor+result.len())?.as_str();
+ result += elem
+ .compile(compiler, document, cursor + result.len())?
+ .as_str();
}
result += "</a>";
@@ -135,9 +142,13 @@ impl RegexRule for LinkRule {
return reports;
}
- if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics)
+ if let Some((sems, tokens)) =
+ Semantics::from_source(token.source(), &state.shared.semantics)
{
- sems.add(display.range().start-1..display.range().start, tokens.link_display_sep);
+ sems.add(
+ display.range().start - 1..display.range().start,
+ tokens.link_display_sep,
+ );
}
let source = Rc::new(VirtualSource::new(
Token::new(display.range(), token.source()),
@@ -212,20 +223,18 @@ impl RegexRule for LinkRule {
}),
);
- //if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
- // RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
- // .ok()
- // .unwrap()
- //}) {
- // let name = matches.get(1).unwrap().range();
- // sems.add(token.source(), name.start-1..name.start, sems.token.link_name_sep);
- // sems.add(token.source(), name.clone(), sems.token.link_name);
- // sems.add(token.source(), name.end..name.end+1, sems.token.link_name_sep);
- // let url = matches.get(2).unwrap().range();
- // sems.add(token.source(), url.start-1..url.start, sems.token.link_url_sep);
- // sems.add(token.source(), url.clone(), sems.token.link_url);
- // sems.add(token.source(), url.end..url.end+1, sems.token.link_url_sep);
- //}
+ if let Some((sems, tokens)) =
+ Semantics::from_source(token.source(), &state.shared.semantics)
+ {
+ sems.add(
+ matches.get(1).unwrap().end()..matches.get(1).unwrap().end() + 1,
+ tokens.link_display_sep,
+ );
+ let url = matches.get(2).unwrap().range();
+ sems.add(url.start - 1..url.start, tokens.link_url_sep);
+ sems.add(url.clone(), tokens.link_url);
+ sems.add(url.end..url.end + 1, tokens.link_url_sep);
+ }
reports
}
@@ -257,9 +266,7 @@ impl RegexRule for LinkRule {
});
return;
}
- Ok(mut paragraph) => {
- std::mem::take(&mut paragraph.content)
- }
+ Ok(mut paragraph) => std::mem::take(&mut paragraph.content),
};
ctx.state.push(
@@ -290,7 +297,8 @@ mod tests {
use crate::parser::langparser::LangParser;
use crate::parser::parser::Parser;
use crate::parser::source::SourceFile;
- use crate::{validate_document, validate_semantics};
+ use crate::validate_document;
+ use crate::validate_semantics;
use super::*;
@@ -353,8 +361,7 @@ nml.link.push("**BOLD link**", "another url")
}
#[test]
- fn semantics()
- {
+ fn semantics() {
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
@@ -364,29 +371,20 @@ nml.link.push("**BOLD link**", "another url")
None,
));
let parser = LangParser::default();
- let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
-
- println!("{:#?}", state.shared.semantics);
- /*
- let source = Rc::new(SourceFile::with_content(
- "".to_string(),
- r#"
-[link](url)
- "#
- .to_string(),
+ let (_, state) = parser.parse(
+ ParserState::new_with_semantics(&parser, None),
+ source.clone(),
None,
- ));
- let parser = LangParser::default();
- let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
+ );
validate_semantics!(state, source.clone(), 0,
- link_name_sep { delta_line == 1, delta_start == 0, length == 1 };
- link_name { delta_line == 0, delta_start == 1, length == 4 };
- link_name_sep { delta_line == 0, delta_start == 4, length == 1 };
- link_url_sep { delta_line == 0, delta_start == 1, length == 1 };
- link_url { delta_line == 0, delta_start == 1, length == 3 };
- link_url_sep { delta_line == 0, delta_start == 3, length == 1 };
- );
- */
+ link_display_sep { delta_line == 1, delta_start == 0, length == 1 };
+ style_marker { delta_line == 0, delta_start == 3, length == 2 };
+ style_marker { delta_line == 0, delta_start == 3, length == 2 };
+ link_display_sep { delta_line == 0, delta_start == 3, length == 1 };
+ link_url_sep { delta_line == 0, delta_start == 1, length == 1 };
+ link_url { delta_line == 0, delta_start == 1, length == 3 };
+ link_url_sep { delta_line == 0, delta_start == 3, length == 1 };
+ );
}
}
diff --git a/src/elements/reference.rs b/src/elements/reference.rs
index b73cb3e..6cc1aaf 100644
--- a/src/elements/reference.rs
+++ b/src/elements/reference.rs
@@ -21,6 +21,7 @@ use crate::document::document::Document;
use crate::document::element::ElemKind;
use crate::document::element::Element;
use crate::document::references::validate_refname;
+use crate::lsp::semantic::Semantics;
use crate::parser::parser::ParserState;
use crate::parser::parser::ReportColors;
use crate::parser::rule::RegexRule;
@@ -311,7 +312,7 @@ impl RegexRule for ReferenceRule {
.downcast_rc::<reference_style::ExternalReferenceStyle>()
.unwrap();
- // §{#refname}
+ // &{#refname}
if refdoc.is_empty() {
state.push(
document,
@@ -322,7 +323,7 @@ impl RegexRule for ReferenceRule {
style,
}),
);
- // §{docname#refname}
+ // &{docname#refname}
} else {
state.push(
document,
@@ -335,25 +336,26 @@ impl RegexRule for ReferenceRule {
);
}
- /*
- if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
- RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
- .ok()
- .unwrap()
- }) {
+ if let Some((sems, tokens)) =
+ Semantics::from_source(token.source(), &state.shared.semantics)
+ {
let link = matches.get(1).unwrap().range();
- sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator);
- sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep);
+ sems.add(link.start - 2..link.start - 1, tokens.reference_operator);
+ sems.add(link.start - 1..link.start, tokens.reference_link_sep);
- if !refdoc.is_empty()
- {
- sems.add(token.source(), link.start.. refdoc.len()+link.start, sems.token.reference_doc);
+ if !refdoc.is_empty() {
+ sems.add(link.start..refdoc.len() + link.start, tokens.reference_doc);
}
- sems.add(token.source(), refdoc.len()+link.start.. refdoc.len()+link.start+1, sems.token.reference_doc_sep);
- sems.add(token.source(), refdoc.len()+link.start+1..link.end, sems.token.reference_link);
- sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep);
+ sems.add(
+ refdoc.len() + link.start..refdoc.len() + link.start + 1,
+ tokens.reference_doc_sep,
+ );
+ sems.add(
+ refdoc.len() + link.start + 1..link.end,
+ tokens.reference_link,
+ );
+ sems.add(link.end..link.end + 1, tokens.reference_link_sep);
}
- */
} else {
state.push(
document,
@@ -363,35 +365,26 @@ impl RegexRule for ReferenceRule {
caption,
}),
);
- /*
- if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
- RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
- .ok()
- .unwrap()
- }) {
+
+ if let Some((sems, tokens)) =
+ Semantics::from_source(token.source(), &state.shared.semantics)
+ {
let link = matches.get(1).unwrap().range();
- sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator);
- sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep);
- sems.add(token.source(), link.clone(), sems.token.reference_link);
- sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep);
+ sems.add(link.start - 2..link.start - 1, tokens.reference_operator);
+ sems.add(link.start - 1..link.start, tokens.reference_link_sep);
+ sems.add(link.clone(), tokens.reference_link);
+ sems.add(link.end..link.end + 1, tokens.reference_link_sep);
}
- */
}
- /*
- if let Some(sems) = state.shared.semantics.as_ref().map(|sems| {
- RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source()))
- .ok()
- .unwrap()
- }) {
- if let Some(props) = matches.get(2).map(|m| m.range())
- {
- sems.add(token.source(), props.start-1..props.start, sems.token.reference_props_sep);
- sems.add(token.source(), props.clone(), sems.token.reference_props);
- sems.add(token.source(), props.end..props.end+1, sems.token.reference_props_sep);
- }
+ if let (Some((sems, tokens)), Some(props)) = (
+ Semantics::from_source(token.source(), &state.shared.semantics),
+ matches.get(2).map(|m| m.range()),
+ ) {
+ sems.add(props.start - 1..props.start, tokens.reference_props_sep);
+ sems.add(props.clone(), tokens.reference_props);
+ sems.add(props.end..props.end + 1, tokens.reference_props_sep);
}
- */
reports
}
@@ -446,9 +439,9 @@ mod tests {
r#"
#{ref} Referenceable section
-§{ref}[caption=Section]
-§{ref}[caption=Another]
-§{ref2}[caption=Before]
+&{ref}[caption=Section]
+&{ref}[caption=Another]
+&{ref2}[caption=Before]
#{ref2} Another section
"#
@@ -475,9 +468,9 @@ mod tests {
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
-§{DocA#ref}[caption=Section]
-§{DocB#ref}
-§{#ref}[caption='ref' from any document]
+&{DocA#ref}[caption=Section]
+&{DocB#ref}
+&{#ref}[caption='ref' from any document]
"#
.to_string(),
None,
@@ -510,8 +503,8 @@ mod tests {
@html.page_title = 1
@compiler.output = b.html
-§{#ref}
-§{a#ref}
+&{#ref}
+&{a#ref}
#{ref2} Another Referenceable section
"#
.into(),
@@ -523,10 +516,10 @@ mod tests {
"format_specific": "[SPECIFIC {refdoc}:{refname}]"
}
-§{#ref}[caption=from 0]
-§{#ref}
-§{#ref2}[caption=from 1]
-§{b#ref2}
+&{#ref}[caption=from 0]
+&{#ref}
+&{#ref2}[caption=from 1]
+&{b#ref2}
"#
.into(),
],
diff --git a/src/elements/section.rs b/src/elements/section.rs
index e15cd97..6ba21b8 100644
--- a/src/elements/section.rs
+++ b/src/elements/section.rs
@@ -552,7 +552,9 @@ nml.section.push("6", 6, "", "refname")
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
-#{📫} test
+# First section
+##{📫}+ test
+#{refname}*+ Another section
"#
.to_string(),
None,
@@ -560,7 +562,6 @@ nml.section.push("6", 6, "", "refname")
let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
- println!("{:#?}", state.shared.semantics);
validate_semantics!(state, source.clone(), 0,
section_heading { delta_line == 1, delta_start == 0, length == 1 };
section_name { delta_line == 0, delta_start == 1 };
diff --git a/src/lsp/semantic.rs b/src/lsp/semantic.rs
index 745c855..d4f2457 100644
--- a/src/lsp/semantic.rs
+++ b/src/lsp/semantic.rs
@@ -134,7 +134,7 @@ impl Tokens {
link_display_sep: token!("macro"),
link_url_sep: token!("macro"),
- link_url: token!("operator", "readonly", "abstract", "abstract"),
+ link_url: token!("function", "readonly", "abstract", "abstract"),
style_marker: token!("operator"),
@@ -170,10 +170,8 @@ pub struct SemanticsData {
pub tokens: RefCell<Vec<SemanticToken>>,
}
-impl SemanticsData
-{
- pub fn new(source: Rc<dyn Source>) -> Self
- {
+impl SemanticsData {
+ pub fn new(source: Rc<dyn Source>) -> Self {
Self {
cursor: RefCell::new(LineCursor::new(source)),
tokens: RefCell::new(vec![]),
@@ -192,9 +190,8 @@ impl<'a> Semantics<'a> {
fn from_source_impl(
source: Rc<dyn Source>,
semantics: &'a Option<RefCell<SemanticsHolder>>,
- range: Range<usize>)
- -> Option<(Self, Ref<'a, Tokens>)>
- {
+ range: Range<usize>,
+ ) -> Option<(Self, Ref<'a, Tokens>)> {
if let Some(location) = source
.clone()
.downcast_rc::<VirtualSource>()
@@ -228,7 +225,7 @@ impl<'a> Semantics<'a> {
}
return None;
}
-
+
pub fn from_source(
source: Rc<dyn Source>,
semantics: &'a Option<RefCell<SemanticsHolder>>,
@@ -238,12 +235,13 @@ impl<'a> Semantics<'a> {
}
let range = source.location().map_or_else(
|| 0..source.content().len(),
- |location| location.range.clone());
+ |location| location.range.clone(),
+ );
return Self::from_source_impl(source, semantics, range);
}
pub fn add(&self, range: Range<usize>, token: (u32, u32)) {
- let range = self.range.start+range.start..self.range.start+range.end;
+ let range = self.range.start + range.start..self.range.start + range.end;
let mut tokens = self.sems.tokens.borrow_mut();
let mut cursor = self.sems.cursor.borrow_mut();
let mut current = cursor.clone();
@@ -270,8 +268,7 @@ impl<'a> Semantics<'a> {
token_type: token.0,
token_modifiers_bitset: token.1,
});
- if cursor.pos + len == range.end
- {
+ if cursor.pos + len == range.end {
break;
}
current = cursor.clone();
diff --git a/src/parser/source.rs b/src/parser/source.rs
index c2edcab..3e14ae3 100644
--- a/src/parser/source.rs
+++ b/src/parser/source.rs
@@ -5,6 +5,7 @@ use std::rc::Rc;
use downcast_rs::impl_downcast;
use downcast_rs::Downcast;
+use unicode_width::UnicodeWidthChar;
/// Trait for source content
pub trait Source: Downcast + Debug {
@@ -24,17 +25,13 @@ impl core::fmt::Display for dyn Source {
}
impl std::cmp::PartialEq for dyn Source {
- fn eq(&self, other: &Self) -> bool {
- self.name() == other.name()
- }
+ fn eq(&self, other: &Self) -> bool { self.name() == other.name() }
}
impl std::cmp::Eq for dyn Source {}
impl std::hash::Hash for dyn Source {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.name().hash(state)
- }
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.name().hash(state) }
}
#[derive(Debug)]
@@ -48,12 +45,7 @@ impl SourceFile {
// TODO: Create a SourceFileRegistry holding already loaded files to avoid reloading them
pub fn new(path: String, location: Option<Token>) -> Result<Self, String> {
match fs::read_to_string(&path) {
- Err(_) => {
- Err(format!(
- "Unable to read file content: `{}`",
- path
- ))
- }
+ Err(_) => Err(format!("Unable to read file content: `{}`", path)),
Ok(content) => Ok(Self {
location,
path,
@@ -70,22 +62,13 @@ impl SourceFile {
}
}
- pub fn path(&self) -> &String
- {
- &self.path
- }
+ pub fn path(&self) -> &String { &self.path }
}
impl Source for SourceFile {
- fn location(&self) -> Option<&Token> {
- self.location.as_ref()
- }
- fn name(&self) -> &String {
- &self.path
- }
- fn content(&self) -> &String {
- &self.content
- }
+ fn location(&self) -> Option<&Token> { self.location.as_ref() }
+ fn name(&self) -> &String { &self.path }
+ fn content(&self) -> &String { &self.content }
}
#[derive(Debug)]
@@ -106,15 +89,9 @@ impl VirtualSource {
}
impl Source for VirtualSource {
- fn location(&self) -> Option<&Token> {
- Some(&self.location)
- }
- fn name(&self) -> &String {
- &self.name
- }
- fn content(&self) -> &String {
- &self.content
- }
+ fn location(&self) -> Option<&Token> { Some(&self.location) }
+ fn name(&self) -> &String { &self.name }
+ fn content(&self) -> &String { &self.content }
}
#[derive(Debug)]
@@ -124,9 +101,7 @@ pub struct Cursor {
}
impl Cursor {
- pub fn new(pos: usize, source: Rc<dyn Source>) -> Self {
- Self { pos, source }
- }
+ pub fn new(pos: usize, source: Rc<dyn Source>) -> Self { Self { pos, source } }
/// Creates [`cursor`] at [`new_pos`] in the same [`file`]
pub fn at(&self, new_pos: usize) -> Self {
@@ -145,9 +120,7 @@ impl Clone for Cursor {
}
}
- fn clone_from(&mut self, source: &Self) {
- *self = source.clone()
- }
+ fn clone_from(&mut self, source: &Self) { *self = source.clone() }
}
/// Cursor type used for the language server
@@ -165,8 +138,7 @@ pub struct LineCursor {
impl LineCursor {
/// Creates a [`LineCursor`] at the begining of the source
- pub fn new(source: Rc<dyn Source>) -> LineCursor
- {
+ pub fn new(source: Rc<dyn Source>) -> LineCursor {
Self {
pos: 0,
line: 0,
@@ -182,14 +154,9 @@ impl LineCursor {
pub fn move_to(&mut self, pos: usize) {
if self.pos < pos {
let start = self.pos;
- let mut it = self.source.content().as_str()[start..]
- .chars()
- .peekable();
+ let mut it = self.source.content().as_str()[start..].chars().peekable();
- let mut prev = self.source.content().as_str()[..start]
- .chars()
- .rev()
- .next();
+ let mut prev = self.source.content().as_str()[..start].chars().rev().next();
while self.pos < pos {
let c = it.next().unwrap();
let len = c.len_utf8();
@@ -197,8 +164,8 @@ impl LineCursor {
if self.pos != start && prev == Some('\n') {
self.line += 1;
self.line_pos = 0;
- }
- self.line_pos += 1;
+ }
+ self.line_pos += c.width().unwrap_or(1);
self.pos += len;
prev = Some(c);
}
@@ -214,9 +181,7 @@ impl LineCursor {
.rev()
.peekable();
- let mut prev = self.source.content().as_str()[start..]
- .chars()
- .next();
+ let mut prev = self.source.content().as_str()[start..].chars().next();
while self.pos > pos {
let c = it.next().unwrap();
let len = c.len_utf8();
@@ -224,8 +189,8 @@ impl LineCursor {
if self.pos != start && prev == Some('\n') {
self.line -= 1;
self.line_pos = 0;
- }
- self.line_pos -= 1;
+ }
+ self.line_pos -= c.width().unwrap_or(1);
self.pos -= len;
prev = Some(c);
}
@@ -247,13 +212,9 @@ pub struct Token {
}
impl Token {
- pub fn new(range: Range<usize>, source: Rc<dyn Source>) -> Self {
- Self { range, source }
- }
+ pub fn new(range: Range<usize>, source: Rc<dyn Source>) -> Self { Self { range, source } }
- pub fn source(&self) -> Rc<dyn Source> {
- self.source.clone()
- }
+ pub fn source(&self) -> Rc<dyn Source> { self.source.clone() }
/// Construct Token from a range
pub fn from(start: &Cursor, end: &Cursor) -> Self {
@@ -265,11 +226,7 @@ impl Token {
}
}
- pub fn start(&self) -> usize {
- self.range.start
- }
+ pub fn start(&self) -> usize { self.range.start }
- pub fn end(&self) -> usize {
- self.range.end
- }
+ pub fn end(&self) -> usize { self.range.end }
}