ef3d0c3e/nml
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix bugs

Browse source
This commit is contained in:
ef3d0c3e 2024-10-19 22:02:10 +02:00
parent 8e45b001a3
commit 02b34d5424
7 changed files with 149 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -40,6 +40,7 @@ tower-lsp = "0.20.0"
unicode-segmentation = "1.11.0" unicode-segmentation = "1.11.0"
walkdir = "2.5.0" walkdir = "2.5.0"
runtime-format = "0.1.3" runtime-format = "0.1.3"
unicode-width = "0.2.0"
[dev-dependencies] [dev-dependencies]
rand = "0.8.5" rand = "0.8.5"

32
src/elements/comment.rs
View file

@ -26,7 +26,12 @@ impl Element for Comment {
fn location(&self) -> &Token { &self.location } fn location(&self) -> &Token { &self.location }
fn kind(&self) -> ElemKind { ElemKind::Invisible } fn kind(&self) -> ElemKind { ElemKind::Invisible }
fn element_name(&self) -> &'static str { "Comment" } fn element_name(&self) -> &'static str { "Comment" }
fn compile(&self, _compiler: &Compiler, _document: &dyn Document, _cursor: usize) -> Result<String, String> { fn compile(
&self,
_compiler: &Compiler,
_document: &dyn Document,
_cursor: usize,
) -> Result<String, String> {
Ok("".to_string()) Ok("".to_string())
} }
} }
@ -90,10 +95,11 @@ impl RegexRule for CommentRule {
}), }),
); );
if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics) if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{ {
let comment = matches.get(1).unwrap().range(); let comment = matches.get(1).unwrap().range();
sems.add(comment.start-2..comment.end, tokens.comment); sems.add(comment.start - 2..comment.end, tokens.comment);
} }
reports reports
@ -108,7 +114,8 @@ mod tests {
use crate::parser::langparser::LangParser; use crate::parser::langparser::LangParser;
use crate::parser::parser::Parser; use crate::parser::parser::Parser;
use crate::parser::source::SourceFile; use crate::parser::source::SourceFile;
use crate::{validate_document, validate_semantics}; use crate::validate_document;
use crate::validate_semantics;
use super::*; use super::*;
@ -137,20 +144,23 @@ COMMENT ::Test
} }
#[test] #[test]
fn semantic() fn semantic() {
{
let source = Rc::new(SourceFile::with_content( let source = Rc::new(SourceFile::with_content(
"".to_string(), "".to_string(),
r#" r#"
::Test ::Test
::Another ::Another
:: Another :: Another
"# "#
.to_string(), .to_string(),
None, None,
)); ));
let parser = LangParser::default(); let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); let (_, state) = parser.parse(
ParserState::new_with_semantics(&parser, None),
source.clone(),
None,
);
validate_semantics!(state, source.clone(), 0, validate_semantics!(state, source.clone(), 0,
comment { delta_line == 1, delta_start == 0, length == 6 }; comment { delta_line == 1, delta_start == 0, length == 6 };

88
src/elements/link.rs
View file

@ -38,7 +38,12 @@ impl Element for Link {
fn location(&self) -> &Token { &self.location } fn location(&self) -> &Token { &self.location }
fn kind(&self) -> ElemKind { ElemKind::Inline } fn kind(&self) -> ElemKind { ElemKind::Inline }
fn element_name(&self) -> &'static str { "Link" } fn element_name(&self) -> &'static str { "Link" }
fn compile(&self, compiler: &Compiler, document: &dyn Document, cursor: usize) -> Result<String, String> { fn compile(
&self,
compiler: &Compiler,
document: &dyn Document,
cursor: usize,
) -> Result<String, String> {
match compiler.target() { match compiler.target() {
Target::HTML => { Target::HTML => {
let mut result = format!( let mut result = format!(
@ -47,7 +52,9 @@ impl Element for Link {
); );
for elem in &self.display { for elem in &self.display {
result += elem.compile(compiler, document, cursor+result.len())?.as_str(); result += elem
.compile(compiler, document, cursor + result.len())?
.as_str();
} }
result += "</a>"; result += "</a>";
@ -135,9 +142,13 @@ impl RegexRule for LinkRule {
return reports; return reports;
} }
if let Some((sems, tokens)) = Semantics::from_source(token.source(), &state.shared.semantics) if let Some((sems, tokens)) =
Semantics::from_source(token.source(), &state.shared.semantics)
{ {
sems.add(display.range().start-1..display.range().start, tokens.link_display_sep); sems.add(
display.range().start - 1..display.range().start,
tokens.link_display_sep,
);
} }
let source = Rc::new(VirtualSource::new( let source = Rc::new(VirtualSource::new(
Token::new(display.range(), token.source()), Token::new(display.range(), token.source()),
@ -212,20 +223,18 @@ impl RegexRule for LinkRule {
}), }),
); );
//if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { if let Some((sems, tokens)) =
// RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) Semantics::from_source(token.source(), &state.shared.semantics)
// .ok() {
// .unwrap() sems.add(
//}) { matches.get(1).unwrap().end()..matches.get(1).unwrap().end() + 1,
// let name = matches.get(1).unwrap().range(); tokens.link_display_sep,
// sems.add(token.source(), name.start-1..name.start, sems.token.link_name_sep); );
// sems.add(token.source(), name.clone(), sems.token.link_name); let url = matches.get(2).unwrap().range();
// sems.add(token.source(), name.end..name.end+1, sems.token.link_name_sep); sems.add(url.start - 1..url.start, tokens.link_url_sep);
// let url = matches.get(2).unwrap().range(); sems.add(url.clone(), tokens.link_url);
// sems.add(token.source(), url.start-1..url.start, sems.token.link_url_sep); sems.add(url.end..url.end + 1, tokens.link_url_sep);
// sems.add(token.source(), url.clone(), sems.token.link_url); }
// sems.add(token.source(), url.end..url.end+1, sems.token.link_url_sep);
//}
reports reports
} }
@ -257,9 +266,7 @@ impl RegexRule for LinkRule {
}); });
return; return;
} }
Ok(mut paragraph) => { Ok(mut paragraph) => std::mem::take(&mut paragraph.content),
std::mem::take(&mut paragraph.content)
}
}; };
ctx.state.push( ctx.state.push(
@ -290,7 +297,8 @@ mod tests {
use crate::parser::langparser::LangParser; use crate::parser::langparser::LangParser;
use crate::parser::parser::Parser; use crate::parser::parser::Parser;
use crate::parser::source::SourceFile; use crate::parser::source::SourceFile;
use crate::{validate_document, validate_semantics}; use crate::validate_document;
use crate::validate_semantics;
use super::*; use super::*;
@ -353,8 +361,7 @@ nml.link.push("**BOLD link**", "another url")
} }
#[test] #[test]
fn semantics() fn semantics() {
{
let source = Rc::new(SourceFile::with_content( let source = Rc::new(SourceFile::with_content(
"".to_string(), "".to_string(),
r#" r#"
@ -364,29 +371,20 @@ nml.link.push("**BOLD link**", "another url")
None, None,
)); ));
let parser = LangParser::default(); let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); let (_, state) = parser.parse(
ParserState::new_with_semantics(&parser, None),
println!("{:#?}", state.shared.semantics); source.clone(),
/*
let source = Rc::new(SourceFile::with_content(
"".to_string(),
r#"
[link](url)
"#
.to_string(),
None, None,
)); );
let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
validate_semantics!(state, source.clone(), 0, validate_semantics!(state, source.clone(), 0,
link_name_sep { delta_line == 1, delta_start == 0, length == 1 }; link_display_sep { delta_line == 1, delta_start == 0, length == 1 };
link_name { delta_line == 0, delta_start == 1, length == 4 }; style_marker { delta_line == 0, delta_start == 3, length == 2 };
link_name_sep { delta_line == 0, delta_start == 4, length == 1 }; style_marker { delta_line == 0, delta_start == 3, length == 2 };
link_url_sep { delta_line == 0, delta_start == 1, length == 1 }; link_display_sep { delta_line == 0, delta_start == 3, length == 1 };
link_url { delta_line == 0, delta_start == 1, length == 3 }; link_url_sep { delta_line == 0, delta_start == 1, length == 1 };
link_url_sep { delta_line == 0, delta_start == 3, length == 1 }; link_url { delta_line == 0, delta_start == 1, length == 3 };
); link_url_sep { delta_line == 0, delta_start == 3, length == 1 };
*/ );
} }
} }

99
src/elements/reference.rs
View file

@ -21,6 +21,7 @@ use crate::document::document::Document;
use crate::document::element::ElemKind; use crate::document::element::ElemKind;
use crate::document::element::Element; use crate::document::element::Element;
use crate::document::references::validate_refname; use crate::document::references::validate_refname;
use crate::lsp::semantic::Semantics;
use crate::parser::parser::ParserState; use crate::parser::parser::ParserState;
use crate::parser::parser::ReportColors; use crate::parser::parser::ReportColors;
use crate::parser::rule::RegexRule; use crate::parser::rule::RegexRule;
@ -311,7 +312,7 @@ impl RegexRule for ReferenceRule {
.downcast_rc::<reference_style::ExternalReferenceStyle>() .downcast_rc::<reference_style::ExternalReferenceStyle>()
.unwrap(); .unwrap();
// §{#refname} // &{#refname}
if refdoc.is_empty() { if refdoc.is_empty() {
state.push( state.push(
document, document,
@ -322,7 +323,7 @@ impl RegexRule for ReferenceRule {
style, style,
}), }),
); );
// §{docname#refname} // &{docname#refname}
} else { } else {
state.push( state.push(
document, document,
@ -335,25 +336,26 @@ impl RegexRule for ReferenceRule {
); );
} }
/* if let Some((sems, tokens)) =
if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { Semantics::from_source(token.source(), &state.shared.semantics)
RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) {
.ok()
.unwrap()
}) {
let link = matches.get(1).unwrap().range(); let link = matches.get(1).unwrap().range();
sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator); sems.add(link.start - 2..link.start - 1, tokens.reference_operator);
sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep); sems.add(link.start - 1..link.start, tokens.reference_link_sep);
if !refdoc.is_empty() if !refdoc.is_empty() {
{ sems.add(link.start..refdoc.len() + link.start, tokens.reference_doc);
sems.add(token.source(), link.start.. refdoc.len()+link.start, sems.token.reference_doc);
} }
sems.add(token.source(), refdoc.len()+link.start.. refdoc.len()+link.start+1, sems.token.reference_doc_sep); sems.add(
sems.add(token.source(), refdoc.len()+link.start+1..link.end, sems.token.reference_link); refdoc.len() + link.start..refdoc.len() + link.start + 1,
sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep); tokens.reference_doc_sep,
);
sems.add(
refdoc.len() + link.start + 1..link.end,
tokens.reference_link,
);
sems.add(link.end..link.end + 1, tokens.reference_link_sep);
} }
*/
} else { } else {
state.push( state.push(
document, document,
@ -363,35 +365,26 @@ impl RegexRule for ReferenceRule {
caption, caption,
}), }),
); );
/*
if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { if let Some((sems, tokens)) =
RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) Semantics::from_source(token.source(), &state.shared.semantics)
.ok() {
.unwrap()
}) {
let link = matches.get(1).unwrap().range(); let link = matches.get(1).unwrap().range();
sems.add(token.source(), link.start-2..link.start-1, sems.token.reference_operator); sems.add(link.start - 2..link.start - 1, tokens.reference_operator);
sems.add(token.source(), link.start-1..link.start, sems.token.reference_link_sep); sems.add(link.start - 1..link.start, tokens.reference_link_sep);
sems.add(token.source(), link.clone(), sems.token.reference_link); sems.add(link.clone(), tokens.reference_link);
sems.add(token.source(), link.end..link.end+1, sems.token.reference_link_sep); sems.add(link.end..link.end + 1, tokens.reference_link_sep);
} }
*/
} }
/* if let (Some((sems, tokens)), Some(props)) = (
if let Some(sems) = state.shared.semantics.as_ref().map(|sems| { Semantics::from_source(token.source(), &state.shared.semantics),
RefMut::filter_map(sems.borrow_mut(), |sems| sems.get_mut(&token.source())) matches.get(2).map(|m| m.range()),
.ok() ) {
.unwrap() sems.add(props.start - 1..props.start, tokens.reference_props_sep);
}) { sems.add(props.clone(), tokens.reference_props);
if let Some(props) = matches.get(2).map(|m| m.range()) sems.add(props.end..props.end + 1, tokens.reference_props_sep);
{
sems.add(token.source(), props.start-1..props.start, sems.token.reference_props_sep);
sems.add(token.source(), props.clone(), sems.token.reference_props);
sems.add(token.source(), props.end..props.end+1, sems.token.reference_props_sep);
}
} }
*/
reports reports
} }
@ -446,9 +439,9 @@ mod tests {
r#" r#"
#{ref} Referenceable section #{ref} Referenceable section
§{ref}[caption=Section] &{ref}[caption=Section]
§{ref}[caption=Another] &{ref}[caption=Another]
§{ref2}[caption=Before] &{ref2}[caption=Before]
#{ref2} Another section #{ref2} Another section
"# "#
@ -475,9 +468,9 @@ mod tests {
let source = Rc::new(SourceFile::with_content( let source = Rc::new(SourceFile::with_content(
"".to_string(), "".to_string(),
r#" r#"
§{DocA#ref}[caption=Section] &{DocA#ref}[caption=Section]
§{DocB#ref} &{DocB#ref}
§{#ref}[caption='ref' from any document] &{#ref}[caption='ref' from any document]
"# "#
.to_string(), .to_string(),
None, None,
@ -510,8 +503,8 @@ mod tests {
@html.page_title = 1 @html.page_title = 1
@compiler.output = b.html @compiler.output = b.html
§{#ref} &{#ref}
§{a#ref} &{a#ref}
#{ref2} Another Referenceable section #{ref2} Another Referenceable section
"# "#
.into(), .into(),
@ -523,10 +516,10 @@ mod tests {
"format_specific": "[SPECIFIC {refdoc}:{refname}]" "format_specific": "[SPECIFIC {refdoc}:{refname}]"
} }
§{#ref}[caption=from 0] &{#ref}[caption=from 0]
§{#ref} &{#ref}
§{#ref2}[caption=from 1] &{#ref2}[caption=from 1]
§{b#ref2} &{b#ref2}
"# "#
.into(), .into(),
], ],

5
src/elements/section.rs
View file

@ -552,7 +552,9 @@ nml.section.push("6", 6, "", "refname")
let source = Rc::new(SourceFile::with_content( let source = Rc::new(SourceFile::with_content(
"".to_string(), "".to_string(),
r#" r#"
#{📫} test # First section
##{📫}+ test
#{refname}*+ Another section
"# "#
.to_string(), .to_string(),
None, None,
@ -560,7 +562,6 @@ nml.section.push("6", 6, "", "refname")
let parser = LangParser::default(); let parser = LangParser::default();
let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None); let (_, state) = parser.parse(ParserState::new_with_semantics(&parser, None), source.clone(), None);
println!("{:#?}", state.shared.semantics);
validate_semantics!(state, source.clone(), 0, validate_semantics!(state, source.clone(), 0,
section_heading { delta_line == 1, delta_start == 0, length == 1 }; section_heading { delta_line == 1, delta_start == 0, length == 1 };
section_name { delta_line == 0, delta_start == 1 }; section_name { delta_line == 0, delta_start == 1 };

23
src/lsp/semantic.rs
View file

@ -134,7 +134,7 @@ impl Tokens {
link_display_sep: token!("macro"), link_display_sep: token!("macro"),
link_url_sep: token!("macro"), link_url_sep: token!("macro"),
link_url: token!("operator", "readonly", "abstract", "abstract"), link_url: token!("function", "readonly", "abstract", "abstract"),
style_marker: token!("operator"), style_marker: token!("operator"),
@ -170,10 +170,8 @@ pub struct SemanticsData {
pub tokens: RefCell<Vec<SemanticToken>>, pub tokens: RefCell<Vec<SemanticToken>>,
} }
impl SemanticsData impl SemanticsData {
{ pub fn new(source: Rc<dyn Source>) -> Self {
pub fn new(source: Rc<dyn Source>) -> Self
{
Self { Self {
cursor: RefCell::new(LineCursor::new(source)), cursor: RefCell::new(LineCursor::new(source)),
tokens: RefCell::new(vec![]), tokens: RefCell::new(vec![]),
@ -192,9 +190,8 @@ impl<'a> Semantics<'a> {
fn from_source_impl( fn from_source_impl(
source: Rc<dyn Source>, source: Rc<dyn Source>,
semantics: &'a Option<RefCell<SemanticsHolder>>, semantics: &'a Option<RefCell<SemanticsHolder>>,
range: Range<usize>) range: Range<usize>,
-> Option<(Self, Ref<'a, Tokens>)> ) -> Option<(Self, Ref<'a, Tokens>)> {
{
if let Some(location) = source if let Some(location) = source
.clone() .clone()
.downcast_rc::<VirtualSource>() .downcast_rc::<VirtualSource>()
@ -228,7 +225,7 @@ impl<'a> Semantics<'a> {
} }
return None; return None;
} }
pub fn from_source( pub fn from_source(
source: Rc<dyn Source>, source: Rc<dyn Source>,
semantics: &'a Option<RefCell<SemanticsHolder>>, semantics: &'a Option<RefCell<SemanticsHolder>>,
@ -238,12 +235,13 @@ impl<'a> Semantics<'a> {
} }
let range = source.location().map_or_else( let range = source.location().map_or_else(
|| 0..source.content().len(), || 0..source.content().len(),
|location| location.range.clone()); |location| location.range.clone(),
);
return Self::from_source_impl(source, semantics, range); return Self::from_source_impl(source, semantics, range);
} }
pub fn add(&self, range: Range<usize>, token: (u32, u32)) { pub fn add(&self, range: Range<usize>, token: (u32, u32)) {
let range = self.range.start+range.start..self.range.start+range.end; let range = self.range.start + range.start..self.range.start + range.end;
let mut tokens = self.sems.tokens.borrow_mut(); let mut tokens = self.sems.tokens.borrow_mut();
let mut cursor = self.sems.cursor.borrow_mut(); let mut cursor = self.sems.cursor.borrow_mut();
let mut current = cursor.clone(); let mut current = cursor.clone();
@ -270,8 +268,7 @@ impl<'a> Semantics<'a> {
token_type: token.0, token_type: token.0,
token_modifiers_bitset: token.1, token_modifiers_bitset: token.1,
}); });
if cursor.pos + len == range.end if cursor.pos + len == range.end {
{
break; break;
} }
current = cursor.clone(); current = cursor.clone();

93
src/parser/source.rs
View file

@ -5,6 +5,7 @@ use std::rc::Rc;
use downcast_rs::impl_downcast; use downcast_rs::impl_downcast;
use downcast_rs::Downcast; use downcast_rs::Downcast;
use unicode_width::UnicodeWidthChar;
/// Trait for source content /// Trait for source content
pub trait Source: Downcast + Debug { pub trait Source: Downcast + Debug {
@ -24,17 +25,13 @@ impl core::fmt::Display for dyn Source {
} }
impl std::cmp::PartialEq for dyn Source { impl std::cmp::PartialEq for dyn Source {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool { self.name() == other.name() }
self.name() == other.name()
}
} }
impl std::cmp::Eq for dyn Source {} impl std::cmp::Eq for dyn Source {}
impl std::hash::Hash for dyn Source { impl std::hash::Hash for dyn Source {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) { fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.name().hash(state) }
self.name().hash(state)
}
} }
#[derive(Debug)] #[derive(Debug)]
@ -48,12 +45,7 @@ impl SourceFile {
// TODO: Create a SourceFileRegistry holding already loaded files to avoid reloading them // TODO: Create a SourceFileRegistry holding already loaded files to avoid reloading them
pub fn new(path: String, location: Option<Token>) -> Result<Self, String> { pub fn new(path: String, location: Option<Token>) -> Result<Self, String> {
match fs::read_to_string(&path) { match fs::read_to_string(&path) {
Err(_) => { Err(_) => Err(format!("Unable to read file content: `{}`", path)),
Err(format!(
"Unable to read file content: `{}`",
path
))
}
Ok(content) => Ok(Self { Ok(content) => Ok(Self {
location, location,
path, path,
@ -70,22 +62,13 @@ impl SourceFile {
} }
} }
pub fn path(&self) -> &String pub fn path(&self) -> &String { &self.path }
{
&self.path
}
} }
impl Source for SourceFile { impl Source for SourceFile {
fn location(&self) -> Option<&Token> { fn location(&self) -> Option<&Token> { self.location.as_ref() }
self.location.as_ref() fn name(&self) -> &String { &self.path }
} fn content(&self) -> &String { &self.content }
fn name(&self) -> &String {
&self.path
}
fn content(&self) -> &String {
&self.content
}
} }
#[derive(Debug)] #[derive(Debug)]
@ -106,15 +89,9 @@ impl VirtualSource {
} }
impl Source for VirtualSource { impl Source for VirtualSource {
fn location(&self) -> Option<&Token> { fn location(&self) -> Option<&Token> { Some(&self.location) }
Some(&self.location) fn name(&self) -> &String { &self.name }
} fn content(&self) -> &String { &self.content }
fn name(&self) -> &String {
&self.name
}
fn content(&self) -> &String {
&self.content
}
} }
#[derive(Debug)] #[derive(Debug)]
@ -124,9 +101,7 @@ pub struct Cursor {
} }
impl Cursor { impl Cursor {
pub fn new(pos: usize, source: Rc<dyn Source>) -> Self { pub fn new(pos: usize, source: Rc<dyn Source>) -> Self { Self { pos, source } }
Self { pos, source }
}
/// Creates [`cursor`] at [`new_pos`] in the same [`file`] /// Creates [`cursor`] at [`new_pos`] in the same [`file`]
pub fn at(&self, new_pos: usize) -> Self { pub fn at(&self, new_pos: usize) -> Self {
@ -145,9 +120,7 @@ impl Clone for Cursor {
} }
} }
fn clone_from(&mut self, source: &Self) { fn clone_from(&mut self, source: &Self) { *self = source.clone() }
*self = source.clone()
}
} }
/// Cursor type used for the language server /// Cursor type used for the language server
@ -165,8 +138,7 @@ pub struct LineCursor {
impl LineCursor { impl LineCursor {
/// Creates a [`LineCursor`] at the begining of the source /// Creates a [`LineCursor`] at the begining of the source
pub fn new(source: Rc<dyn Source>) -> LineCursor pub fn new(source: Rc<dyn Source>) -> LineCursor {
{
Self { Self {
pos: 0, pos: 0,
line: 0, line: 0,
@ -182,14 +154,9 @@ impl LineCursor {
pub fn move_to(&mut self, pos: usize) { pub fn move_to(&mut self, pos: usize) {
if self.pos < pos { if self.pos < pos {
let start = self.pos; let start = self.pos;
let mut it = self.source.content().as_str()[start..] let mut it = self.source.content().as_str()[start..].chars().peekable();
.chars()
.peekable();
let mut prev = self.source.content().as_str()[..start] let mut prev = self.source.content().as_str()[..start].chars().rev().next();
.chars()
.rev()
.next();
while self.pos < pos { while self.pos < pos {
let c = it.next().unwrap(); let c = it.next().unwrap();
let len = c.len_utf8(); let len = c.len_utf8();
@ -197,8 +164,8 @@ impl LineCursor {
if self.pos != start && prev == Some('\n') { if self.pos != start && prev == Some('\n') {
self.line += 1; self.line += 1;
self.line_pos = 0; self.line_pos = 0;
} }
self.line_pos += 1; self.line_pos += c.width().unwrap_or(1);
self.pos += len; self.pos += len;
prev = Some(c); prev = Some(c);
} }
@ -214,9 +181,7 @@ impl LineCursor {
.rev() .rev()
.peekable(); .peekable();
let mut prev = self.source.content().as_str()[start..] let mut prev = self.source.content().as_str()[start..].chars().next();
.chars()
.next();
while self.pos > pos { while self.pos > pos {
let c = it.next().unwrap(); let c = it.next().unwrap();
let len = c.len_utf8(); let len = c.len_utf8();
@ -224,8 +189,8 @@ impl LineCursor {
if self.pos != start && prev == Some('\n') { if self.pos != start && prev == Some('\n') {
self.line -= 1; self.line -= 1;
self.line_pos = 0; self.line_pos = 0;
} }
self.line_pos -= 1; self.line_pos -= c.width().unwrap_or(1);
self.pos -= len; self.pos -= len;
prev = Some(c); prev = Some(c);
} }
@ -247,13 +212,9 @@ pub struct Token {
} }
impl Token { impl Token {
pub fn new(range: Range<usize>, source: Rc<dyn Source>) -> Self { pub fn new(range: Range<usize>, source: Rc<dyn Source>) -> Self { Self { range, source } }
Self { range, source }
}
pub fn source(&self) -> Rc<dyn Source> { pub fn source(&self) -> Rc<dyn Source> { self.source.clone() }
self.source.clone()
}
/// Construct Token from a range /// Construct Token from a range
pub fn from(start: &Cursor, end: &Cursor) -> Self { pub fn from(start: &Cursor, end: &Cursor) -> Self {
@ -265,11 +226,7 @@ impl Token {
} }
} }
pub fn start(&self) -> usize { pub fn start(&self) -> usize { self.range.start }
self.range.start
}
pub fn end(&self) -> usize { pub fn end(&self) -> usize { self.range.end }
self.range.end
}
} }