mod interlinksolver; mod token; mod tree; use std::{iter::Enumerate, rc::Rc, str::Chars}; pub use interlinksolver::{EchoInterlinkSolver, InterlinkSolver}; use token::{IndexedToken, Token}; pub use tree::{Document, LinkKind, NodeStyle, TreeNode}; pub fn process<'raw>(raw: &'raw str, solver: Rc) -> Document<'raw> { let mut r1 = round1(raw); round2(raw, &mut r1); Document { nodes: round3(raw, &r1), solver, } } //MARK: ✨ Passes ✨ /// Round 1 tokenizes to `Token` but does not produce `Bold` or properly /// run `Token::Escape`. round2() is the next step fn round1<'raw>(raw: &'raw str) -> Vec> { let mut chars = CollectingCharIter::from(raw); let mut toks = vec![]; loop { let (idx, ch) = match chars.next() { Some(pair) => pair, None => { chars.append_slice(None, &mut toks); break; } }; match ch { ' ' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::Space)); } '\\' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::Escape)); } '`' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::Code)); } '*' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::Italic)); } '%' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::Percent)); } '|' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::Pipe)); } '[' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::LinkOpen)); } ']' => { chars.append_slice(Some(idx), &mut toks); toks.push(IndexedToken::new(idx, Token::LinkClose)); } '\r' => { chars.append_slice(Some(idx), &mut toks); // We are ignoring carriage returns here } _ch => { chars.set_raw_if_empty(); } } } toks } /// Folds two subsequent `Token::Itlalic` into `Token::Bold` and correctly maps /// `Token::Escape`. fn round2<'raw>(raw: &'raw str, toks: &mut Vec>) { let mut idx = 0; loop { // We alawys want there to be a token after our idx if idx >= toks.len() - 1 { break; } let indicies: [usize; 2] = [toks[idx].idx, toks[idx + 1].idx]; let pattern: [Token<'raw>; 2] = [toks[idx].tok, toks[idx + 1].tok]; match pattern { [Token::Italic, Token::Italic] => { toks[idx] = IndexedToken { idx: indicies[0], tok: Token::Bold, }; toks.remove(idx + 1); continue; } [Token::LinkOpen, Token::LinkOpen] => { toks[idx] = IndexedToken { idx: indicies[0], tok: Token::InterlinkOpen, }; toks.remove(idx + 1); continue; } [Token::LinkClose, Token::LinkClose] => { toks[idx] = IndexedToken { idx: indicies[0], tok: Token::InterlinkClose, }; toks.remove(idx + 1); continue; } _ => (), } idx += 1; } // Collapse subsequent Token::Text and remove Token::Escape idx = 0; loop { if idx >= toks.len() - 1 { break; } let (pair_idx, pair) = split_token_pair([toks[idx], toks[idx + 1]]); if pair[0] == Token::Escape { if pair[1].is_text() { // Erroneous escape, remove it and leave the backslash in the text let tok_idx = pair_idx[0]; let len = 1 + pair[1].len(); let tok = Token::Text(&raw[tok_idx..tok_idx + len]); toks[idx + 1] = IndexedToken::new(tok_idx, tok); toks.remove(idx); idx = idx.saturating_sub(1); continue; } else if pair[1].is_space() { // Erroneous escape, replace it with a Text using it's index let escape_idx = pair_idx[0]; let tok = Token::Text(&raw[escape_idx..escape_idx + 1]); toks[idx] = IndexedToken::new(escape_idx, tok); continue; } else { toks[idx + 1] = IndexedToken { idx: pair_idx[1], tok: Token::Text(&raw[pair_idx[1]..pair_idx[1] + pair[1].len()]), }; } } idx += 1; } idx = 0; loop { if toks[idx].tok == Token::Escape { toks.remove(idx); // Don't increment idx, we removed an element so the vec slid down continue; } idx += 1; if idx == toks.len() { break; } } } fn round3<'raw, 'tok>(raw: &'raw str, toks: &'tok [IndexedToken<'raw>]) -> Vec> { round3_intralevel(raw, toks) } const TEXT_STYLE_LIMIT: &[Token<'static>] = &[Token::LinkOpen, Token::InterlinkOpen]; fn round3_intralevel<'raw, 'tok>( raw: &'raw str, toks: &'tok [IndexedToken<'raw>], ) -> Vec> { let mut doc = vec![]; let text_or_space = |it: &IndexedToken<'raw>| it.tok.is_text() || it.tok.is_space(); let mut curr = &toks[..]; loop { if curr.is_empty() { break doc; } let tok = curr[0]; macro_rules! decay { () => {{ doc.push(TreeNode::Text(tok.tok.as_str())); curr = &curr[1..]; continue; }}; } match tok.tok { Token::Percent | Token::LinkClose | Token::InterlinkClose => decay!(), Token::Text(_) | Token::Space => { let FindCloseRet { group, remaining } = consume_while(&curr[1..], text_or_space); match group.last() { None => decay!(), Some(last) => { let end_idx = last.idx + last.tok.len(); doc.push(TreeNode::Text(&raw[tok.idx..end_idx])); curr = remaining; continue; } } } Token::Escape => unreachable!(), Token::Pipe => match find_or_until(&curr[1..], Token::Pipe, TEXT_STYLE_LIMIT) { None => decay!(), Some(FindCloseRet { group, remaining }) => { let inner = round3_intralevel(raw, group); if let Some(pair) = remaining.get(..2) { let (_idx, pair) = split_token_pair([pair[0], pair[1]]); if let [Token::Percent, Token::Text(txt)] = pair { curr = &remaining[2..]; doc.push(TreeNode::ColoredText { color: txt, children: inner, }); continue; } } curr = remaining; doc.push(TreeNode::ColoredText { color: "", children: inner, }); continue; } }, Token::InterlinkOpen => match find_close(&curr[1..], Token::InterlinkClose) { None => decay!(), Some(FindCloseRet { group, remaining }) => { let inner = round3_intralevel(raw, group); curr = remaining; doc.push(TreeNode::Interlink { children: inner }); continue; } }, Token::LinkOpen => match find_close(&curr[1..], Token::LinkClose) { None => decay!(), Some(FindCloseRet { group, remaining }) => { curr = remaining; let mut splits = group.splitn(2, |tok| tok.tok == Token::Pipe); let first = splits.next(); let second = splits.next(); match (first, second) { (Some(one), None) => { doc.push(TreeNode::Link { kind: LinkKind::External, name: None, children: round3_intralevel(raw, one), }); } (Some(one), Some(two)) => { doc.push(TreeNode::Link { kind: LinkKind::External, name: Some(round3_intralevel(raw, one)), children: round3_intralevel(raw, two), }); } _ => unreachable!(), } continue; } }, Token::Bold | Token::Code | Token::Italic => match find_close(&curr[1..], tok.tok) { None => decay!(), Some(FindCloseRet { group, remaining }) => { let inner = round3_intralevel(raw, group); curr = remaining; let style = match tok.tok { Token::Bold => NodeStyle::Bold, Token::Code => NodeStyle::Code, Token::Italic => NodeStyle::Italic, _ => unreachable!(), }; doc.push(TreeNode::StyledText { style, children: inner, }); continue; } }, } } } fn split_token_pair<'raw>(it: [IndexedToken<'raw>; 2]) -> ([usize; 2], [Token<'raw>; 2]) { ([it[0].idx, it[1].idx], [it[0].tok, it[1].tok]) } fn find_close<'raw, 'tok>( toks: &'tok [IndexedToken<'raw>], close: Token<'raw>, ) -> Option> { match toks.iter().position(|tok| tok.tok.eq(&close)) { None => None, Some(idx) => { let group = &toks[..idx]; let remaining = &toks[idx + 1..]; Some(FindCloseRet { group, remaining }) } } } fn find_or_until<'raw, 'tok>( toks: &'tok [IndexedToken<'raw>], looking: Token<'raw>, limit: &[Token<'raw>], ) -> Option> { match toks .iter() .position(|tok| limit.contains(&tok.tok) || tok.tok == looking) { None => None, Some(idx) => { if toks[idx].tok == looking { let group = &toks[..idx]; let remaining = &toks[idx + 1..]; Some(FindCloseRet { group, remaining }) } else { None } } } } fn consume_while<'raw, 'tok, F>( toks: &'tok [IndexedToken<'raw>], valid: F, ) -> FindCloseRet<'raw, 'tok> where F: Fn(&IndexedToken<'raw>) -> bool, { match toks.iter().position(|it| !valid(it)) { None => FindCloseRet { group: toks, remaining: &[], }, Some(idx) => { let group = &toks[..idx]; let remaining = &toks[idx..]; FindCloseRet { group, remaining } } } } struct FindCloseRet<'raw, 'tok> { group: &'tok [IndexedToken<'raw>], remaining: &'tok [IndexedToken<'raw>], } //MARK: CollectingCharIter struct CollectingCharIter<'raw> { raw: &'raw str, chars: Enumerate>, raw_slice_start: Option, last: Option<(usize, char)>, current: Option<(usize, char)>, } impl<'raw> CollectingCharIter<'raw> { pub fn take_slice(&mut self, end: Option) -> Option> { let end = if let Some(end_idx) = end { end_idx } else { self.raw.len() }; if let Some(start) = self.raw_slice_start.take() { Some(IndexedToken { idx: start, tok: Token::Text(&self.raw[start..end]), }) } else { None } } pub fn append_slice(&mut self, end: Option, tokens: &mut Vec>) { if let Some(slice) = self.take_slice(end) { tokens.push(slice); } } pub fn set_raw_if_empty(&mut self) { if self.raw_slice_start.is_none() { if let Some((start, _)) = self.current { self.raw_slice_start = Some(start); } } } } impl<'raw> From<&'raw str> for CollectingCharIter<'raw> { fn from(raw: &'raw str) -> Self { Self { raw: raw, chars: raw.chars().enumerate(), raw_slice_start: None, last: None, current: None, } } } impl<'raw> Iterator for CollectingCharIter<'raw> { type Item = (usize, char); fn next(&mut self) -> Option { let next = self.chars.next(); self.last = self.current.take(); self.current = next; next } } //MARK: Test #[cfg(test)] mod test { use std::{borrow::Cow, rc::Rc}; use crate::lineup::{ EchoInterlinkSolver, IndexedToken, InterlinkSolver, NodeStyle, Token, TreeNode, process, round1, round2, round3, tree::LinkKind, }; fn only_tokens<'raw>(it: Vec>) -> Vec> { it.into_iter().map(|it| it.tok).collect() } #[test] fn round1_textonly() { let raw = "Raw Text!"; let r1 = round1(raw); assert_eq!( only_tokens(r1), &[Token::Text("Raw"), Token::Space, Token::Text("Text!")] ) } #[test] fn round1_single_asterisk() { let raw = "Raw*Text!"; let r1 = round1(raw); assert_eq!(r1[0].tok, Token::Text("Raw")); assert_eq!(r1[0].idx, 0); assert_eq!(r1[1].tok, Token::Italic); assert_eq!(r1[1].idx, 3); assert_eq!(r1[2].tok, Token::Text("Text!")); assert_eq!(r1[2].idx, 4); } #[test] fn round2_finds_bold() { let raw = "**"; let mut r1 = vec![ IndexedToken::new(0, Token::Italic), IndexedToken::new(1, Token::Italic), ]; round2(raw, &mut r1); assert_eq!(r1, vec![IndexedToken::new(0, Token::Bold)]); } #[test] fn round2_finds_escape() { // 012 345 let raw = "\\* \\**"; let mut r1 = vec![ IndexedToken::new(0, Token::Escape), IndexedToken::new(1, Token::Italic), IndexedToken::new(2, Token::Space), IndexedToken::new(3, Token::Escape), IndexedToken::new(4, Token::Italic), IndexedToken::new(5, Token::Italic), ]; round2(raw, &mut r1); assert_eq!( r1, vec![ IndexedToken::new(1, Token::Text("*")), IndexedToken::new(2, Token::Space), IndexedToken::new(4, Token::Text("**")) ] ) } #[test] fn round2_erroneous_escape() { let raw = "\\wrong escape"; let mut r1 = round1(raw); round2(raw, &mut r1); assert_eq!( only_tokens(r1), &[Token::Text("\\wrong"), Token::Space, Token::Text("escape")] ) } #[test] fn round2_middle_erreoneous_escape() { let raw = "wrong \\escape"; let mut r1 = round1(raw); round2(raw, &mut r1); assert_eq!( only_tokens(r1), &[Token::Text("wrong"), Token::Space, Token::Text("\\escape")] ) } #[test] fn round2_complex() { // 0 1 2 // 01234567890123456789 0123456789 let raw = "Start **bold** and \\* escape"; let mut r1 = round1(raw); round2(raw, &mut r1); assert_eq!( r1, vec![ IndexedToken::new(0, Token::Text("Start")), IndexedToken::new(5, Token::Space), IndexedToken::new(6, Token::Bold), IndexedToken::new(8, Token::Text("bold")), IndexedToken::new(12, Token::Bold), IndexedToken::new(14, Token::Space), IndexedToken::new(15, Token::Text("and")), IndexedToken::new(18, Token::Space), IndexedToken::new(20, Token::Text("*")), IndexedToken::new(21, Token::Space), IndexedToken::new(22, Token::Text("escape")) ] ) } #[test] fn round3_textonly() { let raw = "Raw text!"; let mut r1 = round1(raw); round2(raw, &mut r1); let tree = round3(raw, &r1); assert_eq!(tree, vec![TreeNode::Text(raw)]); } #[test] fn round3_italics() { let raw = "*italisized!*"; let mut r1 = round1(raw); round2(raw, &mut r1); let tree = round3(raw, &r1); assert_eq!( tree, vec![TreeNode::StyledText { style: NodeStyle::Italic, children: vec![TreeNode::Text("italisized!")] }] ) } #[test] fn round3_bold_in_italics() { let raw = "*italics **bold** after*"; let mut r1 = round1(raw); round2(raw, &mut r1); let tree = round3(raw, &r1); assert_eq!( tree, vec![TreeNode::StyledText { style: NodeStyle::Italic, children: vec![ TreeNode::Text("italics "), TreeNode::StyledText { style: NodeStyle::Bold, children: vec![TreeNode::Text("bold")] }, TreeNode::Text(" after") ] }] ) } #[test] fn round3_interlink() { let raw = "[[link]]"; let mut r1 = round1(raw); round2(raw, &mut r1); let tree = round3(raw, &r1); assert_eq!( tree, vec![TreeNode::Interlink { children: vec![TreeNode::Text("link")] }] ) } #[test] fn round3_link() { let raw = "[named|link]"; let mut r1 = round1(raw); round2(raw, &mut r1); let tree = round3(raw, &r1); assert_eq!( tree, vec![TreeNode::Link { kind: LinkKind::External, name: Some(vec![TreeNode::Text("named")]), children: vec![TreeNode::Text("link")] }] ) } #[test] fn round3_colored_text() { let raw = "some |text|%red that's red"; let mut r1 = round1(raw); round2(raw, &mut r1); let tree = round3(raw, &r1); assert_eq!( tree, &[ TreeNode::Text("some "), TreeNode::ColoredText { color: "red", children: vec![TreeNode::Text("text")] }, TreeNode::Text(" that's red") ] ) } #[test] fn treenode_html_text() { let node = TreeNode::Text("Raw Text!"); let rcsolve = Rc::new(EchoInterlinkSolver); let actual = node.html(rcsolve); let expected = "Raw Text!"; assert_eq!(actual, expected) } #[test] fn treenode_html_italics() { let node = TreeNode::StyledText { style: NodeStyle::Italic, children: vec![TreeNode::Text("italics!")], }; let rcsolve = Rc::new(EchoInterlinkSolver); let actual = node.html(rcsolve); let expected = "italics!"; assert_eq!(actual, expected) } #[test] fn treenode_html_interlink() { let node = TreeNode::Interlink { children: vec![TreeNode::Text("link!")], }; let rcsolve = Rc::new(EchoInterlinkSolver); let actual = node.html(rcsolve); let expected = "link!"; assert_eq!(actual, expected) } struct Solver; impl InterlinkSolver for Solver { fn link_title<'n>(&self, name: &'n str) -> Cow<'n, str> { Cow::Owned(name.to_uppercase()) } fn link_location<'n>(&self, name: &'n str) -> Cow<'n, str> { Cow::Owned(format!("/quark/{name}")) } } #[test] fn treenode_html_interlink_with_solver() { let node = TreeNode::Interlink { children: vec![TreeNode::Text("link!")], }; let solver = Rc::new(Solver); let actual = node.html(solver); let expected = "LINK!"; assert_eq!(actual, expected) } #[test] fn treenode_html_link() { let node = TreeNode::Link { kind: LinkKind::External, name: Some(vec![TreeNode::Text("dreamy.place")]), children: vec![TreeNode::Text("https://dreamy.place")], }; let actual = node.html(Rc::new(Solver)); let expected = "dreamy.place"; assert_eq!(actual, expected) } #[test] fn treenode_html_colored_text() { let node = TreeNode::ColoredText { color: "red", children: vec![TreeNode::Text("red text")], }; let actual = node.html(Rc::new(Solver)); let expected = "red text"; assert_eq!(actual, expected) } #[test] fn document_html_complex() { let raw = "*italics **bold** after*"; let expected = "italics bold after"; let rcsolve = Rc::new(EchoInterlinkSolver); let doc = process(raw, rcsolve); let html = doc.into_html(); assert_eq!(html, expected) } }