diff options
| author | gennyble <gen@nyble.dev> | 2026-08-27 18:34:36 -0500 |
|---|---|---|
| committer | gennyble <gen@nyble.dev> | 2026-08-27 18:34:36 -0500 |
| commit | a0e4115f0f122e635481c6d6efbedcf9c9d60909 (patch) | |
| tree | 9eed0321d6672cb6760fe587fa6e409d4620c29d /markup/src | |
| parent | 9baee25012df19a43585982ee497513817e3c6b6 (diff) | |
markup
Diffstat (limited to 'markup/src')
| -rw-r--r-- | markup/src/lib.rs | 4 | ||||
| -rw-r--r-- | markup/src/lineup.rs | 757 | ||||
| -rwxr-xr-x | markup/src/state.rs | 17 |
3 files changed, 776 insertions, 2 deletions
diff --git a/markup/src/lib.rs b/markup/src/lib.rs index c4d7918..23ca432 100644 --- a/markup/src/lib.rs +++ b/markup/src/lib.rs @@ -1,11 +1,13 @@ mod intra; +mod lineup; mod state; -use std::ops::Deref; +use std::{cell::Ref, ops::Deref}; use bempline::{Document, Options}; use crate::state::State; +pub use lineup::InterlinkSolver; pub fn process(raw: &str) -> Markup { let mut state = State::default(); diff --git a/markup/src/lineup.rs b/markup/src/lineup.rs new file mode 100644 index 0000000..6610e77 --- /dev/null +++ b/markup/src/lineup.rs @@ -0,0 +1,757 @@ +use core::fmt; +use std::{borrow::Cow, cell::Ref, iter::Enumerate, ops::Deref, rc::Rc, str::Chars}; + +pub fn process<'raw>(raw: &'raw str, solver: Rc<dyn InterlinkSolver>) -> Document<'raw> { + let mut r1 = round1(raw); + round2(raw, &mut r1); + let tree = round3(raw, &r1); + + Document { + nodes: tree, + solver, + } +} + +/// Round 1 tokenizes to `Token` but does not produce `Bold` or properly +/// run `Token::Escape`. round2() is the next step +fn round1<'raw>(raw: &'raw str) -> Vec<IndexedToken<'raw>> { + let mut chars = CollectingCharIter::from(raw); + + let mut toks = vec![]; + loop { + let (idx, ch) = match chars.next() { + Some(pair) => pair, + None => { + chars.append_slice(None, &mut toks); + break; + } + }; + + match ch { + '\\' => { + chars.append_slice(Some(idx), &mut toks); + toks.push(IndexedToken::new(idx, Token::Escape)); + } + '`' => { + chars.append_slice(Some(idx), &mut toks); + toks.push(IndexedToken::new(idx, Token::Code)); + } + '*' => { + chars.append_slice(Some(idx), &mut toks); + toks.push(IndexedToken::new(idx, Token::Italic)); + } + '[' => { + chars.append_slice(Some(idx), &mut toks); + toks.push(IndexedToken::new(idx, Token::InterlinkOpen)); + } + ']' => { + chars.append_slice(Some(idx), &mut toks); + toks.push(IndexedToken::new(idx, Token::InterlinkClose)); + } + _ch => { + chars.set_raw_if_empty(); + } + } + } + + toks +} + +/// Folds two subsequent `Token::Itlalic` into `Token::Bold` and correctly maps +/// `Token::Escape`. Additionally, combines successive `Token::Text` into +/// single slices. +fn round2<'raw>(raw: &'raw str, toks: &mut Vec<IndexedToken<'raw>>) { + let mut idx = 0; + loop { + // We alawys want there to be a token after our idx + if idx >= toks.len() - 1 { + break; + } + + let indicies: [usize; 2] = [toks[idx].idx, toks[idx + 1].idx]; + let pattern: [Token<'raw>; 2] = [toks[idx].tok, toks[idx + 1].tok]; + + match pattern { + [Token::Italic, Token::Italic] => { + toks[idx] = IndexedToken { + idx: indicies[0], + tok: Token::Bold, + }; + toks.remove(idx + 1); + continue; + } + _ => (), + } + + idx += 1; + } + + // Collapse subsequent Token::Text and remove Token::Escape + idx = 0; + loop { + if idx >= toks.len() - 1 { + break; + } + + let (pair_idx, pair) = split_token_pair([toks[idx], toks[idx + 1]]); + + if pair[0] == Token::Escape { + if pair[1].is_text() { + // Erroneous escape, remove it and leave the backslash in the text + let tok_idx = pair_idx[0]; + let len = 1 + pair[1].len(); + let tok = Token::Text(&raw[tok_idx..tok_idx + len]); + toks[idx + 1] = IndexedToken::new(tok_idx, tok); + toks.remove(idx); + idx = idx.saturating_sub(1); + continue; + } else { + toks[idx + 1] = IndexedToken { + idx: pair_idx[1], + tok: Token::Text(&raw[pair_idx[1]..pair_idx[1] + pair[1].len()]), + }; + } + } + + if let [Token::Text(text1), Token::Text(text2)] = pair { + let first_index = toks[idx].idx; + let length = text1.len() + text2.len(); + toks[idx] = IndexedToken { + idx: first_index, + tok: Token::Text(&raw[first_index..first_index + length]), + }; + + // Remove the second text, as it's been combined into the first + toks.remove(idx + 1); + + // Continue here so we use the same index and compare the modified + // Token::Text with the next token + continue; + } + + idx += 1; + } + + idx = 0; + loop { + if toks[idx].tok == Token::Escape { + toks.remove(idx); + + // Don't increment idx, we removed an element so the vec slid down + continue; + } + + idx += 1; + if idx == toks.len() { + break; + } + } +} + +fn split_token_pair<'raw>(it: [IndexedToken<'raw>; 2]) -> ([usize; 2], [Token<'raw>; 2]) { + ([it[0].idx, it[1].idx], [it[0].tok, it[1].tok]) +} + +fn round3<'raw, 'tok>(raw: &'raw str, toks: &'tok [IndexedToken<'raw>]) -> Vec<TreeNode<'raw>> { + let mut doc = vec![]; + + let mut curr = &toks[..]; + loop { + if curr.is_empty() { + break doc; + } + + let tok = curr[0]; + if let Token::Text(str) = tok.tok { + doc.push(TreeNode::Text(str)); + curr = &curr[1..]; + continue; + } + + match tok.tok { + Token::Text(str) => { + doc.push(TreeNode::Text(str)); + curr = &curr[1..]; + continue; + } + Token::Escape => { + unreachable!() + } + Token::InterlinkClose => { + doc.push(TreeNode::Text("]")); + curr = &curr[1..]; + continue; + } + Token::InterlinkOpen => match find_close(&curr[1..], Token::InterlinkClose) { + None => { + doc.push(TreeNode::Text(tok.tok.as_str())); + curr = &curr[1..]; + continue; + } + Some(FindCloseRet { group, remaining }) => { + let inner = round3(raw, group); + curr = remaining; + doc.push(TreeNode::Interlink { children: inner }); + continue; + } + }, + Token::Bold | Token::Code | Token::Italic => match find_close(&curr[1..], tok.tok) { + None => { + doc.push(TreeNode::Text(tok.tok.as_str())); + curr = &curr[1..]; + continue; + } + Some(FindCloseRet { group, remaining }) => { + let inner = round3(raw, group); + curr = remaining; + + let style = match tok.tok { + Token::Bold => NodeStyle::Bold, + Token::Code => NodeStyle::Code, + Token::Italic => NodeStyle::Italic, + _ => unreachable!(), + }; + + doc.push(TreeNode::StyledText { + style, + children: inner, + }); + continue; + } + }, + } + } +} + +fn find_close<'raw, 'tok>( + toks: &'tok [IndexedToken<'raw>], + close: Token<'raw>, +) -> Option<FindCloseRet<'raw, 'tok>> { + match toks.iter().position(|tok| tok.tok.eq(&close)) { + None => None, + Some(idx) => { + let group = &toks[..idx]; + let remaining = &toks[idx + 1..]; + + Some(FindCloseRet { group, remaining }) + } + } +} + +struct FindCloseRet<'raw, 'tok> { + group: &'tok [IndexedToken<'raw>], + remaining: &'tok [IndexedToken<'raw>], +} + +pub trait InterlinkSolver { + fn link_title<'n>(&self, name: &'n str) -> Cow<'n, str>; + fn link_location<'n>(&self, name: &'n str) -> Cow<'n, str>; +} + +pub struct EchoInterlinkSolver; +impl InterlinkSolver for EchoInterlinkSolver { + fn link_title<'n>(&self, name: &'n str) -> Cow<'n, str> { + Cow::Borrowed(name) + } + + fn link_location<'n>(&self, name: &'n str) -> Cow<'n, str> { + Cow::Borrowed(name) + } +} + +//MARK: Tree +pub struct Document<'raw> { + nodes: Vec<TreeNode<'raw>>, + solver: Rc<dyn InterlinkSolver>, +} + +impl<'raw> Document<'raw> { + pub fn set_interlink_solver(&mut self, solver: Rc<dyn InterlinkSolver>) { + self.solver = solver; + } + + pub fn html(self) -> String { + TreeNode::html_nodes(&self.nodes, Rc::clone(&self.solver)) + } +} + +#[derive(Clone, Debug, PartialEq)] +enum TreeNode<'raw> { + Text(&'raw str), + Interlink { + children: Vec<TreeNode<'raw>>, + }, + StyledText { + style: NodeStyle, + children: Vec<TreeNode<'raw>>, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum NodeStyle { + Bold, + Code, + Italic, +} + +impl<'raw> TreeNode<'raw> { + pub fn html(&self, solver: Rc<dyn InterlinkSolver>) -> Cow<'raw, str> { + match self { + TreeNode::Text(str) => Cow::Borrowed(str), + TreeNode::Interlink { children } => { + let txt = Self::text_nodes(&children, Rc::clone(&solver)); + + let title = solver.link_title(&txt); + let location = solver.link_location(&txt); + + Cow::Owned(format!("<a href=\"{location}\">{title}</a>")) + } + TreeNode::StyledText { style, children } => { + let (open, close) = match style { + NodeStyle::Bold => ("<b>", "</b>"), + NodeStyle::Code => ("<code>", "</code>"), + NodeStyle::Italic => ("<i>", "</i>"), + }; + + let inner = Self::html_nodes(children.as_slice(), solver); + Cow::Owned(format!("{open}{inner}{close}")) + } + } + } + + pub fn text(&self, solver: Rc<dyn InterlinkSolver>) -> Cow<'raw, str> { + match self { + TreeNode::Text(str) => Cow::Borrowed(str), + TreeNode::Interlink { children } => { + let text = Self::text_nodes(children, Rc::clone(&solver)); + Cow::Owned(solver.link_title(&text).into_owned()) + } + TreeNode::StyledText { style: _, children } => { + Cow::Owned(Self::text_nodes(children, solver)) + } + } + } + + fn text_nodes(nodes: &[TreeNode<'raw>], solver: Rc<dyn InterlinkSolver>) -> String { + let mut ret = String::new(); + for node in nodes { + let node_txt = &node.text(Rc::clone(&solver)); + if !node_txt.is_empty() { + ret.push_str(node_txt); + } + } + + ret + } + + fn html_nodes(nodes: &[TreeNode<'raw>], solver: Rc<dyn InterlinkSolver>) -> String { + let mut ret = String::new(); + for node in nodes { + ret.push_str(&node.html(Rc::clone(&solver))); + } + + ret + } +} + +//MARK: CollectingCharIter +struct CollectingCharIter<'raw> { + raw: &'raw str, + chars: Enumerate<Chars<'raw>>, + raw_slice_start: Option<usize>, + + last: Option<(usize, char)>, + current: Option<(usize, char)>, +} + +impl<'raw> CollectingCharIter<'raw> { + pub fn take_slice(&mut self, end: Option<usize>) -> Option<IndexedToken<'raw>> { + let end = if let Some(end_idx) = end { + end_idx + } else { + self.raw.len() + }; + + if let Some(start) = self.raw_slice_start.take() { + Some(IndexedToken { + idx: start, + tok: Token::Text(&self.raw[start..end]), + }) + } else { + None + } + } + + pub fn append_slice(&mut self, end: Option<usize>, tokens: &mut Vec<IndexedToken<'raw>>) { + if let Some(slice) = self.take_slice(end) { + tokens.push(slice); + } + } + + pub fn set_raw_if_empty(&mut self) { + if self.raw_slice_start.is_none() { + if let Some((start, _)) = self.current { + self.raw_slice_start = Some(start); + } + } + } + + pub fn last(&self) -> Option<(usize, char)> { + self.last + } +} + +impl<'raw> From<&'raw str> for CollectingCharIter<'raw> { + fn from(raw: &'raw str) -> Self { + Self { + raw: raw, + chars: raw.chars().enumerate(), + raw_slice_start: None, + + last: None, + current: None, + } + } +} + +impl<'raw> Iterator for CollectingCharIter<'raw> { + type Item = (usize, char); + + fn next(&mut self) -> Option<Self::Item> { + let next = self.chars.next(); + self.last = self.current.take(); + self.current = next; + next + } +} + +struct Round2Tokens<'raw>(Vec<IndexedToken<'raw>>); + +//MARK: Tokens +#[derive(Copy, Clone, Debug, PartialEq)] +struct IndexedToken<'raw> { + idx: usize, + tok: Token<'raw>, +} + +impl<'raw> IndexedToken<'raw> { + pub fn new(idx: usize, tok: Token<'raw>) -> Self { + Self { idx, tok } + } +} + +#[derive(Copy, Clone, Debug, PartialEq)] +enum Token<'raw> { + /// Raw, no markup + Text(&'raw str), + /// backtick + Code, + /// asterisk + Italic, + /// Double-asterisk + Bold, + /// backslash + Escape, + /// Interlink Opening [ + InterlinkOpen, + /// Interlink Closing ] + InterlinkClose, +} + +impl<'raw> Token<'raw> { + pub fn is_text(&self) -> bool { + if let Self::Text(_) = self { + true + } else { + false + } + } + + pub fn len(&self) -> usize { + match self { + Self::Text(txt) => txt.len(), + Self::Code + | Self::Italic + | Self::Escape + | Self::InterlinkOpen + | Self::InterlinkClose => 1, + Self::Bold => 2, + } + } + + pub fn as_str(&self) -> &'raw str { + match self { + Token::Bold => "**", + Token::Code => "`", + Token::Escape => "\\", + Token::InterlinkClose => "]", + Token::InterlinkOpen => "[", + Token::Italic => "*", + Token::Text(str) => str, + } + } +} + +impl<'raw> fmt::Display for Token<'raw> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + Token::Bold => "**", + Token::Code => "`", + Token::Escape => "\\", + Token::InterlinkClose => "]", + Token::InterlinkOpen => "[", + Token::Italic => "*", + Token::Text(str) => str, + }; + + write!(f, "{str}") + } +} + +//MARK: Test +#[cfg(test)] +mod test { + use std::{borrow::Cow, rc::Rc}; + + use crate::lineup::{ + EchoInterlinkSolver, IndexedToken, InterlinkSolver, NodeStyle, Token, TreeNode, process, + round1, round2, round3, + }; + + #[test] + fn round1_textonly() { + let raw = "Raw Text!"; + let r1 = round1(raw); + + assert_eq!(r1[0].tok, Token::Text("Raw Text!")) + } + + #[test] + fn round1_single_asterisk() { + let raw = "Raw*Text!"; + let r1 = round1(raw); + + assert_eq!(r1[0].tok, Token::Text("Raw")); + assert_eq!(r1[0].idx, 0); + + assert_eq!(r1[1].tok, Token::Italic); + assert_eq!(r1[1].idx, 3); + + assert_eq!(r1[2].tok, Token::Text("Text!")); + assert_eq!(r1[2].idx, 4); + } + + #[test] + fn round2_collapse_texts() { + let raw = "RawText!"; + let mut r1 = vec![ + IndexedToken::new(0, Token::Text(&raw[..3])), + IndexedToken::new(3, Token::Text(&raw[3..])), + ]; + + round2(raw, &mut r1); + + assert_eq!(r1, vec![IndexedToken::new(0, Token::Text(raw))]); + } + + #[test] + fn round2_finds_bold() { + let raw = "**"; + let mut r1 = vec![ + IndexedToken::new(0, Token::Italic), + IndexedToken::new(1, Token::Italic), + ]; + + round2(raw, &mut r1); + + assert_eq!(r1, vec![IndexedToken::new(0, Token::Bold)]); + } + + #[test] + fn round2_finds_escape() { + // 012 345 + let raw = "\\* \\**"; + let mut r1 = vec![ + IndexedToken::new(0, Token::Escape), + IndexedToken::new(1, Token::Italic), + IndexedToken::new(2, Token::Text(&raw[2..3])), + IndexedToken::new(3, Token::Escape), + IndexedToken::new(4, Token::Italic), + IndexedToken::new(5, Token::Italic), + ]; + + round2(raw, &mut r1); + + assert_eq!( + r1, + vec![ + IndexedToken::new(1, Token::Text("* ")), + IndexedToken::new(4, Token::Text("**")) + ] + ) + } + + #[test] + fn round2_erroneous_escape() { + let raw = "\\wrong escape"; + let mut r1 = round1(raw); + round2(raw, &mut r1); + + assert_eq!(r1[0].tok, Token::Text("\\wrong escape")) + } + + #[test] + fn round2_middle_erreoneous_escape() { + let raw = "wrong \\escape"; + let mut r1 = round1(raw); + round2(raw, &mut r1); + + assert_eq!( + r1, + vec![IndexedToken::new(0, Token::Text("wrong \\escape"))] + ) + } + + #[test] + fn round2_complex() { + // 0 1 2 + // 01234567890123456789 0123456789 + let raw = "Start **bold** and \\* escape"; + let mut r1 = round1(raw); + round2(raw, &mut r1); + + assert_eq!( + r1, + vec![ + IndexedToken::new(0, Token::Text("Start ")), + IndexedToken::new(6, Token::Bold), + IndexedToken::new(8, Token::Text("bold")), + IndexedToken::new(12, Token::Bold), + IndexedToken::new(14, Token::Text(" and ")), + IndexedToken::new(20, Token::Text("* escape")) + ] + ) + } + + #[test] + fn round3_textonly() { + let raw = "Raw text!"; + let mut r1 = round1(raw); + round2(raw, &mut r1); + + let tree = round3(raw, &r1); + + assert_eq!(tree, vec![TreeNode::Text(raw)]); + } + + #[test] + fn round3_italics() { + let raw = "*italisized!*"; + let mut r1 = round1(raw); + round2(raw, &mut r1); + + let tree = round3(raw, &r1); + + assert_eq!( + tree, + vec![TreeNode::StyledText { + style: NodeStyle::Italic, + children: vec![TreeNode::Text("italisized!")] + }] + ) + } + + #[test] + fn round3_bold_in_italics() { + let raw = "*italics **bold** after*"; + let mut r1 = round1(raw); + round2(raw, &mut r1); + + let tree = round3(raw, &r1); + + assert_eq!( + tree, + vec![TreeNode::StyledText { + style: NodeStyle::Italic, + children: vec![ + TreeNode::Text("italics "), + TreeNode::StyledText { + style: NodeStyle::Bold, + children: vec![TreeNode::Text("bold")] + }, + TreeNode::Text(" after") + ] + }] + ) + } + + #[test] + fn treenode_html_text() { + let node = TreeNode::Text("Raw Text!"); + let rcsolve = Rc::new(EchoInterlinkSolver); + let actual = node.html(rcsolve); + let expected = "Raw Text!"; + + assert_eq!(actual, expected) + } + + #[test] + fn treenode_html_italics() { + let node = TreeNode::StyledText { + style: NodeStyle::Italic, + children: vec![TreeNode::Text("italics!")], + }; + let rcsolve = Rc::new(EchoInterlinkSolver); + let actual = node.html(rcsolve); + let expected = "<i>italics!</i>"; + + assert_eq!(actual, expected) + } + + #[test] + fn treenode_html_interlink() { + let node = TreeNode::Interlink { + children: vec![TreeNode::Text("link!")], + }; + let rcsolve = Rc::new(EchoInterlinkSolver); + let actual = node.html(rcsolve); + let expected = "<a href=\"link!\">link!</a>"; + + assert_eq!(actual, expected) + } + + struct Solver; + impl InterlinkSolver for Solver { + fn link_title<'n>(&self, name: &'n str) -> Cow<'n, str> { + Cow::Owned(name.to_uppercase()) + } + + fn link_location<'n>(&self, name: &'n str) -> Cow<'n, str> { + Cow::Owned(format!("/quark/{name}")) + } + } + + #[test] + fn treenode_html_interlink_with_solver() { + let node = TreeNode::Interlink { + children: vec![TreeNode::Text("link!")], + }; + + let solver = Rc::new(Solver); + let actual = node.html(solver); + let expected = "<a href=\"/quark/link!\">LINK!</a>"; + + assert_eq!(actual, expected) + } + + #[test] + fn document_html_complex() { + let raw = "*italics **bold** after*"; + let expected = "<i>italics <b>bold</b> after</i>"; + let rcsolve = Rc::new(EchoInterlinkSolver); + let doc = process(raw, rcsolve); + let html = doc.html(); + + assert_eq!(html, expected) + } +} diff --git a/markup/src/state.rs b/markup/src/state.rs index 5931376..64522b6 100755 --- a/markup/src/state.rs +++ b/markup/src/state.rs @@ -1,4 +1,6 @@ -use crate::{Fill, Markup, intra}; +use std::{cell::RefCell, rc::Rc}; + +use crate::{Fill, InterlinkSolver, Markup, intra, lineup::EchoInterlinkSolver}; pub(crate) struct State { active_id: Option<String>, @@ -10,6 +12,8 @@ pub(crate) struct State { fills: Vec<Fill>, escaped_html: bool, last_blank: bool, + + solver: Rc<dyn InterlinkSolver>, } impl Default for State { @@ -24,6 +28,8 @@ impl Default for State { fills: vec![], escaped_html: false, last_blank: true, + + solver: Rc::new(EchoInterlinkSolver), } } } @@ -78,6 +84,15 @@ impl State { && (!string.starts_with('<') || (string.starts_with('<') && self.escaped_html)) } + pub fn with_interlink_solver<Solver: InterlinkSolver + 'static>( + mut self, + solver: Solver, + ) -> Self { + self.solver = Rc::new(solver); + + self + } + pub fn process_line(&mut self, line: &str) { if self.parse_command(line) { // Don't set last_blank; a command does not break up a text block |
