//! Basic syntax highlighting functionality. //! //! This module uses librustc_ast's lexer to provide token-based highlighting for //! the HTML documentation generated by rustdoc. //! //! Use the `render_with_highlighting` to highlight some rust code. use std::collections::VecDeque; use std::fmt::{Display, Write}; use rustc_data_structures::fx::FxIndexMap; use rustc_lexer::{Cursor, LiteralKind, TokenKind}; use rustc_span::edition::Edition; use rustc_span::symbol::Symbol; use rustc_span::{BytePos, DUMMY_SP, Span}; use super::format::{self, write_str}; use crate::clean::PrimitiveType; use crate::html::escape::EscapeBodyText; use crate::html::render::{Context, LinkFromSrc}; /// This type is needed in case we want to render links on items to allow to go to their definition. pub(crate) struct HrefContext<'a, 'tcx> { pub(crate) context: &'a Context<'tcx>, /// This span contains the current file we're going through. pub(crate) file_span: Span, /// This field is used to know "how far" from the top of the directory we are to link to either /// documentation pages or other source pages. pub(crate) root_path: &'a str, /// This field is used to calculate precise local URLs. pub(crate) current_href: String, } /// Decorations are represented as a map from CSS class to vector of character ranges. /// Each range will be wrapped in a span with that class. #[derive(Default)] pub(crate) struct DecorationInfo(pub(crate) FxIndexMap<&'static str, Vec<(u32, u32)>>); #[derive(Eq, PartialEq, Clone, Copy)] pub(crate) enum Tooltip { Ignore, CompileFail, ShouldPanic, Edition(Edition), None, } /// Highlights `src` as an inline example, returning the HTML output. pub(crate) fn render_example_with_highlighting( src: &str, out: &mut String, tooltip: Tooltip, playground_button: Option<&str>, extra_classes: &[String], ) { write_header(out, "rust-example-rendered", None, tooltip, extra_classes); write_code(out, src, None, None, None); write_footer(out, playground_button); } fn write_header( out: &mut String, class: &str, extra_content: Option<&str>, tooltip: Tooltip, extra_classes: &[String], ) { write_str( out, format_args!( "
",
if extra_classes.is_empty() { "" } else { " " },
extra_classes.join(" ")
),
);
} else {
write_str(
out,
format_args!(
"",
if extra_classes.is_empty() { "" } else { " " },
extra_classes.join(" ")
),
);
}
write_str(out, format_args!(""));
}
/// Check if two `Class` can be merged together. In the following rules, "unclassified" means `None`
/// basically (since it's `Option`). The following rules apply:
///
/// * If two `Class` have the same variant, then they can be merged.
/// * If the other `Class` is unclassified and only contains white characters (backline,
/// whitespace, etc), it can be merged.
/// * `Class::Ident` is considered the same as unclassified (because it doesn't have an associated
/// CSS class).
fn can_merge(class1: Option, class2: Option, text: &str) -> bool {
match (class1, class2) {
(Some(c1), Some(c2)) => c1.is_equal_to(c2),
(Some(Class::Ident(_)), None) | (None, Some(Class::Ident(_))) => true,
(Some(Class::Macro(_)), _) => false,
(Some(_), None) | (None, Some(_)) => text.trim().is_empty(),
(None, None) => true,
}
}
/// This type is used as a conveniency to prevent having to pass all its fields as arguments into
/// the various functions (which became its methods).
struct TokenHandler<'a, 'tcx, F: Write> {
out: &'a mut F,
/// It contains the closing tag and the associated `Class`.
closing_tags: Vec<(&'static str, Class)>,
/// This is used because we don't automatically generate the closing tag on `ExitSpan` in
/// case an `EnterSpan` event with the same class follows.
pending_exit_span: Option,
/// `current_class` and `pending_elems` are used to group HTML elements with same `class`
/// attributes to reduce the DOM size.
current_class: Option,
/// We need to keep the `Class` for each element because it could contain a `Span` which is
/// used to generate links.
pending_elems: Vec<(&'a str, Option)>,
href_context: Option>,
write_line_number: fn(&mut F, u32, &'static str),
}
impl TokenHandler<'_, '_, F> {
fn handle_exit_span(&mut self) {
// We can't get the last `closing_tags` element using `pop()` because `closing_tags` is
// being used in `write_pending_elems`.
let class = self.closing_tags.last().expect("ExitSpan without EnterSpan").1;
// We flush everything just in case...
self.write_pending_elems(Some(class));
exit_span(self.out, self.closing_tags.pop().expect("ExitSpan without EnterSpan").0);
self.pending_exit_span = None;
}
/// Write all the pending elements sharing a same (or at mergeable) `Class`.
///
/// If there is a "parent" (if a `EnterSpan` event was encountered) and the parent can be merged
/// with the elements' class, then we simply write the elements since the `ExitSpan` event will
/// close the tag.
///
/// Otherwise, if there is only one pending element, we let the `string` function handle both
/// opening and closing the tag, otherwise we do it into this function.
///
/// It returns `true` if `current_class` must be set to `None` afterwards.
fn write_pending_elems(&mut self, current_class: Option) -> bool {
if self.pending_elems.is_empty() {
return false;
}
if let Some((_, parent_class)) = self.closing_tags.last()
&& can_merge(current_class, Some(*parent_class), "")
{
for (text, class) in self.pending_elems.iter() {
string(
self.out,
EscapeBodyText(text),
*class,
&self.href_context,
false,
self.write_line_number,
);
}
} else {
// We only want to "open" the tag ourselves if we have more than one pending and if the
// current parent tag is not the same as our pending content.
let close_tag = if self.pending_elems.len() > 1
&& let Some(current_class) = current_class
// `PreludeTy` can never include more than an ident so it should not generate
// a wrapping `span`.
&& !matches!(current_class, Class::PreludeTy(_))
{
Some(enter_span(self.out, current_class, &self.href_context))
} else {
None
};
for (text, class) in self.pending_elems.iter() {
string(
self.out,
EscapeBodyText(text),
*class,
&self.href_context,
close_tag.is_none(),
self.write_line_number,
);
}
if let Some(close_tag) = close_tag {
exit_span(self.out, close_tag);
}
}
self.pending_elems.clear();
true
}
#[inline]
fn write_line_number(&mut self, line: u32, extra: &'static str) {
(self.write_line_number)(self.out, line, extra);
}
}
impl Drop for TokenHandler<'_, '_, F> {
/// When leaving, we need to flush all pending data to not have missing content.
fn drop(&mut self) {
if self.pending_exit_span.is_some() {
self.handle_exit_span();
} else {
self.write_pending_elems(self.current_class);
}
}
}
fn write_scraped_line_number(out: &mut impl Write, line: u32, extra: &'static str) {
// https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#data-nosnippet-attr
// Do not show "1 2 3 4 5 ..." in web search results.
write!(out, "{extra}{line}",).unwrap();
}
fn write_line_number(out: &mut impl Write, line: u32, extra: &'static str) {
// https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#data-nosnippet-attr
// Do not show "1 2 3 4 5 ..." in web search results.
write!(out, "{extra}{line}",).unwrap();
}
fn empty_line_number(out: &mut impl Write, _: u32, extra: &'static str) {
out.write_str(extra).unwrap();
}
#[derive(Clone, Copy)]
pub(super) struct LineInfo {
pub(super) start_line: u32,
max_lines: u32,
pub(super) is_scraped_example: bool,
}
impl LineInfo {
pub(super) fn new(max_lines: u32) -> Self {
Self { start_line: 1, max_lines: max_lines + 1, is_scraped_example: false }
}
pub(super) fn new_scraped(max_lines: u32, start_line: u32) -> Self {
Self {
start_line: start_line + 1,
max_lines: max_lines + start_line + 1,
is_scraped_example: true,
}
}
}
/// Convert the given `src` source code into HTML by adding classes for highlighting.
///
/// This code is used to render code blocks (in the documentation) as well as the source code pages.
///
/// Some explanations on the last arguments:
///
/// In case we are rendering a code block and not a source code file, `href_context` will be `None`.
/// To put it more simply: if `href_context` is `None`, the code won't try to generate links to an
/// item definition.
///
/// More explanations about spans and how we use them here are provided in the
pub(super) fn write_code(
out: &mut impl Write,
src: &str,
href_context: Option>,
decoration_info: Option<&DecorationInfo>,
line_info: Option,
) {
// This replace allows to fix how the code source with DOS backline characters is displayed.
let src = src.replace("\r\n", "\n");
let mut token_handler = TokenHandler {
out,
closing_tags: Vec::new(),
pending_exit_span: None,
current_class: None,
pending_elems: Vec::new(),
href_context,
write_line_number: match line_info {
Some(line_info) => {
if line_info.is_scraped_example {
write_scraped_line_number
} else {
write_line_number
}
}
None => empty_line_number,
},
};
let (mut line, max_lines) = if let Some(line_info) = line_info {
token_handler.write_line_number(line_info.start_line, "");
(line_info.start_line, line_info.max_lines)
} else {
(0, u32::MAX)
};
Classifier::new(
&src,
token_handler.href_context.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP),
decoration_info,
)
.highlight(&mut |highlight| {
match highlight {
Highlight::Token { text, class } => {
// If we received a `ExitSpan` event and then have a non-compatible `Class`, we
// need to close the ``.
let need_current_class_update = if let Some(pending) =
token_handler.pending_exit_span
&& !can_merge(Some(pending), class, text)
{
token_handler.handle_exit_span();
true
// If the two `Class` are different, time to flush the current content and start
// a new one.
} else if !can_merge(token_handler.current_class, class, text) {
token_handler.write_pending_elems(token_handler.current_class);
true
} else {
token_handler.current_class.is_none()
};
if need_current_class_update {
token_handler.current_class = class.map(Class::dummy);
}
if text == "\n" {
line += 1;
if line < max_lines {
token_handler.pending_elems.push((text, Some(Class::Backline(line))));
}
} else {
token_handler.pending_elems.push((text, class));
}
}
Highlight::EnterSpan { class } => {
let mut should_add = true;
if let Some(pending_exit_span) = token_handler.pending_exit_span {
if class.is_equal_to(pending_exit_span) {
should_add = false;
} else {
token_handler.handle_exit_span();
}
} else {
// We flush everything just in case...
if token_handler.write_pending_elems(token_handler.current_class) {
token_handler.current_class = None;
}
}
if should_add {
let closing_tag =
enter_span(token_handler.out, class, &token_handler.href_context);
token_handler.closing_tags.push((closing_tag, class));
}
token_handler.current_class = None;
token_handler.pending_exit_span = None;
}
Highlight::ExitSpan => {
token_handler.current_class = None;
token_handler.pending_exit_span = Some(
token_handler
.closing_tags
.last()
.as_ref()
.expect("ExitSpan without EnterSpan")
.1,
);
}
};
});
}
fn write_footer(out: &mut String, playground_button: Option<&str>) {
write_str(out, format_args_nl!(" {}