about summary refs log tree commit diff
path: root/src/tools/clippy/clippy_utils
diff options
context:
space:
mode:
authorPhilipp Krones <hello@philkrones.com>2023-05-20 15:39:26 +0200
committerPhilipp Krones <hello@philkrones.com>2023-05-20 15:39:26 +0200
commit12cbb6ced8a848d1de68e7f0fe842217b9563ccc (patch)
treec3bbf48a283d582da6730ae6829b6bd1d8defaf1 /src/tools/clippy/clippy_utils
parent25f084d5e007b9561d155be0b1a2e369c8e4b0ef (diff)
parent435a8ad86c7a33bd7ffb91c59039943408d3b6aa (diff)
downloadrust-12cbb6ced8a848d1de68e7f0fe842217b9563ccc.tar.gz
rust-12cbb6ced8a848d1de68e7f0fe842217b9563ccc.zip
Merge commit '435a8ad86c7a33bd7ffb91c59039943408d3b6aa' into clippyup
Diffstat (limited to 'src/tools/clippy/clippy_utils')
-rw-r--r--src/tools/clippy/clippy_utils/src/consts.rs110
-rw-r--r--src/tools/clippy/clippy_utils/src/hir_utils.rs206
-rw-r--r--src/tools/clippy/clippy_utils/src/lib.rs50
-rw-r--r--src/tools/clippy/clippy_utils/src/source.rs52
4 files changed, 315 insertions, 103 deletions
diff --git a/src/tools/clippy/clippy_utils/src/consts.rs b/src/tools/clippy/clippy_utils/src/consts.rs
index 8075881e3bb..fb772644c0d 100644
--- a/src/tools/clippy/clippy_utils/src/consts.rs
+++ b/src/tools/clippy/clippy_utils/src/consts.rs
@@ -1,18 +1,21 @@
 #![allow(clippy::float_cmp)]
 
+use crate::source::{get_source_text, walk_span_to_context};
 use crate::{clip, is_direct_expn_of, sext, unsext};
 use if_chain::if_chain;
 use rustc_ast::ast::{self, LitFloatType, LitKind};
 use rustc_data_structures::sync::Lrc;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::{BinOp, BinOpKind, Block, Expr, ExprKind, HirId, Item, ItemKind, Node, QPath, UnOp};
+use rustc_lexer::tokenize;
 use rustc_lint::LateContext;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::ty::SubstsRef;
 use rustc_middle::ty::{self, EarlyBinder, FloatTy, ScalarInt, Ty, TyCtxt};
+use rustc_middle::ty::{List, SubstsRef};
 use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::Symbol;
+use rustc_span::SyntaxContext;
 use std::cmp::Ordering::{self, Equal};
 use std::hash::{Hash, Hasher};
 use std::iter;
@@ -210,8 +213,7 @@ pub fn lit_to_mir_constant(lit: &LitKind, ty: Option<Ty<'_>>) -> Constant {
     match *lit {
         LitKind::Str(ref is, _) => Constant::Str(is.to_string()),
         LitKind::Byte(b) => Constant::Int(u128::from(b)),
-        LitKind::ByteStr(ref s, _) => Constant::Binary(Lrc::clone(s)),
-        LitKind::CStr(ref s, _) => Constant::Binary(Lrc::clone(s)),
+        LitKind::ByteStr(ref s, _) | LitKind::CStr(ref s, _) => Constant::Binary(Lrc::clone(s)),
         LitKind::Char(c) => Constant::Char(c),
         LitKind::Int(n, _) => Constant::Int(n),
         LitKind::Float(ref is, LitFloatType::Suffixed(fty)) => match fty {
@@ -228,27 +230,46 @@ pub fn lit_to_mir_constant(lit: &LitKind, ty: Option<Ty<'_>>) -> Constant {
     }
 }
 
+/// The source of a constant value.
+pub enum ConstantSource {
+    /// The value is determined solely from the expression.
+    Local,
+    /// The value is dependent on a defined constant.
+    Constant,
+}
+impl ConstantSource {
+    pub fn is_local(&self) -> bool {
+        matches!(self, Self::Local)
+    }
+}
+
+/// Attempts to evaluate the expression as a constant.
 pub fn constant<'tcx>(
     lcx: &LateContext<'tcx>,
     typeck_results: &ty::TypeckResults<'tcx>,
     e: &Expr<'_>,
-) -> Option<(Constant, bool)> {
-    let mut cx = ConstEvalLateContext {
-        lcx,
-        typeck_results,
-        param_env: lcx.param_env,
-        needed_resolution: false,
-        substs: ty::List::empty(),
-    };
-    cx.expr(e).map(|cst| (cst, cx.needed_resolution))
+) -> Option<Constant> {
+    ConstEvalLateContext::new(lcx, typeck_results).expr(e)
+}
+
+/// Attempts to evaluate the expression as a constant.
+pub fn constant_with_source<'tcx>(
+    lcx: &LateContext<'tcx>,
+    typeck_results: &ty::TypeckResults<'tcx>,
+    e: &Expr<'_>,
+) -> Option<(Constant, ConstantSource)> {
+    let mut ctxt = ConstEvalLateContext::new(lcx, typeck_results);
+    let res = ctxt.expr(e);
+    res.map(|x| (x, ctxt.source))
 }
 
+/// Attempts to evaluate an expression only if it's value is not dependent on other items.
 pub fn constant_simple<'tcx>(
     lcx: &LateContext<'tcx>,
     typeck_results: &ty::TypeckResults<'tcx>,
     e: &Expr<'_>,
 ) -> Option<Constant> {
-    constant(lcx, typeck_results, e).and_then(|(cst, res)| if res { None } else { Some(cst) })
+    constant_with_source(lcx, typeck_results, e).and_then(|(c, s)| s.is_local().then_some(c))
 }
 
 pub fn constant_full_int<'tcx>(
@@ -297,29 +318,25 @@ impl Ord for FullInt {
     }
 }
 
-/// Creates a `ConstEvalLateContext` from the given `LateContext` and `TypeckResults`.
-pub fn constant_context<'a, 'tcx>(
-    lcx: &'a LateContext<'tcx>,
-    typeck_results: &'a ty::TypeckResults<'tcx>,
-) -> ConstEvalLateContext<'a, 'tcx> {
-    ConstEvalLateContext {
-        lcx,
-        typeck_results,
-        param_env: lcx.param_env,
-        needed_resolution: false,
-        substs: ty::List::empty(),
-    }
-}
-
 pub struct ConstEvalLateContext<'a, 'tcx> {
     lcx: &'a LateContext<'tcx>,
     typeck_results: &'a ty::TypeckResults<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    needed_resolution: bool,
+    source: ConstantSource,
     substs: SubstsRef<'tcx>,
 }
 
 impl<'a, 'tcx> ConstEvalLateContext<'a, 'tcx> {
+    fn new(lcx: &'a LateContext<'tcx>, typeck_results: &'a ty::TypeckResults<'tcx>) -> Self {
+        Self {
+            lcx,
+            typeck_results,
+            param_env: lcx.param_env,
+            source: ConstantSource::Local,
+            substs: List::empty(),
+        }
+    }
+
     /// Simple constant folding: Insert an expression, get a constant or none.
     pub fn expr(&mut self, e: &Expr<'_>) -> Option<Constant> {
         match e.kind {
@@ -454,11 +471,9 @@ impl<'a, 'tcx> ConstEvalLateContext<'a, 'tcx> {
                     .const_eval_resolve(self.param_env, mir::UnevaluatedConst::new(def_id, substs), None)
                     .ok()
                     .map(|val| rustc_middle::mir::ConstantKind::from_value(val, ty))?;
-                let result = miri_to_const(self.lcx.tcx, result);
-                if result.is_some() {
-                    self.needed_resolution = true;
-                }
-                result
+                let result = miri_to_const(self.lcx.tcx, result)?;
+                self.source = ConstantSource::Constant;
+                Some(result)
             },
             // FIXME: cover all usable cases.
             _ => None,
@@ -492,8 +507,33 @@ impl<'a, 'tcx> ConstEvalLateContext<'a, 'tcx> {
 
     /// A block can only yield a constant if it only has one constant expression.
     fn block(&mut self, block: &Block<'_>) -> Option<Constant> {
-        if block.stmts.is_empty() {
-            block.expr.as_ref().and_then(|b| self.expr(b))
+        if block.stmts.is_empty()
+            && let Some(expr) = block.expr
+        {
+            // Try to detect any `cfg`ed statements or empty macro expansions.
+            let span = block.span.data();
+            if span.ctxt == SyntaxContext::root() {
+                if let Some(expr_span) = walk_span_to_context(expr.span, span.ctxt)
+                    && let expr_lo = expr_span.lo()
+                    && expr_lo >= span.lo
+                    && let Some(src) = get_source_text(self.lcx, span.lo..expr_lo)
+                    && let Some(src) = src.as_str()
+                {
+                    use rustc_lexer::TokenKind::{Whitespace, LineComment, BlockComment, Semi, OpenBrace};
+                    if !tokenize(src)
+                        .map(|t| t.kind)
+                        .filter(|t| !matches!(t, Whitespace | LineComment { .. } | BlockComment { .. } | Semi))
+                        .eq([OpenBrace])
+                    {
+                        self.source = ConstantSource::Constant;
+                    }
+                } else {
+                    // Unable to access the source. Assume a non-local dependency.
+                    self.source = ConstantSource::Constant;
+                }
+            }
+
+            self.expr(expr)
         } else {
             None
         }
diff --git a/src/tools/clippy/clippy_utils/src/hir_utils.rs b/src/tools/clippy/clippy_utils/src/hir_utils.rs
index 9b7408d5133..a49246a7832 100644
--- a/src/tools/clippy/clippy_utils/src/hir_utils.rs
+++ b/src/tools/clippy/clippy_utils/src/hir_utils.rs
@@ -1,6 +1,7 @@
 use crate::consts::constant_simple;
 use crate::macros::macro_backtrace;
-use crate::source::snippet_opt;
+use crate::source::{get_source_text, snippet_opt, walk_span_to_context, SpanRange};
+use crate::tokenize_with_text;
 use rustc_ast::ast::InlineAsmTemplatePiece;
 use rustc_data_structures::fx::FxHasher;
 use rustc_hir::def::Res;
@@ -13,8 +14,9 @@ use rustc_hir::{
 use rustc_lexer::{tokenize, TokenKind};
 use rustc_lint::LateContext;
 use rustc_middle::ty::TypeckResults;
-use rustc_span::{sym, Symbol};
+use rustc_span::{sym, BytePos, ExpnKind, MacroKind, Symbol, SyntaxContext};
 use std::hash::{Hash, Hasher};
+use std::ops::Range;
 
 /// Callback that is called when two expressions are not equal in the sense of `SpanlessEq`, but
 /// other conditions would make them equal.
@@ -65,6 +67,8 @@ impl<'a, 'tcx> SpanlessEq<'a, 'tcx> {
     pub fn inter_expr(&mut self) -> HirEqInterExpr<'_, 'a, 'tcx> {
         HirEqInterExpr {
             inner: self,
+            left_ctxt: SyntaxContext::root(),
+            right_ctxt: SyntaxContext::root(),
             locals: HirIdMap::default(),
         }
     }
@@ -92,6 +96,8 @@ impl<'a, 'tcx> SpanlessEq<'a, 'tcx> {
 
 pub struct HirEqInterExpr<'a, 'b, 'tcx> {
     inner: &'a mut SpanlessEq<'b, 'tcx>,
+    left_ctxt: SyntaxContext,
+    right_ctxt: SyntaxContext,
 
     // When binding are declared, the binding ID in the left expression is mapped to the one on the
     // right. For example, when comparing `{ let x = 1; x + 2 }` and `{ let y = 1; y + 2 }`,
@@ -126,52 +132,88 @@ impl HirEqInterExpr<'_, '_, '_> {
     }
 
     /// Checks whether two blocks are the same.
+    #[expect(clippy::similar_names)]
     fn eq_block(&mut self, left: &Block<'_>, right: &Block<'_>) -> bool {
-        match (left.stmts, left.expr, right.stmts, right.expr) {
-            ([], None, [], None) => {
-                // For empty blocks, check to see if the tokens are equal. This will catch the case where a macro
-                // expanded to nothing, or the cfg attribute was used.
-                let (Some(left), Some(right)) = (
-                    snippet_opt(self.inner.cx, left.span),
-                    snippet_opt(self.inner.cx, right.span),
-                ) else { return true };
-                let mut left_pos = 0;
-                let left = tokenize(&left)
-                    .map(|t| {
-                        let end = left_pos + t.len as usize;
-                        let s = &left[left_pos..end];
-                        left_pos = end;
-                        (t, s)
-                    })
-                    .filter(|(t, _)| {
-                        !matches!(
-                            t.kind,
-                            TokenKind::LineComment { .. } | TokenKind::BlockComment { .. } | TokenKind::Whitespace
-                        )
-                    })
-                    .map(|(_, s)| s);
-                let mut right_pos = 0;
-                let right = tokenize(&right)
-                    .map(|t| {
-                        let end = right_pos + t.len as usize;
-                        let s = &right[right_pos..end];
-                        right_pos = end;
-                        (t, s)
-                    })
-                    .filter(|(t, _)| {
-                        !matches!(
-                            t.kind,
-                            TokenKind::LineComment { .. } | TokenKind::BlockComment { .. } | TokenKind::Whitespace
-                        )
-                    })
-                    .map(|(_, s)| s);
-                left.eq(right)
-            },
-            _ => {
-                over(left.stmts, right.stmts, |l, r| self.eq_stmt(l, r))
-                    && both(&left.expr, &right.expr, |l, r| self.eq_expr(l, r))
-            },
+        use TokenKind::{BlockComment, LineComment, Semi, Whitespace};
+        if left.stmts.len() != right.stmts.len() {
+            return false;
+        }
+        let lspan = left.span.data();
+        let rspan = right.span.data();
+        if lspan.ctxt != SyntaxContext::root() && rspan.ctxt != SyntaxContext::root() {
+            // Don't try to check in between statements inside macros.
+            return over(left.stmts, right.stmts, |left, right| self.eq_stmt(left, right))
+                && both(&left.expr, &right.expr, |left, right| self.eq_expr(left, right));
+        }
+        if lspan.ctxt != rspan.ctxt {
+            return false;
         }
+
+        let mut lstart = lspan.lo;
+        let mut rstart = rspan.lo;
+
+        for (left, right) in left.stmts.iter().zip(right.stmts) {
+            if !self.eq_stmt(left, right) {
+                return false;
+            }
+
+            // Try to detect any `cfg`ed statements or empty macro expansions.
+            let Some(lstmt_span) = walk_span_to_context(left.span, lspan.ctxt) else {
+                return false;
+            };
+            let Some(rstmt_span) = walk_span_to_context(right.span, rspan.ctxt) else {
+                return false;
+            };
+            let lstmt_span = lstmt_span.data();
+            let rstmt_span = rstmt_span.data();
+
+            if lstmt_span.lo < lstart && rstmt_span.lo < rstart {
+                // Can happen when macros expand to multiple statements, or rearrange statements.
+                // Nothing in between the statements to check in this case.
+                continue;
+            }
+            if lstmt_span.lo < lstart || rstmt_span.lo < rstart {
+                // Only one of the blocks had a weird macro.
+                return false;
+            }
+            if !eq_span_tokens(self.inner.cx, lstart..lstmt_span.lo, rstart..rstmt_span.lo, |t| {
+                !matches!(t, Whitespace | LineComment { .. } | BlockComment { .. } | Semi)
+            }) {
+                return false;
+            }
+
+            lstart = lstmt_span.hi;
+            rstart = rstmt_span.hi;
+        }
+
+        let (lend, rend) = match (left.expr, right.expr) {
+            (Some(left), Some(right)) => {
+                if !self.eq_expr(left, right) {
+                    return false;
+                }
+                let Some(lexpr_span) = walk_span_to_context(left.span, lspan.ctxt) else {
+                    return false;
+                };
+                let Some(rexpr_span) = walk_span_to_context(right.span, rspan.ctxt) else {
+                    return false;
+                };
+                (lexpr_span.lo(), rexpr_span.lo())
+            },
+            (None, None) => (lspan.hi, rspan.hi),
+            (Some(_), None) | (None, Some(_)) => return false,
+        };
+
+        if lend < lstart && rend < rstart {
+            // Can happen when macros rearrange the input.
+            // Nothing in between the statements to check in this case.
+            return true;
+        } else if lend < lstart || rend < rstart {
+            // Only one of the blocks had a weird macro
+            return false;
+        }
+        eq_span_tokens(self.inner.cx, lstart..lend, rstart..rend, |t| {
+            !matches!(t, Whitespace | LineComment { .. } | BlockComment { .. } | Semi)
+        })
     }
 
     fn should_ignore(&mut self, expr: &Expr<'_>) -> bool {
@@ -207,7 +249,7 @@ impl HirEqInterExpr<'_, '_, '_> {
 
     #[expect(clippy::similar_names)]
     pub fn eq_expr(&mut self, left: &Expr<'_>, right: &Expr<'_>) -> bool {
-        if !self.inner.allow_side_effects && left.span.ctxt() != right.span.ctxt() {
+        if !self.check_ctxt(left.span.ctxt(), right.span.ctxt()) {
             return false;
         }
 
@@ -440,6 +482,45 @@ impl HirEqInterExpr<'_, '_, '_> {
     fn eq_type_binding(&mut self, left: &TypeBinding<'_>, right: &TypeBinding<'_>) -> bool {
         left.ident.name == right.ident.name && self.eq_ty(left.ty(), right.ty())
     }
+
+    fn check_ctxt(&mut self, left: SyntaxContext, right: SyntaxContext) -> bool {
+        if self.left_ctxt == left && self.right_ctxt == right {
+            return true;
+        } else if self.left_ctxt == left || self.right_ctxt == right {
+            // Only one context has changed. This can only happen if the two nodes are written differently.
+            return false;
+        } else if left != SyntaxContext::root() {
+            let mut left_data = left.outer_expn_data();
+            let mut right_data = right.outer_expn_data();
+            loop {
+                use TokenKind::{BlockComment, LineComment, Whitespace};
+                if left_data.macro_def_id != right_data.macro_def_id
+                    || (matches!(left_data.kind, ExpnKind::Macro(MacroKind::Bang, name) if name == sym::cfg)
+                        && !eq_span_tokens(self.inner.cx, left_data.call_site, right_data.call_site, |t| {
+                            !matches!(t, Whitespace | LineComment { .. } | BlockComment { .. })
+                        }))
+                {
+                    // Either a different chain of macro calls, or different arguments to the `cfg` macro.
+                    return false;
+                }
+                let left_ctxt = left_data.call_site.ctxt();
+                let right_ctxt = right_data.call_site.ctxt();
+                if left_ctxt == SyntaxContext::root() && right_ctxt == SyntaxContext::root() {
+                    break;
+                }
+                if left_ctxt == SyntaxContext::root() || right_ctxt == SyntaxContext::root() {
+                    // Different lengths for the expansion stack. This can only happen if nodes are written differently,
+                    // or shouldn't be compared to start with.
+                    return false;
+                }
+                left_data = left_ctxt.outer_expn_data();
+                right_data = right_ctxt.outer_expn_data();
+            }
+        }
+        self.left_ctxt = left;
+        self.right_ctxt = right;
+        true
+    }
 }
 
 /// Some simple reductions like `{ return }` => `return`
@@ -1038,3 +1119,34 @@ pub fn hash_expr(cx: &LateContext<'_>, e: &Expr<'_>) -> u64 {
     h.hash_expr(e);
     h.finish()
 }
+
+#[expect(clippy::similar_names)]
+fn eq_span_tokens(
+    cx: &LateContext<'_>,
+    left: impl SpanRange,
+    right: impl SpanRange,
+    pred: impl Fn(TokenKind) -> bool,
+) -> bool {
+    fn f(cx: &LateContext<'_>, left: Range<BytePos>, right: Range<BytePos>, pred: impl Fn(TokenKind) -> bool) -> bool {
+        if let Some(lsrc) = get_source_text(cx, left)
+            && let Some(lsrc) = lsrc.as_str()
+            && let Some(rsrc) = get_source_text(cx, right)
+            && let Some(rsrc) = rsrc.as_str()
+        {
+            let pred = |t: &(_, _)| pred(t.0);
+            let map = |(_, x)| x;
+
+            let ltok = tokenize_with_text(lsrc)
+                .filter(pred)
+                .map(map);
+            let rtok = tokenize_with_text(rsrc)
+                .filter(pred)
+                .map(map);
+            ltok.eq(rtok)
+        } else {
+            // Unable to access the source. Conservatively assume the blocks aren't equal.
+            false
+        }
+    }
+    f(cx, left.into_range(), right.into_range(), pred)
+}
diff --git a/src/tools/clippy/clippy_utils/src/lib.rs b/src/tools/clippy/clippy_utils/src/lib.rs
index 964104fc31d..575c29a6b6f 100644
--- a/src/tools/clippy/clippy_utils/src/lib.rs
+++ b/src/tools/clippy/clippy_utils/src/lib.rs
@@ -1,5 +1,6 @@
 #![feature(array_chunks)]
 #![feature(box_patterns)]
+#![feature(if_let_guard)]
 #![feature(let_chains)]
 #![feature(lint_reasons)]
 #![feature(never_type)]
@@ -76,6 +77,7 @@ use std::sync::OnceLock;
 use std::sync::{Mutex, MutexGuard};
 
 use if_chain::if_chain;
+use itertools::Itertools;
 use rustc_ast::ast::{self, LitKind, RangeLimits};
 use rustc_ast::Attribute;
 use rustc_data_structures::fx::FxHashMap;
@@ -282,6 +284,15 @@ pub fn is_wild(pat: &Pat<'_>) -> bool {
     matches!(pat.kind, PatKind::Wild)
 }
 
+/// Checks if the given `QPath` belongs to a type alias.
+pub fn is_ty_alias(qpath: &QPath<'_>) -> bool {
+    match *qpath {
+        QPath::Resolved(_, path) => matches!(path.res, Res::Def(DefKind::TyAlias, ..)),
+        QPath::TypeRelative(ty, _) if let TyKind::Path(qpath) = ty.kind => { is_ty_alias(&qpath) },
+        _ => false,
+    }
+}
+
 /// Checks if the method call given in `expr` belongs to the given trait.
 /// This is a deprecated function, consider using [`is_trait_method`].
 pub fn match_trait_method(cx: &LateContext<'_>, expr: &Expr<'_>, path: &[&str]) -> bool {
@@ -1488,7 +1499,7 @@ pub fn is_range_full(cx: &LateContext<'_>, expr: &Expr<'_>, container_path: Opti
                 && let const_val = cx.tcx.valtree_to_const_val((bnd_ty, min_val.to_valtree()))
                 && let min_const_kind = ConstantKind::from_value(const_val, bnd_ty)
                 && let Some(min_const) = miri_to_const(cx.tcx, min_const_kind)
-                && let Some((start_const, _)) = constant(cx, cx.typeck_results(), start)
+                && let Some(start_const) = constant(cx, cx.typeck_results(), start)
             {
                 start_const == min_const
             } else {
@@ -1504,7 +1515,7 @@ pub fn is_range_full(cx: &LateContext<'_>, expr: &Expr<'_>, container_path: Opti
                         && let const_val = cx.tcx.valtree_to_const_val((bnd_ty, max_val.to_valtree()))
                         && let max_const_kind = ConstantKind::from_value(const_val, bnd_ty)
                         && let Some(max_const) = miri_to_const(cx.tcx, max_const_kind)
-                        && let Some((end_const, _)) = constant(cx, cx.typeck_results(), end)
+                        && let Some(end_const) = constant(cx, cx.typeck_results(), end)
                     {
                         end_const == max_const
                     } else {
@@ -1536,7 +1547,7 @@ pub fn is_integer_const(cx: &LateContext<'_>, e: &Expr<'_>, value: u128) -> bool
         return true;
     }
     let enclosing_body = cx.tcx.hir().enclosing_body_owner(e.hir_id);
-    if let Some((Constant::Int(v), _)) = constant(cx, cx.tcx.typeck(enclosing_body), e) {
+    if let Some(Constant::Int(v)) = constant(cx, cx.tcx.typeck(enclosing_body), e) {
         return value == v;
     }
     false
@@ -2480,6 +2491,17 @@ pub fn walk_to_expr_usage<'tcx, T>(
     None
 }
 
+/// Tokenizes the input while keeping the text associated with each token.
+pub fn tokenize_with_text(s: &str) -> impl Iterator<Item = (TokenKind, &str)> {
+    let mut pos = 0;
+    tokenize(s).map(move |t| {
+        let end = pos + t.len;
+        let range = pos as usize..end as usize;
+        pos = end;
+        (t.kind, s.get(range).unwrap_or_default())
+    })
+}
+
 /// Checks whether a given span has any comment token
 /// This checks for all types of comment: line "//", block "/**", doc "///" "//!"
 pub fn span_contains_comment(sm: &SourceMap, span: Span) -> bool {
@@ -2496,23 +2518,11 @@ pub fn span_contains_comment(sm: &SourceMap, span: Span) -> bool {
 /// Comments are returned wrapped with their relevant delimiters
 pub fn span_extract_comment(sm: &SourceMap, span: Span) -> String {
     let snippet = sm.span_to_snippet(span).unwrap_or_default();
-    let mut comments_buf: Vec<String> = Vec::new();
-    let mut index: usize = 0;
-
-    for token in tokenize(&snippet) {
-        let token_range = index..(index + token.len as usize);
-        index += token.len as usize;
-        match token.kind {
-            TokenKind::BlockComment { .. } | TokenKind::LineComment { .. } => {
-                if let Some(comment) = snippet.get(token_range) {
-                    comments_buf.push(comment.to_string());
-                }
-            },
-            _ => (),
-        }
-    }
-
-    comments_buf.join("\n")
+    let res = tokenize_with_text(&snippet)
+        .filter(|(t, _)| matches!(t, TokenKind::BlockComment { .. } | TokenKind::LineComment { .. }))
+        .map(|(_, s)| s)
+        .join("\n");
+    res
 }
 
 pub fn span_find_starting_semi(sm: &SourceMap, span: Span) -> Span {
diff --git a/src/tools/clippy/clippy_utils/src/source.rs b/src/tools/clippy/clippy_utils/src/source.rs
index 62fa37660fa..0f60290644a 100644
--- a/src/tools/clippy/clippy_utils/src/source.rs
+++ b/src/tools/clippy/clippy_utils/src/source.rs
@@ -2,14 +2,64 @@
 
 #![allow(clippy::module_name_repetitions)]
 
+use rustc_data_structures::sync::Lrc;
 use rustc_errors::Applicability;
 use rustc_hir::{Expr, ExprKind};
 use rustc_lint::{LateContext, LintContext};
 use rustc_session::Session;
-use rustc_span::hygiene;
 use rustc_span::source_map::{original_sp, SourceMap};
+use rustc_span::{hygiene, SourceFile};
 use rustc_span::{BytePos, Pos, Span, SpanData, SyntaxContext, DUMMY_SP};
 use std::borrow::Cow;
+use std::ops::Range;
+
+/// A type which can be converted to the range portion of a `Span`.
+pub trait SpanRange {
+    fn into_range(self) -> Range<BytePos>;
+}
+impl SpanRange for Span {
+    fn into_range(self) -> Range<BytePos> {
+        let data = self.data();
+        data.lo..data.hi
+    }
+}
+impl SpanRange for SpanData {
+    fn into_range(self) -> Range<BytePos> {
+        self.lo..self.hi
+    }
+}
+impl SpanRange for Range<BytePos> {
+    fn into_range(self) -> Range<BytePos> {
+        self
+    }
+}
+
+pub struct SourceFileRange {
+    pub sf: Lrc<SourceFile>,
+    pub range: Range<usize>,
+}
+impl SourceFileRange {
+    /// Attempts to get the text from the source file. This can fail if the source text isn't
+    /// loaded.
+    pub fn as_str(&self) -> Option<&str> {
+        self.sf.src.as_ref().and_then(|x| x.get(self.range.clone()))
+    }
+}
+
+/// Gets the source file, and range in the file, of the given span. Returns `None` if the span
+/// extends through multiple files, or is malformed.
+pub fn get_source_text(cx: &impl LintContext, sp: impl SpanRange) -> Option<SourceFileRange> {
+    fn f(sm: &SourceMap, sp: Range<BytePos>) -> Option<SourceFileRange> {
+        let start = sm.lookup_byte_offset(sp.start);
+        let end = sm.lookup_byte_offset(sp.end);
+        if !Lrc::ptr_eq(&start.sf, &end.sf) || start.pos > end.pos {
+            return None;
+        }
+        let range = start.pos.to_usize()..end.pos.to_usize();
+        Some(SourceFileRange { sf: start.sf, range })
+    }
+    f(cx.sess().source_map(), sp.into_range())
+}
 
 /// Like `snippet_block`, but add braces if the expr is not an `ExprKind::Block`.
 pub fn expr_block<T: LintContext>(