diff options
Diffstat (limited to 'src')
56 files changed, 1235 insertions, 800 deletions
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 43561fa4f2f..f939f79e4f4 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -662,7 +662,7 @@ impl Step for RustdocJSNotStd { target: self.target, mode: "js-doc-test", suite: "rustdoc-js", - path: None, + path: "src/test/rustdoc-js", compare_mode: None, }); } else { @@ -698,7 +698,7 @@ impl Step for RustdocUi { target: self.target, mode: "ui", suite: "rustdoc-ui", - path: Some("src/test/rustdoc-ui"), + path: "src/test/rustdoc-ui", compare_mode: None, }) } @@ -843,7 +843,7 @@ macro_rules! test_definitions { target: self.target, mode: $mode, suite: $suite, - path: Some($path), + path: $path, compare_mode: $compare_mode, }) } @@ -926,7 +926,7 @@ struct Compiletest { target: Interned<String>, mode: &'static str, suite: &'static str, - path: Option<&'static str>, + path: &'static str, compare_mode: Option<&'static str>, } @@ -949,7 +949,7 @@ impl Step for Compiletest { let suite = self.suite; // Path for test suite - let suite_path = self.path.unwrap_or(""); + let suite_path = self.path; // Skip codegen tests if they aren't enabled in configuration. if !builder.config.codegen_tests && suite == "codegen" { @@ -1051,10 +1051,10 @@ impl Step for Compiletest { cmd.arg("--docck-python").arg(builder.python()); if builder.config.build.ends_with("apple-darwin") { - // Force /usr/bin/python on macOS for LLDB tests because we're loading the + // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the // LLDB plugin's compiled module which only works with the system python // (namely not Homebrew-installed python) - cmd.arg("--lldb-python").arg("/usr/bin/python"); + cmd.arg("--lldb-python").arg("/usr/bin/python3"); } else { cmd.arg("--lldb-python").arg(builder.python()); } diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 79a49fc48be..74b7469ea27 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -63,7 +63,7 @@ jobs: - job: macOS timeoutInMinutes: 600 pool: - vmImage: macos-10.13 + vmImage: macos-10.15 steps: - template: steps/run.yml strategy: diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index c39f75aba89..ee9425aa1c5 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -51,10 +51,6 @@ steps: displayName: Install clang condition: and(succeeded(), not(variables.SKIP_JOB)) -- bash: src/ci/scripts/switch-xcode.sh - displayName: Switch to Xcode 9.3 - condition: and(succeeded(), not(variables.SKIP_JOB)) - - bash: src/ci/scripts/install-wix.sh displayName: Install wix condition: and(succeeded(), not(variables.SKIP_JOB)) diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml index b6177b2cc9b..f8ddf0eb46c 100644 --- a/src/ci/azure-pipelines/try.yml +++ b/src/ci/azure-pipelines/try.yml @@ -25,7 +25,7 @@ jobs: # - job: macOS # timeoutInMinutes: 600 # pool: -# vmImage: macos-10.13 +# vmImage: macos-10.15 # steps: # - template: steps/run.yml # strategy: diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index e16a4814197..c242f5d4562 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -19,9 +19,7 @@ if isMacOS; then # native clang is configured to use the correct path, but our custom one # doesn't. This sets the SDKROOT environment variable to the SDK so that # our own clang can figure out the correct include path on its own. - if ! [[ -d "/usr/include" ]]; then - ciCommandSetEnv SDKROOT "$(xcrun --sdk macosx --show-sdk-path)" - fi + ciCommandSetEnv SDKROOT "$(xcrun --sdk macosx --show-sdk-path)" # Configure `AR` specifically so rustbuild doesn't try to infer it as # `clang-ar` by accident. diff --git a/src/ci/scripts/switch-xcode.sh b/src/ci/scripts/switch-xcode.sh deleted file mode 100755 index 2cbb2ddbc70..00000000000 --- a/src/ci/scripts/switch-xcode.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Switch to XCode 9.3 on OSX since it seems to be the last version that supports -# i686-apple-darwin. We'll eventually want to upgrade this and it will probably -# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. - -set -euo pipefail -IFS=$'\n\t' - -source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" - -if isMacOS; then - sudo xcode-select --switch /Applications/Xcode_9.3.app -fi diff --git a/src/liballoc/string.rs b/src/liballoc/string.rs index 3cb1f259a0b..f5afea15d65 100644 --- a/src/liballoc/string.rs +++ b/src/liballoc/string.rs @@ -2106,18 +2106,11 @@ impl ops::DerefMut for String { } } -/// An error when parsing a `String`. +/// A type alias for [`Infallible`]. /// -/// This `enum` is slightly awkward: it will never actually exist. This error is -/// part of the type signature of the implementation of [`FromStr`] on -/// [`String`]. The return type of [`from_str`], requires that an error be -/// defined, but, given that a [`String`] can always be made into a new -/// [`String`] without error, this type will never actually be returned. As -/// such, it is only here to satisfy said signature, and is useless otherwise. +/// This alias exists for backwards compatibility, and may be eventually deprecated. /// -/// [`FromStr`]: ../../std/str/trait.FromStr.html -/// [`String`]: struct.String.html -/// [`from_str`]: ../../std/str/trait.FromStr.html#tymethod.from_str +/// [`Infallible`]: ../../core/convert/enum.Infallible.html #[stable(feature = "str_parse_error", since = "1.5.0")] pub type ParseError = core::convert::Infallible; @@ -2125,7 +2118,7 @@ pub type ParseError = core::convert::Infallible; impl FromStr for String { type Err = core::convert::Infallible; #[inline] - fn from_str(s: &str) -> Result<String, ParseError> { + fn from_str(s: &str) -> Result<String, Self::Err> { Ok(String::from(s)) } } diff --git a/src/libcore/alloc.rs b/src/libcore/alloc.rs index 71f7f971eab..a04e75bc7ce 100644 --- a/src/libcore/alloc.rs +++ b/src/libcore/alloc.rs @@ -241,11 +241,13 @@ impl Layout { #[unstable(feature = "alloc_layout_extra", issue = "55724")] #[inline] pub fn repeat(&self, n: usize) -> Result<(Self, usize), LayoutErr> { - // This cannot overflow. Quoting from the invariant of Layout: - // > `size`, when rounded up to the nearest multiple of `align`, - // > must not overflow (i.e., the rounded value must be less than - // > `usize::MAX`) - let padded_size = self.size() + self.padding_needed_for(self.align()); + // Warning, removing the checked_add here led to segfaults in #67174. Further + // analysis in #69225 seems to indicate that this is an LTO-related + // miscompilation, so #67174 might be able to be reapplied in the future. + let padded_size = self + .size() + .checked_add(self.padding_needed_for(self.align())) + .ok_or(LayoutErr { private: () })?; let alloc_size = padded_size.checked_mul(n).ok_or(LayoutErr { private: () })?; unsafe { diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs index e5b8412e117..7c4acb0edb7 100644 --- a/src/libcore/str/mod.rs +++ b/src/libcore/str/mod.rs @@ -1499,7 +1499,7 @@ fn contains_nonascii(x: usize) -> bool { /// Walks through `v` checking that it's a valid UTF-8 sequence, /// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. -#[inline] +#[inline(always)] fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { let mut index = 0; let len = v.len(); diff --git a/src/librustc/dep_graph/dep_node.rs b/src/librustc/dep_graph/dep_node.rs index 29b94986a5f..eb7e2871bfc 100644 --- a/src/librustc/dep_graph/dep_node.rs +++ b/src/librustc/dep_graph/dep_node.rs @@ -76,10 +76,6 @@ macro_rules! erase { ($x:tt) => {{}}; } -macro_rules! replace { - ($x:tt with $($y:tt)*) => ($($y)*) -} - macro_rules! is_anon_attr { (anon) => { true @@ -99,19 +95,18 @@ macro_rules! is_eval_always_attr { } macro_rules! contains_anon_attr { - ($($attr:ident),*) => ({$(is_anon_attr!($attr) | )* false}); + ($($attr:ident $(($($attr_args:tt)*))* ),*) => ({$(is_anon_attr!($attr) | )* false}); } macro_rules! contains_eval_always_attr { - ($($attr:ident),*) => ({$(is_eval_always_attr!($attr) | )* false}); + ($($attr:ident $(($($attr_args:tt)*))* ),*) => ({$(is_eval_always_attr!($attr) | )* false}); } macro_rules! define_dep_nodes { (<$tcx:tt> $( - [$($attr:ident),* ] + [$($attrs:tt)*] $variant:ident $(( $tuple_arg_ty:ty $(,)? ))* - $({ $($struct_arg_name:ident : $struct_arg_ty:ty),* })* ,)* ) => ( #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, @@ -126,7 +121,7 @@ macro_rules! define_dep_nodes { match *self { $( DepKind :: $variant => { - if contains_anon_attr!($($attr),*) { + if contains_anon_attr!($($attrs)*) { return false; } @@ -136,13 +131,6 @@ macro_rules! define_dep_nodes { ::CAN_RECONSTRUCT_QUERY_KEY; })* - // struct args - $({ - - return <( $($struct_arg_ty,)* ) as DepNodeParams> - ::CAN_RECONSTRUCT_QUERY_KEY; - })* - true } )* @@ -152,7 +140,7 @@ macro_rules! define_dep_nodes { pub fn is_anon(&self) -> bool { match *self { $( - DepKind :: $variant => { contains_anon_attr!($($attr),*) } + DepKind :: $variant => { contains_anon_attr!($($attrs)*) } )* } } @@ -160,7 +148,7 @@ macro_rules! define_dep_nodes { pub fn is_eval_always(&self) -> bool { match *self { $( - DepKind :: $variant => { contains_eval_always_attr!($($attr), *) } + DepKind :: $variant => { contains_eval_always_attr!($($attrs)*) } )* } } @@ -176,12 +164,6 @@ macro_rules! define_dep_nodes { return true; })* - // struct args - $({ - $(erase!($struct_arg_name);)* - return true; - })* - false } )* @@ -189,11 +171,43 @@ macro_rules! define_dep_nodes { } } - pub enum DepConstructor<$tcx> { + pub struct DepConstructor; + + impl DepConstructor { $( - $variant $(( $tuple_arg_ty ))* - $({ $($struct_arg_name : $struct_arg_ty),* })* - ),* + #[inline(always)] + #[allow(unreachable_code, non_snake_case)] + pub fn $variant<'tcx>(_tcx: TyCtxt<'tcx>, $(arg: $tuple_arg_ty)*) -> DepNode { + // tuple args + $({ + erase!($tuple_arg_ty); + let hash = DepNodeParams::to_fingerprint(&arg, _tcx); + let dep_node = DepNode { + kind: DepKind::$variant, + hash + }; + + #[cfg(debug_assertions)] + { + if !dep_node.kind.can_reconstruct_query_key() && + (_tcx.sess.opts.debugging_opts.incremental_info || + _tcx.sess.opts.debugging_opts.query_dep_graph) + { + _tcx.dep_graph.register_dep_node_debug_str(dep_node, || { + arg.to_debug_str(_tcx) + }); + } + } + + return dep_node; + })* + + DepNode { + kind: DepKind::$variant, + hash: Fingerprint::ZERO, + } + } + )* } #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, @@ -204,75 +218,6 @@ macro_rules! define_dep_nodes { } impl DepNode { - #[allow(unreachable_code, non_snake_case)] - pub fn new<'tcx>(tcx: TyCtxt<'tcx>, - dep: DepConstructor<'tcx>) - -> DepNode - { - match dep { - $( - DepConstructor :: $variant $(( replace!(($tuple_arg_ty) with arg) ))* - $({ $($struct_arg_name),* })* - => - { - // tuple args - $({ - erase!($tuple_arg_ty); - let hash = DepNodeParams::to_fingerprint(&arg, tcx); - let dep_node = DepNode { - kind: DepKind::$variant, - hash - }; - - #[cfg(debug_assertions)] - { - if !dep_node.kind.can_reconstruct_query_key() && - (tcx.sess.opts.debugging_opts.incremental_info || - tcx.sess.opts.debugging_opts.query_dep_graph) - { - tcx.dep_graph.register_dep_node_debug_str(dep_node, || { - arg.to_debug_str(tcx) - }); - } - } - - return dep_node; - })* - - // struct args - $({ - let tupled_args = ( $($struct_arg_name,)* ); - let hash = DepNodeParams::to_fingerprint(&tupled_args, - tcx); - let dep_node = DepNode { - kind: DepKind::$variant, - hash - }; - - #[cfg(debug_assertions)] - { - if !dep_node.kind.can_reconstruct_query_key() && - (tcx.sess.opts.debugging_opts.incremental_info || - tcx.sess.opts.debugging_opts.query_dep_graph) - { - tcx.dep_graph.register_dep_node_debug_str(dep_node, || { - tupled_args.to_debug_str(tcx) - }); - } - } - - return dep_node; - })* - - DepNode { - kind: DepKind::$variant, - hash: Fingerprint::ZERO, - } - } - )* - } - } - /// Construct a DepNode from the given DepKind and DefPathHash. This /// method will assert that the given DepKind actually requires a /// single DefId/DefPathHash parameter. diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 258723bb39d..531a45b120c 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -1122,6 +1122,7 @@ impl CurrentDepGraph { } impl DepGraphData { + #[inline(never)] fn read_index(&self, source: DepNodeIndex) { ty::tls::with_context_opt(|icx| { let icx = if let Some(icx) = icx { icx } else { return }; diff --git a/src/librustc/mir/mono.rs b/src/librustc/mir/mono.rs index 6da7c09c7df..9a3ddfb0e82 100644 --- a/src/librustc/mir/mono.rs +++ b/src/librustc/mir/mono.rs @@ -362,7 +362,7 @@ impl<'tcx> CodegenUnit<'tcx> { } pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode { - DepNode::new(tcx, DepConstructor::CompileCodegenUnit(self.name())) + DepConstructor::CompileCodegenUnit(tcx, self.name()) } } diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index 5a415fa954f..e59738d8886 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -2,7 +2,7 @@ use crate::arena::Arena; use crate::dep_graph::DepGraph; -use crate::dep_graph::{self, DepConstructor, DepNode}; +use crate::dep_graph::{self, DepConstructor}; use crate::hir::exports::Export; use crate::hir::map as hir_map; use crate::hir::map::DefPathHash; @@ -1347,7 +1347,7 @@ impl<'tcx> TyCtxt<'tcx> { // We cannot use the query versions of crates() and crate_hash(), since // those would need the DepNodes that we are allocating here. for cnum in self.cstore.crates_untracked() { - let dep_node = DepNode::new(self, DepConstructor::CrateMetadata(cnum)); + let dep_node = DepConstructor::CrateMetadata(self, cnum); let crate_hash = self.cstore.crate_hash_untracked(cnum); self.dep_graph.with_task( dep_node, @@ -1688,6 +1688,7 @@ pub mod tls { /// Gets the pointer to the current `ImplicitCtxt`. #[cfg(not(parallel_compiler))] + #[inline] fn get_tlv() -> usize { TLV.with(|tlv| tlv.get()) } diff --git a/src/librustc/ty/query/caches.rs b/src/librustc/ty/query/caches.rs new file mode 100644 index 00000000000..efc2804bd4d --- /dev/null +++ b/src/librustc/ty/query/caches.rs @@ -0,0 +1,112 @@ +use crate::dep_graph::DepNodeIndex; +use crate::ty::query::config::QueryAccessors; +use crate::ty::query::plumbing::{QueryLookup, QueryState, QueryStateShard}; +use crate::ty::TyCtxt; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sharded::Sharded; +use std::default::Default; +use std::hash::Hash; + +pub(crate) trait CacheSelector<K, V> { + type Cache: QueryCache<K, V>; +} + +pub(crate) trait QueryCache<K, V>: Default { + type Sharded: Default; + + /// Checks if the query is already computed and in the cache. + /// It returns the shard index and a lock guard to the shard, + /// which will be used if the query is not in the cache and we need + /// to compute it. + fn lookup<'tcx, R, GetCache, OnHit, OnMiss, Q>( + &self, + state: &'tcx QueryState<'tcx, Q>, + get_cache: GetCache, + key: K, + // `on_hit` can be called while holding a lock to the query state shard. + on_hit: OnHit, + on_miss: OnMiss, + ) -> R + where + Q: QueryAccessors<'tcx>, + GetCache: for<'a> Fn(&'a mut QueryStateShard<'tcx, Q>) -> &'a mut Self::Sharded, + OnHit: FnOnce(&V, DepNodeIndex) -> R, + OnMiss: FnOnce(K, QueryLookup<'tcx, Q>) -> R; + + fn complete( + &self, + tcx: TyCtxt<'tcx>, + lock_sharded_storage: &mut Self::Sharded, + key: K, + value: V, + index: DepNodeIndex, + ); + + fn iter<R, L>( + &self, + shards: &Sharded<L>, + get_shard: impl Fn(&mut L) -> &mut Self::Sharded, + f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R, + ) -> R; +} + +pub struct DefaultCacheSelector; + +impl<K: Eq + Hash, V: Clone> CacheSelector<K, V> for DefaultCacheSelector { + type Cache = DefaultCache; +} + +#[derive(Default)] +pub struct DefaultCache; + +impl<K: Eq + Hash, V: Clone> QueryCache<K, V> for DefaultCache { + type Sharded = FxHashMap<K, (V, DepNodeIndex)>; + + #[inline(always)] + fn lookup<'tcx, R, GetCache, OnHit, OnMiss, Q>( + &self, + state: &'tcx QueryState<'tcx, Q>, + get_cache: GetCache, + key: K, + on_hit: OnHit, + on_miss: OnMiss, + ) -> R + where + Q: QueryAccessors<'tcx>, + GetCache: for<'a> Fn(&'a mut QueryStateShard<'tcx, Q>) -> &'a mut Self::Sharded, + OnHit: FnOnce(&V, DepNodeIndex) -> R, + OnMiss: FnOnce(K, QueryLookup<'tcx, Q>) -> R, + { + let mut lookup = state.get_lookup(&key); + let lock = &mut *lookup.lock; + + let result = get_cache(lock).raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + + if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) } + } + + #[inline] + fn complete( + &self, + _: TyCtxt<'tcx>, + lock_sharded_storage: &mut Self::Sharded, + key: K, + value: V, + index: DepNodeIndex, + ) { + lock_sharded_storage.insert(key, (value, index)); + } + + fn iter<R, L>( + &self, + shards: &Sharded<L>, + get_shard: impl Fn(&mut L) -> &mut Self::Sharded, + f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R, + ) -> R { + let mut shards = shards.lock_shards(); + let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect(); + let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1)); + f(Box::new(results)) + } +} diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs index dbb6a1080e6..e0e1ca374d9 100644 --- a/src/librustc/ty/query/config.rs +++ b/src/librustc/ty/query/config.rs @@ -1,15 +1,15 @@ use crate::dep_graph::SerializedDepNodeIndex; use crate::dep_graph::{DepKind, DepNode}; +use crate::ty::query::caches::QueryCache; use crate::ty::query::plumbing::CycleError; use crate::ty::query::queries; -use crate::ty::query::{Query, QueryCache}; +use crate::ty::query::{Query, QueryState}; use crate::ty::TyCtxt; use rustc_data_structures::profiling::ProfileCategory; use rustc_hir::def_id::{CrateNum, DefId}; use crate::ich::StableHashingContext; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::sharded::Sharded; use std::borrow::Cow; use std::fmt::Debug; use std::hash::Hash; @@ -30,10 +30,12 @@ pub(crate) trait QueryAccessors<'tcx>: QueryConfig<'tcx> { const ANON: bool; const EVAL_ALWAYS: bool; + type Cache: QueryCache<Self::Key, Self::Value>; + fn query(key: Self::Key) -> Query<'tcx>; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_cache<'a>(tcx: TyCtxt<'tcx>) -> &'a Sharded<QueryCache<'tcx, Self>>; + fn query_state<'a>(tcx: TyCtxt<'tcx>) -> &'a QueryState<'tcx, Self>; fn to_dep_node(tcx: TyCtxt<'tcx>, key: &Self::Key) -> DepNode; @@ -61,7 +63,10 @@ pub(crate) trait QueryDescription<'tcx>: QueryAccessors<'tcx> { } } -impl<'tcx, M: QueryAccessors<'tcx, Key = DefId>> QueryDescription<'tcx> for M { +impl<'tcx, M: QueryAccessors<'tcx, Key = DefId>> QueryDescription<'tcx> for M +where + <M as QueryAccessors<'tcx>>::Cache: QueryCache<DefId, <M as QueryConfig<'tcx>>::Value>, +{ default fn describe(tcx: TyCtxt<'_>, def_id: DefId) -> Cow<'static, str> { if !tcx.sess.verbose() { format!("processing `{}`", tcx.def_path_str(def_id)).into() diff --git a/src/librustc/ty/query/keys.rs b/src/librustc/ty/query/keys.rs index c1c88e96f94..09fb307a1ce 100644 --- a/src/librustc/ty/query/keys.rs +++ b/src/librustc/ty/query/keys.rs @@ -4,6 +4,7 @@ use crate::infer::canonical::Canonical; use crate::mir; use crate::traits; use crate::ty::fast_reject::SimplifiedType; +use crate::ty::query::caches::DefaultCacheSelector; use crate::ty::subst::SubstsRef; use crate::ty::{self, Ty, TyCtxt}; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LOCAL_CRATE}; @@ -12,7 +13,9 @@ use rustc_span::{Span, DUMMY_SP}; /// The `Key` trait controls what types can legally be used as the key /// for a query. -pub(super) trait Key { +pub trait Key { + type CacheSelector; + /// Given an instance of this key, what crate is it referring to? /// This is used to find the provider. fn query_crate(&self) -> CrateNum; @@ -23,6 +26,8 @@ pub(super) trait Key { } impl<'tcx> Key for ty::InstanceDef<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -33,6 +38,8 @@ impl<'tcx> Key for ty::InstanceDef<'tcx> { } impl<'tcx> Key for ty::Instance<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -43,6 +50,8 @@ impl<'tcx> Key for ty::Instance<'tcx> { } impl<'tcx> Key for mir::interpret::GlobalId<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.instance.query_crate() } @@ -53,6 +62,8 @@ impl<'tcx> Key for mir::interpret::GlobalId<'tcx> { } impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -63,6 +74,8 @@ impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> { } impl Key for CrateNum { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { *self } @@ -72,6 +85,8 @@ impl Key for CrateNum { } impl Key for DefIndex { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -81,6 +96,8 @@ impl Key for DefIndex { } impl Key for DefId { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.krate } @@ -90,6 +107,8 @@ impl Key for DefId { } impl Key for (DefId, DefId) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.0.krate } @@ -99,6 +118,8 @@ impl Key for (DefId, DefId) { } impl Key for (CrateNum, DefId) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.0 } @@ -108,6 +129,8 @@ impl Key for (CrateNum, DefId) { } impl Key for (DefId, SimplifiedType) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.0.krate } @@ -117,6 +140,8 @@ impl Key for (DefId, SimplifiedType) { } impl<'tcx> Key for SubstsRef<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -126,6 +151,8 @@ impl<'tcx> Key for SubstsRef<'tcx> { } impl<'tcx> Key for (DefId, SubstsRef<'tcx>) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.0.krate } @@ -135,6 +162,8 @@ impl<'tcx> Key for (DefId, SubstsRef<'tcx>) { } impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.1.def_id().krate } @@ -144,6 +173,8 @@ impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) { } impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -153,6 +184,8 @@ impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) { } impl<'tcx> Key for ty::PolyTraitRef<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.def_id().krate } @@ -162,6 +195,8 @@ impl<'tcx> Key for ty::PolyTraitRef<'tcx> { } impl<'tcx> Key for &'tcx ty::Const<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -171,6 +206,8 @@ impl<'tcx> Key for &'tcx ty::Const<'tcx> { } impl<'tcx> Key for Ty<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -180,6 +217,8 @@ impl<'tcx> Key for Ty<'tcx> { } impl<'tcx> Key for ty::ParamEnv<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -189,6 +228,8 @@ impl<'tcx> Key for ty::ParamEnv<'tcx> { } impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { self.value.query_crate() } @@ -198,6 +239,8 @@ impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> { } impl<'tcx> Key for traits::Environment<'tcx> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -207,6 +250,8 @@ impl<'tcx> Key for traits::Environment<'tcx> { } impl Key for Symbol { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -218,6 +263,8 @@ impl Key for Symbol { /// Canonical query goals correspond to abstract trait operations that /// are not tied to any crate in particular. impl<'tcx, T> Key for Canonical<'tcx, T> { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } @@ -228,6 +275,8 @@ impl<'tcx, T> Key for Canonical<'tcx, T> { } impl Key for (Symbol, u32, u32) { + type CacheSelector = DefaultCacheSelector; + fn query_crate(&self) -> CrateNum { LOCAL_CRATE } diff --git a/src/librustc/ty/query/mod.rs b/src/librustc/ty/query/mod.rs index 21698cd7374..381a7b1f03f 100644 --- a/src/librustc/ty/query/mod.rs +++ b/src/librustc/ty/query/mod.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::{self, DepNode}; +use crate::dep_graph::{self, DepConstructor, DepNode}; use crate::hir::exports::Export; use crate::infer::canonical::{self, Canonical}; use crate::lint::LintLevelMap; @@ -52,7 +52,6 @@ use rustc_target::spec::PanicStrategy; use rustc_attr as attr; use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; -use std::any::type_name; use std::borrow::Cow; use std::convert::TryFrom; use std::ops::Deref; @@ -64,6 +63,9 @@ mod plumbing; use self::plumbing::*; pub use self::plumbing::{force_from_dep_node, CycleError}; +mod stats; +pub use self::stats::print_stats; + mod job; #[cfg(parallel_compiler)] pub use self::job::handle_deadlock; @@ -76,6 +78,9 @@ use self::keys::Key; mod values; use self::values::Value; +mod caches; +use self::caches::CacheSelector; + mod config; use self::config::QueryAccessors; pub use self::config::QueryConfig; diff --git a/src/librustc/ty/query/on_disk_cache.rs b/src/librustc/ty/query/on_disk_cache.rs index 45d95e97a9c..b92081ff7c0 100644 --- a/src/librustc/ty/query/on_disk_cache.rs +++ b/src/librustc/ty/query/on_disk_cache.rs @@ -1035,20 +1035,22 @@ where .prof .extra_verbose_generic_activity("encode_query_results_for", ::std::any::type_name::<Q>()); - let shards = Q::query_cache(tcx).lock_shards(); - assert!(shards.iter().all(|shard| shard.active.is_empty())); - for (key, entry) in shards.iter().flat_map(|shard| shard.results.iter()) { - if Q::cache_on_disk(tcx, key.clone(), Some(&entry.value)) { - let dep_node = SerializedDepNodeIndex::new(entry.index.index()); - - // Record position of the cache entry. - query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - - // Encode the type check tables with the `SerializedDepNodeIndex` - // as tag. - encoder.encode_tagged(dep_node, &entry.value)?; - } - } + let state = Q::query_state(tcx); + assert!(state.all_inactive()); + + state.iter_results(|results| { + for (key, value, dep_node) in results { + if Q::cache_on_disk(tcx, key.clone(), Some(&value)) { + let dep_node = SerializedDepNodeIndex::new(dep_node.index()); + + // Record position of the cache entry. + query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - Ok(()) + // Encode the type check tables with the `SerializedDepNodeIndex` + // as tag. + encoder.encode_tagged(dep_node, &value)?; + } + } + Ok(()) + }) } diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 8b787915de6..a61256b9fcb 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -3,7 +3,8 @@ //! manage the caches, and so forth. use crate::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; -use crate::ty::query::config::{QueryConfig, QueryDescription}; +use crate::ty::query::caches::QueryCache; +use crate::ty::query::config::{QueryAccessors, QueryDescription}; use crate::ty::query::job::{QueryInfo, QueryJob, QueryJobId, QueryShardJobId}; use crate::ty::query::Query; use crate::ty::tls; @@ -12,10 +13,8 @@ use crate::ty::{self, TyCtxt}; #[cfg(not(parallel_compiler))] use rustc_data_structures::cold_path; use rustc_data_structures::fx::{FxHashMap, FxHasher}; -#[cfg(parallel_compiler)] -use rustc_data_structures::profiling::TimingGuard; use rustc_data_structures::sharded::Sharded; -use rustc_data_structures::sync::Lock; +use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, FatalError, Handler, Level}; use rustc_span::source_map::DUMMY_SP; @@ -25,26 +24,50 @@ use std::hash::{Hash, Hasher}; use std::mem; use std::num::NonZeroU32; use std::ptr; +#[cfg(debug_assertions)] +use std::sync::atomic::{AtomicUsize, Ordering}; -pub struct QueryCache<'tcx, D: QueryConfig<'tcx> + ?Sized> { - pub(super) results: FxHashMap<D::Key, QueryValue<D::Value>>, +pub(crate) struct QueryStateShard<'tcx, D: QueryAccessors<'tcx> + ?Sized> { + pub(super) cache: <<D as QueryAccessors<'tcx>>::Cache as QueryCache<D::Key, D::Value>>::Sharded, pub(super) active: FxHashMap<D::Key, QueryResult<'tcx>>, /// Used to generate unique ids for active jobs. pub(super) jobs: u32, +} - #[cfg(debug_assertions)] - pub(super) cache_hits: usize, +impl<'tcx, Q: QueryAccessors<'tcx>> QueryStateShard<'tcx, Q> { + fn get_cache( + &mut self, + ) -> &mut <<Q as QueryAccessors<'tcx>>::Cache as QueryCache<Q::Key, Q::Value>>::Sharded { + &mut self.cache + } } -pub(super) struct QueryValue<T> { - pub(super) value: T, - pub(super) index: DepNodeIndex, +impl<'tcx, Q: QueryAccessors<'tcx>> Default for QueryStateShard<'tcx, Q> { + fn default() -> QueryStateShard<'tcx, Q> { + QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 } + } +} + +pub(crate) struct QueryState<'tcx, D: QueryAccessors<'tcx> + ?Sized> { + pub(super) cache: D::Cache, + pub(super) shards: Sharded<QueryStateShard<'tcx, D>>, + #[cfg(debug_assertions)] + pub(super) cache_hits: AtomicUsize, } -impl<T> QueryValue<T> { - pub(super) fn new(value: T, dep_node_index: DepNodeIndex) -> QueryValue<T> { - QueryValue { value, index: dep_node_index } +impl<'tcx, Q: QueryAccessors<'tcx>> QueryState<'tcx, Q> { + pub(super) fn get_lookup<K: Hash>(&'tcx self, key: &K) -> QueryLookup<'tcx, Q> { + // We compute the key's hash once and then use it for both the + // shard lookup and the hashmap lookup. This relies on the fact + // that both of them use `FxHasher`. + let mut hasher = FxHasher::default(); + key.hash(&mut hasher); + let key_hash = hasher.finish(); + + let shard = self.shards.get_shard_index_by_hash(key_hash); + let lock = self.shards.get_shard_by_index(shard).lock(); + QueryLookup { key_hash, shard, lock } } } @@ -58,142 +81,134 @@ pub(super) enum QueryResult<'tcx> { Poisoned, } -impl<'tcx, M: QueryConfig<'tcx>> Default for QueryCache<'tcx, M> { - fn default() -> QueryCache<'tcx, M> { - QueryCache { - results: FxHashMap::default(), - active: FxHashMap::default(), - jobs: 0, +impl<'tcx, M: QueryAccessors<'tcx>> QueryState<'tcx, M> { + pub fn iter_results<R>( + &self, + f: impl for<'a> FnOnce( + Box<dyn Iterator<Item = (&'a M::Key, &'a M::Value, DepNodeIndex)> + 'a>, + ) -> R, + ) -> R { + self.cache.iter(&self.shards, |shard| &mut shard.cache, f) + } + pub fn all_inactive(&self) -> bool { + let shards = self.shards.lock_shards(); + shards.iter().all(|shard| shard.active.is_empty()) + } +} + +impl<'tcx, M: QueryAccessors<'tcx>> Default for QueryState<'tcx, M> { + fn default() -> QueryState<'tcx, M> { + QueryState { + cache: M::Cache::default(), + shards: Default::default(), #[cfg(debug_assertions)] - cache_hits: 0, + cache_hits: AtomicUsize::new(0), } } } +/// Values used when checking a query cache which can be reused on a cache-miss to execute the query. +pub(crate) struct QueryLookup<'tcx, Q: QueryAccessors<'tcx>> { + pub(super) key_hash: u64, + pub(super) shard: usize, + pub(super) lock: LockGuard<'tcx, QueryStateShard<'tcx, Q>>, +} + /// A type representing the responsibility to execute the job in the `job` field. /// This will poison the relevant query if dropped. -pub(super) struct JobOwner<'a, 'tcx, Q: QueryDescription<'tcx>> { - cache: &'a Sharded<QueryCache<'tcx, Q>>, +pub(super) struct JobOwner<'tcx, Q: QueryDescription<'tcx>> { + tcx: TyCtxt<'tcx>, key: Q::Key, id: QueryJobId, } -impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { +impl<'tcx, Q: QueryDescription<'tcx>> JobOwner<'tcx, Q> { /// Either gets a `JobOwner` corresponding the query, allowing us to /// start executing the query, or returns with the result of the query. - /// If the query is executing elsewhere, this will wait for it. + /// This function assumes that `try_get_cached` is already called and returned `lookup`. + /// If the query is executing elsewhere, this will wait for it and return the result. /// If the query panicked, this will silently panic. /// /// This function is inlined because that results in a noticeable speed-up /// for some compile-time benchmarks. #[inline(always)] - pub(super) fn try_get(tcx: TyCtxt<'tcx>, span: Span, key: &Q::Key) -> TryGetJob<'a, 'tcx, Q> { - // Handling the `query_blocked_prof_timer` is a bit weird because of the - // control flow in this function: Blocking is implemented by - // awaiting a running job and, once that is done, entering the loop below - // again from the top. In that second iteration we will hit the - // cache which provides us with the information we need for - // finishing the "query-blocked" event. - // - // We thus allocate `query_blocked_prof_timer` outside the loop, - // initialize it during the first iteration and finish it during the - // second iteration. - #[cfg(parallel_compiler)] - let mut query_blocked_prof_timer: Option<TimingGuard<'_>> = None; - - let cache = Q::query_cache(tcx); - loop { - // We compute the key's hash once and then use it for both the - // shard lookup and the hashmap lookup. This relies on the fact - // that both of them use `FxHasher`. - let mut state = FxHasher::default(); - key.hash(&mut state); - let key_hash = state.finish(); - - let shard = cache.get_shard_index_by_hash(key_hash); - let mut lock_guard = cache.get_shard_by_index(shard).lock(); - let lock = &mut *lock_guard; - - if let Some((_, value)) = - lock.results.raw_entry().from_key_hashed_nocheck(key_hash, key) - { - if unlikely!(tcx.prof.enabled()) { - tcx.prof.query_cache_hit(value.index.into()); + pub(super) fn try_start( + tcx: TyCtxt<'tcx>, + span: Span, + key: &Q::Key, + mut lookup: QueryLookup<'tcx, Q>, + ) -> TryGetJob<'tcx, Q> { + let lock = &mut *lookup.lock; + + let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) { + Entry::Occupied(mut entry) => { + match entry.get_mut() { + QueryResult::Started(job) => { + // For parallel queries, we'll block and wait until the query running + // in another thread has completed. Record how long we wait in the + // self-profiler. + let _query_blocked_prof_timer = if cfg!(parallel_compiler) { + Some(tcx.prof.query_blocked()) + } else { + None + }; - #[cfg(parallel_compiler)] - { - if let Some(prof_timer) = query_blocked_prof_timer.take() { - prof_timer.finish_with_query_invocation_id(value.index.into()); - } - } - } + // Create the id of the job we're waiting for + let id = QueryJobId::new(job.id, lookup.shard, Q::dep_kind()); - let result = (value.value.clone(), value.index); - #[cfg(debug_assertions)] - { - lock.cache_hits += 1; + (job.latch(id), _query_blocked_prof_timer) + } + QueryResult::Poisoned => FatalError.raise(), } - return TryGetJob::JobCompleted(result); } + Entry::Vacant(entry) => { + // No job entry for this query. Return a new one to be started later. - let latch = match lock.active.entry((*key).clone()) { - Entry::Occupied(mut entry) => { - match entry.get_mut() { - QueryResult::Started(job) => { - // For parallel queries, we'll block and wait until the query running - // in another thread has completed. Record how long we wait in the - // self-profiler. - #[cfg(parallel_compiler)] - { - query_blocked_prof_timer = Some(tcx.prof.query_blocked()); - } - - // Create the id of the job we're waiting for - let id = QueryJobId::new(job.id, shard, Q::dep_kind()); - - job.latch(id) - } - QueryResult::Poisoned => FatalError.raise(), - } - } - Entry::Vacant(entry) => { - // No job entry for this query. Return a new one to be started later. + // Generate an id unique within this shard. + let id = lock.jobs.checked_add(1).unwrap(); + lock.jobs = id; + let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); + + let global_id = QueryJobId::new(id, lookup.shard, Q::dep_kind()); - // Generate an id unique within this shard. - let id = lock.jobs.checked_add(1).unwrap(); - lock.jobs = id; - let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); + let job = tls::with_related_context(tcx, |icx| QueryJob::new(id, span, icx.query)); - let global_id = QueryJobId::new(id, shard, Q::dep_kind()); + entry.insert(QueryResult::Started(job)); - let job = - tls::with_related_context(tcx, |icx| QueryJob::new(id, span, icx.query)); + let owner = JobOwner { tcx, id: global_id, key: (*key).clone() }; + return TryGetJob::NotYetStarted(owner); + } + }; + mem::drop(lookup.lock); - entry.insert(QueryResult::Started(job)); + // If we are single-threaded we know that we have cycle error, + // so we just return the error. + #[cfg(not(parallel_compiler))] + return TryGetJob::Cycle(cold_path(|| { + Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)) + })); - let owner = JobOwner { cache, id: global_id, key: (*key).clone() }; - return TryGetJob::NotYetStarted(owner); - } - }; - mem::drop(lock_guard); + // With parallel queries we might just have to wait on some other + // thread. + #[cfg(parallel_compiler)] + { + let result = latch.wait_on(tcx, span); - // If we are single-threaded we know that we have cycle error, - // so we just return the error. - #[cfg(not(parallel_compiler))] - return TryGetJob::Cycle(cold_path(|| { - Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)) - })); + if let Err(cycle) = result { + return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle)); + } - // With parallel queries we might just have to wait on some other - // thread. - #[cfg(parallel_compiler)] - { - let result = latch.wait_on(tcx, span); + let cached = tcx.try_get_cached::<Q, _, _, _>( + (*key).clone(), + |value, index| (value.clone(), index), + |_, _| panic!("value must be in cache after waiting"), + ); - if let Err(cycle) = result { - return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle)); - } + if let Some(prof_timer) = _query_blocked_prof_timer.take() { + prof_timer.finish_with_query_invocation_id(cached.1.into()); } + + return TryGetJob::JobCompleted(cached); } } @@ -203,19 +218,20 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { pub(super) fn complete(self, result: &Q::Value, dep_node_index: DepNodeIndex) { // We can move out of `self` here because we `mem::forget` it below let key = unsafe { ptr::read(&self.key) }; - let cache = self.cache; + let tcx = self.tcx; // Forget ourself so our destructor won't poison the query mem::forget(self); - let value = QueryValue::new(result.clone(), dep_node_index); let job = { - let mut lock = cache.get_shard_by_value(&key).lock(); + let state = Q::query_state(tcx); + let result = result.clone(); + let mut lock = state.shards.get_shard_by_value(&key).lock(); let job = match lock.active.remove(&key).unwrap() { QueryResult::Started(job) => job, QueryResult::Poisoned => panic!(), }; - lock.results.insert(key, value); + state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index); job }; @@ -233,12 +249,13 @@ where (result, diagnostics.into_inner()) } -impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> { +impl<'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'tcx, Q> { #[inline(never)] #[cold] fn drop(&mut self) { // Poison the query so jobs waiting on it panic. - let shard = self.cache.get_shard_by_value(&self.key); + let state = Q::query_state(self.tcx); + let shard = state.shards.get_shard_by_value(&self.key); let job = { let mut shard = shard.lock(); let job = match shard.active.remove(&self.key).unwrap() { @@ -261,14 +278,15 @@ pub struct CycleError<'tcx> { pub(super) cycle: Vec<QueryInfo<'tcx>>, } -/// The result of `try_get_lock`. -pub(super) enum TryGetJob<'a, 'tcx, D: QueryDescription<'tcx>> { +/// The result of `try_start`. +pub(super) enum TryGetJob<'tcx, D: QueryDescription<'tcx>> { /// The query is not yet started. Contains a guard to the cache eventually used to start it. - NotYetStarted(JobOwner<'a, 'tcx, D>), + NotYetStarted(JobOwner<'tcx, D>), /// The query was already completed. /// Returns the result of the query and its dep-node index /// if it succeeded or a cycle error if it failed. + #[cfg(parallel_compiler)] JobCompleted((D::Value, DepNodeIndex)), /// Trying to execute the query resulted in a cycle. @@ -396,13 +414,72 @@ impl<'tcx> TyCtxt<'tcx> { eprintln!("end of query stack"); } + /// Checks if the query is already computed and in the cache. + /// It returns the shard index and a lock guard to the shard, + /// which will be used if the query is not in the cache and we need + /// to compute it. + #[inline(always)] + fn try_get_cached<Q, R, OnHit, OnMiss>( + self, + key: Q::Key, + // `on_hit` can be called while holding a lock to the query cache + on_hit: OnHit, + on_miss: OnMiss, + ) -> R + where + Q: QueryDescription<'tcx> + 'tcx, + OnHit: FnOnce(&Q::Value, DepNodeIndex) -> R, + OnMiss: FnOnce(Q::Key, QueryLookup<'tcx, Q>) -> R, + { + let state = Q::query_state(self); + + state.cache.lookup( + state, + QueryStateShard::<Q>::get_cache, + key, + |value, index| { + if unlikely!(self.prof.enabled()) { + self.prof.query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + state.cache_hits.fetch_add(1, Ordering::Relaxed); + } + on_hit(value, index) + }, + on_miss, + ) + } + #[inline(never)] - pub(super) fn get_query<Q: QueryDescription<'tcx>>(self, span: Span, key: Q::Key) -> Q::Value { + pub(super) fn get_query<Q: QueryDescription<'tcx> + 'tcx>( + self, + span: Span, + key: Q::Key, + ) -> Q::Value { debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); - let job = match JobOwner::try_get(self, span, &key) { + self.try_get_cached::<Q, _, _, _>( + key, + |value, index| { + self.dep_graph.read_index(index); + value.clone() + }, + |key, lookup| self.try_execute_query::<Q>(span, key, lookup), + ) + } + + #[inline(always)] + pub(super) fn try_execute_query<Q: QueryDescription<'tcx>>( + self, + span: Span, + key: Q::Key, + lookup: QueryLookup<'tcx, Q>, + ) -> Q::Value { + let job = match JobOwner::try_start(self, span, &key, lookup) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, + #[cfg(parallel_compiler)] TryGetJob::JobCompleted((v, index)) => { self.dep_graph.read_index(index); return v; @@ -560,7 +637,7 @@ impl<'tcx> TyCtxt<'tcx> { fn force_query_with_job<Q: QueryDescription<'tcx>>( self, key: Q::Key, - job: JobOwner<'_, 'tcx, Q>, + job: JobOwner<'tcx, Q>, dep_node: DepNode, ) -> (Q::Value, DepNodeIndex) { // If the following assertion triggers, it can have two reasons: @@ -615,7 +692,7 @@ impl<'tcx> TyCtxt<'tcx> { /// side-effects -- e.g., in order to report errors for erroneous programs. /// /// Note: The optimization is only available during incr. comp. - pub(super) fn ensure_query<Q: QueryDescription<'tcx>>(self, key: Q::Key) -> () { + pub(super) fn ensure_query<Q: QueryDescription<'tcx> + 'tcx>(self, key: Q::Key) -> () { if Q::EVAL_ALWAYS { let _ = self.get_query::<Q>(DUMMY_SP, key); return; @@ -643,14 +720,30 @@ impl<'tcx> TyCtxt<'tcx> { } #[allow(dead_code)] - fn force_query<Q: QueryDescription<'tcx>>(self, key: Q::Key, span: Span, dep_node: DepNode) { + fn force_query<Q: QueryDescription<'tcx> + 'tcx>( + self, + key: Q::Key, + span: Span, + dep_node: DepNode, + ) { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - let job = match JobOwner::try_get(self, span, &key) { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) | TryGetJob::JobCompleted(_) => return, - }; - self.force_query_with_job::<Q>(key, job, dep_node); + + self.try_get_cached::<Q, _, _, _>( + key, + |_, _| { + // Cache hit, do nothing + }, + |key, lookup| { + let job = match JobOwner::try_start(self, span, &key, lookup) { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => return, + }; + self.force_query_with_job::<Q>(key, job, dep_node); + }, + ); } } @@ -659,17 +752,17 @@ macro_rules! handle_cycle_error { $tcx.report_cycle($error).emit(); Value::from_cycle_error($tcx) }}; - ([fatal_cycle$(, $modifiers:ident)*][$tcx:expr, $error:expr]) => {{ + ([fatal_cycle $($rest:tt)*][$tcx:expr, $error:expr]) => {{ $tcx.report_cycle($error).emit(); $tcx.sess.abort_if_errors(); unreachable!() }}; - ([cycle_delay_bug$(, $modifiers:ident)*][$tcx:expr, $error:expr]) => {{ + ([cycle_delay_bug $($rest:tt)*][$tcx:expr, $error:expr]) => {{ $tcx.report_cycle($error).delay_as_bug(); Value::from_cycle_error($tcx) }}; - ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => { - handle_cycle_error!([$($modifiers),*][$($args)*]) + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { + handle_cycle_error!([$($($modifiers)*)*][$($args)*]) }; } @@ -677,11 +770,11 @@ macro_rules! is_anon { ([]) => {{ false }}; - ([anon$(, $modifiers:ident)*]) => {{ + ([anon $($rest:tt)*]) => {{ true }}; - ([$other:ident$(, $modifiers:ident)*]) => { - is_anon!([$($modifiers),*]) + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => { + is_anon!([$($($modifiers)*)*]) }; } @@ -689,11 +782,23 @@ macro_rules! is_eval_always { ([]) => {{ false }}; - ([eval_always$(, $modifiers:ident)*]) => {{ + ([eval_always $($rest:tt)*]) => {{ true }}; - ([$other:ident$(, $modifiers:ident)*]) => { - is_eval_always!([$($modifiers),*]) + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*]) => { + is_eval_always!([$($($modifiers)*)*]) + }; +} + +macro_rules! query_storage { + ([][$K:ty, $V:ty]) => { + <<$K as Key>::CacheSelector as CacheSelector<$K, $V>>::Cache + }; + ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => { + $ty + }; + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { + query_storage!([$($($modifiers)*)*][$($args)*]) }; } @@ -701,11 +806,11 @@ macro_rules! hash_result { ([][$hcx:expr, $result:expr]) => {{ dep_graph::hash_result($hcx, &$result) }}; - ([no_hash$(, $modifiers:ident)*][$hcx:expr, $result:expr]) => {{ + ([no_hash $($rest:tt)*][$hcx:expr, $result:expr]) => {{ None }}; - ([$other:ident$(, $modifiers:ident)*][$($args:tt)*]) => { - hash_result!([$($modifiers),*][$($args)*]) + ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { + hash_result!([$($($modifiers)*)*][$($args)*]) }; } @@ -725,7 +830,6 @@ macro_rules! define_queries_inner { [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => { use std::mem; - use rustc_data_structures::sharded::Sharded; use crate::{ rustc_data_structures::stable_hasher::HashStable, rustc_data_structures::stable_hasher::StableHasher, @@ -760,11 +864,11 @@ macro_rules! define_queries_inner { $( // We use try_lock_shards here since we are called from the // deadlock handler, and this shouldn't be locked. - let shards = self.$name.try_lock_shards()?; + let shards = self.$name.shards.try_lock_shards()?; let shards = shards.iter().enumerate(); jobs.extend(shards.flat_map(|(shard_id, shard)| { shard.active.iter().filter_map(move |(k, v)| { - if let QueryResult::Started(ref job) = *v { + if let QueryResult::Started(ref job) = *v { let id = QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), @@ -776,111 +880,15 @@ macro_rules! define_queries_inner { query: queries::$name::query(k.clone()) }; Some((id, QueryJobInfo { info, job: job.clone() })) - } else { - None - } + } else { + None + } }) })); )* Some(jobs) } - - pub fn print_stats(&self) { - let mut queries = Vec::new(); - - #[derive(Clone)] - struct QueryStats { - name: &'static str, - cache_hits: usize, - key_size: usize, - key_type: &'static str, - value_size: usize, - value_type: &'static str, - entry_count: usize, - } - - fn stats<'tcx, Q: QueryConfig<'tcx>>( - name: &'static str, - map: &Sharded<QueryCache<'tcx, Q>>, - ) -> QueryStats { - let map = map.lock_shards(); - QueryStats { - name, - #[cfg(debug_assertions)] - cache_hits: map.iter().map(|shard| shard.cache_hits).sum(), - #[cfg(not(debug_assertions))] - cache_hits: 0, - key_size: mem::size_of::<Q::Key>(), - key_type: type_name::<Q::Key>(), - value_size: mem::size_of::<Q::Value>(), - value_type: type_name::<Q::Value>(), - entry_count: map.iter().map(|shard| shard.results.len()).sum(), - } - } - - $( - queries.push(stats::<queries::$name<'_>>( - stringify!($name), - &self.$name, - )); - )* - - if cfg!(debug_assertions) { - let hits: usize = queries.iter().map(|s| s.cache_hits).sum(); - let results: usize = queries.iter().map(|s| s.entry_count).sum(); - println!("\nQuery cache hit rate: {}", hits as f64 / (hits + results) as f64); - } - - let mut query_key_sizes = queries.clone(); - query_key_sizes.sort_by_key(|q| q.key_size); - println!("\nLarge query keys:"); - for q in query_key_sizes.iter().rev() - .filter(|q| q.key_size > 8) { - println!( - " {} - {} x {} - {}", - q.name, - q.key_size, - q.entry_count, - q.key_type - ); - } - - let mut query_value_sizes = queries.clone(); - query_value_sizes.sort_by_key(|q| q.value_size); - println!("\nLarge query values:"); - for q in query_value_sizes.iter().rev() - .filter(|q| q.value_size > 8) { - println!( - " {} - {} x {} - {}", - q.name, - q.value_size, - q.entry_count, - q.value_type - ); - } - - if cfg!(debug_assertions) { - let mut query_cache_hits = queries.clone(); - query_cache_hits.sort_by_key(|q| q.cache_hits); - println!("\nQuery cache hits:"); - for q in query_cache_hits.iter().rev() { - println!( - " {} - {} ({}%)", - q.name, - q.cache_hits, - q.cache_hits as f64 / (q.cache_hits + q.entry_count) as f64 - ); - } - } - - let mut query_value_count = queries.clone(); - query_value_count.sort_by_key(|q| q.entry_count); - println!("\nQuery value count:"); - for q in query_value_count.iter().rev() { - println!(" {} - {}", q.name, q.entry_count); - } - } } #[allow(nonstandard_style)] @@ -956,7 +964,6 @@ macro_rules! define_queries_inner { $(impl<$tcx> QueryConfig<$tcx> for queries::$name<$tcx> { type Key = $K; type Value = $V; - const NAME: &'static str = stringify!($name); const CATEGORY: ProfileCategory = $category; } @@ -965,22 +972,22 @@ macro_rules! define_queries_inner { const ANON: bool = is_anon!([$($modifiers)*]); const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]); + type Cache = query_storage!([$($modifiers)*][$K, $V]); + #[inline(always)] fn query(key: Self::Key) -> Query<'tcx> { Query::$name(key) } #[inline(always)] - fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a Sharded<QueryCache<$tcx, Self>> { + fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<$tcx, Self> { &tcx.queries.$name } #[allow(unused)] #[inline(always)] fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode { - use crate::dep_graph::DepConstructor::*; - - DepNode::new(tcx, $node(*key)) + DepConstructor::$node(tcx, *key) } #[inline(always)] @@ -1132,7 +1139,7 @@ macro_rules! define_queries_struct { providers: IndexVec<CrateNum, Providers<$tcx>>, fallback_extern_providers: Box<Providers<$tcx>>, - $($(#[$attr])* $name: Sharded<QueryCache<$tcx, queries::$name<$tcx>>>,)* + $($(#[$attr])* $name: QueryState<$tcx, queries::$name<$tcx>>,)* } }; } diff --git a/src/librustc/ty/query/profiling_support.rs b/src/librustc/ty/query/profiling_support.rs index 79b32ba83ae..99ada34d59e 100644 --- a/src/librustc/ty/query/profiling_support.rs +++ b/src/librustc/ty/query/profiling_support.rs @@ -1,11 +1,10 @@ use crate::hir::map::definitions::DefPathData; use crate::ty::context::TyCtxt; -use crate::ty::query::config::QueryConfig; -use crate::ty::query::plumbing::QueryCache; +use crate::ty::query::config::QueryAccessors; +use crate::ty::query::plumbing::QueryState; use measureme::{StringComponent, StringId}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfiler; -use rustc_data_structures::sharded::Sharded; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE}; use std::fmt::Debug; use std::io::Write; @@ -161,10 +160,10 @@ where pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, Q>( tcx: TyCtxt<'tcx>, query_name: &'static str, - query_cache: &Sharded<QueryCache<'tcx, Q>>, + query_state: &QueryState<'tcx, Q>, string_cache: &mut QueryKeyStringCache, ) where - Q: QueryConfig<'tcx>, + Q: QueryAccessors<'tcx>, { tcx.prof.with_profiler(|profiler| { let event_id_builder = profiler.event_id_builder(); @@ -181,20 +180,8 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, Q>( // need to invoke queries itself, we cannot keep the query caches // locked while doing so. Instead we copy out the // `(query_key, dep_node_index)` pairs and release the lock again. - let query_keys_and_indices = { - let shards = query_cache.lock_shards(); - let len = shards.iter().map(|shard| shard.results.len()).sum(); - - let mut query_keys_and_indices = Vec::with_capacity(len); - - for shard in &shards { - query_keys_and_indices.extend( - shard.results.iter().map(|(q_key, q_val)| (q_key.clone(), q_val.index)), - ); - } - - query_keys_and_indices - }; + let query_keys_and_indices: Vec<_> = query_state + .iter_results(|results| results.map(|(k, _, i)| (k.clone(), i)).collect()); // Now actually allocate the strings. If allocating the strings // generates new entries in the query cache, we'll miss them but @@ -218,18 +205,14 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, Q>( let query_name = profiler.get_or_alloc_cached_string(query_name); let event_id = event_id_builder.from_label(query_name).to_string_id(); - let shards = query_cache.lock_shards(); + query_state.iter_results(|results| { + let query_invocation_ids: Vec<_> = results.map(|v| v.2.into()).collect(); - for shard in shards.iter() { - let query_invocation_ids = shard - .results - .values() - .map(|v| v.index) - .map(|dep_node_index| dep_node_index.into()); - - profiler - .bulk_map_query_invocation_id_to_single_string(query_invocation_ids, event_id); - } + profiler.bulk_map_query_invocation_id_to_single_string( + query_invocation_ids.into_iter(), + event_id, + ); + }); } }); } diff --git a/src/librustc/ty/query/stats.rs b/src/librustc/ty/query/stats.rs new file mode 100644 index 00000000000..d257320d4ea --- /dev/null +++ b/src/librustc/ty/query/stats.rs @@ -0,0 +1,139 @@ +use crate::ty::query::config::QueryAccessors; +use crate::ty::query::plumbing::QueryState; +use crate::ty::query::queries; +use crate::ty::TyCtxt; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; + +use std::any::type_name; +use std::mem; +#[cfg(debug_assertions)] +use std::sync::atomic::Ordering; + +trait KeyStats { + fn key_stats(&self, stats: &mut QueryStats); +} + +impl<T> KeyStats for T { + default fn key_stats(&self, _: &mut QueryStats) {} +} + +impl KeyStats for DefId { + fn key_stats(&self, stats: &mut QueryStats) { + if self.krate == LOCAL_CRATE { + stats.local_def_id_keys = Some(stats.local_def_id_keys.unwrap_or(0) + 1); + } + } +} + +#[derive(Clone)] +struct QueryStats { + name: &'static str, + cache_hits: usize, + key_size: usize, + key_type: &'static str, + value_size: usize, + value_type: &'static str, + entry_count: usize, + local_def_id_keys: Option<usize>, +} + +fn stats<'tcx, Q: QueryAccessors<'tcx>>( + name: &'static str, + map: &QueryState<'tcx, Q>, +) -> QueryStats { + let mut stats = QueryStats { + name, + #[cfg(debug_assertions)] + cache_hits: map.cache_hits.load(Ordering::Relaxed), + #[cfg(not(debug_assertions))] + cache_hits: 0, + key_size: mem::size_of::<Q::Key>(), + key_type: type_name::<Q::Key>(), + value_size: mem::size_of::<Q::Value>(), + value_type: type_name::<Q::Value>(), + entry_count: map.iter_results(|results| results.count()), + local_def_id_keys: None, + }; + map.iter_results(|results| { + for (key, _, _) in results { + key.key_stats(&mut stats) + } + }); + stats +} + +pub fn print_stats(tcx: TyCtxt<'_>) { + let queries = query_stats(tcx); + + if cfg!(debug_assertions) { + let hits: usize = queries.iter().map(|s| s.cache_hits).sum(); + let results: usize = queries.iter().map(|s| s.entry_count).sum(); + println!("\nQuery cache hit rate: {}", hits as f64 / (hits + results) as f64); + } + + let mut query_key_sizes = queries.clone(); + query_key_sizes.sort_by_key(|q| q.key_size); + println!("\nLarge query keys:"); + for q in query_key_sizes.iter().rev().filter(|q| q.key_size > 8) { + println!(" {} - {} x {} - {}", q.name, q.key_size, q.entry_count, q.key_type); + } + + let mut query_value_sizes = queries.clone(); + query_value_sizes.sort_by_key(|q| q.value_size); + println!("\nLarge query values:"); + for q in query_value_sizes.iter().rev().filter(|q| q.value_size > 8) { + println!(" {} - {} x {} - {}", q.name, q.value_size, q.entry_count, q.value_type); + } + + if cfg!(debug_assertions) { + let mut query_cache_hits = queries.clone(); + query_cache_hits.sort_by_key(|q| q.cache_hits); + println!("\nQuery cache hits:"); + for q in query_cache_hits.iter().rev() { + println!( + " {} - {} ({}%)", + q.name, + q.cache_hits, + q.cache_hits as f64 / (q.cache_hits + q.entry_count) as f64 + ); + } + } + + let mut query_value_count = queries.clone(); + query_value_count.sort_by_key(|q| q.entry_count); + println!("\nQuery value count:"); + for q in query_value_count.iter().rev() { + println!(" {} - {}", q.name, q.entry_count); + } + + let mut def_id_density: Vec<_> = + queries.iter().filter(|q| q.local_def_id_keys.is_some()).collect(); + def_id_density.sort_by_key(|q| q.local_def_id_keys.unwrap()); + println!("\nLocal DefId density:"); + let total = tcx.hir().definitions().def_index_count() as f64; + for q in def_id_density.iter().rev() { + let local = q.local_def_id_keys.unwrap(); + println!(" {} - {} = ({}%)", q.name, local, (local as f64 * 100.0) / total); + } +} + +macro_rules! print_stats { + (<$tcx:tt> $($category:tt { + $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)* + },)*) => { + fn query_stats(tcx: TyCtxt<'_>) -> Vec<QueryStats> { + let mut queries = Vec::new(); + + $($( + queries.push(stats::<queries::$name<'_>>( + stringify!($name), + &tcx.queries.$name, + )); + )*)* + + queries + } + } +} + +rustc_query_append! { [print_stats!][<'tcx>] } diff --git a/src/librustc_data_structures/profiling.rs b/src/librustc_data_structures/profiling.rs index 1d0ac4f4907..f2c80510f22 100644 --- a/src/librustc_data_structures/profiling.rs +++ b/src/librustc_data_structures/profiling.rs @@ -81,6 +81,7 @@ //! //! [mm]: https://github.com/rust-lang/measureme/ +use crate::cold_path; use crate::fx::FxHashMap; use std::borrow::Borrow; @@ -531,9 +532,11 @@ impl<'a> TimingGuard<'a> { #[inline] pub fn finish_with_query_invocation_id(self, query_invocation_id: QueryInvocationId) { if let Some(guard) = self.0 { - let event_id = StringId::new_virtual(query_invocation_id.0); - let event_id = EventId::from_virtual(event_id); - guard.finish_with_override_event_id(event_id); + cold_path(|| { + let event_id = StringId::new_virtual(query_invocation_id.0); + let event_id = EventId::from_virtual(event_id); + guard.finish_with_override_event_id(event_id); + }); } } diff --git a/src/librustc_data_structures/stable_hasher.rs b/src/librustc_data_structures/stable_hasher.rs index 0a26bf3bdc9..a98e77cebd8 100644 --- a/src/librustc_data_structures/stable_hasher.rs +++ b/src/librustc_data_structures/stable_hasher.rs @@ -27,6 +27,7 @@ pub trait StableHasherResult: Sized { } impl StableHasher { + #[inline] pub fn new() -> Self { StableHasher { state: SipHasher128::new_with_keys(0, 0) } } diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index 0b4a337051f..96a2ac08f2c 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -696,8 +696,8 @@ impl<'tcx> QueryContext<'tcx> { ty::tls::enter_global(self.0, |tcx| f(tcx)) } - pub fn print_stats(&self) { - self.0.queries.print_stats() + pub fn print_stats(&mut self) { + self.enter(|tcx| ty::query::print_stats(tcx)) } } diff --git a/src/librustc_interface/queries.rs b/src/librustc_interface/queries.rs index 720d162ac81..0c77ab57500 100644 --- a/src/librustc_interface/queries.rs +++ b/src/librustc_interface/queries.rs @@ -340,7 +340,7 @@ impl Compiler { if self.session().opts.debugging_opts.query_stats { if let Ok(gcx) = queries.global_ctxt() { - gcx.peek().print_stats(); + gcx.peek_mut().print_stats(); } } diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs index 294cdb7643f..6362f3c2c49 100644 --- a/src/librustc_macros/src/query.rs +++ b/src/librustc_macros/src/query.rs @@ -33,6 +33,9 @@ enum QueryModifier { /// The description of the query. Desc(Option<Ident>, Punctuated<Expr, Token![,]>), + /// Use this type for the in-memory cache. + Storage(Type), + /// Cache the query to disk if the `Expr` returns true. Cache(Option<(IdentOrWild, IdentOrWild)>, Block), @@ -106,6 +109,9 @@ impl Parse for QueryModifier { let id = args.parse()?; let block = input.parse()?; Ok(QueryModifier::LoadCached(tcx, id, block)) + } else if modifier == "storage" { + let ty = input.parse()?; + Ok(QueryModifier::Storage(ty)) } else if modifier == "fatal_cycle" { Ok(QueryModifier::FatalCycle) } else if modifier == "cycle_delay_bug" { @@ -198,6 +204,9 @@ struct QueryModifiers { /// The description of the query. desc: Option<(Option<Ident>, Punctuated<Expr, Token![,]>)>, + /// Use this type for the in-memory cache. + storage: Option<Type>, + /// Cache the query to disk if the `Block` returns true. cache: Option<(Option<(IdentOrWild, IdentOrWild)>, Block)>, @@ -226,6 +235,7 @@ struct QueryModifiers { /// Process query modifiers into a struct, erroring on duplicates fn process_modifiers(query: &mut Query) -> QueryModifiers { let mut load_cached = None; + let mut storage = None; let mut cache = None; let mut desc = None; let mut fatal_cycle = false; @@ -242,6 +252,12 @@ fn process_modifiers(query: &mut Query) -> QueryModifiers { } load_cached = Some((tcx, id, block)); } + QueryModifier::Storage(ty) => { + if storage.is_some() { + panic!("duplicate modifier `storage` for query `{}`", query.name); + } + storage = Some(ty); + } QueryModifier::Cache(args, expr) => { if cache.is_some() { panic!("duplicate modifier `cache` for query `{}`", query.name); @@ -294,6 +310,7 @@ fn process_modifiers(query: &mut Query) -> QueryModifiers { } QueryModifiers { load_cached, + storage, cache, desc, fatal_cycle, @@ -451,6 +468,10 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { if modifiers.fatal_cycle { attributes.push(quote! { fatal_cycle }); }; + // Pass on the storage modifier + if let Some(ref ty) = modifiers.storage { + attributes.push(quote! { storage(#ty) }); + }; // Pass on the cycle_delay_bug modifier if modifiers.cycle_delay_bug { attributes.push(quote! { cycle_delay_bug }); diff --git a/src/librustc_mir/const_eval/eval_queries.rs b/src/librustc_mir/const_eval/eval_queries.rs index 4d5464f774f..1cec5d30e9b 100644 --- a/src/librustc_mir/const_eval/eval_queries.rs +++ b/src/librustc_mir/const_eval/eval_queries.rs @@ -72,8 +72,8 @@ fn eval_body_using_ecx<'mir, 'tcx>( Ok(ret) } -/// The `InterpCx` is only meant to be used to do field and index projections into constants for -/// `simd_shuffle` and const patterns in match arms. +/// The `InterpCx` is only meant to be used to do field and index projections into promoteds +/// and const patterns in match arms. /// /// The function containing the `match` that is currently being analyzed may have generic bounds /// that inform us about the generic bounds of the constant. E.g., using an associated constant diff --git a/src/librustc_mir/dataflow/generic/mod.rs b/src/librustc_mir/dataflow/generic/mod.rs index ea643042c5f..23b22550a3b 100644 --- a/src/librustc_mir/dataflow/generic/mod.rs +++ b/src/librustc_mir/dataflow/generic/mod.rs @@ -395,5 +395,16 @@ impl<T: Idx> GenKill<T> for BitSet<T> { } } +// For compatibility with old framework +impl<T: Idx> GenKill<T> for crate::dataflow::GenKillSet<T> { + fn gen(&mut self, elem: T) { + self.gen(elem); + } + + fn kill(&mut self, elem: T) { + self.kill(elem); + } +} + #[cfg(test)] mod tests; diff --git a/src/librustc_mir/dataflow/impls/borrowed_locals.rs b/src/librustc_mir/dataflow/impls/borrowed_locals.rs index 63834d0ecda..95a676c0892 100644 --- a/src/librustc_mir/dataflow/impls/borrowed_locals.rs +++ b/src/librustc_mir/dataflow/impls/borrowed_locals.rs @@ -1,102 +1,274 @@ pub use super::*; -use crate::dataflow::{BitDenotation, GenKillSet}; +use crate::dataflow::generic::{AnalysisDomain, GenKill, GenKillAnalysis}; use rustc::mir::visit::Visitor; use rustc::mir::*; +use rustc::ty::{ParamEnv, TyCtxt}; +use rustc_span::DUMMY_SP; -/// This calculates if any part of a MIR local could have previously been borrowed. -/// This means that once a local has been borrowed, its bit will be set -/// from that point and onwards, until we see a StorageDead statement for the local, -/// at which points there is no memory associated with the local, so it cannot be borrowed. -/// This is used to compute which locals are live during a yield expression for -/// immovable generators. -#[derive(Copy, Clone)] -pub struct HaveBeenBorrowedLocals<'a, 'tcx> { - body: &'a Body<'tcx>, +pub type MaybeMutBorrowedLocals<'mir, 'tcx> = MaybeBorrowedLocals<MutBorrow<'mir, 'tcx>>; + +/// A dataflow analysis that tracks whether a pointer or reference could possibly exist that points +/// to a given local. +/// +/// The `K` parameter determines what kind of borrows are tracked. By default, +/// `MaybeBorrowedLocals` looks for *any* borrow of a local. If you are only interested in borrows +/// that might allow mutation, use the `MaybeMutBorrowedLocals` type alias instead. +/// +/// At present, this is used as a very limited form of alias analysis. For example, +/// `MaybeBorrowedLocals` is used to compute which locals are live during a yield expression for +/// immovable generators. `MaybeMutBorrowedLocals` is used during const checking to prove that a +/// local has not been mutated via indirect assignment (e.g., `*p = 42`), the side-effects of a +/// function call or inline assembly. +pub struct MaybeBorrowedLocals<K = AnyBorrow> { + kind: K, + ignore_borrow_on_drop: bool, +} + +impl MaybeBorrowedLocals { + /// A dataflow analysis that records whether a pointer or reference exists that may alias the + /// given local. + pub fn all_borrows() -> Self { + MaybeBorrowedLocals { kind: AnyBorrow, ignore_borrow_on_drop: false } + } +} + +impl MaybeMutBorrowedLocals<'mir, 'tcx> { + /// A dataflow analysis that records whether a pointer or reference exists that may *mutably* + /// alias the given local. + /// + /// This includes `&mut` and pointers derived from an `&mut`, as well as shared borrows of + /// types with interior mutability. + pub fn mut_borrows_only( + tcx: TyCtxt<'tcx>, + body: &'mir mir::Body<'tcx>, + param_env: ParamEnv<'tcx>, + ) -> Self { + MaybeBorrowedLocals { + kind: MutBorrow { body, tcx, param_env }, + ignore_borrow_on_drop: false, + } + } } -impl<'a, 'tcx> HaveBeenBorrowedLocals<'a, 'tcx> { - pub fn new(body: &'a Body<'tcx>) -> Self { - HaveBeenBorrowedLocals { body } +impl<K> MaybeBorrowedLocals<K> { + /// During dataflow analysis, ignore the borrow that may occur when a place is dropped. + /// + /// Drop terminators may call custom drop glue (`Drop::drop`), which takes `&mut self` as a + /// parameter. In the general case, a drop impl could launder that reference into the + /// surrounding environment through a raw pointer, thus creating a valid `*mut` pointing to the + /// dropped local. We are not yet willing to declare this particular case UB, so we must treat + /// all dropped locals as mutably borrowed for now. See discussion on [#61069]. + /// + /// In some contexts, we know that this borrow will never occur. For example, during + /// const-eval, custom drop glue cannot be run. Code that calls this should document the + /// assumptions that justify ignoring `Drop` terminators in this way. + /// + /// [#61069]: https://github.com/rust-lang/rust/pull/61069 + pub fn unsound_ignore_borrow_on_drop(self) -> Self { + MaybeBorrowedLocals { ignore_borrow_on_drop: true, ..self } } - pub fn body(&self) -> &Body<'tcx> { - self.body + fn transfer_function<'a, T>(&'a self, trans: &'a mut T) -> TransferFunction<'a, T, K> { + TransferFunction { + kind: &self.kind, + trans, + ignore_borrow_on_drop: self.ignore_borrow_on_drop, + } } } -impl<'a, 'tcx> BitDenotation<'tcx> for HaveBeenBorrowedLocals<'a, 'tcx> { +impl<K> AnalysisDomain<'tcx> for MaybeBorrowedLocals<K> +where + K: BorrowAnalysisKind<'tcx>, +{ type Idx = Local; - fn name() -> &'static str { - "has_been_borrowed_locals" + + const NAME: &'static str = K::ANALYSIS_NAME; + + fn bits_per_block(&self, body: &mir::Body<'tcx>) -> usize { + body.local_decls().len() + } + + fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut BitSet<Self::Idx>) { + // No locals are aliased on function entry + } +} + +impl<K> GenKillAnalysis<'tcx> for MaybeBorrowedLocals<K> +where + K: BorrowAnalysisKind<'tcx>, +{ + fn statement_effect( + &self, + trans: &mut impl GenKill<Self::Idx>, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + self.transfer_function(trans).visit_statement(statement, location); } - fn bits_per_block(&self) -> usize { - self.body.local_decls.len() + + fn terminator_effect( + &self, + trans: &mut impl GenKill<Self::Idx>, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + self.transfer_function(trans).visit_terminator(terminator, location); } - fn start_block_effect(&self, _on_entry: &mut BitSet<Local>) { - // Nothing is borrowed on function entry + fn call_return_effect( + &self, + _trans: &mut impl GenKill<Self::Idx>, + _block: mir::BasicBlock, + _func: &mir::Operand<'tcx>, + _args: &[mir::Operand<'tcx>], + _dest_place: &mir::Place<'tcx>, + ) { } +} + +impl<K> BottomValue for MaybeBorrowedLocals<K> { + // bottom = unborrowed + const BOTTOM_VALUE: bool = false; +} - fn statement_effect(&self, trans: &mut GenKillSet<Local>, loc: Location) { - let stmt = &self.body[loc.block].statements[loc.statement_index]; +/// A `Visitor` that defines the transfer function for `MaybeBorrowedLocals`. +struct TransferFunction<'a, T, K> { + trans: &'a mut T, + kind: &'a K, + ignore_borrow_on_drop: bool, +} - BorrowedLocalsVisitor { trans }.visit_statement(stmt, loc); +impl<T, K> Visitor<'tcx> for TransferFunction<'a, T, K> +where + T: GenKill<Local>, + K: BorrowAnalysisKind<'tcx>, +{ + fn visit_statement(&mut self, stmt: &Statement<'tcx>, location: Location) { + self.super_statement(stmt, location); - // StorageDead invalidates all borrows and raw pointers to a local - match stmt.kind { - StatementKind::StorageDead(l) => trans.kill(l), - _ => (), + // When we reach a `StorageDead` statement, we can assume that any pointers to this memory + // are now invalid. + if let StatementKind::StorageDead(local) = stmt.kind { + self.trans.kill(local); } } - fn terminator_effect(&self, trans: &mut GenKillSet<Local>, loc: Location) { - let terminator = self.body[loc.block].terminator(); - BorrowedLocalsVisitor { trans }.visit_terminator(terminator, loc); - match &terminator.kind { - // Drop terminators borrows the location - TerminatorKind::Drop { location, .. } - | TerminatorKind::DropAndReplace { location, .. } => { - if let Some(local) = find_local(location) { - trans.gen(local); + fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: Location) { + self.super_rvalue(rvalue, location); + + match rvalue { + mir::Rvalue::AddressOf(mt, borrowed_place) => { + if !borrowed_place.is_indirect() && self.kind.in_address_of(*mt, borrowed_place) { + self.trans.gen(borrowed_place.local); } } - _ => (), + + mir::Rvalue::Ref(_, kind, borrowed_place) => { + if !borrowed_place.is_indirect() && self.kind.in_ref(*kind, borrowed_place) { + self.trans.gen(borrowed_place.local); + } + } + + mir::Rvalue::Cast(..) + | mir::Rvalue::Use(..) + | mir::Rvalue::Repeat(..) + | mir::Rvalue::Len(..) + | mir::Rvalue::BinaryOp(..) + | mir::Rvalue::CheckedBinaryOp(..) + | mir::Rvalue::NullaryOp(..) + | mir::Rvalue::UnaryOp(..) + | mir::Rvalue::Discriminant(..) + | mir::Rvalue::Aggregate(..) => {} } } - fn propagate_call_return( - &self, - _in_out: &mut BitSet<Local>, - _call_bb: mir::BasicBlock, - _dest_bb: mir::BasicBlock, - _dest_place: &mir::Place<'tcx>, - ) { - // Nothing to do when a call returns successfully + fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) { + self.super_terminator(terminator, location); + + match terminator.kind { + mir::TerminatorKind::Drop { location: dropped_place, .. } + | mir::TerminatorKind::DropAndReplace { location: dropped_place, .. } => { + // See documentation for `unsound_ignore_borrow_on_drop` for an explanation. + if !self.ignore_borrow_on_drop { + self.trans.gen(dropped_place.local); + } + } + + TerminatorKind::Abort + | TerminatorKind::Assert { .. } + | TerminatorKind::Call { .. } + | TerminatorKind::FalseEdges { .. } + | TerminatorKind::FalseUnwind { .. } + | TerminatorKind::GeneratorDrop + | TerminatorKind::Goto { .. } + | TerminatorKind::Resume + | TerminatorKind::Return + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::Unreachable + | TerminatorKind::Yield { .. } => {} + } } } -impl<'a, 'tcx> BottomValue for HaveBeenBorrowedLocals<'a, 'tcx> { - // bottom = unborrowed - const BOTTOM_VALUE: bool = false; +pub struct AnyBorrow; + +pub struct MutBorrow<'mir, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'mir Body<'tcx>, + param_env: ParamEnv<'tcx>, +} + +impl MutBorrow<'mir, 'tcx> { + /// `&` and `&raw` only allow mutation if the borrowed place is `!Freeze`. + /// + /// This assumes that it is UB to take the address of a struct field whose type is + /// `Freeze`, then use pointer arithmetic to derive a pointer to a *different* field of + /// that same struct whose type is `!Freeze`. If we decide that this is not UB, we will + /// have to check the type of the borrowed **local** instead of the borrowed **place** + /// below. See [rust-lang/unsafe-code-guidelines#134]. + /// + /// [rust-lang/unsafe-code-guidelines#134]: https://github.com/rust-lang/unsafe-code-guidelines/issues/134 + fn shared_borrow_allows_mutation(&self, place: &Place<'tcx>) -> bool { + !place.ty(self.body, self.tcx).ty.is_freeze(self.tcx, self.param_env, DUMMY_SP) + } } -struct BorrowedLocalsVisitor<'gk> { - trans: &'gk mut GenKillSet<Local>, +pub trait BorrowAnalysisKind<'tcx> { + const ANALYSIS_NAME: &'static str; + + fn in_address_of(&self, mt: Mutability, place: &Place<'tcx>) -> bool; + fn in_ref(&self, kind: mir::BorrowKind, place: &Place<'tcx>) -> bool; } -fn find_local(place: &Place<'_>) -> Option<Local> { - if !place.is_indirect() { Some(place.local) } else { None } +impl BorrowAnalysisKind<'tcx> for AnyBorrow { + const ANALYSIS_NAME: &'static str = "maybe_borrowed_locals"; + + fn in_ref(&self, _: mir::BorrowKind, _: &Place<'_>) -> bool { + true + } + fn in_address_of(&self, _: Mutability, _: &Place<'_>) -> bool { + true + } } -impl<'tcx> Visitor<'tcx> for BorrowedLocalsVisitor<'_> { - fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { - if let Rvalue::Ref(_, _, ref place) = *rvalue { - if let Some(local) = find_local(place) { - self.trans.gen(local); +impl BorrowAnalysisKind<'tcx> for MutBorrow<'mir, 'tcx> { + const ANALYSIS_NAME: &'static str = "maybe_mut_borrowed_locals"; + + fn in_ref(&self, kind: mir::BorrowKind, place: &Place<'tcx>) -> bool { + match kind { + mir::BorrowKind::Mut { .. } => true, + mir::BorrowKind::Shared | mir::BorrowKind::Shallow | mir::BorrowKind::Unique => { + self.shared_borrow_allows_mutation(place) } } + } - self.super_rvalue(rvalue, location) + fn in_address_of(&self, mt: Mutability, place: &Place<'tcx>) -> bool { + match mt { + Mutability::Mut => true, + Mutability::Not => self.shared_borrow_allows_mutation(place), + } } } diff --git a/src/librustc_mir/dataflow/impls/indirect_mutation.rs b/src/librustc_mir/dataflow/impls/indirect_mutation.rs deleted file mode 100644 index 85bf342c8a3..00000000000 --- a/src/librustc_mir/dataflow/impls/indirect_mutation.rs +++ /dev/null @@ -1,136 +0,0 @@ -use rustc::mir::visit::Visitor; -use rustc::mir::{self, Local, Location}; -use rustc::ty::{self, TyCtxt}; -use rustc_index::bit_set::BitSet; -use rustc_span::DUMMY_SP; - -use crate::dataflow::{self, GenKillSet}; - -/// Whether a borrow to a `Local` has been created that could allow that `Local` to be mutated -/// indirectly. This could either be a mutable reference (`&mut`) or a shared borrow if the type of -/// that `Local` allows interior mutability. Operations that can mutate local's indirectly include: -/// assignments through a pointer (`*p = 42`), function calls, drop terminators and inline assembly. -/// -/// If this returns false for a `Local` at a given statement (or terminator), that `Local` could -/// not possibly have been mutated indirectly prior to that statement. -#[derive(Copy, Clone)] -pub struct IndirectlyMutableLocals<'mir, 'tcx> { - body: &'mir mir::Body<'tcx>, - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, -} - -impl<'mir, 'tcx> IndirectlyMutableLocals<'mir, 'tcx> { - pub fn new( - tcx: TyCtxt<'tcx>, - body: &'mir mir::Body<'tcx>, - param_env: ty::ParamEnv<'tcx>, - ) -> Self { - IndirectlyMutableLocals { body, tcx, param_env } - } - - fn transfer_function<'a>( - &self, - trans: &'a mut GenKillSet<Local>, - ) -> TransferFunction<'a, 'mir, 'tcx> { - TransferFunction { body: self.body, tcx: self.tcx, param_env: self.param_env, trans } - } -} - -impl<'mir, 'tcx> dataflow::BitDenotation<'tcx> for IndirectlyMutableLocals<'mir, 'tcx> { - type Idx = Local; - - fn name() -> &'static str { - "mut_borrowed_locals" - } - - fn bits_per_block(&self) -> usize { - self.body.local_decls.len() - } - - fn start_block_effect(&self, _entry_set: &mut BitSet<Local>) { - // Nothing is borrowed on function entry - } - - fn statement_effect(&self, trans: &mut GenKillSet<Local>, loc: Location) { - let stmt = &self.body[loc.block].statements[loc.statement_index]; - self.transfer_function(trans).visit_statement(stmt, loc); - } - - fn terminator_effect(&self, trans: &mut GenKillSet<Local>, loc: Location) { - let terminator = self.body[loc.block].terminator(); - self.transfer_function(trans).visit_terminator(terminator, loc); - } - - fn propagate_call_return( - &self, - _in_out: &mut BitSet<Local>, - _call_bb: mir::BasicBlock, - _dest_bb: mir::BasicBlock, - _dest_place: &mir::Place<'tcx>, - ) { - // Nothing to do when a call returns successfully - } -} - -impl<'mir, 'tcx> dataflow::BottomValue for IndirectlyMutableLocals<'mir, 'tcx> { - // bottom = unborrowed - const BOTTOM_VALUE: bool = false; -} - -/// A `Visitor` that defines the transfer function for `IndirectlyMutableLocals`. -struct TransferFunction<'a, 'mir, 'tcx> { - trans: &'a mut GenKillSet<Local>, - body: &'mir mir::Body<'tcx>, - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, -} - -impl<'tcx> TransferFunction<'_, '_, 'tcx> { - /// Returns `true` if this borrow would allow mutation of the `borrowed_place`. - fn borrow_allows_mutation( - &self, - kind: mir::BorrowKind, - borrowed_place: &mir::Place<'tcx>, - ) -> bool { - match kind { - mir::BorrowKind::Mut { .. } => true, - - mir::BorrowKind::Shared | mir::BorrowKind::Shallow | mir::BorrowKind::Unique => { - !borrowed_place.ty(self.body, self.tcx).ty.is_freeze( - self.tcx, - self.param_env, - DUMMY_SP, - ) - } - } - } -} - -impl<'tcx> Visitor<'tcx> for TransferFunction<'_, '_, 'tcx> { - fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: Location) { - if let mir::Rvalue::Ref(_, kind, ref borrowed_place) = *rvalue { - if self.borrow_allows_mutation(kind, borrowed_place) { - if !borrowed_place.is_indirect() { - self.trans.gen(borrowed_place.local); - } - } - } - - self.super_rvalue(rvalue, location); - } - - fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) { - // This method purposely does nothing except call `super_terminator`. It exists solely to - // document the subtleties around drop terminators. - - self.super_terminator(terminator, location); - - if let mir::TerminatorKind::Drop { location: _, .. } - | mir::TerminatorKind::DropAndReplace { location: _, .. } = &terminator.kind - { - // Although drop terminators mutably borrow the location being dropped, that borrow - // cannot live beyond the drop terminator because the dropped location is invalidated. - } - } -} diff --git a/src/librustc_mir/dataflow/impls/mod.rs b/src/librustc_mir/dataflow/impls/mod.rs index 5b2264c2a65..acea3185781 100644 --- a/src/librustc_mir/dataflow/impls/mod.rs +++ b/src/librustc_mir/dataflow/impls/mod.rs @@ -20,11 +20,9 @@ use super::drop_flag_effects_for_location; use super::on_lookup_result_bits; mod borrowed_locals; -mod indirect_mutation; mod storage_liveness; pub use self::borrowed_locals::*; -pub use self::indirect_mutation::IndirectlyMutableLocals; pub use self::storage_liveness::*; pub(super) mod borrows; diff --git a/src/librustc_mir/dataflow/impls/storage_liveness.rs b/src/librustc_mir/dataflow/impls/storage_liveness.rs index 040c13e8210..7508d71945e 100644 --- a/src/librustc_mir/dataflow/impls/storage_liveness.rs +++ b/src/librustc_mir/dataflow/impls/storage_liveness.rs @@ -1,8 +1,8 @@ pub use super::*; +use crate::dataflow::generic::{Results, ResultsRefCursor}; use crate::dataflow::BitDenotation; -use crate::dataflow::HaveBeenBorrowedLocals; -use crate::dataflow::{DataflowResults, DataflowResultsCursor, DataflowResultsRefCursor}; +use crate::dataflow::MaybeBorrowedLocals; use rustc::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor}; use rustc::mir::*; use std::cell::RefCell; @@ -69,22 +69,23 @@ impl<'a, 'tcx> BottomValue for MaybeStorageLive<'a, 'tcx> { const BOTTOM_VALUE: bool = false; } +type BorrowedLocalsResults<'a, 'tcx> = ResultsRefCursor<'a, 'a, 'tcx, MaybeBorrowedLocals>; + /// Dataflow analysis that determines whether each local requires storage at a /// given location; i.e. whether its storage can go away without being observed. pub struct RequiresStorage<'mir, 'tcx> { body: ReadOnlyBodyAndCache<'mir, 'tcx>, - borrowed_locals: - RefCell<DataflowResultsRefCursor<'mir, 'tcx, HaveBeenBorrowedLocals<'mir, 'tcx>>>, + borrowed_locals: RefCell<BorrowedLocalsResults<'mir, 'tcx>>, } impl<'mir, 'tcx: 'mir> RequiresStorage<'mir, 'tcx> { pub fn new( body: ReadOnlyBodyAndCache<'mir, 'tcx>, - borrowed_locals: &'mir DataflowResults<'tcx, HaveBeenBorrowedLocals<'mir, 'tcx>>, + borrowed_locals: &'mir Results<'tcx, MaybeBorrowedLocals>, ) -> Self { RequiresStorage { body, - borrowed_locals: RefCell::new(DataflowResultsCursor::new(borrowed_locals, *body)), + borrowed_locals: RefCell::new(ResultsRefCursor::new(*body, borrowed_locals)), } } @@ -111,11 +112,12 @@ impl<'mir, 'tcx> BitDenotation<'tcx> for RequiresStorage<'mir, 'tcx> { } fn before_statement_effect(&self, sets: &mut GenKillSet<Self::Idx>, loc: Location) { - // If we borrow or assign to a place then it needs storage for that - // statement. - self.check_for_borrow(sets, loc); - let stmt = &self.body[loc.block].statements[loc.statement_index]; + + // If a place is borrowed in a statement, it needs storage for that statement. + self.borrowed_locals.borrow().analysis().statement_effect(sets, stmt, loc); + + // If a place is assigned to in a statement, it needs storage for that statement. match stmt.kind { StatementKind::StorageDead(l) => sets.kill(l), StatementKind::Assign(box (ref place, _)) @@ -138,12 +140,13 @@ impl<'mir, 'tcx> BitDenotation<'tcx> for RequiresStorage<'mir, 'tcx> { } fn before_terminator_effect(&self, sets: &mut GenKillSet<Local>, loc: Location) { - self.check_for_borrow(sets, loc); + let terminator = self.body[loc.block].terminator(); - if let TerminatorKind::Call { destination: Some((Place { local, .. }, _)), .. } = - self.body[loc.block].terminator().kind - { - sets.gen(local); + // If a place is borrowed in a terminator, it needs storage for that terminator. + self.borrowed_locals.borrow().analysis().terminator_effect(sets, terminator, loc); + + if let TerminatorKind::Call { destination: Some((place, _)), .. } = terminator.kind { + sets.gen(place.local); } } @@ -179,14 +182,6 @@ impl<'mir, 'tcx> RequiresStorage<'mir, 'tcx> { let mut visitor = MoveVisitor { sets, borrowed_locals: &self.borrowed_locals }; visitor.visit_location(self.body, loc); } - - /// Gen locals that are newly borrowed. This includes borrowing any part of - /// a local (we rely on this behavior of `HaveBeenBorrowedLocals`). - fn check_for_borrow(&self, sets: &mut GenKillSet<Local>, loc: Location) { - let mut borrowed_locals = self.borrowed_locals.borrow_mut(); - borrowed_locals.seek(loc); - borrowed_locals.each_gen_bit(|l| sets.gen(l)); - } } impl<'mir, 'tcx> BottomValue for RequiresStorage<'mir, 'tcx> { @@ -195,8 +190,7 @@ impl<'mir, 'tcx> BottomValue for RequiresStorage<'mir, 'tcx> { } struct MoveVisitor<'a, 'mir, 'tcx> { - borrowed_locals: - &'a RefCell<DataflowResultsRefCursor<'mir, 'tcx, HaveBeenBorrowedLocals<'mir, 'tcx>>>, + borrowed_locals: &'a RefCell<BorrowedLocalsResults<'mir, 'tcx>>, sets: &'a mut GenKillSet<Local>, } @@ -204,7 +198,7 @@ impl<'a, 'mir: 'a, 'tcx> Visitor<'tcx> for MoveVisitor<'a, 'mir, 'tcx> { fn visit_local(&mut self, local: &Local, context: PlaceContext, loc: Location) { if PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) == context { let mut borrowed_locals = self.borrowed_locals.borrow_mut(); - borrowed_locals.seek(loc); + borrowed_locals.seek_before(loc); if !borrowed_locals.contains(*local) { self.sets.kill(*local); } diff --git a/src/librustc_mir/dataflow/mod.rs b/src/librustc_mir/dataflow/mod.rs index 7cd7fc309b6..41bac894e48 100644 --- a/src/librustc_mir/dataflow/mod.rs +++ b/src/librustc_mir/dataflow/mod.rs @@ -23,8 +23,7 @@ pub(crate) use self::drop_flag_effects::*; pub use self::impls::borrows::Borrows; pub use self::impls::DefinitelyInitializedPlaces; pub use self::impls::EverInitializedPlaces; -pub use self::impls::HaveBeenBorrowedLocals; -pub use self::impls::IndirectlyMutableLocals; +pub use self::impls::{MaybeBorrowedLocals, MaybeMutBorrowedLocals}; pub use self::impls::{MaybeInitializedPlaces, MaybeUninitializedPlaces}; pub use self::impls::{MaybeStorageLive, RequiresStorage}; diff --git a/src/librustc_mir/transform/check_consts/resolver.rs b/src/librustc_mir/transform/check_consts/resolver.rs index b804dc4b5b6..3e14cc6d32a 100644 --- a/src/librustc_mir/transform/check_consts/resolver.rs +++ b/src/librustc_mir/transform/check_consts/resolver.rs @@ -15,7 +15,7 @@ use crate::dataflow::{self as old_dataflow, generic as dataflow}; /// `FlowSensitiveAnalysis`. /// /// This transfer does nothing when encountering an indirect assignment. Consumers should rely on -/// the `IndirectlyMutableLocals` dataflow pass to see if a `Local` may have become qualified via +/// the `MaybeMutBorrowedLocals` dataflow pass to see if a `Local` may have become qualified via /// an indirect assignment or function call. struct TransferFunction<'a, 'mir, 'tcx, Q> { item: &'a Item<'mir, 'tcx>, diff --git a/src/librustc_mir/transform/check_consts/validation.rs b/src/librustc_mir/transform/check_consts/validation.rs index 1a1c34e1c67..1553f826c7e 100644 --- a/src/librustc_mir/transform/check_consts/validation.rs +++ b/src/librustc_mir/transform/check_consts/validation.rs @@ -16,17 +16,19 @@ use rustc_span::Span; use std::borrow::Cow; use std::ops::Deref; -use self::old_dataflow::IndirectlyMutableLocals; use super::ops::{self, NonConstOp}; use super::qualifs::{self, HasMutInterior, NeedsDrop}; use super::resolver::FlowSensitiveAnalysis; use super::{is_lang_panic_fn, ConstKind, Item, Qualif}; use crate::const_eval::{is_const_fn, is_unstable_const_fn}; -use crate::dataflow::{self as old_dataflow, generic as dataflow}; -use dataflow::Analysis; +use crate::dataflow::generic::{self as dataflow, Analysis}; +use crate::dataflow::MaybeMutBorrowedLocals; +// We are using `MaybeMutBorrowedLocals` as a proxy for whether an item may have been mutated +// through a pointer prior to the given point. This is okay even though `MaybeMutBorrowedLocals` +// kills locals upon `StorageDead` because a local will never be used after a `StorageDead`. pub type IndirectlyMutableResults<'mir, 'tcx> = - old_dataflow::DataflowResultsCursor<'mir, 'tcx, IndirectlyMutableLocals<'mir, 'tcx>>; + dataflow::ResultsCursor<'mir, 'tcx, MaybeMutBorrowedLocals<'mir, 'tcx>>; struct QualifCursor<'a, 'mir, 'tcx, Q: Qualif> { cursor: dataflow::ResultsCursor<'mir, 'tcx, FlowSensitiveAnalysis<'a, 'mir, 'tcx, Q>>, @@ -59,7 +61,7 @@ pub struct Qualifs<'a, 'mir, 'tcx> { impl Qualifs<'a, 'mir, 'tcx> { fn indirectly_mutable(&mut self, local: Local, location: Location) -> bool { - self.indirectly_mutable.seek(location); + self.indirectly_mutable.seek_before(location); self.indirectly_mutable.get().contains(local) } @@ -135,22 +137,21 @@ impl Deref for Validator<'_, 'mir, 'tcx> { impl Validator<'a, 'mir, 'tcx> { pub fn new(item: &'a Item<'mir, 'tcx>) -> Self { + let Item { tcx, body, def_id, param_env, .. } = *item; + let needs_drop = QualifCursor::new(NeedsDrop, item); let has_mut_interior = QualifCursor::new(HasMutInterior, item); - let dead_unwinds = BitSet::new_empty(item.body.basic_blocks().len()); - let indirectly_mutable = old_dataflow::do_dataflow( - item.tcx, - &*item.body, - item.def_id, - &item.tcx.get_attrs(item.def_id), - &dead_unwinds, - old_dataflow::IndirectlyMutableLocals::new(item.tcx, *item.body, item.param_env), - |_, local| old_dataflow::DebugFormatted::new(&local), - ); - - let indirectly_mutable = - old_dataflow::DataflowResultsCursor::new(indirectly_mutable, *item.body); + // We can use `unsound_ignore_borrow_on_drop` here because custom drop impls are not + // allowed in a const. + // + // FIXME(ecstaticmorse): Someday we want to allow custom drop impls. How do we do this + // without breaking stable code? + let indirectly_mutable = MaybeMutBorrowedLocals::mut_borrows_only(tcx, *body, param_env) + .unsound_ignore_borrow_on_drop() + .into_engine(tcx, *body, def_id) + .iterate_to_fixpoint() + .into_results_cursor(*body); let qualifs = Qualifs { needs_drop, has_mut_interior, indirectly_mutable }; diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs index d5d9a8c83cd..b3dc87d1a16 100644 --- a/src/librustc_mir/transform/generator.rs +++ b/src/librustc_mir/transform/generator.rs @@ -49,9 +49,10 @@ //! For generators with state 1 (returned) and state 2 (poisoned) it does nothing. //! Otherwise it drops all the values in scope at the last suspension point. +use crate::dataflow::generic::{Analysis, ResultsCursor}; use crate::dataflow::{do_dataflow, DataflowResultsCursor, DebugFormatted}; use crate::dataflow::{DataflowResults, DataflowResultsConsumer, FlowAtLocation}; -use crate::dataflow::{HaveBeenBorrowedLocals, MaybeStorageLive, RequiresStorage}; +use crate::dataflow::{MaybeBorrowedLocals, MaybeStorageLive, RequiresStorage}; use crate::transform::no_landing_pads::no_landing_pads; use crate::transform::simplify; use crate::transform::{MirPass, MirSource}; @@ -471,17 +472,10 @@ fn locals_live_across_suspend_points( // Calculate the MIR locals which have been previously // borrowed (even if they are still active). - let borrowed_locals_analysis = HaveBeenBorrowedLocals::new(body_ref); - let borrowed_locals_results = do_dataflow( - tcx, - body_ref, - def_id, - &[], - &dead_unwinds, - borrowed_locals_analysis, - |bd, p| DebugFormatted::new(&bd.body().local_decls[p]), - ); - let mut borrowed_locals_cursor = DataflowResultsCursor::new(&borrowed_locals_results, body_ref); + let borrowed_locals_results = + MaybeBorrowedLocals::all_borrows().into_engine(tcx, body_ref, def_id).iterate_to_fixpoint(); + + let mut borrowed_locals_cursor = ResultsCursor::new(body_ref, &borrowed_locals_results); // Calculate the MIR locals that we actually need to keep storage around // for. @@ -521,7 +515,7 @@ fn locals_live_across_suspend_points( // If a borrow is converted to a raw reference, we must also assume that it lives // forever. Note that the final liveness is still bounded by the storage liveness // of the local, which happens using the `intersect` operation below. - borrowed_locals_cursor.seek(loc); + borrowed_locals_cursor.seek_before(loc); liveness.outs[block].union(borrowed_locals_cursor.get()); } diff --git a/src/librustc_mir/transform/promote_consts.rs b/src/librustc_mir/transform/promote_consts.rs index a5d59860c3d..a11ee57f465 100644 --- a/src/librustc_mir/transform/promote_consts.rs +++ b/src/librustc_mir/transform/promote_consts.rs @@ -24,7 +24,6 @@ use rustc_span::{Span, DUMMY_SP}; use syntax::ast::LitKind; use rustc_index::vec::{Idx, IndexVec}; -use rustc_target::spec::abi::Abi; use std::cell::Cell; use std::{cmp, iter, mem, usize}; @@ -106,11 +105,10 @@ pub enum Candidate { /// Promotion of the `x` in `[x; 32]`. Repeat(Location), - /// Currently applied to function calls where the callee has the unstable - /// `#[rustc_args_required_const]` attribute as well as the SIMD shuffle - /// intrinsic. The intrinsic requires the arguments are indeed constant and - /// the attribute currently provides the semantic requirement that arguments - /// must be constant. + /// Function calls where the callee has the unstable + /// `#[rustc_args_required_const]` attribute. The attribute requires that + /// the arguments be constant, usually because they are encoded as an + /// immediate operand in a platform intrinsic. Argument { bb: BasicBlock, index: usize }, } @@ -218,17 +216,6 @@ impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> { if let TerminatorKind::Call { ref func, .. } = *kind { if let ty::FnDef(def_id, _) = func.ty(self.body, self.tcx).kind { - let fn_sig = self.tcx.fn_sig(def_id); - if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = fn_sig.abi() { - let name = self.tcx.item_name(def_id); - // FIXME(eddyb) use `#[rustc_args_required_const(2)]` for shuffles. - if name.as_str().starts_with("simd_shuffle") { - self.candidates.push(Candidate::Argument { bb: location.block, index: 2 }); - - return; // Don't double count `simd_shuffle` candidates - } - } - if let Some(constant_args) = args_required_const(self.tcx, def_id) { for index in constant_args { self.candidates.push(Candidate::Argument { bb: location.block, index }); @@ -730,8 +717,7 @@ pub fn validate_candidates( .filter(|&candidate| { validator.explicit = candidate.forces_explicit_promotion(); - // FIXME(eddyb) also emit the errors for shuffle indices - // and `#[rustc_args_required_const]` arguments here. + // FIXME(eddyb) also emit the errors for `#[rustc_args_required_const]` arguments here. let is_promotable = validator.validate_candidate(candidate).is_ok(); match candidate { diff --git a/src/librustc_mir/transform/rustc_peek.rs b/src/librustc_mir/transform/rustc_peek.rs index 7d8506eb281..6176cf8bc0f 100644 --- a/src/librustc_mir/transform/rustc_peek.rs +++ b/src/librustc_mir/transform/rustc_peek.rs @@ -12,9 +12,8 @@ use rustc_index::bit_set::BitSet; use crate::dataflow::generic::{Analysis, Results, ResultsCursor}; use crate::dataflow::move_paths::{HasMoveData, MoveData}; use crate::dataflow::move_paths::{LookupResult, MovePathIndex}; -use crate::dataflow::IndirectlyMutableLocals; +use crate::dataflow::MaybeMutBorrowedLocals; use crate::dataflow::MoveDataParamEnv; -use crate::dataflow::{do_dataflow, DebugFormatted}; use crate::dataflow::{ DefinitelyInitializedPlaces, MaybeInitializedPlaces, MaybeUninitializedPlaces, }; @@ -24,7 +23,6 @@ pub struct SanityCheck; impl<'tcx> MirPass<'tcx> for SanityCheck { fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut BodyAndCache<'tcx>) { use crate::dataflow::has_rustc_mir_with; - let def_id = src.def_id(); if !tcx.has_attr(def_id, sym::rustc_mir) { debug!("skipping rustc_peek::SanityCheck on {}", tcx.def_path_str(def_id)); @@ -37,7 +35,6 @@ impl<'tcx> MirPass<'tcx> for SanityCheck { let param_env = tcx.param_env(def_id); let move_data = MoveData::gather_moves(body, tcx, param_env).unwrap(); let mdpe = MoveDataParamEnv { move_data: move_data, param_env: param_env }; - let dead_unwinds = BitSet::new_empty(body.basic_blocks().len()); let flow_inits = MaybeInitializedPlaces::new(tcx, body, &mdpe) .into_engine(tcx, body, def_id) @@ -48,15 +45,9 @@ impl<'tcx> MirPass<'tcx> for SanityCheck { let flow_def_inits = DefinitelyInitializedPlaces::new(tcx, body, &mdpe) .into_engine(tcx, body, def_id) .iterate_to_fixpoint(); - let _flow_indirectly_mut = do_dataflow( - tcx, - body, - def_id, - &attributes, - &dead_unwinds, - IndirectlyMutableLocals::new(tcx, body, param_env), - |_, i| DebugFormatted::new(&i), - ); + let flow_mut_borrowed = MaybeMutBorrowedLocals::mut_borrows_only(tcx, body, param_env) + .into_engine(tcx, body, def_id) + .iterate_to_fixpoint(); if has_rustc_mir_with(&attributes, sym::rustc_peek_maybe_init).is_some() { sanity_check_via_rustc_peek(tcx, body, def_id, &attributes, &flow_inits); @@ -67,12 +58,9 @@ impl<'tcx> MirPass<'tcx> for SanityCheck { if has_rustc_mir_with(&attributes, sym::rustc_peek_definite_init).is_some() { sanity_check_via_rustc_peek(tcx, body, def_id, &attributes, &flow_def_inits); } - // FIXME: Uncomment these as analyses are migrated to the new framework - /* if has_rustc_mir_with(&attributes, sym::rustc_peek_indirectly_mutable).is_some() { - sanity_check_via_rustc_peek(tcx, body, def_id, &attributes, &flow_indirectly_mut); + sanity_check_via_rustc_peek(tcx, body, def_id, &attributes, &flow_mut_borrowed); } - */ if has_rustc_mir_with(&attributes, sym::stop_after_dataflow).is_some() { tcx.sess.fatal("stop_after_dataflow ended compilation"); } @@ -276,8 +264,7 @@ where } } -/* FIXME: Add this back once `IndirectlyMutableLocals` uses the new dataflow framework. -impl<'tcx> RustcPeekAt<'tcx> for IndirectlyMutableLocals<'_, 'tcx> { +impl<'tcx> RustcPeekAt<'tcx> for MaybeMutBorrowedLocals<'_, 'tcx> { fn peek_at( &self, tcx: TyCtxt<'tcx>, @@ -298,4 +285,3 @@ impl<'tcx> RustcPeekAt<'tcx> for IndirectlyMutableLocals<'_, 'tcx> { } } } -*/ diff --git a/src/librustc_session/session.rs b/src/librustc_session/session.rs index 6f003043aa9..2fb7977dce9 100644 --- a/src/librustc_session/session.rs +++ b/src/librustc_session/session.rs @@ -392,6 +392,7 @@ impl Session { ); } + #[inline] pub fn source_map(&self) -> &source_map::SourceMap { self.parse_sess.source_map() } diff --git a/src/librustc_span/symbol.rs b/src/librustc_span/symbol.rs index 3c419334d42..97708d91d7e 100644 --- a/src/librustc_span/symbol.rs +++ b/src/librustc_span/symbol.rs @@ -993,6 +993,7 @@ impl Encodable for Symbol { } impl Decodable for Symbol { + #[inline] fn decode<D: Decoder>(d: &mut D) -> Result<Symbol, D::Error> { Ok(Symbol::intern(&d.read_str()?)) } @@ -1031,6 +1032,7 @@ impl Interner { } } + #[inline] pub fn intern(&mut self, string: &str) -> Symbol { if let Some(&name) = self.names.get(string) { return name; diff --git a/src/librustc_target/spec/thumb_base.rs b/src/librustc_target/spec/thumb_base.rs index 204f4723b70..99ab996be95 100644 --- a/src/librustc_target/spec/thumb_base.rs +++ b/src/librustc_target/spec/thumb_base.rs @@ -50,6 +50,9 @@ pub fn opts() -> TargetOptions { // until we figure a way to add the pretty printers without requiring a volatile load cf. // rust-lang/rust#44993. emit_debug_gdb_scripts: false, + // LLVM is eager to trash the link register when calling `noreturn` functions, which + // breaks debugging. Preserve LR by default to prevent that from happening. + eliminate_frame_pointer: false, ..Default::default() } } diff --git a/src/libserialize/serialize.rs b/src/libserialize/serialize.rs index 19283ffc438..8c6548cd3c5 100644 --- a/src/libserialize/serialize.rs +++ b/src/libserialize/serialize.rs @@ -36,6 +36,7 @@ pub trait Encoder { fn emit_str(&mut self, v: &str) -> Result<(), Self::Error>; // Compound types: + #[inline] fn emit_enum<F>(&mut self, _name: &str, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -57,6 +58,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_enum_variant_arg<F>(&mut self, _a_idx: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -89,6 +91,7 @@ pub trait Encoder { self.emit_enum_variant_arg(f_idx, f) } + #[inline] fn emit_struct<F>(&mut self, _name: &str, _len: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -96,6 +99,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_struct_field<F>( &mut self, _f_name: &str, @@ -108,6 +112,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_tuple<F>(&mut self, _len: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -115,6 +120,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_tuple_arg<F>(&mut self, _idx: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -164,6 +170,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_seq_elt<F>(&mut self, _idx: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -179,6 +186,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_map_elt_key<F>(&mut self, _idx: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -186,6 +194,7 @@ pub trait Encoder { f(self) } + #[inline] fn emit_map_elt_val<F>(&mut self, _idx: usize, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, @@ -218,6 +227,7 @@ pub trait Decoder { fn read_str(&mut self) -> Result<Cow<'_, str>, Self::Error>; // Compound types: + #[inline] fn read_enum<T, F>(&mut self, _name: &str, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -225,6 +235,7 @@ pub trait Decoder { f(self) } + #[inline] fn read_enum_variant<T, F>(&mut self, _names: &[&str], mut f: F) -> Result<T, Self::Error> where F: FnMut(&mut Self, usize) -> Result<T, Self::Error>, @@ -233,6 +244,7 @@ pub trait Decoder { f(self, disr) } + #[inline] fn read_enum_variant_arg<T, F>(&mut self, _a_idx: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -259,6 +271,7 @@ pub trait Decoder { self.read_enum_variant_arg(f_idx, f) } + #[inline] fn read_struct<T, F>(&mut self, _s_name: &str, _len: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -266,6 +279,7 @@ pub trait Decoder { f(self) } + #[inline] fn read_struct_field<T, F>( &mut self, _f_name: &str, @@ -278,6 +292,7 @@ pub trait Decoder { f(self) } + #[inline] fn read_tuple<T, F>(&mut self, _len: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -285,6 +300,7 @@ pub trait Decoder { f(self) } + #[inline] fn read_tuple_arg<T, F>(&mut self, _a_idx: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -328,6 +344,7 @@ pub trait Decoder { f(self, len) } + #[inline] fn read_seq_elt<T, F>(&mut self, _idx: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -343,6 +360,7 @@ pub trait Decoder { f(self, len) } + #[inline] fn read_map_elt_key<T, F>(&mut self, _idx: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, @@ -350,6 +368,7 @@ pub trait Decoder { f(self) } + #[inline] fn read_map_elt_val<T, F>(&mut self, _idx: usize, f: F) -> Result<T, Self::Error> where F: FnOnce(&mut Self) -> Result<T, Self::Error>, diff --git a/src/libstd/fs.rs b/src/libstd/fs.rs index cff7bbe5ef1..09be3f13050 100644 --- a/src/libstd/fs.rs +++ b/src/libstd/fs.rs @@ -844,10 +844,7 @@ impl OpenOptions { self } - /// Sets the option for creating a new file. - /// - /// This option indicates whether a new file will be created if the file - /// does not yet already exist. + /// Sets the option to create a new file, or open it if it already exists. /// /// In order for the file to be created, [`write`] or [`append`] access must /// be used. @@ -868,11 +865,10 @@ impl OpenOptions { self } - /// Sets the option to always create a new file. + /// Sets the option to create a new file, failing if it already exists. /// - /// This option indicates whether a new file will be created. - /// No file is allowed to exist at the target location, also no (dangling) - /// symlink. + /// No file is allowed to exist at the target location, also no (dangling) symlink. In this + /// way, if the call succeeds, the file returned is guaranteed to be new. /// /// This option is useful because it is atomic. Otherwise between checking /// whether a file exists and creating a new one, the file may have been diff --git a/src/test/incremental/issue-61530.rs b/src/test/incremental/issue-61530.rs index 06b2957ac62..c9c600efed8 100644 --- a/src/test/incremental/issue-61530.rs +++ b/src/test/incremental/issue-61530.rs @@ -1,4 +1,4 @@ -#![feature(repr_simd, platform_intrinsics)] +#![feature(repr_simd, platform_intrinsics, rustc_attrs)] // revisions:rpass1 rpass2 @@ -6,6 +6,7 @@ struct I32x2(i32, i32); extern "platform-intrinsic" { + #[rustc_args_required_const(2)] fn simd_shuffle2<T, U>(x: T, y: T, idx: [u32; 2]) -> U; } diff --git a/src/test/ui/issues/issue-38074.rs b/src/test/ui/issues/issue-38074.rs index 214d6752cef..8f7905b31ec 100644 --- a/src/test/ui/issues/issue-38074.rs +++ b/src/test/ui/issues/issue-38074.rs @@ -1,9 +1,10 @@ // run-pass // ignore-emscripten FIXME(#45351) -#![feature(platform_intrinsics, repr_simd)] +#![feature(platform_intrinsics, repr_simd, rustc_attrs)] extern "platform-intrinsic" { + #[rustc_args_required_const(2)] fn simd_shuffle2<T, U>(x: T, y: T, idx: [u32; 2]) -> U; } diff --git a/src/test/ui/issues/issue-69225-layout-repeated-checked-add.rs b/src/test/ui/issues/issue-69225-layout-repeated-checked-add.rs new file mode 100644 index 00000000000..7f43e4d1a51 --- /dev/null +++ b/src/test/ui/issues/issue-69225-layout-repeated-checked-add.rs @@ -0,0 +1,31 @@ +// Ensure we appropriately error instead of overflowing a calculation when creating a new Alloc +// Layout + +// run-fail +// compile-flags: -C opt-level=3 +// error-pattern: index out of bounds: the len is 0 but the index is 16777216 +// ignore-wasm no panic or subprocess support +// ignore-emscripten no panic or subprocess support + +fn do_test(x: usize) { + let arr = vec![vec![0u8; 3]]; + + let mut z = Vec::new(); + for arr_ref in arr { + for y in 0..x { + for _ in 0..1 { + z.extend(std::iter::repeat(0).take(x)); + let a = y * x; + let b = (y + 1) * x - 1; + let slice = &arr_ref[a..b]; + eprintln!("{} {} {} {}", a, b, arr_ref.len(), slice.len()); + eprintln!("{:?}", slice[1 << 24]); + } + } + } +} + +fn main() { + do_test(1); + do_test(2); +} diff --git a/src/test/ui/mir-dataflow/indirect-mutation-offset.rs b/src/test/ui/mir-dataflow/indirect-mutation-offset.rs index 884c83b6616..caa307e269f 100644 --- a/src/test/ui/mir-dataflow/indirect-mutation-offset.rs +++ b/src/test/ui/mir-dataflow/indirect-mutation-offset.rs @@ -1,6 +1,11 @@ // compile-flags: -Zunleash-the-miri-inside-of-you -// ignore-test Temporarily ignored while this analysis is migrated to the new framework. +// This test demonstrates a shortcoming of the `MaybeMutBorrowedLocals` analysis. It does not +// handle code that takes a reference to one field of a struct, then use pointer arithmetic to +// transform it to another field of that same struct that may have interior mutability. For now, +// this is UB, but this may change in the future. See [rust-lang/unsafe-code-guidelines#134]. +// +// [rust-lang/unsafe-code-guidelines#134]: https://github.com/rust-lang/unsafe-code-guidelines/issues/134 #![feature(core_intrinsics, rustc_attrs, const_raw_ptr_deref)] diff --git a/src/test/ui/mir-dataflow/indirect-mutation-offset.stderr b/src/test/ui/mir-dataflow/indirect-mutation-offset.stderr index 0ae9a40c96a..8d3548ececd 100644 --- a/src/test/ui/mir-dataflow/indirect-mutation-offset.stderr +++ b/src/test/ui/mir-dataflow/indirect-mutation-offset.stderr @@ -1,5 +1,5 @@ error: rustc_peek: bit not set - --> $DIR/indirect-mutation-offset.rs:34:14 + --> $DIR/indirect-mutation-offset.rs:41:14 | LL | unsafe { rustc_peek(x) }; | ^^^^^^^^^^^^^ diff --git a/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.rs b/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.rs index 5929d05f4de..c9c9ab879f2 100644 --- a/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.rs +++ b/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.rs @@ -42,9 +42,13 @@ extern "platform-intrinsic" { fn simd_insert<T, E>(x: T, idx: u32, y: E) -> T; fn simd_extract<T, E>(x: T, idx: u32) -> E; + #[rustc_args_required_const(2)] fn simd_shuffle2<T, U>(x: T, y: T, idx: [u32; 2]) -> U; + #[rustc_args_required_const(2)] fn simd_shuffle3<T, U>(x: T, y: T, idx: [u32; 3]) -> U; + #[rustc_args_required_const(2)] fn simd_shuffle4<T, U>(x: T, y: T, idx: [u32; 4]) -> U; + #[rustc_args_required_const(2)] fn simd_shuffle8<T, U>(x: T, y: T, idx: [u32; 8]) -> U; } diff --git a/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.stderr b/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.stderr index 78022c0c8bd..29916f85902 100644 --- a/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.stderr +++ b/src/test/ui/simd-intrinsic/simd-intrinsic-generic-elements.stderr @@ -1,89 +1,89 @@ error[E0511]: invalid monomorphization of `simd_insert` intrinsic: expected SIMD input type, found non-SIMD `i32` - --> $DIR/simd-intrinsic-generic-elements.rs:55:9 + --> $DIR/simd-intrinsic-generic-elements.rs:59:9 | LL | simd_insert(0, 0, 0); | ^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_insert` intrinsic: expected inserted type `i32` (element of input `i32x4`), found `f64` - --> $DIR/simd-intrinsic-generic-elements.rs:57:9 + --> $DIR/simd-intrinsic-generic-elements.rs:61:9 | LL | simd_insert(x, 0, 1.0); | ^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_extract` intrinsic: expected return type `i32` (element of input `i32x4`), found `f32` - --> $DIR/simd-intrinsic-generic-elements.rs:59:9 + --> $DIR/simd-intrinsic-generic-elements.rs:63:9 | LL | simd_extract::<_, f32>(x, 0); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle2` intrinsic: expected SIMD input type, found non-SIMD `i32` - --> $DIR/simd-intrinsic-generic-elements.rs:62:9 + --> $DIR/simd-intrinsic-generic-elements.rs:66:9 | LL | simd_shuffle2::<i32, i32>(0, 0, [0; 2]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle3` intrinsic: expected SIMD input type, found non-SIMD `i32` - --> $DIR/simd-intrinsic-generic-elements.rs:64:9 + --> $DIR/simd-intrinsic-generic-elements.rs:68:9 | LL | simd_shuffle3::<i32, i32>(0, 0, [0; 3]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle4` intrinsic: expected SIMD input type, found non-SIMD `i32` - --> $DIR/simd-intrinsic-generic-elements.rs:66:9 + --> $DIR/simd-intrinsic-generic-elements.rs:70:9 | LL | simd_shuffle4::<i32, i32>(0, 0, [0; 4]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle8` intrinsic: expected SIMD input type, found non-SIMD `i32` - --> $DIR/simd-intrinsic-generic-elements.rs:68:9 + --> $DIR/simd-intrinsic-generic-elements.rs:72:9 | LL | simd_shuffle8::<i32, i32>(0, 0, [0; 8]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle2` intrinsic: expected return element type `i32` (element of input `i32x4`), found `f32x2` with element type `f32` - --> $DIR/simd-intrinsic-generic-elements.rs:71:9 + --> $DIR/simd-intrinsic-generic-elements.rs:75:9 | LL | simd_shuffle2::<_, f32x2>(x, x, [0; 2]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle3` intrinsic: expected return element type `i32` (element of input `i32x4`), found `f32x3` with element type `f32` - --> $DIR/simd-intrinsic-generic-elements.rs:73:9 + --> $DIR/simd-intrinsic-generic-elements.rs:77:9 | LL | simd_shuffle3::<_, f32x3>(x, x, [0; 3]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle4` intrinsic: expected return element type `i32` (element of input `i32x4`), found `f32x4` with element type `f32` - --> $DIR/simd-intrinsic-generic-elements.rs:75:9 + --> $DIR/simd-intrinsic-generic-elements.rs:79:9 | LL | simd_shuffle4::<_, f32x4>(x, x, [0; 4]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle8` intrinsic: expected return element type `i32` (element of input `i32x4`), found `f32x8` with element type `f32` - --> $DIR/simd-intrinsic-generic-elements.rs:77:9 + --> $DIR/simd-intrinsic-generic-elements.rs:81:9 | LL | simd_shuffle8::<_, f32x8>(x, x, [0; 8]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle2` intrinsic: expected return type of length 2, found `i32x8` with length 8 - --> $DIR/simd-intrinsic-generic-elements.rs:80:9 + --> $DIR/simd-intrinsic-generic-elements.rs:84:9 | LL | simd_shuffle2::<_, i32x8>(x, x, [0; 2]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle3` intrinsic: expected return type of length 3, found `i32x4` with length 4 - --> $DIR/simd-intrinsic-generic-elements.rs:82:9 + --> $DIR/simd-intrinsic-generic-elements.rs:86:9 | LL | simd_shuffle3::<_, i32x4>(x, x, [0; 3]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle4` intrinsic: expected return type of length 4, found `i32x3` with length 3 - --> $DIR/simd-intrinsic-generic-elements.rs:84:9 + --> $DIR/simd-intrinsic-generic-elements.rs:88:9 | LL | simd_shuffle4::<_, i32x3>(x, x, [0; 4]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shuffle8` intrinsic: expected return type of length 8, found `i32x2` with length 2 - --> $DIR/simd-intrinsic-generic-elements.rs:86:9 + --> $DIR/simd-intrinsic-generic-elements.rs:90:9 | LL | simd_shuffle8::<_, i32x2>(x, x, [0; 8]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557-ice.rs b/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557-ice.rs index 4c09ae25c5f..b03b0ef5089 100644 --- a/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557-ice.rs +++ b/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557-ice.rs @@ -3,9 +3,10 @@ // // run-pass // compile-flags: -Zmir-opt-level=3 -#![feature(platform_intrinsics, repr_simd)] +#![feature(platform_intrinsics, repr_simd, rustc_attrs)] extern "platform-intrinsic" { + #[rustc_args_required_const(2)] fn simd_shuffle2<T, U>(x: T, y: T, idx: [u32; 2]) -> U; } diff --git a/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557.rs b/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557.rs index 7a0d955686b..2741dc13336 100644 --- a/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557.rs +++ b/src/test/ui/simd-intrinsic/simd-intrinsic-inlining-issue67557.rs @@ -3,9 +3,10 @@ // // run-pass // compile-flags: -Zmir-opt-level=3 -#![feature(platform_intrinsics, repr_simd)] +#![feature(platform_intrinsics, repr_simd, rustc_attrs)] extern "platform-intrinsic" { + #[rustc_args_required_const(2)] fn simd_shuffle2<T, U>(x: T, y: T, idx: [u32; 2]) -> U; } diff --git a/src/test/ui/simd/simd-intrinsic-generic-elements.rs b/src/test/ui/simd/simd-intrinsic-generic-elements.rs index ea3d4b18944..abff59fea78 100644 --- a/src/test/ui/simd/simd-intrinsic-generic-elements.rs +++ b/src/test/ui/simd/simd-intrinsic-generic-elements.rs @@ -1,7 +1,7 @@ // run-pass // ignore-emscripten FIXME(#45351) hits an LLVM assert -#![feature(repr_simd, platform_intrinsics)] +#![feature(repr_simd, platform_intrinsics, rustc_attrs)] #[repr(simd)] #[derive(Copy, Clone, Debug, PartialEq)] @@ -25,9 +25,13 @@ extern "platform-intrinsic" { fn simd_insert<T, E>(x: T, idx: u32, y: E) -> T; fn simd_extract<T, E>(x: T, idx: u32) -> E; + #[rustc_args_required_const(2)] fn simd_shuffle2<T, U>(x: T, y: T, idx: [u32; 2]) -> U; + #[rustc_args_required_const(2)] fn simd_shuffle3<T, U>(x: T, y: T, idx: [u32; 3]) -> U; + #[rustc_args_required_const(2)] fn simd_shuffle4<T, U>(x: T, y: T, idx: [u32; 4]) -> U; + #[rustc_args_required_const(2)] fn simd_shuffle8<T, U>(x: T, y: T, idx: [u32; 8]) -> U; } diff --git a/src/tools/cargo b/src/tools/cargo -Subproject 3c53211c3d7fee4f430f170115af5baad17a3da +Subproject e02974078a692d7484f510eaec0e88d1b6cc020 diff --git a/src/tools/clippy b/src/tools/clippy -Subproject b91ae16eb1ab03b996de5ffc44db054244c1d2f +Subproject 2855b2143972df7102333193aa3c83ddce227e3 diff --git a/src/tools/linkchecker/linkcheck.sh b/src/tools/linkchecker/linkcheck.sh new file mode 100755 index 00000000000..bbccc17e494 --- /dev/null +++ b/src/tools/linkchecker/linkcheck.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# +# This is a script that can be used in each book's CI to validate links using +# the same tool as rust-lang/rust. +# +# This requires the rust-docs rustup component to be installed in the nightly +# toolchain. +# +# Usage: +# ./linkcheck.sh <name-of-book> +# +# Options: +# +# -i "Iterative" mode. The script will not clean up after it is done so +# you can inspect the result, and re-run more quickly. +# +# --all Check all books. This can help make sure you don't break links +# from other books into your book. + +set -e + +if [ ! -f book.toml ] && [ ! -f src/SUMMARY.md ] +then + echo "Run command in root directory of the book." + exit 1 +fi + +html_dir="$(rustc +nightly --print sysroot)/share/doc/rust/html" + +if [ ! -d "$html_dir" ] +then + echo "HTML docs are missing from sysroot: $html_dir" + echo "Make sure the nightly rust-docs rustup component is installed." + exit 1 +fi + +book_name="" +# Iterative will avoid cleaning up, so you can quickly run it repeatedly. +iterative=0 +# If "1", test all books, else only this book. +all_books=0 + +while [ "$1" != "" ] +do + case "$1" in + -i) + iterative=1 + ;; + --all) + all_books=1 + ;; + *) + if [ -n "$book_name" ] + then + echo "only one argument allowed" + exit 1 + fi + book_name="$1" + ;; + esac + shift +done + +if [ -z "$book_name" ] +then + echo "usage: $0 <name-of-book>" + exit 1 +fi + +if [ ! -d "$html_dir/$book_name" ] +then + echo "book name \"$book_name\" not found in sysroot \"$html_dir\"" + exit 1 +fi + +if [ "$iterative" = "0" ] +then + echo "Cleaning old directories..." + rm -rf linkcheck linkchecker +fi + +if [ ! -e "linkchecker/main.rs" ] || [ "$iterative" = "0" ] +then + echo "Downloading linkchecker source..." + mkdir linkchecker + curl -o linkchecker/Cargo.toml \ + https://raw.githubusercontent.com/rust-lang/rust/master/src/tools/linkchecker/Cargo.toml + curl -o linkchecker/main.rs \ + https://raw.githubusercontent.com/rust-lang/rust/master/src/tools/linkchecker/main.rs +fi + +echo "Building book \"$book_name\"..." +mdbook build + +cp -R "$html_dir" linkcheck +rm -rf "linkcheck/$book_name" +cp -R book "linkcheck/$book_name" + +if [ "$all_books" = "1" ] +then + check_path="linkcheck" +else + check_path="linkcheck/$book_name" +fi +echo "Running linkchecker on \"$check_path\"..." +cargo run --manifest-path=linkchecker/Cargo.toml -- "$check_path" + +if [ "$iterative" = "0" ] +then + rm -rf linkcheck linkchecker +fi + +echo "Link check completed successfully!" |