diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 50 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder/autodiff.rs | 24 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 39 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 299 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs | 42 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/unused.rs | 170 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 17 |
11 files changed, 353 insertions, 346 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 71059338151..8294e29d07d 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -17,14 +17,13 @@ use rustc_target::callconv::{ use rustc_target::spec::SanitizerSet; use smallvec::SmallVec; -use crate::attributes::llfn_attrs_from_instance; +use crate::attributes::{self, llfn_attrs_from_instance}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use crate::{attributes, llvm_util}; trait ArgAttributesExt { fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); @@ -437,7 +436,6 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| { if cx.sess().opts.optimize != config::OptLevel::No - && llvm_util::get_version() >= (19, 0, 0) && matches!(scalar.primitive(), Primitive::Int(..)) // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected @@ -571,19 +569,6 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } _ => {} } - if bx.cx.sess().opts.optimize != config::OptLevel::No - && llvm_util::get_version() < (19, 0, 0) - && let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr - && matches!(scalar.primitive(), Primitive::Int(..)) - // If the value is a boolean, the range is 0..2 and that ultimately - // become 0..0 when the type becomes i1, which would be rejected - // by the LLVM verifier. - && !scalar.is_bool() - // LLVM also rejects full range. - && !scalar.is_always_valid(bx) - { - bx.range_metadata(callsite, scalar.valid_range(bx)); - } for arg in self.args.iter() { match &arg.mode { PassMode::Ignore => {} diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 3d7afa17bdf..e8c42d16733 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -407,30 +407,28 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // Do not set sanitizer attributes for naked functions. to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); - if llvm_util::get_version() >= (19, 0, 0) { - // For non-naked functions, set branch protection attributes on aarch64. - if let Some(BranchProtection { bti, pac_ret }) = - cx.sess().opts.unstable_opts.branch_protection - { - assert!(cx.sess().target.arch == "aarch64"); - if bti { - to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); - } - if let Some(PacRet { leaf, pc, key }) = pac_ret { - if pc { - to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); - } - to_add.push(llvm::CreateAttrStringValue( - cx.llcx, - "sign-return-address", - if leaf { "all" } else { "non-leaf" }, - )); - to_add.push(llvm::CreateAttrStringValue( - cx.llcx, - "sign-return-address-key", - if key == PAuthKey::A { "a_key" } else { "b_key" }, - )); + // For non-naked functions, set branch protection attributes on aarch64. + if let Some(BranchProtection { bti, pac_ret }) = + cx.sess().opts.unstable_opts.branch_protection + { + assert!(cx.sess().target.arch == "aarch64"); + if bti { + to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); + } + if let Some(PacRet { leaf, pc, key }) = pac_ret { + if pc { + to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); } + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "sign-return-address", + if leaf { "all" } else { "non-leaf" }, + )); + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "sign-return-address-key", + if key == PAuthKey::A { "a_key" } else { "b_key" }, + )); } } } @@ -510,12 +508,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), })) - // HACK: LLVM versions 19+ do not have the FPMR feature and treat it as always enabled - // It only exists as a feature in LLVM 18, cannot be passed down for any other version - .chain(match &*cx.tcx.sess.target.arch { - "aarch64" if llvm_util::get_version().0 == 18 => vec!["+fpmr".to_string()], - _ => vec![], - }) .collect::<Vec<String>>(); if cx.tcx.sess.target.is_like_wasm { diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index bf6138142b6..76d431a4975 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -439,12 +439,9 @@ fn report_inline_asm( let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) { SpanData::default() } else { - let lo = BytePos::from_u32(cookie as u32); - let hi = BytePos::from_u32((cookie >> 32) as u32); SpanData { - lo, - // LLVM version < 19 silently truncates the cookie to 32 bits in some situations. - hi: if hi.to_u32() != 0 { hi } else { lo }, + lo: BytePos::from_u32(cookie as u32), + hi: BytePos::from_u32((cookie >> 32) as u32), ctxt: SyntaxContext::root(), parent: None, } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 297f104d124..35134e9f5a0 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -30,6 +30,7 @@ use smallvec::SmallVec; use tracing::{debug, instrument}; use crate::abi::FnAbiLlvmExt; +use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, FullCx, GenericCx, SCx}; use crate::llvm::{ @@ -38,7 +39,6 @@ use crate::llvm::{ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use crate::{attributes, llvm_util}; #[must_use] pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> { @@ -927,11 +927,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug_assert_ne!(self.val_ty(val), dest_ty); let trunc = self.trunc(val, dest_ty); - if llvm_util::get_version() >= (19, 0, 0) { - unsafe { - if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNUW(trunc, True); - } + unsafe { + if llvm::LLVMIsAInstruction(trunc).is_some() { + llvm::LLVMSetNUW(trunc, True); } } trunc @@ -941,11 +939,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug_assert_ne!(self.val_ty(val), dest_ty); let trunc = self.trunc(val, dest_ty); - if llvm_util::get_version() >= (19, 0, 0) { - unsafe { - if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNSW(trunc, True); - } + unsafe { + if llvm::LLVMIsAInstruction(trunc).is_some() { + llvm::LLVMSetNSW(trunc, True); } } trunc @@ -1899,10 +1895,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { hash: &'ll Value, bitmap_bits: &'ll Value, ) { - assert!( - crate::llvm_util::get_version() >= (19, 0, 0), - "MCDC intrinsics require LLVM 19 or later" - ); self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]); } @@ -1914,10 +1906,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { bitmap_index: &'ll Value, mcdc_temp: &'ll Value, ) { - assert!( - crate::llvm_util::get_version() >= (19, 0, 0), - "MCDC intrinsics require LLVM 19 or later" - ); let args = &[fn_name, hash, bitmap_index, mcdc_temp]; self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args); } @@ -1929,10 +1917,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { - assert!( - crate::llvm_util::get_version() >= (19, 0, 0), - "MCDC intrinsics require LLVM 19 or later" - ); let align = self.tcx.data_layout.i32_align.abi; let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align); let new_tv_index = self.add(current_tv_index, cond_index); diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 7d264ba4d00..5e7ef27143b 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -201,7 +201,23 @@ fn compute_enzyme_fn_ty<'ll>( } if attrs.width == 1 { - todo!("Handle sret for scalar ad"); + // Enzyme returns a struct of style: + // `{ original_ret(if requested), float, float, ... }` + let mut struct_elements = vec![]; + if attrs.has_primal_ret() { + struct_elements.push(inner_ret_ty); + } + // Next, we push the list of active floats, since they will be lowered to `enzyme_out`, + // and therefore part of the return struct. + let param_tys = cx.func_params_types(fn_ty); + for (act, param_ty) in attrs.input_activity.iter().zip(param_tys) { + if matches!(act, DiffActivity::Active) { + // Now find the float type at position i based on the fn_ty, + // to know what (f16/f32/f64/...) to add to the struct. + struct_elements.push(param_ty); + } + } + ret_ty = cx.type_struct(&struct_elements, false); } else { // First we check if we also have to deal with the primal return. match attrs.mode { @@ -388,7 +404,11 @@ fn generate_enzyme_call<'ll>( // now store the result of the enzyme call into the sret pointer. let sret_ptr = outer_args[0]; let call_ty = cx.val_ty(call); - assert_eq!(cx.type_kind(call_ty), TypeKind::Array); + if attrs.width == 1 { + assert_eq!(cx.type_kind(call_ty), TypeKind::Struct); + } else { + assert_eq!(cx.type_kind(call_ty), TypeKind::Array); + } llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr); } builder.ret_void(); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 3be8cd5f6ac..4ec69995518 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -164,23 +164,6 @@ pub(crate) unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (19, 0, 0) { - if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { - // LLVM 19 sets -Fn32 in its data layout string for 64-bit ARM - // Earlier LLVMs leave this default, so remove it. - // See https://github.com/llvm/llvm-project/pull/90702 - target_data_layout = target_data_layout.replace("-Fn32", ""); - } - } - - if llvm_version < (19, 0, 0) { - if sess.target.arch == "loongarch64" { - // LLVM 19 updates the LoongArch64 data layout. - // See https://github.com/llvm/llvm-project/pull/93814 - target_data_layout = target_data_layout.replace("-n32:64", "-n64"); - } - } - if llvm_version < (20, 0, 0) { if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { // LLVM 20 defines three additional address spaces for alternate @@ -328,6 +311,22 @@ pub(crate) unsafe fn create_module<'ll>( pfe.prefix().into(), ); } + + // Add "kcfi-arity" module flag if KCFI arity indicator is enabled. (See + // https://github.com/llvm/llvm-project/pull/117121.) + if sess.is_sanitizer_kcfi_arity_enabled() { + // KCFI arity indicator requires LLVM 21.0.0 or later. + if llvm_version < (21, 0, 0) { + tcx.dcx().emit_err(crate::errors::SanitizerKcfiArityRequiresLLVM2100); + } + + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "kcfi-arity", + 1, + ); + } } // Control Flow Guard is currently only supported by MSVC and LLVM on Windows. @@ -1202,10 +1201,8 @@ impl<'ll> CodegenCx<'ll, '_> { if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); - if crate::llvm_util::get_version() >= (19, 0, 0) { - ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); - ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); - } + ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); + ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); } ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 9a2473d6cf2..55b1e728b70 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -5,15 +5,11 @@ use rustc_abi::Align; use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, }; -use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; -use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_data_structures::fx::FxIndexMap; use rustc_index::IndexVec; -use rustc_middle::mir; -use rustc_middle::mir::mono::MonoItemPartitions; -use rustc_middle::ty::{self, TyCtxt}; +use rustc_middle::ty::TyCtxt; use rustc_session::RemapFileNameExt; use rustc_session::config::RemapPathScopeComponents; -use rustc_span::def_id::DefIdSet; use rustc_span::{SourceFile, StableSourceFileId}; use tracing::debug; @@ -24,6 +20,7 @@ use crate::llvm; mod covfun; mod spans; +mod unused; /// Generates and exports the coverage map, which is embedded in special /// linker sections in the final binary. @@ -56,13 +53,6 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { None => return, }; - // The order of entries in this global file table needs to be deterministic, - // and ideally should also be independent of the details of stable-hashing, - // because coverage tests snapshots (`.cov-map`) can observe the order and - // would need to be re-blessed if it changes. As long as those requirements - // are satisfied, the order can be arbitrary. - let mut global_file_table = GlobalFileTable::new(); - let mut covfun_records = instances_used .iter() .copied() @@ -70,18 +60,13 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { // order that doesn't depend on the stable-hash-based order in which // instances were visited during codegen. .sorted_by_cached_key(|&instance| tcx.symbol_name(instance).name) - .filter_map(|instance| prepare_covfun_record(tcx, &mut global_file_table, instance, true)) + .filter_map(|instance| prepare_covfun_record(tcx, instance, true)) .collect::<Vec<_>>(); // In a single designated CGU, also prepare covfun records for functions // in this crate that were instrumented for coverage, but are unused. if cx.codegen_unit.is_code_coverage_dead_code_cgu() { - let mut unused_instances = gather_unused_function_instances(cx); - // Sort the unused instances by symbol name, for the same reason as the used ones. - unused_instances.sort_by_cached_key(|&instance| tcx.symbol_name(instance).name); - covfun_records.extend(unused_instances.into_iter().filter_map(|instance| { - prepare_covfun_record(tcx, &mut global_file_table, instance, false) - })); + unused::prepare_covfun_records_for_unused_functions(cx, &mut covfun_records); } // If there are no covfun records for this CGU, don't generate a covmap record. @@ -93,91 +78,88 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { return; } - // Encode all filenames referenced by coverage mappings in this CGU. - let filenames_buffer = global_file_table.make_filenames_buffer(tcx); - // The `llvm-cov` tool uses this hash to associate each covfun record with - // its corresponding filenames table, since the final binary will typically - // contain multiple covmap records from different compilation units. - let filenames_hash = llvm_cov::hash_bytes(&filenames_buffer); - - let mut unused_function_names = vec![]; + // Prepare the global file table for this CGU, containing all paths needed + // by one or more covfun records. + let global_file_table = + GlobalFileTable::build(tcx, covfun_records.iter().flat_map(|c| c.all_source_files())); for covfun in &covfun_records { - unused_function_names.extend(covfun.mangled_function_name_if_unused()); - - covfun::generate_covfun_record(cx, filenames_hash, covfun) - } - - // For unused functions, we need to take their mangled names and store them - // in a specially-named global array. LLVM's `InstrProfiling` pass will - // detect this global and include those names in its `__llvm_prf_names` - // section. (See `llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp`.) - if !unused_function_names.is_empty() { - assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); - - let name_globals = unused_function_names - .into_iter() - .map(|mangled_function_name| cx.const_str(mangled_function_name).0) - .collect::<Vec<_>>(); - let initializer = cx.const_array(cx.type_ptr(), &name_globals); - - let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), c"__llvm_coverage_names"); - llvm::set_global_constant(array, true); - llvm::set_linkage(array, llvm::Linkage::InternalLinkage); - llvm::set_initializer(array, initializer); + covfun::generate_covfun_record(cx, &global_file_table, covfun) } // Generate the coverage map header, which contains the filenames used by // this CGU's coverage mappings, and store it in a well-known global. // (This is skipped if we returned early due to having no covfun records.) - generate_covmap_record(cx, covmap_version, &filenames_buffer); + generate_covmap_record(cx, covmap_version, &global_file_table.filenames_buffer); } -/// Maps "global" (per-CGU) file ID numbers to their underlying source files. +/// Maps "global" (per-CGU) file ID numbers to their underlying source file paths. +#[derive(Debug)] struct GlobalFileTable { /// This "raw" table doesn't include the working dir, so a file's /// global ID is its index in this set **plus one**. - raw_file_table: FxIndexMap<StableSourceFileId, Arc<SourceFile>>, + raw_file_table: FxIndexMap<StableSourceFileId, String>, + + /// The file table in encoded form (possibly compressed), which can be + /// included directly in this CGU's `__llvm_covmap` record. + filenames_buffer: Vec<u8>, + + /// Truncated hash of the bytes in `filenames_buffer`. + /// + /// The `llvm-cov` tool uses this hash to associate each covfun record with + /// its corresponding filenames table, since the final binary will typically + /// contain multiple covmap records from different compilation units. + filenames_hash: u64, } impl GlobalFileTable { - fn new() -> Self { - Self { raw_file_table: FxIndexMap::default() } - } + /// Builds a "global file table" for this CGU, mapping numeric IDs to + /// path strings. + fn build<'a>(tcx: TyCtxt<'_>, all_files: impl Iterator<Item = &'a SourceFile>) -> Self { + let mut raw_file_table = FxIndexMap::default(); + + for file in all_files { + raw_file_table.entry(file.stable_id).or_insert_with(|| { + file.name + .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) + .to_string_lossy() + .into_owned() + }); + } - fn global_file_id_for_file(&mut self, file: &Arc<SourceFile>) -> GlobalFileId { - // Ensure the given file has a table entry, and get its index. - let entry = self.raw_file_table.entry(file.stable_id); - let raw_id = entry.index(); - entry.or_insert_with(|| Arc::clone(file)); + // FIXME(Zalathar): Consider sorting the file table here, but maybe + // only after adding filename support to coverage-dump, so that the + // table order isn't directly visible in `.coverage-map` snapshots. - // The raw file table doesn't include an entry for the working dir - // (which has ID 0), so add 1 to get the correct ID. - GlobalFileId::from_usize(raw_id + 1) - } + let mut table = Vec::with_capacity(raw_file_table.len() + 1); - fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> { - let mut table = Vec::with_capacity(self.raw_file_table.len() + 1); - - // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) - // requires setting the first filename to the compilation directory. - // Since rustc generates coverage maps with relative paths, the - // compilation directory can be combined with the relative paths - // to get absolute paths, if needed. - table.push( - tcx.sess - .opts - .working_dir - .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) - .to_string_lossy(), - ); + // Since version 6 of the LLVM coverage mapping format, the first entry + // in the global file table is treated as a base directory, used to + // resolve any other entries that are stored as relative paths. + let base_dir = tcx + .sess + .opts + .working_dir + .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) + .to_string_lossy(); + table.push(base_dir.as_ref()); // Add the regular entries after the base directory. - table.extend(self.raw_file_table.values().map(|file| { - file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy() - })); + table.extend(raw_file_table.values().map(|name| name.as_str())); - llvm_cov::write_filenames_to_buffer(&table) + // Encode the file table into a buffer, and get the hash of its encoded + // bytes, so that we can embed that hash in `__llvm_covfun` records. + let filenames_buffer = llvm_cov::write_filenames_to_buffer(&table); + let filenames_hash = llvm_cov::hash_bytes(&filenames_buffer); + + Self { raw_file_table, filenames_buffer, filenames_hash } + } + + fn get_existing_id(&self, file: &SourceFile) -> Option<GlobalFileId> { + let raw_id = self.raw_file_table.get_index_of(&file.stable_id)?; + // The raw file table doesn't include an entry for the base dir + // (which has ID 0), so add 1 to get the correct ID. + Some(GlobalFileId::from_usize(raw_id + 1)) } } @@ -193,26 +175,31 @@ rustc_index::newtype_index! { struct LocalFileId {} } -/// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) -/// file IDs. +/// Holds a mapping from "local" (per-function) file IDs to their corresponding +/// source files. #[derive(Debug, Default)] struct VirtualFileMapping { - local_to_global: IndexVec<LocalFileId, GlobalFileId>, - global_to_local: FxIndexMap<GlobalFileId, LocalFileId>, + local_file_table: IndexVec<LocalFileId, Arc<SourceFile>>, } impl VirtualFileMapping { - fn local_id_for_global(&mut self, global_file_id: GlobalFileId) -> LocalFileId { - *self - .global_to_local - .entry(global_file_id) - .or_insert_with(|| self.local_to_global.push(global_file_id)) + fn push_file(&mut self, source_file: &Arc<SourceFile>) -> LocalFileId { + self.local_file_table.push(Arc::clone(source_file)) } - fn to_vec(&self) -> Vec<u32> { - // This clone could be avoided by transmuting `&[GlobalFileId]` to `&[u32]`, - // but it isn't hot or expensive enough to justify the extra unsafety. - self.local_to_global.iter().map(|&global| GlobalFileId::as_u32(global)).collect() + /// Resolves all of the filenames in this local file mapping to a list of + /// global file IDs in its CGU, for inclusion in this function's + /// `__llvm_covfun` record. + /// + /// The global file IDs are returned as `u32` to make FFI easier. + fn resolve_all(&self, global_file_table: &GlobalFileTable) -> Option<Vec<u32>> { + self.local_file_table + .iter() + .map(|file| try { + let id = global_file_table.get_existing_id(file)?; + GlobalFileId::as_u32(id) + }) + .collect::<Option<Vec<_>>>() } } @@ -249,121 +236,3 @@ fn generate_covmap_record<'ll>(cx: &CodegenCx<'ll, '_>, version: u32, filenames_ cx.add_used_global(covmap_global); } - -/// Each CGU will normally only emit coverage metadata for the functions that it actually generates. -/// But since we don't want unused functions to disappear from coverage reports, we also scan for -/// functions that were instrumented but are not participating in codegen. -/// -/// These unused functions don't need to be codegenned, but we do need to add them to the function -/// coverage map (in a single designated CGU) so that we still emit coverage mappings for them. -/// We also end up adding their symbol names to a special global array that LLVM will include in -/// its embedded coverage data. -fn gather_unused_function_instances<'tcx>(cx: &CodegenCx<'_, 'tcx>) -> Vec<ty::Instance<'tcx>> { - assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); - - let tcx = cx.tcx; - let usage = prepare_usage_sets(tcx); - - let is_unused_fn = |def_id: LocalDefId| -> bool { - // Usage sets expect `DefId`, so convert from `LocalDefId`. - let d: DefId = LocalDefId::to_def_id(def_id); - // To be potentially eligible for "unused function" mappings, a definition must: - // - Be eligible for coverage instrumentation - // - Not participate directly in codegen (or have lost all its coverage statements) - // - Not have any coverage statements inlined into codegenned functions - tcx.is_eligible_for_coverage(def_id) - && (!usage.all_mono_items.contains(&d) || usage.missing_own_coverage.contains(&d)) - && !usage.used_via_inlining.contains(&d) - }; - - // FIXME(#79651): Consider trying to filter out dummy instantiations of - // unused generic functions from library crates, because they can produce - // "unused instantiation" in coverage reports even when they are actually - // used by some downstream crate in the same binary. - - tcx.mir_keys(()) - .iter() - .copied() - .filter(|&def_id| is_unused_fn(def_id)) - .map(|def_id| make_dummy_instance(tcx, def_id)) - .collect::<Vec<_>>() -} - -struct UsageSets<'tcx> { - all_mono_items: &'tcx DefIdSet, - used_via_inlining: FxHashSet<DefId>, - missing_own_coverage: FxHashSet<DefId>, -} - -/// Prepare sets of definitions that are relevant to deciding whether something -/// is an "unused function" for coverage purposes. -fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> { - let MonoItemPartitions { all_mono_items, codegen_units, .. } = - tcx.collect_and_partition_mono_items(()); - - // Obtain a MIR body for each function participating in codegen, via an - // arbitrary instance. - let mut def_ids_seen = FxHashSet::default(); - let def_and_mir_for_all_mono_fns = codegen_units - .iter() - .flat_map(|cgu| cgu.items().keys()) - .filter_map(|item| match item { - mir::mono::MonoItem::Fn(instance) => Some(instance), - mir::mono::MonoItem::Static(_) | mir::mono::MonoItem::GlobalAsm(_) => None, - }) - // We only need one arbitrary instance per definition. - .filter(move |instance| def_ids_seen.insert(instance.def_id())) - .map(|instance| { - // We don't care about the instance, just its underlying MIR. - let body = tcx.instance_mir(instance.def); - (instance.def_id(), body) - }); - - // Functions whose coverage statements were found inlined into other functions. - let mut used_via_inlining = FxHashSet::default(); - // Functions that were instrumented, but had all of their coverage statements - // removed by later MIR transforms (e.g. UnreachablePropagation). - let mut missing_own_coverage = FxHashSet::default(); - - for (def_id, body) in def_and_mir_for_all_mono_fns { - let mut saw_own_coverage = false; - - // Inspect every coverage statement in the function's MIR. - for stmt in body - .basic_blocks - .iter() - .flat_map(|block| &block.statements) - .filter(|stmt| matches!(stmt.kind, mir::StatementKind::Coverage(_))) - { - if let Some(inlined) = stmt.source_info.scope.inlined_instance(&body.source_scopes) { - // This coverage statement was inlined from another function. - used_via_inlining.insert(inlined.def_id()); - } else { - // Non-inlined coverage statements belong to the enclosing function. - saw_own_coverage = true; - } - } - - if !saw_own_coverage && body.function_coverage_info.is_some() { - missing_own_coverage.insert(def_id); - } - } - - UsageSets { all_mono_items, used_via_inlining, missing_own_coverage } -} - -fn make_dummy_instance<'tcx>(tcx: TyCtxt<'tcx>, local_def_id: LocalDefId) -> ty::Instance<'tcx> { - let def_id = local_def_id.to_def_id(); - - // Make a dummy instance that fills in all generics with placeholders. - ty::Instance::new( - def_id, - ty::GenericArgs::for_item(tcx, def_id, |param, _| { - if let ty::GenericParamDefKind::Lifetime = param.kind { - tcx.lifetimes.re_erased.into() - } else { - tcx.mk_param_from_def(param) - } - }), - ) -} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index 048e1988c32..7bdbc685952 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -5,6 +5,7 @@ //! [^win]: On Windows the section name is `.lcovfun`. use std::ffi::CString; +use std::sync::Arc; use rustc_abi::Align; use rustc_codegen_ssa::traits::{ @@ -15,7 +16,7 @@ use rustc_middle::mir::coverage::{ MappingKind, Op, }; use rustc_middle::ty::{Instance, TyCtxt}; -use rustc_span::Span; +use rustc_span::{SourceFile, Span}; use rustc_target::spec::HasTargetSpec; use tracing::debug; @@ -38,16 +39,15 @@ pub(crate) struct CovfunRecord<'tcx> { } impl<'tcx> CovfunRecord<'tcx> { - /// FIXME(Zalathar): Make this the responsibility of the code that determines - /// which functions are unused. - pub(crate) fn mangled_function_name_if_unused(&self) -> Option<&'tcx str> { - (!self.is_used).then_some(self.mangled_function_name) + /// Iterator that yields all source files referred to by this function's + /// coverage mappings. Used to build the global file table for the CGU. + pub(crate) fn all_source_files(&self) -> impl Iterator<Item = &SourceFile> { + self.virtual_file_mapping.local_file_table.iter().map(Arc::as_ref) } } pub(crate) fn prepare_covfun_record<'tcx>( tcx: TyCtxt<'tcx>, - global_file_table: &mut GlobalFileTable, instance: Instance<'tcx>, is_used: bool, ) -> Option<CovfunRecord<'tcx>> { @@ -65,7 +65,7 @@ pub(crate) fn prepare_covfun_record<'tcx>( regions: ffi::Regions::default(), }; - fill_region_tables(tcx, global_file_table, fn_cov_info, ids_info, &mut covfun); + fill_region_tables(tcx, fn_cov_info, ids_info, &mut covfun); if covfun.regions.has_no_regions() { debug!(?covfun, "function has no mappings to embed; skipping"); @@ -100,7 +100,6 @@ fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec<ffi::CounterExpression /// Populates the mapping region tables in the current function's covfun record. fn fill_region_tables<'tcx>( tcx: TyCtxt<'tcx>, - global_file_table: &mut GlobalFileTable, fn_cov_info: &'tcx FunctionCoverageInfo, ids_info: &'tcx CoverageIdsInfo, covfun: &mut CovfunRecord<'tcx>, @@ -114,11 +113,7 @@ fn fill_region_tables<'tcx>( }; let source_file = source_map.lookup_source_file(first_span.lo()); - // Look up the global file ID for that file. - let global_file_id = global_file_table.global_file_id_for_file(&source_file); - - // Associate that global file ID with a local file ID for this function. - let local_file_id = covfun.virtual_file_mapping.local_id_for_global(global_file_id); + let local_file_id = covfun.virtual_file_mapping.push_file(&source_file); // In rare cases, _all_ of a function's spans are discarded, and coverage // codegen needs to handle that gracefully to avoid #133606. @@ -187,7 +182,7 @@ fn fill_region_tables<'tcx>( /// as a global variable in the `__llvm_covfun` section. pub(crate) fn generate_covfun_record<'tcx>( cx: &CodegenCx<'_, 'tcx>, - filenames_hash: u64, + global_file_table: &GlobalFileTable, covfun: &CovfunRecord<'tcx>, ) { let &CovfunRecord { @@ -199,12 +194,19 @@ pub(crate) fn generate_covfun_record<'tcx>( ref regions, } = covfun; + let Some(local_file_table) = virtual_file_mapping.resolve_all(global_file_table) else { + debug_assert!( + false, + "all local files should be present in the global file table: \ + global_file_table = {global_file_table:?}, \ + virtual_file_mapping = {virtual_file_mapping:?}" + ); + return; + }; + // Encode the function's coverage mappings into a buffer. - let coverage_mapping_buffer = llvm_cov::write_function_mappings_to_buffer( - &virtual_file_mapping.to_vec(), - expressions, - regions, - ); + let coverage_mapping_buffer = + llvm_cov::write_function_mappings_to_buffer(&local_file_table, expressions, regions); // A covfun record consists of four target-endian integers, followed by the // encoded mapping data in bytes. Note that the length field is 32 bits. @@ -217,7 +219,7 @@ pub(crate) fn generate_covfun_record<'tcx>( cx.const_u64(func_name_hash), cx.const_u32(coverage_mapping_buffer.len() as u32), cx.const_u64(source_hash), - cx.const_u64(filenames_hash), + cx.const_u64(global_file_table.filenames_hash), cx.const_bytes(&coverage_mapping_buffer), ], // This struct needs to be packed, so that the 32-bit length field diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/unused.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/unused.rs new file mode 100644 index 00000000000..68f60f169b5 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/unused.rs @@ -0,0 +1,170 @@ +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; +use rustc_data_structures::fx::FxHashSet; +use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_middle::mir; +use rustc_middle::mir::mono::MonoItemPartitions; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_span::def_id::DefIdSet; + +use crate::common::CodegenCx; +use crate::coverageinfo::mapgen::covfun::{CovfunRecord, prepare_covfun_record}; +use crate::llvm; + +/// Each CGU will normally only emit coverage metadata for the functions that it actually generates. +/// But since we don't want unused functions to disappear from coverage reports, we also scan for +/// functions that were instrumented but are not participating in codegen. +/// +/// These unused functions don't need to be codegenned, but we do need to add them to the function +/// coverage map (in a single designated CGU) so that we still emit coverage mappings for them. +/// We also end up adding their symbol names to a special global array that LLVM will include in +/// its embedded coverage data. +pub(crate) fn prepare_covfun_records_for_unused_functions<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + covfun_records: &mut Vec<CovfunRecord<'tcx>>, +) { + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); + + let mut unused_instances = gather_unused_function_instances(cx); + // Sort the unused instances by symbol name, so that their order isn't hash-sensitive. + unused_instances.sort_by_key(|instance| instance.symbol_name); + + // Try to create a covfun record for each unused function. + let mut name_globals = Vec::with_capacity(unused_instances.len()); + covfun_records.extend(unused_instances.into_iter().filter_map(|unused| try { + let record = prepare_covfun_record(cx.tcx, unused.instance, false)?; + // If successful, also store its symbol name in a global constant. + name_globals.push(cx.const_str(unused.symbol_name.name).0); + record + })); + + // Store the names of unused functions in a specially-named global array. + // LLVM's `InstrProfilling` pass will detect this array, and include the + // referenced names in its `__llvm_prf_names` section. + // (See `llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp`.) + if !name_globals.is_empty() { + let initializer = cx.const_array(cx.type_ptr(), &name_globals); + + let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), c"__llvm_coverage_names"); + llvm::set_global_constant(array, true); + llvm::set_linkage(array, llvm::Linkage::InternalLinkage); + llvm::set_initializer(array, initializer); + } +} + +/// Holds a dummy function instance along with its symbol name, to avoid having +/// to repeatedly query for the name. +struct UnusedInstance<'tcx> { + instance: ty::Instance<'tcx>, + symbol_name: ty::SymbolName<'tcx>, +} + +fn gather_unused_function_instances<'tcx>(cx: &CodegenCx<'_, 'tcx>) -> Vec<UnusedInstance<'tcx>> { + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); + + let tcx = cx.tcx; + let usage = prepare_usage_sets(tcx); + + let is_unused_fn = |def_id: LocalDefId| -> bool { + // Usage sets expect `DefId`, so convert from `LocalDefId`. + let d: DefId = LocalDefId::to_def_id(def_id); + // To be potentially eligible for "unused function" mappings, a definition must: + // - Be eligible for coverage instrumentation + // - Not participate directly in codegen (or have lost all its coverage statements) + // - Not have any coverage statements inlined into codegenned functions + tcx.is_eligible_for_coverage(def_id) + && (!usage.all_mono_items.contains(&d) || usage.missing_own_coverage.contains(&d)) + && !usage.used_via_inlining.contains(&d) + }; + + // FIXME(#79651): Consider trying to filter out dummy instantiations of + // unused generic functions from library crates, because they can produce + // "unused instantiation" in coverage reports even when they are actually + // used by some downstream crate in the same binary. + + tcx.mir_keys(()) + .iter() + .copied() + .filter(|&def_id| is_unused_fn(def_id)) + .map(|def_id| make_dummy_instance(tcx, def_id)) + .map(|instance| UnusedInstance { instance, symbol_name: tcx.symbol_name(instance) }) + .collect::<Vec<_>>() +} + +struct UsageSets<'tcx> { + all_mono_items: &'tcx DefIdSet, + used_via_inlining: FxHashSet<DefId>, + missing_own_coverage: FxHashSet<DefId>, +} + +/// Prepare sets of definitions that are relevant to deciding whether something +/// is an "unused function" for coverage purposes. +fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> { + let MonoItemPartitions { all_mono_items, codegen_units, .. } = + tcx.collect_and_partition_mono_items(()); + + // Obtain a MIR body for each function participating in codegen, via an + // arbitrary instance. + let mut def_ids_seen = FxHashSet::default(); + let def_and_mir_for_all_mono_fns = codegen_units + .iter() + .flat_map(|cgu| cgu.items().keys()) + .filter_map(|item| match item { + mir::mono::MonoItem::Fn(instance) => Some(instance), + mir::mono::MonoItem::Static(_) | mir::mono::MonoItem::GlobalAsm(_) => None, + }) + // We only need one arbitrary instance per definition. + .filter(move |instance| def_ids_seen.insert(instance.def_id())) + .map(|instance| { + // We don't care about the instance, just its underlying MIR. + let body = tcx.instance_mir(instance.def); + (instance.def_id(), body) + }); + + // Functions whose coverage statements were found inlined into other functions. + let mut used_via_inlining = FxHashSet::default(); + // Functions that were instrumented, but had all of their coverage statements + // removed by later MIR transforms (e.g. UnreachablePropagation). + let mut missing_own_coverage = FxHashSet::default(); + + for (def_id, body) in def_and_mir_for_all_mono_fns { + let mut saw_own_coverage = false; + + // Inspect every coverage statement in the function's MIR. + for stmt in body + .basic_blocks + .iter() + .flat_map(|block| &block.statements) + .filter(|stmt| matches!(stmt.kind, mir::StatementKind::Coverage(_))) + { + if let Some(inlined) = stmt.source_info.scope.inlined_instance(&body.source_scopes) { + // This coverage statement was inlined from another function. + used_via_inlining.insert(inlined.def_id()); + } else { + // Non-inlined coverage statements belong to the enclosing function. + saw_own_coverage = true; + } + } + + if !saw_own_coverage && body.function_coverage_info.is_some() { + missing_own_coverage.insert(def_id); + } + } + + UsageSets { all_mono_items, used_via_inlining, missing_own_coverage } +} + +fn make_dummy_instance<'tcx>(tcx: TyCtxt<'tcx>, local_def_id: LocalDefId) -> ty::Instance<'tcx> { + let def_id = local_def_id.to_def_id(); + + // Make a dummy instance that fills in all generics with placeholders. + ty::Instance::new( + def_id, + ty::GenericArgs::for_item(tcx, def_id, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), + ) +} diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 4c5a78ca74f..ecf108f988f 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -217,3 +217,7 @@ pub(crate) struct MismatchedDataLayout<'a> { pub(crate) struct FixedX18InvalidArch<'a> { pub arch: &'a str, } + +#[derive(Diagnostic)] +#[diag(codegen_llvm_sanitizer_kcfi_arity_requires_llvm_21_0_0)] +pub(crate) struct SanitizerKcfiArityRequiresLLVM2100; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 4e85286ed55..36e35f81392 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -256,7 +256,6 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - ("aarch64", "pauth-lr") if get_version().0 < 19 => None, // Before LLVM 20 those two features were packaged together as b16b16 ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), @@ -270,20 +269,9 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")), ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")), // Filter out features that are not supported by the current LLVM version - ("aarch64", "fpmr") if get_version().0 != 18 => None, + ("aarch64", "fpmr") => None, // only existed in 18 ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), - // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single - // feature called `fast-unaligned-access`. In LLVM 19, it was split back out. - ("riscv32" | "riscv64", "unaligned-scalar-mem" | "unaligned-vector-mem") - if get_version().0 == 18 => - { - Some(LLVMFeature::new("fast-unaligned-access")) - } // Filter out features that are not supported by the current LLVM version - ("riscv32" | "riscv64", "zaamo") if get_version().0 < 19 => None, - ("riscv32" | "riscv64", "zabha") if get_version().0 < 19 => None, - ("riscv32" | "riscv64", "zalrsc") if get_version().0 < 19 => None, - ("riscv32" | "riscv64", "zama16b") if get_version().0 < 19 => None, ("riscv32" | "riscv64", "zacas") if get_version().0 < 20 => None, // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => { @@ -295,10 +283,9 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), // In LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available and SPARC-V8+ ABI used". // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L27-L28 - // Before LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available". + // Before LLVM 19, there was no `v8plus` feature and `v9` means "SPARC-V9 instruction available". // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), - ("sparc", "v8plus") if get_version().0 < 19 => None, ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), // These new `amx` variants and `movrs` were introduced in LLVM20 ("x86", "amx-avx512" | "amx-fp8" | "amx-movrs" | "amx-tf32" | "amx-transpose") |