diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder/autodiff.rs | 214 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs | 33 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs | 45 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs | 20 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs | 44 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 184 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 20 |
14 files changed, 377 insertions, 274 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 668795191a2..f083cfbd7d3 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -728,7 +728,7 @@ impl ThinBuffer { } } - pub unsafe fn from_raw_ptr(ptr: *mut llvm::ThinLTOBuffer) -> ThinBuffer { + pub(crate) unsafe fn from_raw_ptr(ptr: *mut llvm::ThinLTOBuffer) -> ThinBuffer { let mut ptr = NonNull::new(ptr).unwrap(); ThinBuffer(unsafe { ptr.as_mut() }) } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 55d34f5f2ef..297f104d124 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -14,6 +14,7 @@ use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, @@ -1074,6 +1075,35 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } } + fn three_way_compare( + &mut self, + ty: Ty<'tcx>, + lhs: Self::Value, + rhs: Self::Value, + ) -> Option<Self::Value> { + // FIXME: See comment on the definition of `three_way_compare`. + if crate::llvm_util::get_version() < (20, 0, 0) { + return None; + } + + let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) { + (true, 8) => "llvm.scmp.i8.i8", + (true, 16) => "llvm.scmp.i8.i16", + (true, 32) => "llvm.scmp.i8.i32", + (true, 64) => "llvm.scmp.i8.i64", + (true, 128) => "llvm.scmp.i8.i128", + + (false, 8) => "llvm.ucmp.i8.i8", + (false, 16) => "llvm.ucmp.i8.i16", + (false, 32) => "llvm.ucmp.i8.i32", + (false, 64) => "llvm.ucmp.i8.i64", + (false, 128) => "llvm.ucmp.i8.i128", + + _ => bug!("three-way compare unsupported for type {ty:?}"), + }; + Some(self.call_intrinsic(name, &[lhs, rhs])) + } + /* Miscellaneous instructions */ fn memcpy( &mut self, diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 71705ecb4d0..7cd4ee539d8 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -28,6 +28,113 @@ fn get_params(fnc: &Value) -> Vec<&Value> { } } +fn match_args_from_caller_to_enzyme<'ll>( + cx: &SimpleCx<'ll>, + args: &mut Vec<&'ll llvm::Value>, + inputs: &[DiffActivity], + outer_args: &[&'ll llvm::Value], +) { + debug!("matching autodiff arguments"); + // We now handle the issue that Rust level arguments not always match the llvm-ir level + // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on + // llvm-ir level. The number of activities matches the number of Rust level arguments, so we + // need to match those. + // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it + // using iterators and peek()? + let mut outer_pos: usize = 0; + let mut activity_pos = 0; + + let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap(); + let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap(); + let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap(); + let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap(); + + while activity_pos < inputs.len() { + let diff_activity = inputs[activity_pos as usize]; + // Duplicated arguments received a shadow argument, into which enzyme will write the + // gradient. + let (activity, duplicated): (&Metadata, bool) = match diff_activity { + DiffActivity::None => panic!("not a valid input activity"), + DiffActivity::Const => (enzyme_const, false), + DiffActivity::Active => (enzyme_out, false), + DiffActivity::ActiveOnly => (enzyme_out, false), + DiffActivity::Dual => (enzyme_dup, true), + DiffActivity::DualOnly => (enzyme_dupnoneed, true), + DiffActivity::Duplicated => (enzyme_dup, true), + DiffActivity::DuplicatedOnly => (enzyme_dupnoneed, true), + DiffActivity::FakeActivitySize => (enzyme_const, false), + }; + let outer_arg = outer_args[outer_pos]; + args.push(cx.get_metadata_value(activity)); + args.push(outer_arg); + if duplicated { + // We know that duplicated args by construction have a following argument, + // so this can not be out of bounds. + let next_outer_arg = outer_args[outer_pos + 1]; + let next_outer_ty = cx.val_ty(next_outer_arg); + // FIXME(ZuseZ4): We should add support for Vec here too, but it's less urgent since + // vectors behind references (&Vec<T>) are already supported. Users can not pass a + // Vec by value for reverse mode, so this would only help forward mode autodiff. + let slice = { + if activity_pos + 1 >= inputs.len() { + // If there is no arg following our ptr, it also can't be a slice, + // since that would lead to a ptr, int pair. + false + } else { + let next_activity = inputs[activity_pos + 1]; + // We analyze the MIR types and add this dummy activity if we visit a slice. + next_activity == DiffActivity::FakeActivitySize + } + }; + if slice { + // A duplicated slice will have the following two outer_fn arguments: + // (..., ptr1, int1, ptr2, int2, ...). We add the following llvm-ir to our __enzyme call: + // (..., metadata! enzyme_dup, ptr, ptr, int1, ...). + // FIXME(ZuseZ4): We will upstream a safety check later which asserts that + // int2 >= int1, which means the shadow vector is large enough to store the gradient. + assert!(unsafe { + llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Integer + }); + let next_outer_arg2 = outer_args[outer_pos + 2]; + let next_outer_ty2 = cx.val_ty(next_outer_arg2); + assert!(unsafe { + llvm::LLVMRustGetTypeKind(next_outer_ty2) == llvm::TypeKind::Pointer + }); + let next_outer_arg3 = outer_args[outer_pos + 3]; + let next_outer_ty3 = cx.val_ty(next_outer_arg3); + assert!(unsafe { + llvm::LLVMRustGetTypeKind(next_outer_ty3) == llvm::TypeKind::Integer + }); + args.push(next_outer_arg2); + args.push(cx.get_metadata_value(enzyme_const)); + args.push(next_outer_arg); + outer_pos += 4; + activity_pos += 2; + } else { + // A duplicated pointer will have the following two outer_fn arguments: + // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call: + // (..., metadata! enzyme_dup, ptr, ptr, ...). + if matches!(diff_activity, DiffActivity::Duplicated | DiffActivity::DuplicatedOnly) + { + assert!( + unsafe { llvm::LLVMRustGetTypeKind(next_outer_ty) } + == llvm::TypeKind::Pointer + ); + } + // In the case of Dual we don't have assumptions, e.g. f32 would be valid. + args.push(next_outer_arg); + outer_pos += 2; + activity_pos += 1; + } + } else { + // We do not differentiate with resprect to this argument. + // We already added the metadata and argument above, so just increase the counters. + outer_pos += 1; + activity_pos += 1; + } + } +} + /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another /// function with expected naming and calling conventions[^1] which will be /// discovered by the enzyme LLVM pass and its body populated with the differentiated @@ -43,9 +150,6 @@ fn generate_enzyme_call<'ll>( outer_fn: &'ll Value, attrs: AutoDiffAttrs, ) { - let inputs = attrs.input_activity; - let output = attrs.ret_activity; - // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { DiffMode::Forward => "__enzyme_fwddiff", @@ -132,111 +236,13 @@ fn generate_enzyme_call<'ll>( let mut args = Vec::with_capacity(num_args as usize + 1); args.push(fn_to_diff); - let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap(); - let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap(); - let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap(); - let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap(); let enzyme_primal_ret = cx.create_metadata("enzyme_primal_return".to_string()).unwrap(); - - match output { - DiffActivity::Dual => { - args.push(cx.get_metadata_value(enzyme_primal_ret)); - } - DiffActivity::Active => { - args.push(cx.get_metadata_value(enzyme_primal_ret)); - } - _ => {} + if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { + args.push(cx.get_metadata_value(enzyme_primal_ret)); } - debug!("matching autodiff arguments"); - // We now handle the issue that Rust level arguments not always match the llvm-ir level - // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on - // llvm-ir level. The number of activities matches the number of Rust level arguments, so we - // need to match those. - // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it - // using iterators and peek()? - let mut outer_pos: usize = 0; - let mut activity_pos = 0; let outer_args: Vec<&llvm::Value> = get_params(outer_fn); - while activity_pos < inputs.len() { - let diff_activity = inputs[activity_pos as usize]; - // Duplicated arguments received a shadow argument, into which enzyme will write the - // gradient. - let (activity, duplicated): (&Metadata, bool) = match diff_activity { - DiffActivity::None => panic!("not a valid input activity"), - DiffActivity::Const => (enzyme_const, false), - DiffActivity::Active => (enzyme_out, false), - DiffActivity::ActiveOnly => (enzyme_out, false), - DiffActivity::Dual => (enzyme_dup, true), - DiffActivity::DualOnly => (enzyme_dupnoneed, true), - DiffActivity::Duplicated => (enzyme_dup, true), - DiffActivity::DuplicatedOnly => (enzyme_dupnoneed, true), - DiffActivity::FakeActivitySize => (enzyme_const, false), - }; - let outer_arg = outer_args[outer_pos]; - args.push(cx.get_metadata_value(activity)); - args.push(outer_arg); - if duplicated { - // We know that duplicated args by construction have a following argument, - // so this can not be out of bounds. - let next_outer_arg = outer_args[outer_pos + 1]; - let next_outer_ty = cx.val_ty(next_outer_arg); - // FIXME(ZuseZ4): We should add support for Vec here too, but it's less urgent since - // vectors behind references (&Vec<T>) are already supported. Users can not pass a - // Vec by value for reverse mode, so this would only help forward mode autodiff. - let slice = { - if activity_pos + 1 >= inputs.len() { - // If there is no arg following our ptr, it also can't be a slice, - // since that would lead to a ptr, int pair. - false - } else { - let next_activity = inputs[activity_pos + 1]; - // We analyze the MIR types and add this dummy activity if we visit a slice. - next_activity == DiffActivity::FakeActivitySize - } - }; - if slice { - // A duplicated slice will have the following two outer_fn arguments: - // (..., ptr1, int1, ptr2, int2, ...). We add the following llvm-ir to our __enzyme call: - // (..., metadata! enzyme_dup, ptr, ptr, int1, ...). - // FIXME(ZuseZ4): We will upstream a safety check later which asserts that - // int2 >= int1, which means the shadow vector is large enough to store the gradient. - assert!(llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Integer); - let next_outer_arg2 = outer_args[outer_pos + 2]; - let next_outer_ty2 = cx.val_ty(next_outer_arg2); - assert!(llvm::LLVMRustGetTypeKind(next_outer_ty2) == llvm::TypeKind::Pointer); - let next_outer_arg3 = outer_args[outer_pos + 3]; - let next_outer_ty3 = cx.val_ty(next_outer_arg3); - assert!(llvm::LLVMRustGetTypeKind(next_outer_ty3) == llvm::TypeKind::Integer); - args.push(next_outer_arg2); - args.push(cx.get_metadata_value(enzyme_const)); - args.push(next_outer_arg); - outer_pos += 4; - activity_pos += 2; - } else { - // A duplicated pointer will have the following two outer_fn arguments: - // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call: - // (..., metadata! enzyme_dup, ptr, ptr, ...). - if matches!( - diff_activity, - DiffActivity::Duplicated | DiffActivity::DuplicatedOnly - ) { - assert!( - llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer - ); - } - // In the case of Dual we don't have assumptions, e.g. f32 would be valid. - args.push(next_outer_arg); - outer_pos += 2; - activity_pos += 1; - } - } else { - // We do not differentiate with resprect to this argument. - // We already added the metadata and argument above, so just increase the counters. - outer_pos += 1; - activity_pos += 1; - } - } + match_args_from_caller_to_enzyme(&cx, &mut args, &attrs.input_activity, &outer_args); let call = builder.call(enzyme_ty, ad_fn, &args, None); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 926445c780b..f7b096ff976 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1127,6 +1127,18 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128); + ifn!("llvm.scmp.i8.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.scmp.i8.i16", fn(t_i16, t_i16) -> t_i8); + ifn!("llvm.scmp.i8.i32", fn(t_i32, t_i32) -> t_i8); + ifn!("llvm.scmp.i8.i64", fn(t_i64, t_i64) -> t_i8); + ifn!("llvm.scmp.i8.i128", fn(t_i128, t_i128) -> t_i8); + + ifn!("llvm.ucmp.i8.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.ucmp.i8.i16", fn(t_i16, t_i16) -> t_i8); + ifn!("llvm.ucmp.i8.i32", fn(t_i32, t_i32) -> t_i8); + ifn!("llvm.ucmp.i8.i64", fn(t_i64, t_i64) -> t_i8); + ifn!("llvm.ucmp.i8.i128", fn(t_i128, t_i128) -> t_i8); + ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void); ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index b617f4d37f5..f6000e72840 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -146,6 +146,7 @@ pub(crate) struct CoverageSpan { #[derive(Clone, Debug, Default)] pub(crate) struct Regions { pub(crate) code_regions: Vec<CodeRegion>, + pub(crate) expansion_regions: Vec<ExpansionRegion>, pub(crate) branch_regions: Vec<BranchRegion>, pub(crate) mcdc_branch_regions: Vec<MCDCBranchRegion>, pub(crate) mcdc_decision_regions: Vec<MCDCDecisionRegion>, @@ -154,10 +155,16 @@ pub(crate) struct Regions { impl Regions { /// Returns true if none of this structure's tables contain any regions. pub(crate) fn has_no_regions(&self) -> bool { - let Self { code_regions, branch_regions, mcdc_branch_regions, mcdc_decision_regions } = - self; + let Self { + code_regions, + expansion_regions, + branch_regions, + mcdc_branch_regions, + mcdc_decision_regions, + } = self; code_regions.is_empty() + && expansion_regions.is_empty() && branch_regions.is_empty() && mcdc_branch_regions.is_empty() && mcdc_decision_regions.is_empty() @@ -172,6 +179,14 @@ pub(crate) struct CodeRegion { pub(crate) counter: Counter, } +/// Must match the layout of `LLVMRustCoverageExpansionRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct ExpansionRegion { + pub(crate) cov_span: CoverageSpan, + pub(crate) expanded_file_id: u32, +} + /// Must match the layout of `LLVMRustCoverageBranchRegion`. #[derive(Clone, Debug)] #[repr(C)] diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index 2cd7fa3225a..907d6d41a1f 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -63,8 +63,18 @@ pub(crate) fn write_function_mappings_to_buffer( expressions: &[ffi::CounterExpression], regions: &ffi::Regions, ) -> Vec<u8> { - let ffi::Regions { code_regions, branch_regions, mcdc_branch_regions, mcdc_decision_regions } = - regions; + let ffi::Regions { + code_regions, + expansion_regions, + branch_regions, + mcdc_branch_regions, + mcdc_decision_regions, + } = regions; + + // SAFETY: + // - All types are FFI-compatible and have matching representations in Rust/C++. + // - For pointer/length pairs, the pointer and length come from the same vector or slice. + // - C++ code does not retain any pointers after the call returns. llvm::build_byte_buffer(|buffer| unsafe { llvm::LLVMRustCoverageWriteFunctionMappingsToBuffer( virtual_file_mapping.as_ptr(), @@ -73,6 +83,8 @@ pub(crate) fn write_function_mappings_to_buffer( expressions.len(), code_regions.as_ptr(), code_regions.len(), + expansion_regions.as_ptr(), + expansion_regions.len(), branch_regions.as_ptr(), branch_regions.len(), mcdc_branch_regions.as_ptr(), diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index 5b487bc1a8b..048e1988c32 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -120,12 +120,22 @@ fn fill_region_tables<'tcx>( // Associate that global file ID with a local file ID for this function. let local_file_id = covfun.virtual_file_mapping.local_id_for_global(global_file_id); - let ffi::Regions { code_regions, branch_regions, mcdc_branch_regions, mcdc_decision_regions } = - &mut covfun.regions; - - let make_cov_span = - |span: Span| spans::make_coverage_span(local_file_id, source_map, &source_file, span); + // In rare cases, _all_ of a function's spans are discarded, and coverage + // codegen needs to handle that gracefully to avoid #133606. + // It's hard for tests to trigger this organically, so instead we set + // `-Zcoverage-options=discard-all-spans-in-codegen` to force it to occur. let discard_all = tcx.sess.coverage_discard_all_spans_in_codegen(); + let make_coords = |span: Span| { + if discard_all { None } else { spans::make_coords(source_map, &source_file, span) } + }; + + let ffi::Regions { + code_regions, + expansion_regions: _, // FIXME(Zalathar): Fill out support for expansion regions + branch_regions, + mcdc_branch_regions, + mcdc_decision_regions, + } = &mut covfun.regions; // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. @@ -140,17 +150,8 @@ fn fill_region_tables<'tcx>( ffi::Counter::from_term(term) }; - // Convert the `Span` into coordinates that we can pass to LLVM, or - // discard the span if conversion fails. In rare, cases _all_ of a - // function's spans are discarded, and the rest of coverage codegen - // needs to handle that gracefully to avoid a repeat of #133606. - // We don't have a good test case for triggering that organically, so - // instead we set `-Zcoverage-options=discard-all-spans-in-codegen` - // to force it to occur. - let Some(cov_span) = make_cov_span(span) else { continue }; - if discard_all { - continue; - } + let Some(coords) = make_coords(span) else { continue }; + let cov_span = coords.make_coverage_span(local_file_id); match *kind { MappingKind::Code { bcb } => { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs index 3193be31ada..39a59560c9d 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs @@ -5,22 +5,40 @@ use tracing::debug; use crate::coverageinfo::ffi; use crate::coverageinfo::mapgen::LocalFileId; +/// Line and byte-column coordinates of a source code span within some file. +/// The file itself must be tracked separately. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Coords { + /// 1-based starting line of the source code span. + pub(crate) start_line: u32, + /// 1-based starting column (in bytes) of the source code span. + pub(crate) start_col: u32, + /// 1-based ending line of the source code span. + pub(crate) end_line: u32, + /// 1-based ending column (in bytes) of the source code span. High bit must be unset. + pub(crate) end_col: u32, +} + +impl Coords { + /// Attaches a local file ID to these coordinates to produce an `ffi::CoverageSpan`. + pub(crate) fn make_coverage_span(&self, local_file_id: LocalFileId) -> ffi::CoverageSpan { + let &Self { start_line, start_col, end_line, end_col } = self; + let file_id = local_file_id.as_u32(); + ffi::CoverageSpan { file_id, start_line, start_col, end_line, end_col } + } +} + /// Converts the span into its start line and column, and end line and column. /// /// Line numbers and column numbers are 1-based. Unlike most column numbers emitted by /// the compiler, these column numbers are denoted in **bytes**, because that's what /// LLVM's `llvm-cov` tool expects to see in coverage maps. /// -/// Returns `None` if the conversion failed for some reason. This shouldn't happen, +/// Returns `None` if the conversion failed for some reason. This should be uncommon, /// but it's hard to rule out entirely (especially in the presence of complex macros /// or other expansions), and if it does happen then skipping a span or function is /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid. -pub(crate) fn make_coverage_span( - file_id: LocalFileId, - source_map: &SourceMap, - file: &SourceFile, - span: Span, -) -> Option<ffi::CoverageSpan> { +pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) -> Option<Coords> { let span = ensure_non_empty_span(source_map, span)?; let lo = span.lo(); @@ -44,8 +62,7 @@ pub(crate) fn make_coverage_span( start_line = source_map.doctest_offset_line(&file.name, start_line); end_line = source_map.doctest_offset_line(&file.name, end_line); - check_coverage_span(ffi::CoverageSpan { - file_id: file_id.as_u32(), + check_coords(Coords { start_line: start_line as u32, start_col: start_col as u32, end_line: end_line as u32, @@ -80,8 +97,8 @@ fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> { /// it will immediately exit with a fatal error. To prevent that from happening, /// discard regions that are improperly ordered, or might be interpreted in a /// way that makes them improperly ordered. -fn check_coverage_span(cov_span: ffi::CoverageSpan) -> Option<ffi::CoverageSpan> { - let ffi::CoverageSpan { file_id: _, start_line, start_col, end_line, end_col } = cov_span; +fn check_coords(coords: Coords) -> Option<Coords> { + let Coords { start_line, start_col, end_line, end_col } = coords; // Line/column coordinates are supposed to be 1-based. If we ever emit // coordinates of 0, `llvm-cov` might misinterpret them. @@ -94,17 +111,17 @@ fn check_coverage_span(cov_span: ffi::CoverageSpan) -> Option<ffi::CoverageSpan> let is_ordered = (start_line, start_col) <= (end_line, end_col); if all_nonzero && end_col_has_high_bit_unset && is_ordered { - Some(cov_span) + Some(coords) } else { debug!( - ?cov_span, + ?coords, ?all_nonzero, ?end_col_has_high_bit_unset, ?is_ordered, "Skipping source region that would be misinterpreted or rejected by LLVM" ); // If this happens in a debug build, ICE to make it easier to notice. - debug_assert!(false, "Improper source region: {cov_span:?}"); + debug_assert!(false, "Improper source region: {coords:?}"); None } } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index f622646a5d9..425381b0ffa 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -29,7 +29,7 @@ use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; use context::SimpleCx; use errors::{AutoDiffWithoutLTO, ParseTargetMachineConfig}; -pub(crate) use llvm_util::target_features_cfg; +use llvm_util::target_features_cfg; use rustc_ast::expand::allocator::AllocatorKind; use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; @@ -71,9 +71,7 @@ mod debuginfo; mod declare; mod errors; mod intrinsic; -// FIXME(Zalathar): Fix all the unreachable-pub warnings that would occur if -// this isn't pub, then make it not pub. -pub mod llvm; +mod llvm; mod llvm_util; mod mono_item; mod type_; diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 63b2b15c514..51bcc4d123d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -5,17 +5,17 @@ use std::{slice, str}; use rustc_fs_util::path_to_c_string; -pub struct ArchiveRO { +pub(crate) struct ArchiveRO { pub raw: &'static mut super::Archive, } unsafe impl Send for ArchiveRO {} -pub struct Iter<'a> { +pub(crate) struct Iter<'a> { raw: &'a mut super::ArchiveIterator<'a>, } -pub struct Child<'a> { +pub(crate) struct Child<'a> { pub raw: &'a mut super::ArchiveChild<'a>, } diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index 11043b664f5..0e0f2b0eab0 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -3,13 +3,13 @@ use libc::c_uint; use rustc_span::InnerSpan; -pub use self::Diagnostic::*; -pub use self::OptimizationDiagnosticKind::*; +pub(crate) use self::Diagnostic::*; +use self::OptimizationDiagnosticKind::*; use super::{DiagnosticInfo, SMDiagnostic}; use crate::value::Value; #[derive(Copy, Clone, Debug)] -pub enum OptimizationDiagnosticKind { +pub(crate) enum OptimizationDiagnosticKind { OptimizationRemark, OptimizationMissed, OptimizationAnalysis, @@ -19,9 +19,10 @@ pub enum OptimizationDiagnosticKind { OptimizationRemarkOther, } -pub struct OptimizationDiagnostic<'ll> { +pub(crate) struct OptimizationDiagnostic<'ll> { pub kind: OptimizationDiagnosticKind, pub pass_name: String, + #[expect(dead_code)] pub function: &'ll Value, pub line: c_uint, pub column: c_uint, @@ -73,14 +74,14 @@ impl<'ll> OptimizationDiagnostic<'ll> { } } -pub struct SrcMgrDiagnostic { +pub(crate) struct SrcMgrDiagnostic { pub level: super::DiagnosticLevel, pub message: String, pub source: Option<(String, Vec<InnerSpan>)>, } impl SrcMgrDiagnostic { - pub unsafe fn unpack(diag: &SMDiagnostic) -> SrcMgrDiagnostic { + pub(crate) unsafe fn unpack(diag: &SMDiagnostic) -> SrcMgrDiagnostic { // Recover the post-substitution assembly code from LLVM for better // diagnostics. let mut have_source = false; @@ -120,7 +121,7 @@ impl SrcMgrDiagnostic { } #[derive(Clone)] -pub struct InlineAsmDiagnostic { +pub(crate) struct InlineAsmDiagnostic { pub level: super::DiagnosticLevel, pub cookie: u64, pub message: String, @@ -158,7 +159,7 @@ impl InlineAsmDiagnostic { } } -pub enum Diagnostic<'ll> { +pub(crate) enum Diagnostic<'ll> { Optimization(OptimizationDiagnostic<'ll>), InlineAsm(InlineAsmDiagnostic), PGO(&'ll DiagnosticInfo), @@ -166,11 +167,12 @@ pub enum Diagnostic<'ll> { Unsupported(&'ll DiagnosticInfo), /// LLVM has other types that we do not wrap here. + #[expect(dead_code)] UnknownDiagnostic(&'ll DiagnosticInfo), } impl<'ll> Diagnostic<'ll> { - pub unsafe fn unpack(di: &'ll DiagnosticInfo) -> Self { + pub(crate) unsafe fn unpack(di: &'ll DiagnosticInfo) -> Self { use super::DiagnosticKind as Dk; unsafe { diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index f6b23862907..79e4cc8aa77 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -31,20 +31,20 @@ unsafe extern "C" { #[repr(C)] #[derive(Copy, Clone, PartialEq)] -pub enum LLVMRustVerifierFailureAction { +pub(crate) enum LLVMRustVerifierFailureAction { LLVMAbortProcessAction = 0, LLVMPrintMessageAction = 1, LLVMReturnStatusAction = 2, } #[cfg(llvm_enzyme)] -pub use self::Enzyme_AD::*; +pub(crate) use self::Enzyme_AD::*; #[cfg(llvm_enzyme)] -pub mod Enzyme_AD { +pub(crate) mod Enzyme_AD { use libc::c_void; unsafe extern "C" { - pub fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); + pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); } unsafe extern "C" { static mut EnzymePrintPerf: c_void; @@ -56,42 +56,42 @@ pub mod Enzyme_AD { static mut EnzymeInline: c_void; static mut RustTypeRules: c_void; } - pub fn set_print_perf(print: bool) { + pub(crate) fn set_print_perf(print: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintPerf), print as u8); } } - pub fn set_print_activity(print: bool) { + pub(crate) fn set_print_activity(print: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintActivity), print as u8); } } - pub fn set_print_type(print: bool) { + pub(crate) fn set_print_type(print: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintType), print as u8); } } - pub fn set_print(print: bool) { + pub(crate) fn set_print(print: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrint), print as u8); } } - pub fn set_strict_aliasing(strict: bool) { + pub(crate) fn set_strict_aliasing(strict: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymeStrictAliasing), strict as u8); } } - pub fn set_loose_types(loose: bool) { + pub(crate) fn set_loose_types(loose: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(looseTypeAnalysis), loose as u8); } } - pub fn set_inline(val: bool) { + pub(crate) fn set_inline(val: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymeInline), val as u8); } } - pub fn set_rust_rules(val: bool) { + pub(crate) fn set_rust_rules(val: bool) { unsafe { EnzymeSetCLBool(std::ptr::addr_of_mut!(RustTypeRules), val as u8); } @@ -99,34 +99,34 @@ pub mod Enzyme_AD { } #[cfg(not(llvm_enzyme))] -pub use self::Fallback_AD::*; +pub(crate) use self::Fallback_AD::*; #[cfg(not(llvm_enzyme))] -pub mod Fallback_AD { +pub(crate) mod Fallback_AD { #![allow(unused_variables)] - pub fn set_inline(val: bool) { + pub(crate) fn set_inline(val: bool) { unimplemented!() } - pub fn set_print_perf(print: bool) { + pub(crate) fn set_print_perf(print: bool) { unimplemented!() } - pub fn set_print_activity(print: bool) { + pub(crate) fn set_print_activity(print: bool) { unimplemented!() } - pub fn set_print_type(print: bool) { + pub(crate) fn set_print_type(print: bool) { unimplemented!() } - pub fn set_print(print: bool) { + pub(crate) fn set_print(print: bool) { unimplemented!() } - pub fn set_strict_aliasing(strict: bool) { + pub(crate) fn set_strict_aliasing(strict: bool) { unimplemented!() } - pub fn set_loose_types(loose: bool) { + pub(crate) fn set_loose_types(loose: bool) { unimplemented!() } - pub fn set_rust_rules(val: bool) { + pub(crate) fn set_rust_rules(val: bool) { unimplemented!() } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 39087a4d6f4..3ce3761944b 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -32,10 +32,10 @@ use crate::llvm; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. -pub type Bool = c_int; +pub(crate) type Bool = c_int; -pub const True: Bool = 1 as Bool; -pub const False: Bool = 0 as Bool; +pub(crate) const True: Bool = 1 as Bool; +pub(crate) const False: Bool = 0 as Bool; /// Wrapper for a raw enum value returned from LLVM's C APIs. /// @@ -44,7 +44,7 @@ pub const False: Bool = 0 as Bool; /// value and returns it. Instead, return this raw wrapper, then convert to the /// Rust-side enum explicitly. #[repr(transparent)] -pub struct RawEnum<T> { +pub(crate) struct RawEnum<T> { value: u32, /// We don't own or consume a `T`, but we can produce one. _rust_side_type: PhantomData<fn() -> T>, @@ -64,7 +64,7 @@ impl<T: TryFrom<u32>> RawEnum<T> { #[derive(Copy, Clone, PartialEq)] #[repr(C)] #[allow(dead_code)] // Variants constructed by C++. -pub enum LLVMRustResult { +pub(crate) enum LLVMRustResult { Success, Failure, } @@ -83,7 +83,7 @@ pub enum LLVMRustResult { /// C++ API. #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum ModuleFlagMergeBehavior { +pub(crate) enum ModuleFlagMergeBehavior { Error = 1, Warning = 2, Require = 3, @@ -101,7 +101,7 @@ pub enum ModuleFlagMergeBehavior { /// See <https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/CallingConv.h> #[derive(Copy, Clone, PartialEq, Debug, TryFromU32)] #[repr(C)] -pub enum CallConv { +pub(crate) enum CallConv { CCallConv = 0, FastCallConv = 8, ColdCallConv = 9, @@ -126,7 +126,7 @@ pub enum CallConv { /// Must match the layout of `LLVMLinkage`. #[derive(Copy, Clone, PartialEq, TryFromU32)] #[repr(C)] -pub enum Linkage { +pub(crate) enum Linkage { ExternalLinkage = 0, AvailableExternallyLinkage = 1, LinkOnceAnyLinkage = 2, @@ -153,7 +153,7 @@ pub enum Linkage { /// Must match the layout of `LLVMVisibility`. #[repr(C)] #[derive(Copy, Clone, PartialEq, TryFromU32)] -pub enum Visibility { +pub(crate) enum Visibility { Default = 0, Hidden = 1, Protected = 2, @@ -171,8 +171,9 @@ impl Visibility { /// LLVMUnnamedAddr #[repr(C)] -pub enum UnnamedAddr { +pub(crate) enum UnnamedAddr { No, + #[expect(dead_code)] Local, Global, } @@ -180,7 +181,7 @@ pub enum UnnamedAddr { /// LLVMDLLStorageClass #[derive(Copy, Clone)] #[repr(C)] -pub enum DLLStorageClass { +pub(crate) enum DLLStorageClass { #[allow(dead_code)] Default = 0, DllImport = 1, // Function to be imported from DLL. @@ -193,7 +194,8 @@ pub enum DLLStorageClass { /// though it is not ABI compatible (since it's a C++ enum) #[repr(C)] #[derive(Copy, Clone, Debug)] -pub enum AttributeKind { +#[expect(dead_code, reason = "Some variants are unused, but are kept to match the C++")] +pub(crate) enum AttributeKind { AlwaysInline = 0, ByVal = 1, Cold = 2, @@ -241,7 +243,7 @@ pub enum AttributeKind { /// LLVMIntPredicate #[derive(Copy, Clone)] #[repr(C)] -pub enum IntPredicate { +pub(crate) enum IntPredicate { IntEQ = 32, IntNE = 33, IntUGT = 34, @@ -275,7 +277,7 @@ impl IntPredicate { /// LLVMRealPredicate #[derive(Copy, Clone)] #[repr(C)] -pub enum RealPredicate { +pub(crate) enum RealPredicate { RealPredicateFalse = 0, RealOEQ = 1, RealOGT = 2, @@ -321,7 +323,8 @@ impl RealPredicate { /// LLVMTypeKind #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] -pub enum TypeKind { +#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] +pub(crate) enum TypeKind { Void = 0, Half = 1, Float = 2, @@ -373,7 +376,7 @@ impl TypeKind { /// LLVMAtomicRmwBinOp #[derive(Copy, Clone)] #[repr(C)] -pub enum AtomicRmwBinOp { +pub(crate) enum AtomicRmwBinOp { AtomicXchg = 0, AtomicAdd = 1, AtomicSub = 2, @@ -409,7 +412,7 @@ impl AtomicRmwBinOp { /// LLVMAtomicOrdering #[derive(Copy, Clone)] #[repr(C)] -pub enum AtomicOrdering { +pub(crate) enum AtomicOrdering { #[allow(dead_code)] NotAtomic = 0, Unordered = 1, @@ -438,7 +441,7 @@ impl AtomicOrdering { /// LLVMRustFileType #[derive(Copy, Clone)] #[repr(C)] -pub enum FileType { +pub(crate) enum FileType { AssemblyFile, ObjectFile, } @@ -446,7 +449,8 @@ pub enum FileType { /// LLVMMetadataType #[derive(Copy, Clone)] #[repr(C)] -pub enum MetadataType { +#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] +pub(crate) enum MetadataType { MD_dbg = 0, MD_tbaa = 1, MD_prof = 2, @@ -470,7 +474,7 @@ pub enum MetadataType { /// LLVMRustAsmDialect #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum AsmDialect { +pub(crate) enum AsmDialect { Att, Intel, } @@ -478,7 +482,7 @@ pub enum AsmDialect { /// LLVMRustCodeGenOptLevel #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum CodeGenOptLevel { +pub(crate) enum CodeGenOptLevel { None, Less, Default, @@ -487,7 +491,7 @@ pub enum CodeGenOptLevel { /// LLVMRustPassBuilderOptLevel #[repr(C)] -pub enum PassBuilderOptLevel { +pub(crate) enum PassBuilderOptLevel { O0, O1, O2, @@ -499,7 +503,7 @@ pub enum PassBuilderOptLevel { /// LLVMRustOptStage #[derive(PartialEq)] #[repr(C)] -pub enum OptStage { +pub(crate) enum OptStage { PreLinkNoLTO, PreLinkThinLTO, PreLinkFatLTO, @@ -509,7 +513,7 @@ pub enum OptStage { /// LLVMRustSanitizerOptions #[repr(C)] -pub struct SanitizerOptions { +pub(crate) struct SanitizerOptions { pub sanitize_address: bool, pub sanitize_address_recover: bool, pub sanitize_cfi: bool, @@ -530,7 +534,7 @@ pub struct SanitizerOptions { /// LLVMRustRelocModel #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum RelocModel { +pub(crate) enum RelocModel { Static, PIC, DynamicNoPic, @@ -542,7 +546,7 @@ pub enum RelocModel { /// LLVMRustFloatABI #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum FloatAbi { +pub(crate) enum FloatAbi { Default, Soft, Hard, @@ -551,7 +555,7 @@ pub enum FloatAbi { /// LLVMRustCodeModel #[derive(Copy, Clone)] #[repr(C)] -pub enum CodeModel { +pub(crate) enum CodeModel { Tiny, Small, Kernel, @@ -564,7 +568,7 @@ pub enum CodeModel { #[derive(Copy, Clone)] #[repr(C)] #[allow(dead_code)] // Variants constructed by C++. -pub enum DiagnosticKind { +pub(crate) enum DiagnosticKind { Other, InlineAsm, StackSize, @@ -587,7 +591,7 @@ pub enum DiagnosticKind { #[derive(Copy, Clone)] #[repr(C)] #[allow(dead_code)] // Variants constructed by C++. -pub enum DiagnosticLevel { +pub(crate) enum DiagnosticLevel { Error, Warning, Note, @@ -597,7 +601,7 @@ pub enum DiagnosticLevel { /// LLVMRustArchiveKind #[derive(Copy, Clone)] #[repr(C)] -pub enum ArchiveKind { +pub(crate) enum ArchiveKind { K_GNU, K_BSD, K_DARWIN, @@ -607,15 +611,15 @@ pub enum ArchiveKind { unsafe extern "C" { // LLVMRustThinLTOData - pub type ThinLTOData; + pub(crate) type ThinLTOData; // LLVMRustThinLTOBuffer - pub type ThinLTOBuffer; + pub(crate) type ThinLTOBuffer; } /// LLVMRustThinLTOModule #[repr(C)] -pub struct ThinLTOModule { +pub(crate) struct ThinLTOModule { pub identifier: *const c_char, pub data: *const u8, pub len: usize, @@ -624,7 +628,8 @@ pub struct ThinLTOModule { /// LLVMThreadLocalMode #[derive(Copy, Clone)] #[repr(C)] -pub enum ThreadLocalMode { +pub(crate) enum ThreadLocalMode { + #[expect(dead_code)] NotThreadLocal, GeneralDynamic, LocalDynamic, @@ -635,7 +640,7 @@ pub enum ThreadLocalMode { /// LLVMRustChecksumKind #[derive(Copy, Clone)] #[repr(C)] -pub enum ChecksumKind { +pub(crate) enum ChecksumKind { None, MD5, SHA1, @@ -645,7 +650,7 @@ pub enum ChecksumKind { /// LLVMRustMemoryEffects #[derive(Copy, Clone)] #[repr(C)] -pub enum MemoryEffects { +pub(crate) enum MemoryEffects { None, ReadOnly, InaccessibleMemOnly, @@ -654,7 +659,8 @@ pub enum MemoryEffects { /// LLVMOpcode #[derive(Copy, Clone, PartialEq, Eq)] #[repr(C)] -pub enum Opcode { +#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] +pub(crate) enum Opcode { Ret = 1, Br = 2, Switch = 3, @@ -735,48 +741,48 @@ struct InvariantOpaque<'a> { // Opaque pointer types unsafe extern "C" { - pub type Module; - pub type Context; - pub type Type; - pub type Value; - pub type ConstantInt; - pub type Attribute; - pub type Metadata; - pub type BasicBlock; - pub type Comdat; + pub(crate) type Module; + pub(crate) type Context; + pub(crate) type Type; + pub(crate) type Value; + pub(crate) type ConstantInt; + pub(crate) type Attribute; + pub(crate) type Metadata; + pub(crate) type BasicBlock; + pub(crate) type Comdat; } #[repr(C)] -pub struct Builder<'a>(InvariantOpaque<'a>); +pub(crate) struct Builder<'a>(InvariantOpaque<'a>); #[repr(C)] -pub struct PassManager<'a>(InvariantOpaque<'a>); +pub(crate) struct PassManager<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub type TargetMachine; - pub type Archive; + pub(crate) type Archive; } #[repr(C)] -pub struct ArchiveIterator<'a>(InvariantOpaque<'a>); +pub(crate) struct ArchiveIterator<'a>(InvariantOpaque<'a>); #[repr(C)] -pub struct ArchiveChild<'a>(InvariantOpaque<'a>); +pub(crate) struct ArchiveChild<'a>(InvariantOpaque<'a>); unsafe extern "C" { - pub type Twine; - pub type DiagnosticInfo; - pub type SMDiagnostic; + pub(crate) type Twine; + pub(crate) type DiagnosticInfo; + pub(crate) type SMDiagnostic; } #[repr(C)] -pub struct RustArchiveMember<'a>(InvariantOpaque<'a>); +pub(crate) struct RustArchiveMember<'a>(InvariantOpaque<'a>); /// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); #[repr(C)] -pub struct Linker<'a>(InvariantOpaque<'a>); +pub(crate) struct Linker<'a>(InvariantOpaque<'a>); unsafe extern "C" { - pub type DiagnosticHandler; + pub(crate) type DiagnosticHandler; } -pub type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); +pub(crate) type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); -pub mod debuginfo { +pub(crate) mod debuginfo { use std::ptr; use bitflags::bitflags; @@ -793,7 +799,7 @@ pub mod debuginfo { /// builder reference typically has a shorter lifetime than the LLVM /// session (`'ll`) that it participates in. #[repr(C)] - pub struct DIBuilder<'ll>(InvariantOpaque<'ll>); + pub(crate) struct DIBuilder<'ll>(InvariantOpaque<'ll>); /// Owning pointer to a `DIBuilder<'ll>` that will dispose of the builder /// when dropped. Use `.as_ref()` to get the underlying `&DIBuilder` @@ -822,22 +828,22 @@ pub mod debuginfo { } } - pub type DIDescriptor = Metadata; - pub type DILocation = Metadata; - pub type DIScope = DIDescriptor; - pub type DIFile = DIScope; - pub type DILexicalBlock = DIScope; - pub type DISubprogram = DIScope; - pub type DIType = DIDescriptor; - pub type DIBasicType = DIType; - pub type DIDerivedType = DIType; - pub type DICompositeType = DIDerivedType; - pub type DIVariable = DIDescriptor; - pub type DIGlobalVariableExpression = DIDescriptor; - pub type DIArray = DIDescriptor; - pub type DISubrange = DIDescriptor; - pub type DIEnumerator = DIDescriptor; - pub type DITemplateTypeParameter = DIDescriptor; + pub(crate) type DIDescriptor = Metadata; + pub(crate) type DILocation = Metadata; + pub(crate) type DIScope = DIDescriptor; + pub(crate) type DIFile = DIScope; + pub(crate) type DILexicalBlock = DIScope; + pub(crate) type DISubprogram = DIScope; + pub(crate) type DIType = DIDescriptor; + pub(crate) type DIBasicType = DIType; + pub(crate) type DIDerivedType = DIType; + pub(crate) type DICompositeType = DIDerivedType; + pub(crate) type DIVariable = DIDescriptor; + pub(crate) type DIGlobalVariableExpression = DIDescriptor; + pub(crate) type DIArray = DIDescriptor; + pub(crate) type DISubrange = DIDescriptor; + pub(crate) type DIEnumerator = DIDescriptor; + pub(crate) type DITemplateTypeParameter = DIDescriptor; bitflags! { /// Must match the layout of `LLVMDIFlags` in the LLVM-C API. @@ -846,7 +852,7 @@ pub mod debuginfo { /// assertions in `RustWrapper.cpp` used by `fromRust(LLVMDIFlags)`. #[repr(transparent)] #[derive(Clone, Copy, Default)] - pub struct DIFlags: u32 { + pub(crate) struct DIFlags: u32 { const FlagZero = 0; const FlagPrivate = 1; const FlagProtected = 2; @@ -886,7 +892,7 @@ pub mod debuginfo { bitflags! { #[repr(transparent)] #[derive(Clone, Copy, Default)] - pub struct DISPFlags: u32 { + pub(crate) struct DISPFlags: u32 { const SPFlagZero = 0; const SPFlagVirtual = 1; const SPFlagPureVirtual = 2; @@ -900,7 +906,7 @@ pub mod debuginfo { /// LLVMRustDebugEmissionKind #[derive(Copy, Clone)] #[repr(C)] - pub enum DebugEmissionKind { + pub(crate) enum DebugEmissionKind { NoDebug, FullDebug, LineTablesOnly, @@ -932,8 +938,9 @@ pub mod debuginfo { /// LLVMRustDebugNameTableKind #[derive(Clone, Copy)] #[repr(C)] - pub enum DebugNameTableKind { + pub(crate) enum DebugNameTableKind { Default, + #[expect(dead_code)] Gnu, None, } @@ -943,7 +950,7 @@ pub mod debuginfo { bitflags! { #[repr(transparent)] #[derive(Default)] - pub struct AllocKindFlags : u64 { + pub(crate) struct AllocKindFlags : u64 { const Unknown = 0; const Alloc = 1; const Realloc = 1 << 1; @@ -966,19 +973,20 @@ bitflags! { } unsafe extern "C" { - pub type ModuleBuffer; + pub(crate) type ModuleBuffer; } -pub type SelfProfileBeforePassCallback = +pub(crate) type SelfProfileBeforePassCallback = unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); -pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); +pub(crate) type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); -pub type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; -pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; +pub(crate) type GetSymbolsCallback = + unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; +pub(crate) type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; #[derive(Copy, Clone)] #[repr(transparent)] -pub struct MetadataKindId(c_uint); +pub(crate) struct MetadataKindId(c_uint); impl From<MetadataType> for MetadataKindId { fn from(value: MetadataType) -> Self { @@ -2019,6 +2027,8 @@ unsafe extern "C" { NumExpressions: size_t, CodeRegions: *const crate::coverageinfo::ffi::CodeRegion, NumCodeRegions: size_t, + ExpansionRegions: *const crate::coverageinfo::ffi::ExpansionRegion, + NumExpansionRegions: size_t, BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, NumBranchRegions: size_t, MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index a36226b25a2..6ca81c651ed 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -9,18 +9,18 @@ use libc::c_uint; use rustc_abi::{Align, Size, WrappingRange}; use rustc_llvm::RustString; -pub use self::CallConv::*; -pub use self::CodeGenOptSize::*; -pub use self::MetadataType::*; -pub use self::ffi::*; +pub(crate) use self::CallConv::*; +pub(crate) use self::CodeGenOptSize::*; +pub(crate) use self::MetadataType::*; +pub(crate) use self::ffi::*; use crate::common::AsCCharPtr; -pub mod archive_ro; -pub mod diagnostic; -pub mod enzyme_ffi; +pub(crate) mod archive_ro; +pub(crate) mod diagnostic; +pub(crate) mod enzyme_ffi; mod ffi; -pub use self::enzyme_ffi::*; +pub(crate) use self::enzyme_ffi::*; impl LLVMRustResult { pub(crate) fn into_result(self) -> Result<(), ()> { @@ -127,7 +127,7 @@ pub(crate) fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) } #[derive(Copy, Clone)] -pub enum AttributePlace { +pub(crate) enum AttributePlace { ReturnValue, Argument(u32), Function, @@ -145,7 +145,7 @@ impl AttributePlace { #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum CodeGenOptSize { +pub(crate) enum CodeGenOptSize { CodeGenOptSizeNone = 0, CodeGenOptSizeDefault = 1, CodeGenOptSizeAggressive = 2, |