From 93465e6c3106043b8db7089ff7a1a4d610d8f79f Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Wed, 29 Jan 2025 06:46:05 +0900 Subject: Mark condition/carry bit as clobbered in C-SKY inline assembly --- compiler/rustc_codegen_llvm/src/asm.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 3722d4350a2..be5673eddf9 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -286,7 +286,9 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::M68k => { constraints.push("~{ccr}".to_string()); } - InlineAsmArch::CSKY => {} + InlineAsmArch::CSKY => { + constraints.push("~{psr}".to_string()); + } } } if !options.contains(InlineAsmOptions::NOMEM) { -- cgit 1.4.1-3-g733a5 From e9cb36bd0f86bc79587ecf840ff6b2fa7cc94d17 Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Tue, 4 Feb 2025 10:35:05 -0500 Subject: nvptx64: update default alignment to match LLVM 21 This changed in llvm/llvm-project@91cb8f5d3202870602c6bef807bc4c7ae8a32790. The commit itself is mostly about some intrinsic instructions, but as an aside it also mentions something about addrspace for tensor memory, which I believe is what this string is telling us. @rustbot label: +llvm-main --- compiler/rustc_codegen_llvm/src/context.rs | 6 ++++++ compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ba4fd75fb94..ebb4b421ad0 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -194,6 +194,12 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("-i128:128", ""); } } + if llvm_version < (21, 0, 0) { + if sess.target.arch == "nvptx64" { + // LLVM 21 updated the default layout on nvptx: https://github.com/llvm/llvm-project/pull/124961 + target_data_layout = target_data_layout.replace("e-p6:32:32-i64", "e-i64"); + } + } // Ensure the data-layout values hardcoded remain the defaults. { diff --git a/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs b/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs index 289251e906a..80bfa358243 100644 --- a/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs +++ b/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs @@ -5,7 +5,7 @@ use crate::spec::{ pub(crate) fn target() -> Target { Target { arch: "nvptx64".into(), - data_layout: "e-i64:64-i128:128-v16:16-v32:32-n16:32:64".into(), + data_layout: "e-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64".into(), llvm_target: "nvptx64-nvidia-cuda".into(), metadata: crate::spec::TargetMetadata { description: Some("--emit=asm generates PTX code that runs on NVIDIA GPUs".into()), -- cgit 1.4.1-3-g733a5 From 70b9ba3d6e1d64e6b00da707e5b1b5127e63b1cf Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 5 Feb 2025 18:47:23 -0500 Subject: fix fwd-mode autodiff case --- compiler/rustc_codegen_llvm/src/builder/autodiff.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 9e8e4e1c567..474b0940203 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -164,10 +164,10 @@ fn generate_enzyme_call<'ll>( let mut activity_pos = 0; let outer_args: Vec<&llvm::Value> = get_params(outer_fn); while activity_pos < inputs.len() { - let activity = inputs[activity_pos as usize]; + let diff_activity = inputs[activity_pos as usize]; // Duplicated arguments received a shadow argument, into which enzyme will write the // gradient. - let (activity, duplicated): (&Metadata, bool) = match activity { + let (activity, duplicated): (&Metadata, bool) = match diff_activity { DiffActivity::None => panic!("not a valid input activity"), DiffActivity::Const => (enzyme_const, false), DiffActivity::Active => (enzyme_out, false), @@ -222,7 +222,12 @@ fn generate_enzyme_call<'ll>( // A duplicated pointer will have the following two outer_fn arguments: // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call: // (..., metadata! enzyme_dup, ptr, ptr, ...). - assert!(llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer); + if matches!(diff_activity, DiffActivity::Duplicated | DiffActivity::DuplicatedOnly) { + assert!( + llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer + ); + } + // In the case of Dual we don't have assumptions, e.g. f32 would be valid. args.push(next_outer_arg); outer_pos += 2; activity_pos += 1; -- cgit 1.4.1-3-g733a5 From ee7dc06cf181c073b1040669a40bc325d00f8c6d Mon Sep 17 00:00:00 2001 From: Zalathar Date: Mon, 20 Jan 2025 22:43:59 +1100 Subject: coverage: Store BCB node IDs in mappings, and resolve them in codegen Even though the coverage graph itself is no longer available during codegen, its nodes can still be used as opaque IDs. --- .../src/coverageinfo/mapgen/covfun.rs | 29 +++++++----- compiler/rustc_middle/src/mir/coverage.rs | 51 ++++++++++++---------- .../rustc_mir_transform/src/coverage/counters.rs | 6 +-- compiler/rustc_mir_transform/src/coverage/graph.rs | 10 +---- compiler/rustc_mir_transform/src/coverage/mod.rs | 46 ++++++++----------- compiler/rustc_mir_transform/src/coverage/query.rs | 4 +- .../branch_match_arms.main.InstrumentCoverage.diff | 12 ++--- ...instrument_coverage.bar.InstrumentCoverage.diff | 2 +- ...nstrument_coverage.main.InstrumentCoverage.diff | 10 ++--- tests/mir-opt/coverage/instrument_coverage.rs | 4 +- ..._coverage_cleanup.main.CleanupPostBorrowck.diff | 10 ++--- ...t_coverage_cleanup.main.InstrumentCoverage.diff | 10 ++--- 12 files changed, 91 insertions(+), 103 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index 460a4664615..38e7f4f21d4 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -11,7 +11,8 @@ use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, }; use rustc_middle::mir::coverage::{ - CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, MappingKind, Op, + BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, + MappingKind, Op, }; use rustc_middle::ty::{Instance, TyCtxt}; use rustc_span::Span; @@ -140,7 +141,12 @@ fn fill_region_tables<'tcx>( for &Mapping { ref kind, span } in &fn_cov_info.mappings { // If the mapping refers to counters/expressions that were removed by // MIR opts, replace those occurrences with zero. - let kind = kind.map_terms(|term| if is_zero_term(term) { CovTerm::Zero } else { term }); + let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { + let term = + fn_cov_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term"); + let term = if is_zero_term(term) { CovTerm::Zero } else { term }; + ffi::Counter::from_term(term) + }; // Convert the `Span` into coordinates that we can pass to LLVM, or // discard the span if conversion fails. In rare, cases _all_ of a @@ -154,23 +160,22 @@ fn fill_region_tables<'tcx>( continue; } - match kind { - MappingKind::Code(term) => { - code_regions - .push(ffi::CodeRegion { cov_span, counter: ffi::Counter::from_term(term) }); + match *kind { + MappingKind::Code { bcb } => { + code_regions.push(ffi::CodeRegion { cov_span, counter: counter_for_bcb(bcb) }); } - MappingKind::Branch { true_term, false_term } => { + MappingKind::Branch { true_bcb, false_bcb } => { branch_regions.push(ffi::BranchRegion { cov_span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), + true_counter: counter_for_bcb(true_bcb), + false_counter: counter_for_bcb(false_bcb), }); } - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params } => { mcdc_branch_regions.push(ffi::MCDCBranchRegion { cov_span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), + true_counter: counter_for_bcb(true_bcb), + false_counter: counter_for_bcb(false_bcb), mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), }); } diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index 46534697e1d..8123707dc73 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -179,34 +179,19 @@ pub struct Expression { #[derive(TyEncodable, TyDecodable, Hash, HashStable)] pub enum MappingKind { /// Associates a normal region of code with a counter/expression/zero. - Code(CovTerm), + Code { bcb: BasicCoverageBlock }, /// Associates a branch region with separate counters for true and false. - Branch { true_term: CovTerm, false_term: CovTerm }, + Branch { true_bcb: BasicCoverageBlock, false_bcb: BasicCoverageBlock }, /// Associates a branch region with separate counters for true and false. - MCDCBranch { true_term: CovTerm, false_term: CovTerm, mcdc_params: ConditionInfo }, + MCDCBranch { + true_bcb: BasicCoverageBlock, + false_bcb: BasicCoverageBlock, + mcdc_params: ConditionInfo, + }, /// Associates a decision region with a bitmap and number of conditions. MCDCDecision(DecisionInfo), } -impl MappingKind { - /// Returns a copy of this mapping kind, in which all coverage terms have - /// been replaced with ones returned by the given function. - pub fn map_terms(&self, map_fn: impl Fn(CovTerm) -> CovTerm) -> Self { - match *self { - Self::Code(term) => Self::Code(map_fn(term)), - Self::Branch { true_term, false_term } => { - Self::Branch { true_term: map_fn(true_term), false_term: map_fn(false_term) } - } - Self::MCDCBranch { true_term, false_term, mcdc_params } => Self::MCDCBranch { - true_term: map_fn(true_term), - false_term: map_fn(false_term), - mcdc_params, - }, - Self::MCDCDecision(param) => Self::MCDCDecision(param), - } - } -} - #[derive(Clone, Debug)] #[derive(TyEncodable, TyDecodable, Hash, HashStable)] pub struct Mapping { @@ -222,10 +207,14 @@ pub struct Mapping { pub struct FunctionCoverageInfo { pub function_source_hash: u64, pub body_span: Span, + pub num_counters: usize, - pub mcdc_bitmap_bits: usize, pub expressions: IndexVec, + pub mappings: Vec, + pub term_for_bcb: IndexVec>, + + pub mcdc_bitmap_bits: usize, /// The depth of the deepest decision is used to know how many /// temp condbitmaps should be allocated for the function. pub mcdc_num_condition_bitmaps: usize, @@ -329,3 +318,19 @@ impl CoverageIdsInfo { } } } + +rustc_index::newtype_index! { + /// During the `InstrumentCoverage` MIR pass, a BCB is a node in the + /// "coverage graph", which is a refinement of the MIR control-flow graph + /// that merges or omits some blocks that aren't relevant to coverage. + /// + /// After that pass is complete, the coverage graph no longer exists, so a + /// BCB is effectively an opaque ID. + #[derive(HashStable)] + #[encodable] + #[orderable] + #[debug_format = "bcb{}"] + pub struct BasicCoverageBlock { + const START_BCB = 0; + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index adb99a75a9e..50d5bf31829 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -133,7 +133,7 @@ pub(super) struct CoverageCounters { next_counter_id: CounterId, /// Coverage counters/expressions that are associated with individual BCBs. - node_counters: IndexVec>, + pub(crate) node_counters: IndexVec>, /// Table of expression data, associating each expression ID with its /// corresponding operator (+ or -) and its LHS/RHS operands. @@ -203,10 +203,6 @@ impl CoverageCounters { counter } - pub(super) fn term_for_bcb(&self, bcb: BasicCoverageBlock) -> Option { - self.node_counters[bcb] - } - /// Returns an iterator over all the nodes in the coverage graph that /// should have a counter-increment statement injected into MIR, along with /// each site's corresponding counter ID. diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 392b54c8d81..6d84248ddfb 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -8,6 +8,7 @@ use rustc_data_structures::graph::dominators::Dominators; use rustc_data_structures::graph::{self, DirectedGraph, StartNode}; use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; +pub(crate) use rustc_middle::mir::coverage::{BasicCoverageBlock, START_BCB}; use rustc_middle::mir::{self, BasicBlock, Terminator, TerminatorKind}; use tracing::debug; @@ -269,15 +270,6 @@ impl graph::Predecessors for CoverageGraph { } } -rustc_index::newtype_index! { - /// A node in the control-flow graph of CoverageGraph. - #[orderable] - #[debug_format = "bcb{}"] - pub(crate) struct BasicCoverageBlock { - const START_BCB = 0; - } -} - /// `BasicCoverageBlockData` holds the data indexed by a `BasicCoverageBlock`. /// /// A `BasicCoverageBlock` (BCB) represents the maximal-length sequence of MIR `BasicBlock`s without diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 15487d05a30..b1afcc84b6e 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -95,12 +95,13 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: let coverage_counters = counters::make_bcb_counters(&graph, &bcbs_with_counter_mappings); - let mappings = create_mappings(&extracted_mappings, &coverage_counters); + let mappings = create_mappings(&extracted_mappings); if mappings.is_empty() { // No spans could be converted into valid mappings, so skip this function. debug!("no spans could be converted into valid mappings; skipping"); return; } + let term_for_bcb = coverage_counters.node_counters.clone(); inject_coverage_statements(mir_body, &graph, &extracted_mappings, &coverage_counters); @@ -116,26 +117,24 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: mir_body.function_coverage_info = Some(Box::new(FunctionCoverageInfo { function_source_hash: hir_info.function_source_hash, body_span: hir_info.body_span, + num_counters: coverage_counters.num_counters(), - mcdc_bitmap_bits: extracted_mappings.mcdc_bitmap_bits, expressions: coverage_counters.into_expressions(), + mappings, + term_for_bcb, + + mcdc_bitmap_bits: extracted_mappings.mcdc_bitmap_bits, mcdc_num_condition_bitmaps, })); } -/// For each coverage span extracted from MIR, create a corresponding -/// mapping. +/// For each coverage span extracted from MIR, create a corresponding mapping. /// -/// Precondition: All BCBs corresponding to those spans have been given -/// coverage counters. -fn create_mappings( - extracted_mappings: &ExtractedMappings, - coverage_counters: &CoverageCounters, -) -> Vec { - let term_for_bcb = - |bcb| coverage_counters.term_for_bcb(bcb).expect("all BCBs with spans were given counters"); - +/// FIXME(Zalathar): This used to be where BCBs in the extracted mappings were +/// resolved to a `CovTerm`. But that is now handled elsewhere, so this +/// function can potentially be simplified even further. +fn create_mappings(extracted_mappings: &ExtractedMappings) -> Vec { // Fully destructure the mappings struct to make sure we don't miss any kinds. let ExtractedMappings { num_bcbs: _, @@ -150,23 +149,18 @@ fn create_mappings( mappings.extend(code_mappings.iter().map( // Ordinary code mappings are the simplest kind. |&mappings::CodeMapping { span, bcb }| { - let kind = MappingKind::Code(term_for_bcb(bcb)); + let kind = MappingKind::Code { bcb }; Mapping { kind, span } }, )); mappings.extend(branch_pairs.iter().map( |&mappings::BranchPair { span, true_bcb, false_bcb }| { - let true_term = term_for_bcb(true_bcb); - let false_term = term_for_bcb(false_bcb); - let kind = MappingKind::Branch { true_term, false_term }; + let kind = MappingKind::Branch { true_bcb, false_bcb }; Mapping { kind, span } }, )); - let term_for_bcb = - |bcb| coverage_counters.term_for_bcb(bcb).expect("all BCBs with spans were given counters"); - // MCDC branch mappings are appended with their decisions in case decisions were ignored. mappings.extend(mcdc_degraded_branches.iter().map( |&mappings::MCDCBranch { @@ -176,11 +170,7 @@ fn create_mappings( condition_info: _, true_index: _, false_index: _, - }| { - let true_term = term_for_bcb(true_bcb); - let false_term = term_for_bcb(false_bcb); - Mapping { kind: MappingKind::Branch { true_term, false_term }, span } - }, + }| { Mapping { kind: MappingKind::Branch { true_bcb, false_bcb }, span } }, )); for (decision, branches) in mcdc_mappings { @@ -201,12 +191,10 @@ fn create_mappings( true_index: _, false_index: _, }| { - let true_term = term_for_bcb(true_bcb); - let false_term = term_for_bcb(false_bcb); Mapping { kind: MappingKind::MCDCBranch { - true_term, - false_term, + true_bcb, + false_bcb, mcdc_params: condition_info, }, span, diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index a849ed4c3e2..ef2724222ce 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -101,7 +101,9 @@ fn coverage_ids_info<'tcx>( // to any particular point in the control-flow graph. // (Keep this in sync with the injection of `ExpressionUsed` // statements in the `InstrumentCoverage` MIR pass.) - if let MappingKind::Code(CovTerm::Expression(id)) = mapping.kind { + if let MappingKind::Code { bcb } = mapping.kind + && let Some(CovTerm::Expression(id)) = fn_cov_info.term_for_bcb[bcb] + { expressions_seen.remove(id); } } diff --git a/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff index 138586300ce..9647e13f75a 100644 --- a/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff @@ -30,12 +30,12 @@ + coverage ExpressionId(0) => Expression { lhs: Counter(1), op: Add, rhs: Counter(2) }; + coverage ExpressionId(1) => Expression { lhs: Expression(0), op: Add, rhs: Counter(3) }; + coverage ExpressionId(2) => Expression { lhs: Counter(0), op: Subtract, rhs: Expression(1) }; -+ coverage Code(Counter(0)) => $DIR/branch_match_arms.rs:14:1: 15:21 (#0); -+ coverage Code(Counter(1)) => $DIR/branch_match_arms.rs:16:17: 16:33 (#0); -+ coverage Code(Counter(2)) => $DIR/branch_match_arms.rs:17:17: 17:33 (#0); -+ coverage Code(Counter(3)) => $DIR/branch_match_arms.rs:18:17: 18:33 (#0); -+ coverage Code(Expression(2)) => $DIR/branch_match_arms.rs:19:17: 19:33 (#0); -+ coverage Code(Counter(0)) => $DIR/branch_match_arms.rs:21:2: 21:2 (#0); ++ coverage Code { bcb: bcb0 } => $DIR/branch_match_arms.rs:14:1: 15:21 (#0); ++ coverage Code { bcb: bcb1 } => $DIR/branch_match_arms.rs:16:17: 16:33 (#0); ++ coverage Code { bcb: bcb3 } => $DIR/branch_match_arms.rs:17:17: 17:33 (#0); ++ coverage Code { bcb: bcb4 } => $DIR/branch_match_arms.rs:18:17: 18:33 (#0); ++ coverage Code { bcb: bcb5 } => $DIR/branch_match_arms.rs:19:17: 19:33 (#0); ++ coverage Code { bcb: bcb2 } => $DIR/branch_match_arms.rs:21:2: 21:2 (#0); + bb0: { + Coverage::CounterIncrement(0); diff --git a/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff index a91d88984a8..16b7a1a63b4 100644 --- a/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff @@ -5,7 +5,7 @@ let mut _0: bool; + coverage body span: $DIR/instrument_coverage.rs:29:18: 31:2 (#0) -+ coverage Code(Counter(0)) => $DIR/instrument_coverage.rs:29:1: 31:2 (#0); ++ coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:29:1: 31:2 (#0); + bb0: { + Coverage::CounterIncrement(0); diff --git a/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff index d7ea442518e..fe74ad24e91 100644 --- a/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff @@ -9,11 +9,11 @@ + coverage body span: $DIR/instrument_coverage.rs:14:11: 20:2 (#0) + coverage ExpressionId(0) => Expression { lhs: Counter(1), op: Subtract, rhs: Counter(0) }; -+ coverage Code(Counter(0)) => $DIR/instrument_coverage.rs:14:1: 14:11 (#0); -+ coverage Code(Counter(1)) => $DIR/instrument_coverage.rs:16:12: 16:17 (#0); -+ coverage Code(Counter(0)) => $DIR/instrument_coverage.rs:17:13: 17:18 (#0); -+ coverage Code(Expression(0)) => $DIR/instrument_coverage.rs:18:10: 18:10 (#0); -+ coverage Code(Counter(0)) => $DIR/instrument_coverage.rs:20:2: 20:2 (#0); ++ coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:14:1: 14:11 (#0); ++ coverage Code { bcb: bcb1 } => $DIR/instrument_coverage.rs:16:12: 16:17 (#0); ++ coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:17:13: 17:18 (#0); ++ coverage Code { bcb: bcb3 } => $DIR/instrument_coverage.rs:18:10: 18:10 (#0); ++ coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:20:2: 20:2 (#0); + bb0: { + Coverage::CounterIncrement(0); diff --git a/tests/mir-opt/coverage/instrument_coverage.rs b/tests/mir-opt/coverage/instrument_coverage.rs index c49786f9615..20c730645f8 100644 --- a/tests/mir-opt/coverage/instrument_coverage.rs +++ b/tests/mir-opt/coverage/instrument_coverage.rs @@ -8,7 +8,7 @@ // EMIT_MIR instrument_coverage.main.InstrumentCoverage.diff // CHECK-LABEL: fn main() // CHECK: coverage body span: -// CHECK: coverage Code(Counter({{[0-9]+}})) => +// CHECK: coverage Code { bcb: bcb{{[0-9]+}} } => // CHECK: bb0: // CHECK: Coverage::CounterIncrement fn main() { @@ -22,7 +22,7 @@ fn main() { // EMIT_MIR instrument_coverage.bar.InstrumentCoverage.diff // CHECK-LABEL: fn bar() // CHECK: coverage body span: -// CHECK: coverage Code(Counter({{[0-9]+}})) => +// CHECK: coverage Code { bcb: bcb{{[0-9]+}} } => // CHECK: bb0: // CHECK: Coverage::CounterIncrement #[inline(never)] diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff index 2c7ec6e85eb..28a3f3beade 100644 --- a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff +++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff @@ -9,11 +9,11 @@ coverage body span: $DIR/instrument_coverage_cleanup.rs:13:11: 15:2 (#0) coverage ExpressionId(0) => Expression { lhs: Counter(0), op: Subtract, rhs: Counter(1) }; - coverage Code(Counter(0)) => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0); - coverage Code(Expression(0)) => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0); - coverage Code(Counter(1)) => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0); - coverage Code(Counter(0)) => $DIR/instrument_coverage_cleanup.rs:15:2: 15:2 (#0); - coverage Branch { true_term: Expression(0), false_term: Counter(1) } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0); + coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0); + coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0); + coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0); + coverage Code { bcb: bcb2 } => $DIR/instrument_coverage_cleanup.rs:15:2: 15:2 (#0); + coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0); bb0: { Coverage::CounterIncrement(0); diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff index c08265eb0e9..0f1dea02b8d 100644 --- a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff @@ -9,11 +9,11 @@ + coverage body span: $DIR/instrument_coverage_cleanup.rs:13:11: 15:2 (#0) + coverage ExpressionId(0) => Expression { lhs: Counter(0), op: Subtract, rhs: Counter(1) }; -+ coverage Code(Counter(0)) => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0); -+ coverage Code(Expression(0)) => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0); -+ coverage Code(Counter(1)) => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0); -+ coverage Code(Counter(0)) => $DIR/instrument_coverage_cleanup.rs:15:2: 15:2 (#0); -+ coverage Branch { true_term: Expression(0), false_term: Counter(1) } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0); ++ coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0); ++ coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0); ++ coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0); ++ coverage Code { bcb: bcb2 } => $DIR/instrument_coverage_cleanup.rs:15:2: 15:2 (#0); ++ coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0); + bb0: { + Coverage::CounterIncrement(0); -- cgit 1.4.1-3-g733a5 From 20d051ec870739c8f263e5f6f581ca24a5dd56fd Mon Sep 17 00:00:00 2001 From: Zalathar Date: Wed, 22 Jan 2025 13:55:08 +1100 Subject: coverage: Defer part of counter-creation until codegen --- .../src/coverageinfo/mapgen/covfun.rs | 13 +-- .../rustc_codegen_llvm/src/coverageinfo/mod.rs | 19 ++--- compiler/rustc_codegen_llvm/src/lib.rs | 1 + compiler/rustc_middle/src/mir/coverage.rs | 70 ++++++++++------ compiler/rustc_middle/src/mir/pretty.rs | 6 +- compiler/rustc_middle/src/query/mod.rs | 13 ++- .../rustc_mir_transform/src/coverage/counters.rs | 69 +++++----------- .../src/coverage/counters/node_flow.rs | 25 +----- .../rustc_mir_transform/src/coverage/mappings.rs | 61 -------------- compiler/rustc_mir_transform/src/coverage/mod.rs | 70 ++++------------ compiler/rustc_mir_transform/src/coverage/query.rs | 94 ++++++++++++++++------ .../src/coverage/spans/from_mir.rs | 3 +- .../branch_match_arms.main.InstrumentCoverage.diff | 14 ++-- ...instrument_coverage.bar.InstrumentCoverage.diff | 2 +- ...nstrument_coverage.main.InstrumentCoverage.diff | 8 +- tests/mir-opt/coverage/instrument_coverage.rs | 4 +- ..._coverage_cleanup.main.CleanupPostBorrowck.diff | 8 +- ...t_coverage_cleanup.main.InstrumentCoverage.diff | 8 +- .../coverage/instrument_coverage_cleanup.rs | 4 +- 19 files changed, 196 insertions(+), 296 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index 38e7f4f21d4..1290c6efb09 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -54,7 +54,7 @@ pub(crate) fn prepare_covfun_record<'tcx>( let fn_cov_info = tcx.instance_mir(instance.def).function_coverage_info.as_deref()?; let ids_info = tcx.coverage_ids_info(instance.def)?; - let expressions = prepare_expressions(fn_cov_info, ids_info, is_used); + let expressions = prepare_expressions(ids_info, is_used); let mut covfun = CovfunRecord { mangled_function_name: tcx.symbol_name(instance).name, @@ -76,11 +76,7 @@ pub(crate) fn prepare_covfun_record<'tcx>( } /// Convert the function's coverage-counter expressions into a form suitable for FFI. -fn prepare_expressions( - fn_cov_info: &FunctionCoverageInfo, - ids_info: &CoverageIdsInfo, - is_used: bool, -) -> Vec { +fn prepare_expressions(ids_info: &CoverageIdsInfo, is_used: bool) -> Vec { // If any counters or expressions were removed by MIR opts, replace their // terms with zero. let counter_for_term = |term| { @@ -95,7 +91,7 @@ fn prepare_expressions( // producing the final coverage map, so there's no need to do the same // thing on the Rust side unless we're confident we can do much better. // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.) - fn_cov_info + ids_info .expressions .iter() .map(move |&Expression { lhs, op, rhs }| ffi::CounterExpression { @@ -142,8 +138,7 @@ fn fill_region_tables<'tcx>( // If the mapping refers to counters/expressions that were removed by // MIR opts, replace those occurrences with zero. let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { - let term = - fn_cov_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term"); + let term = ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term"); let term = if is_zero_term(term) { CovTerm::Zero } else { term }; ffi::Counter::from_term(term) }; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 021108cd51c..e404db5e196 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -160,17 +160,10 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( "marker statement {kind:?} should have been removed by CleanupPostBorrowck" ), - CoverageKind::CounterIncrement { id } => { - // The number of counters passed to `llvm.instrprof.increment` might - // be smaller than the number originally inserted by the instrumentor, - // if some high-numbered counters were removed by MIR optimizations. - // If so, LLVM's profiler runtime will use fewer physical counters. + CoverageKind::VirtualCounter { bcb } + if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) => + { let num_counters = ids_info.num_counters_after_mir_opts(); - assert!( - num_counters as usize <= function_coverage_info.num_counters, - "num_counters disagreement: query says {num_counters} but function info only has {}", - function_coverage_info.num_counters - ); let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); @@ -182,10 +175,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { ); bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::ExpressionUsed { id: _ } => { - // Expression-used statements are markers that are handled by - // `coverage_ids_info`, so there's nothing to codegen here. - } + // If a BCB doesn't have an associated physical counter, there's nothing to codegen. + CoverageKind::VirtualCounter { .. } => {} CoverageKind::CondBitmapUpdate { index, decision_depth } => { let cond_bitmap = coverage_cx .try_get_mcdc_condition_bitmap(&instance, decision_depth) diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 14346795fda..f0d04b2b644 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -13,6 +13,7 @@ #![feature(extern_types)] #![feature(file_buffered)] #![feature(hash_raw_entry)] +#![feature(if_let_guard)] #![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] #![feature(let_chains)] diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index 8123707dc73..31b9eeb715b 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -2,8 +2,9 @@ use std::fmt::{self, Debug, Formatter}; -use rustc_index::IndexVec; +use rustc_data_structures::fx::FxIndexMap; use rustc_index::bit_set::DenseBitSet; +use rustc_index::{Idx, IndexVec}; use rustc_macros::{HashStable, TyDecodable, TyEncodable}; use rustc_span::Span; @@ -103,23 +104,12 @@ pub enum CoverageKind { /// Should be erased before codegen (at some point after `InstrumentCoverage`). BlockMarker { id: BlockMarkerId }, - /// Marks the point in MIR control flow represented by a coverage counter. + /// Marks its enclosing basic block with the ID of the coverage graph node + /// that it was part of during the `InstrumentCoverage` MIR pass. /// - /// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR. - /// - /// If this statement does not survive MIR optimizations, any mappings that - /// refer to this counter can have those references simplified to zero. - CounterIncrement { id: CounterId }, - - /// Marks the point in MIR control-flow represented by a coverage expression. - /// - /// If this statement does not survive MIR optimizations, any mappings that - /// refer to this expression can have those references simplified to zero. - /// - /// (This is only inserted for expression IDs that are directly used by - /// mappings. Intermediate expressions with no direct mappings are - /// retained/zeroed based on whether they are transitively used.) - ExpressionUsed { id: ExpressionId }, + /// During codegen, this might be lowered to `llvm.instrprof.increment` or + /// to a no-op, depending on the outcome of counter-creation. + VirtualCounter { bcb: BasicCoverageBlock }, /// Marks the point in MIR control flow represented by a evaluated condition. /// @@ -138,8 +128,7 @@ impl Debug for CoverageKind { match self { SpanMarker => write!(fmt, "SpanMarker"), BlockMarker { id } => write!(fmt, "BlockMarker({:?})", id.index()), - CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()), - ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()), + VirtualCounter { bcb } => write!(fmt, "VirtualCounter({bcb:?})"), CondBitmapUpdate { index, decision_depth } => { write!(fmt, "CondBitmapUpdate(index={:?}, depth={:?})", index, decision_depth) } @@ -208,11 +197,12 @@ pub struct FunctionCoverageInfo { pub function_source_hash: u64, pub body_span: Span, - pub num_counters: usize, - pub expressions: IndexVec, + /// Used in conjunction with `priority_list` to create physical counters + /// and counter expressions, after MIR optimizations. + pub node_flow_data: NodeFlowData, + pub priority_list: Vec, pub mappings: Vec, - pub term_for_bcb: IndexVec>, pub mcdc_bitmap_bits: usize, /// The depth of the deepest decision is used to know how many @@ -281,15 +271,18 @@ pub struct MCDCDecisionSpan { pub num_conditions: usize, } -/// Summarizes coverage IDs inserted by the `InstrumentCoverage` MIR pass -/// (for compiler option `-Cinstrument-coverage`), after MIR optimizations -/// have had a chance to potentially remove some of them. +/// Contains information needed during codegen, obtained by inspecting the +/// function's MIR after MIR optimizations. /// -/// Used by the `coverage_ids_info` query. +/// Returned by the `coverage_ids_info` query. #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable)] pub struct CoverageIdsInfo { pub counters_seen: DenseBitSet, pub zero_expressions: DenseBitSet, + + pub phys_counter_for_node: FxIndexMap, + pub term_for_bcb: IndexVec>, + pub expressions: IndexVec, } impl CoverageIdsInfo { @@ -334,3 +327,28 @@ rustc_index::newtype_index! { const START_BCB = 0; } } + +/// Data representing a view of some underlying graph, in which each node's +/// successors have been merged into a single "supernode". +/// +/// The resulting supernodes have no obvious meaning on their own. +/// However, merging successor nodes means that a node's out-edges can all +/// be combined into a single out-edge, whose flow is the same as the flow +/// (execution count) of its corresponding node in the original graph. +/// +/// With all node flows now in the original graph now represented as edge flows +/// in the merged graph, it becomes possible to analyze the original node flows +/// using techniques for analyzing edge flows. +#[derive(Clone, Debug)] +#[derive(TyEncodable, TyDecodable, Hash, HashStable)] +pub struct NodeFlowData { + /// Maps each node to the supernode that contains it, indicated by some + /// arbitrary "root" node that is part of that supernode. + pub supernodes: IndexVec, + /// For each node, stores the single supernode that all of its successors + /// have been merged into. + /// + /// (Note that each node in a supernode can potentially have a _different_ + /// successor supernode from its peers.) + pub succ_supernodes: IndexVec, +} diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 3007b787496..11ebbbe807d 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -619,13 +619,9 @@ fn write_function_coverage_info( function_coverage_info: &coverage::FunctionCoverageInfo, w: &mut dyn io::Write, ) -> io::Result<()> { - let coverage::FunctionCoverageInfo { body_span, expressions, mappings, .. } = - function_coverage_info; + let coverage::FunctionCoverageInfo { body_span, mappings, .. } = function_coverage_info; writeln!(w, "{INDENT}coverage body span: {body_span:?}")?; - for (id, expression) in expressions.iter_enumerated() { - writeln!(w, "{INDENT}coverage {id:?} => {expression:?};")?; - } for coverage::Mapping { kind, span } in mappings { writeln!(w, "{INDENT}coverage {kind:?} => {span:?};")?; } diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 6c442fc1047..228f94a17f8 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -615,9 +615,16 @@ rustc_queries! { feedable } - /// Summarizes coverage IDs inserted by the `InstrumentCoverage` MIR pass - /// (for compiler option `-Cinstrument-coverage`), after MIR optimizations - /// have had a chance to potentially remove some of them. + /// Scans through a function's MIR after MIR optimizations, to prepare the + /// information needed by codegen when `-Cinstrument-coverage` is active. + /// + /// This includes the details of where to insert `llvm.instrprof.increment` + /// intrinsics, and the expression tables to be embedded in the function's + /// coverage metadata. + /// + /// FIXME(Zalathar): This query's purpose has drifted a bit and should + /// probably be renamed, but that can wait until after the potential + /// follow-ups to #136053 have settled down. /// /// Returns `None` for functions that were not instrumented. query coverage_ids_info(key: ty::InstanceKind<'tcx>) -> Option<&'tcx mir::coverage::CoverageIdsInfo> { diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index 50d5bf31829..fa4f80f827b 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -2,7 +2,6 @@ use std::cmp::Ordering; use either::Either; use itertools::Itertools; -use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_data_structures::graph::DirectedGraph; use rustc_index::IndexVec; @@ -11,31 +10,35 @@ use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, use crate::coverage::counters::balanced_flow::BalancedFlowGraph; use crate::coverage::counters::node_flow::{ - CounterTerm, NodeCounters, make_node_counters, node_flow_data_for_balanced_graph, + CounterTerm, NodeCounters, NodeFlowData, node_flow_data_for_balanced_graph, }; use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph}; mod balanced_flow; -mod node_flow; +pub(crate) mod node_flow; mod union_find; -/// Ensures that each BCB node needing a counter has one, by creating physical -/// counters or counter expressions for nodes as required. -pub(super) fn make_bcb_counters( - graph: &CoverageGraph, - bcb_needs_counter: &DenseBitSet, -) -> CoverageCounters { +/// Struct containing the results of [`prepare_bcb_counters_data`]. +pub(crate) struct BcbCountersData { + pub(crate) node_flow_data: NodeFlowData, + pub(crate) priority_list: Vec, +} + +/// Analyzes the coverage graph to create intermediate data structures that +/// will later be used (during codegen) to create physical counters or counter +/// expressions for each BCB node that needs one. +pub(crate) fn prepare_bcb_counters_data(graph: &CoverageGraph) -> BcbCountersData { // Create the derived graphs that are necessary for subsequent steps. let balanced_graph = BalancedFlowGraph::for_graph(graph, |n| !graph[n].is_out_summable); let node_flow_data = node_flow_data_for_balanced_graph(&balanced_graph); - // Use those graphs to determine which nodes get physical counters, and how - // to compute the execution counts of other nodes from those counters. + // Also create a "priority list" of coverage graph nodes, to help determine + // which ones get physical counters or counter expressions. This needs to + // be done now, because the later parts of the counter-creation process + // won't have access to the original coverage graph. let priority_list = make_node_flow_priority_list(graph, balanced_graph); - let node_counters = make_node_counters(&node_flow_data, &priority_list); - // Convert the counters into a form suitable for embedding into MIR. - transcribe_counters(&node_counters, bcb_needs_counter) + BcbCountersData { node_flow_data, priority_list } } /// Arranges the nodes in `balanced_graph` into a list, such that earlier nodes @@ -74,7 +77,7 @@ fn make_node_flow_priority_list( } // Converts node counters into a form suitable for embedding into MIR. -fn transcribe_counters( +pub(crate) fn transcribe_counters( old: &NodeCounters, bcb_needs_counter: &DenseBitSet, ) -> CoverageCounters { @@ -129,7 +132,7 @@ fn transcribe_counters( pub(super) struct CoverageCounters { /// List of places where a counter-increment statement should be injected /// into MIR, each with its corresponding counter ID. - phys_counter_for_node: FxIndexMap, + pub(crate) phys_counter_for_node: FxIndexMap, next_counter_id: CounterId, /// Coverage counters/expressions that are associated with individual BCBs. @@ -137,7 +140,7 @@ pub(super) struct CoverageCounters { /// Table of expression data, associating each expression ID with its /// corresponding operator (+ or -) and its LHS/RHS operands. - expressions: IndexVec, + pub(crate) expressions: IndexVec, /// Remember expressions that have already been created (or simplified), /// so that we don't create unnecessary duplicates. expressions_memo: FxHashMap, @@ -188,12 +191,6 @@ impl CoverageCounters { self.make_expression(lhs, Op::Subtract, rhs_sum) } - pub(super) fn num_counters(&self) -> usize { - let num_counters = self.phys_counter_for_node.len(); - assert_eq!(num_counters, self.next_counter_id.as_usize()); - num_counters - } - fn set_node_counter(&mut self, bcb: BasicCoverageBlock, counter: CovTerm) -> CovTerm { let existing = self.node_counters[bcb].replace(counter); assert!( @@ -202,30 +199,4 @@ impl CoverageCounters { ); counter } - - /// Returns an iterator over all the nodes in the coverage graph that - /// should have a counter-increment statement injected into MIR, along with - /// each site's corresponding counter ID. - pub(super) fn counter_increment_sites( - &self, - ) -> impl Iterator + Captures<'_> { - self.phys_counter_for_node.iter().map(|(&site, &id)| (id, site)) - } - - /// Returns an iterator over the subset of BCB nodes that have been associated - /// with a counter *expression*, along with the ID of that expression. - pub(super) fn bcb_nodes_with_coverage_expressions( - &self, - ) -> impl Iterator + Captures<'_> { - self.node_counters.iter_enumerated().filter_map(|(bcb, &counter)| match counter { - // Yield the BCB along with its associated expression ID. - Some(CovTerm::Expression(id)) => Some((bcb, id)), - // This BCB is associated with a counter or nothing, so skip it. - Some(CovTerm::Counter { .. } | CovTerm::Zero) | None => None, - }) - } - - pub(super) fn into_expressions(self) -> IndexVec { - self.expressions - } } diff --git a/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs b/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs index 9d80b3af42d..91ed54b8b59 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs @@ -9,6 +9,7 @@ use rustc_data_structures::graph; use rustc_index::bit_set::DenseBitSet; use rustc_index::{Idx, IndexSlice, IndexVec}; +pub(crate) use rustc_middle::mir::coverage::NodeFlowData; use rustc_middle::mir::coverage::Op; use crate::coverage::counters::union_find::UnionFind; @@ -16,30 +17,6 @@ use crate::coverage::counters::union_find::UnionFind; #[cfg(test)] mod tests; -/// Data representing a view of some underlying graph, in which each node's -/// successors have been merged into a single "supernode". -/// -/// The resulting supernodes have no obvious meaning on their own. -/// However, merging successor nodes means that a node's out-edges can all -/// be combined into a single out-edge, whose flow is the same as the flow -/// (execution count) of its corresponding node in the original graph. -/// -/// With all node flows now in the original graph now represented as edge flows -/// in the merged graph, it becomes possible to analyze the original node flows -/// using techniques for analyzing edge flows. -#[derive(Debug)] -pub(crate) struct NodeFlowData { - /// Maps each node to the supernode that contains it, indicated by some - /// arbitrary "root" node that is part of that supernode. - supernodes: IndexVec, - /// For each node, stores the single supernode that all of its successors - /// have been merged into. - /// - /// (Note that each node in a supernode can potentially have a _different_ - /// successor supernode from its peers.) - succ_supernodes: IndexVec, -} - /// Creates a "merged" view of an underlying graph. /// /// The given graph is assumed to have [“balanced flow”](balanced-flow), diff --git a/compiler/rustc_mir_transform/src/coverage/mappings.rs b/compiler/rustc_mir_transform/src/coverage/mappings.rs index 8d0d92dc367..d83c0d40a7e 100644 --- a/compiler/rustc_mir_transform/src/coverage/mappings.rs +++ b/compiler/rustc_mir_transform/src/coverage/mappings.rs @@ -1,9 +1,7 @@ use std::collections::BTreeSet; use rustc_data_structures::fx::FxIndexMap; -use rustc_data_structures::graph::DirectedGraph; use rustc_index::IndexVec; -use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::coverage::{ BlockMarkerId, BranchSpan, ConditionId, ConditionInfo, CoverageInfoHi, CoverageKind, }; @@ -63,10 +61,6 @@ const MCDC_MAX_BITMAP_SIZE: usize = i32::MAX as usize; #[derive(Default)] pub(super) struct ExtractedMappings { - /// Store our own copy of [`CoverageGraph::num_nodes`], so that we don't - /// need access to the whole graph when allocating per-BCB data. This is - /// only public so that other code can still use exhaustive destructuring. - pub(super) num_bcbs: usize, pub(super) code_mappings: Vec, pub(super) branch_pairs: Vec, pub(super) mcdc_bitmap_bits: usize, @@ -118,7 +112,6 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>( ); ExtractedMappings { - num_bcbs: graph.num_nodes(), code_mappings, branch_pairs, mcdc_bitmap_bits, @@ -127,60 +120,6 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>( } } -impl ExtractedMappings { - pub(super) fn all_bcbs_with_counter_mappings(&self) -> DenseBitSet { - // Fully destructure self to make sure we don't miss any fields that have mappings. - let Self { - num_bcbs, - code_mappings, - branch_pairs, - mcdc_bitmap_bits: _, - mcdc_degraded_branches, - mcdc_mappings, - } = self; - - // Identify which BCBs have one or more mappings. - let mut bcbs_with_counter_mappings = DenseBitSet::new_empty(*num_bcbs); - let mut insert = |bcb| { - bcbs_with_counter_mappings.insert(bcb); - }; - - for &CodeMapping { span: _, bcb } in code_mappings { - insert(bcb); - } - for &BranchPair { true_bcb, false_bcb, .. } in branch_pairs { - insert(true_bcb); - insert(false_bcb); - } - for &MCDCBranch { true_bcb, false_bcb, .. } in mcdc_degraded_branches - .iter() - .chain(mcdc_mappings.iter().map(|(_, branches)| branches.into_iter()).flatten()) - { - insert(true_bcb); - insert(false_bcb); - } - - // MC/DC decisions refer to BCBs, but don't require those BCBs to have counters. - if bcbs_with_counter_mappings.is_empty() { - debug_assert!( - mcdc_mappings.is_empty(), - "A function with no counter mappings shouldn't have any decisions: {mcdc_mappings:?}", - ); - } - - bcbs_with_counter_mappings - } - - /// Returns the set of BCBs that have one or more `Code` mappings. - pub(super) fn bcbs_with_ordinary_code_mappings(&self) -> DenseBitSet { - let mut bcbs = DenseBitSet::new_empty(self.num_bcbs); - for &CodeMapping { span: _, bcb } in &self.code_mappings { - bcbs.insert(bcb); - } - bcbs - } -} - fn resolve_block_markers( coverage_info_hi: &CoverageInfoHi, mir_body: &mir::Body<'_>, diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index b1afcc84b6e..e195681bc92 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -21,7 +21,7 @@ use rustc_span::Span; use rustc_span::def_id::LocalDefId; use tracing::{debug, debug_span, trace}; -use crate::coverage::counters::CoverageCounters; +use crate::coverage::counters::BcbCountersData; use crate::coverage::graph::CoverageGraph; use crate::coverage::mappings::ExtractedMappings; @@ -82,29 +82,21 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: let extracted_mappings = mappings::extract_all_mapping_info_from_mir(tcx, mir_body, &hir_info, &graph); - //////////////////////////////////////////////////// - // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure - // every coverage span has a `Counter` or `Expression` assigned to its `BasicCoverageBlock` - // and all `Expression` dependencies (operands) are also generated, for any other - // `BasicCoverageBlock`s not already associated with a coverage span. - let bcbs_with_counter_mappings = extracted_mappings.all_bcbs_with_counter_mappings(); - if bcbs_with_counter_mappings.is_empty() { - // No relevant spans were found in MIR, so skip instrumenting this function. - return; - } - - let coverage_counters = counters::make_bcb_counters(&graph, &bcbs_with_counter_mappings); - let mappings = create_mappings(&extracted_mappings); if mappings.is_empty() { // No spans could be converted into valid mappings, so skip this function. debug!("no spans could be converted into valid mappings; skipping"); return; } - let term_for_bcb = coverage_counters.node_counters.clone(); - inject_coverage_statements(mir_body, &graph, &extracted_mappings, &coverage_counters); + // Use the coverage graph to prepare intermediate data that will eventually + // be used to assign physical counters and counter expressions to points in + // the control-flow graph + let BcbCountersData { node_flow_data, priority_list } = + counters::prepare_bcb_counters_data(&graph); + // Inject coverage statements into MIR. + inject_coverage_statements(mir_body, &graph); inject_mcdc_statements(mir_body, &graph, &extracted_mappings); let mcdc_num_condition_bitmaps = extracted_mappings @@ -118,11 +110,10 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: function_source_hash: hir_info.function_source_hash, body_span: hir_info.body_span, - num_counters: coverage_counters.num_counters(), - expressions: coverage_counters.into_expressions(), + node_flow_data, + priority_list, mappings, - term_for_bcb, mcdc_bitmap_bits: extracted_mappings.mcdc_bitmap_bits, mcdc_num_condition_bitmaps, @@ -137,7 +128,6 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: fn create_mappings(extracted_mappings: &ExtractedMappings) -> Vec { // Fully destructure the mappings struct to make sure we don't miss any kinds. let ExtractedMappings { - num_bcbs: _, code_mappings, branch_pairs, mcdc_bitmap_bits: _, @@ -215,41 +205,11 @@ fn create_mappings(extracted_mappings: &ExtractedMappings) -> Vec { mappings } -/// For each BCB node or BCB edge that has an associated coverage counter, -/// inject any necessary coverage statements into MIR. -fn inject_coverage_statements<'tcx>( - mir_body: &mut mir::Body<'tcx>, - graph: &CoverageGraph, - extracted_mappings: &ExtractedMappings, - coverage_counters: &CoverageCounters, -) { - // Inject counter-increment statements into MIR. - for (id, bcb) in coverage_counters.counter_increment_sites() { - let target_bb = graph[bcb].leader_bb(); - inject_statement(mir_body, CoverageKind::CounterIncrement { id }, target_bb); - } - - // For each counter expression that is directly associated with at least one - // span, we inject an "expression-used" statement, so that coverage codegen - // can check whether the injected statement survived MIR optimization. - // (BCB edges can't have spans, so we only need to process BCB nodes here.) - // - // We only do this for ordinary `Code` mappings, because branch and MC/DC - // mappings might have expressions that don't correspond to any single - // point in the control-flow graph. - // - // See the code in `rustc_codegen_llvm::coverageinfo::map_data` that deals - // with "expressions seen" and "zero terms". - let eligible_bcbs = extracted_mappings.bcbs_with_ordinary_code_mappings(); - for (bcb, expression_id) in coverage_counters - .bcb_nodes_with_coverage_expressions() - .filter(|&(bcb, _)| eligible_bcbs.contains(bcb)) - { - inject_statement( - mir_body, - CoverageKind::ExpressionUsed { id: expression_id }, - graph[bcb].leader_bb(), - ); +/// Inject any necessary coverage statements into MIR, so that they influence codegen. +fn inject_coverage_statements<'tcx>(mir_body: &mut mir::Body<'tcx>, graph: &CoverageGraph) { + for (bcb, data) in graph.iter_enumerated() { + let target_bb = data.leader_bb(); + inject_statement(mir_body, CoverageKind::VirtualCounter { bcb }, target_bb); } } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index ef2724222ce..f362f4ccfa2 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -1,9 +1,10 @@ use rustc_data_structures::captures::Captures; +use rustc_index::IndexSlice; use rustc_index::bit_set::DenseBitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::{ - CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, ExpressionId, - FunctionCoverageInfo, MappingKind, Op, + BasicCoverageBlock, CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, + ExpressionId, MappingKind, Op, }; use rustc_middle::mir::{Body, Statement, StatementKind}; use rustc_middle::ty::{self, TyCtxt}; @@ -12,6 +13,9 @@ use rustc_span::def_id::LocalDefId; use rustc_span::sym; use tracing::trace; +use crate::coverage::counters::node_flow::make_node_counters; +use crate::coverage::counters::{CoverageCounters, transcribe_counters}; + /// Registers query/hook implementations related to coverage. pub(crate) fn provide(providers: &mut Providers) { providers.hooks.is_eligible_for_coverage = is_eligible_for_coverage; @@ -89,41 +93,85 @@ fn coverage_ids_info<'tcx>( let mir_body = tcx.instance_mir(instance_def); let fn_cov_info = mir_body.function_coverage_info.as_deref()?; - let mut counters_seen = DenseBitSet::new_empty(fn_cov_info.num_counters); - let mut expressions_seen = DenseBitSet::new_filled(fn_cov_info.expressions.len()); + // Scan through the final MIR to see which BCBs survived MIR opts. + // Any BCB not in this set was optimized away. + let mut bcbs_seen = DenseBitSet::new_empty(fn_cov_info.priority_list.len()); + for kind in all_coverage_in_mir_body(mir_body) { + match *kind { + CoverageKind::VirtualCounter { bcb } => { + bcbs_seen.insert(bcb); + } + _ => {} + } + } + + // Determine the set of BCBs that are referred to by mappings, and therefore + // need a counter. Any node not in this set will only get a counter if it + // is part of the counter expression for a node that is in the set. + let mut bcb_needs_counter = + DenseBitSet::::new_empty(fn_cov_info.priority_list.len()); + for mapping in &fn_cov_info.mappings { + match mapping.kind { + MappingKind::Code { bcb } => { + bcb_needs_counter.insert(bcb); + } + MappingKind::Branch { true_bcb, false_bcb } => { + bcb_needs_counter.insert(true_bcb); + bcb_needs_counter.insert(false_bcb); + } + MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params: _ } => { + bcb_needs_counter.insert(true_bcb); + bcb_needs_counter.insert(false_bcb); + } + MappingKind::MCDCDecision(_) => {} + } + } + + let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &fn_cov_info.priority_list); + let coverage_counters = transcribe_counters(&node_counters, &bcb_needs_counter); + + let mut counters_seen = DenseBitSet::new_empty(coverage_counters.node_counters.len()); + let mut expressions_seen = DenseBitSet::new_filled(coverage_counters.expressions.len()); // For each expression ID that is directly used by one or more mappings, // mark it as not-yet-seen. This indicates that we expect to see a - // corresponding `ExpressionUsed` statement during MIR traversal. + // corresponding `VirtualCounter` statement during MIR traversal. for mapping in fn_cov_info.mappings.iter() { // Currently we only worry about ordinary code mappings. // For branch and MC/DC mappings, expressions might not correspond // to any particular point in the control-flow graph. - // (Keep this in sync with the injection of `ExpressionUsed` - // statements in the `InstrumentCoverage` MIR pass.) if let MappingKind::Code { bcb } = mapping.kind - && let Some(CovTerm::Expression(id)) = fn_cov_info.term_for_bcb[bcb] + && let Some(CovTerm::Expression(id)) = coverage_counters.node_counters[bcb] { expressions_seen.remove(id); } } - for kind in all_coverage_in_mir_body(mir_body) { - match *kind { - CoverageKind::CounterIncrement { id } => { - counters_seen.insert(id); - } - CoverageKind::ExpressionUsed { id } => { - expressions_seen.insert(id); - } - _ => {} + for bcb in bcbs_seen.iter() { + if let Some(&id) = coverage_counters.phys_counter_for_node.get(&bcb) { + counters_seen.insert(id); + } + if let Some(CovTerm::Expression(id)) = coverage_counters.node_counters[bcb] { + expressions_seen.insert(id); } } - let zero_expressions = - identify_zero_expressions(fn_cov_info, &counters_seen, &expressions_seen); + let zero_expressions = identify_zero_expressions( + &coverage_counters.expressions, + &counters_seen, + &expressions_seen, + ); - Some(CoverageIdsInfo { counters_seen, zero_expressions }) + let CoverageCounters { phys_counter_for_node, node_counters, expressions, .. } = + coverage_counters; + + Some(CoverageIdsInfo { + counters_seen, + zero_expressions, + phys_counter_for_node, + term_for_bcb: node_counters, + expressions, + }) } fn all_coverage_in_mir_body<'a, 'tcx>( @@ -150,7 +198,7 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool { /// already being performed by the Rust-side expression renumbering, so that /// the resulting coverage mappings don't get worse. fn identify_zero_expressions( - fn_cov_info: &FunctionCoverageInfo, + expressions: &IndexSlice, counters_seen: &DenseBitSet, expressions_seen: &DenseBitSet, ) -> DenseBitSet { @@ -158,13 +206,13 @@ fn identify_zero_expressions( // have zero as both of their operands, and will therefore always have // a value of zero. Other expressions that refer to these as operands // can have those operands replaced with `CovTerm::Zero`. - let mut zero_expressions = DenseBitSet::new_empty(fn_cov_info.expressions.len()); + let mut zero_expressions = DenseBitSet::new_empty(expressions.len()); // Simplify a copy of each expression based on lower-numbered expressions, // and then update the set of always-zero expressions if necessary. // (By construction, expressions can only refer to other expressions // that have lower IDs, so one pass is sufficient.) - for (id, expression) in fn_cov_info.expressions.iter_enumerated() { + for (id, expression) in expressions.iter_enumerated() { if !expressions_seen.contains(id) { // If an expression was not seen, it must have been optimized away, // so any operand that refers to it can be replaced with zero. diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 26ce743be36..73b68d7155c 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -137,8 +137,7 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option { // These coverage statements should not exist prior to coverage instrumentation. StatementKind::Coverage( - CoverageKind::CounterIncrement { .. } - | CoverageKind::ExpressionUsed { .. } + CoverageKind::VirtualCounter { .. } | CoverageKind::CondBitmapUpdate { .. } | CoverageKind::TestVectorBitmapUpdate { .. }, ) => bug!( diff --git a/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff index 9647e13f75a..e9f48a85f9c 100644 --- a/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff @@ -27,9 +27,6 @@ } + coverage body span: $DIR/branch_match_arms.rs:14:11: 21:2 (#0) -+ coverage ExpressionId(0) => Expression { lhs: Counter(1), op: Add, rhs: Counter(2) }; -+ coverage ExpressionId(1) => Expression { lhs: Expression(0), op: Add, rhs: Counter(3) }; -+ coverage ExpressionId(2) => Expression { lhs: Counter(0), op: Subtract, rhs: Expression(1) }; + coverage Code { bcb: bcb0 } => $DIR/branch_match_arms.rs:14:1: 15:21 (#0); + coverage Code { bcb: bcb1 } => $DIR/branch_match_arms.rs:16:17: 16:33 (#0); + coverage Code { bcb: bcb3 } => $DIR/branch_match_arms.rs:17:17: 17:33 (#0); @@ -38,7 +35,7 @@ + coverage Code { bcb: bcb2 } => $DIR/branch_match_arms.rs:21:2: 21:2 (#0); + bb0: { -+ Coverage::CounterIncrement(0); ++ Coverage::VirtualCounter(bcb0); StorageLive(_1); _1 = Enum::A(const 0_u32); PlaceMention(_1); @@ -52,22 +49,22 @@ } bb2: { -+ Coverage::CounterIncrement(1); ++ Coverage::VirtualCounter(bcb1); falseEdge -> [real: bb8, imaginary: bb3]; } bb3: { -+ Coverage::CounterIncrement(2); ++ Coverage::VirtualCounter(bcb3); falseEdge -> [real: bb7, imaginary: bb4]; } bb4: { -+ Coverage::CounterIncrement(3); ++ Coverage::VirtualCounter(bcb4); falseEdge -> [real: bb6, imaginary: bb5]; } bb5: { -+ Coverage::ExpressionUsed(2); ++ Coverage::VirtualCounter(bcb5); StorageLive(_9); _9 = copy ((_1 as A).0: u32); StorageLive(_10); @@ -124,6 +121,7 @@ } bb13: { ++ Coverage::VirtualCounter(bcb2); StorageDead(_1); return; } diff --git a/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff index 16b7a1a63b4..cbef6de917d 100644 --- a/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff @@ -8,7 +8,7 @@ + coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:29:1: 31:2 (#0); + bb0: { -+ Coverage::CounterIncrement(0); ++ Coverage::VirtualCounter(bcb0); _0 = const true; return; } diff --git a/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff index fe74ad24e91..b166d79a412 100644 --- a/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff @@ -8,7 +8,6 @@ let mut _3: !; + coverage body span: $DIR/instrument_coverage.rs:14:11: 20:2 (#0) -+ coverage ExpressionId(0) => Expression { lhs: Counter(1), op: Subtract, rhs: Counter(0) }; + coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:14:1: 14:11 (#0); + coverage Code { bcb: bcb1 } => $DIR/instrument_coverage.rs:16:12: 16:17 (#0); + coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:17:13: 17:18 (#0); @@ -16,12 +15,12 @@ + coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:20:2: 20:2 (#0); + bb0: { -+ Coverage::CounterIncrement(0); ++ Coverage::VirtualCounter(bcb0); goto -> bb1; } bb1: { -+ Coverage::CounterIncrement(1); ++ Coverage::VirtualCounter(bcb1); falseUnwind -> [real: bb2, unwind: bb6]; } @@ -35,13 +34,14 @@ } bb4: { ++ Coverage::VirtualCounter(bcb2); _0 = const (); StorageDead(_2); return; } bb5: { -+ Coverage::ExpressionUsed(0); ++ Coverage::VirtualCounter(bcb3); _1 = const (); StorageDead(_2); goto -> bb1; diff --git a/tests/mir-opt/coverage/instrument_coverage.rs b/tests/mir-opt/coverage/instrument_coverage.rs index 20c730645f8..48647402d0f 100644 --- a/tests/mir-opt/coverage/instrument_coverage.rs +++ b/tests/mir-opt/coverage/instrument_coverage.rs @@ -10,7 +10,7 @@ // CHECK: coverage body span: // CHECK: coverage Code { bcb: bcb{{[0-9]+}} } => // CHECK: bb0: -// CHECK: Coverage::CounterIncrement +// CHECK: Coverage::VirtualCounter fn main() { loop { if bar() { @@ -24,7 +24,7 @@ fn main() { // CHECK: coverage body span: // CHECK: coverage Code { bcb: bcb{{[0-9]+}} } => // CHECK: bb0: -// CHECK: Coverage::CounterIncrement +// CHECK: Coverage::VirtualCounter #[inline(never)] fn bar() -> bool { true diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff index 28a3f3beade..855f806aae1 100644 --- a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff +++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff @@ -8,7 +8,6 @@ coverage branch { true: BlockMarkerId(0), false: BlockMarkerId(1) } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0) coverage body span: $DIR/instrument_coverage_cleanup.rs:13:11: 15:2 (#0) - coverage ExpressionId(0) => Expression { lhs: Counter(0), op: Subtract, rhs: Counter(1) }; coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0); coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0); coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0); @@ -16,7 +15,7 @@ coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0); bb0: { - Coverage::CounterIncrement(0); + Coverage::VirtualCounter(bcb0); - Coverage::SpanMarker; + nop; StorageLive(_1); @@ -28,7 +27,7 @@ } bb2: { - Coverage::CounterIncrement(1); + Coverage::VirtualCounter(bcb1); - Coverage::BlockMarker(1); + nop; _0 = const (); @@ -36,7 +35,7 @@ } bb3: { - Coverage::ExpressionUsed(0); + Coverage::VirtualCounter(bcb3); - Coverage::BlockMarker(0); + nop; _0 = const (); @@ -44,6 +43,7 @@ } bb4: { + Coverage::VirtualCounter(bcb2); StorageDead(_1); return; } diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff index 0f1dea02b8d..df1f1e8bc50 100644 --- a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff +++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff @@ -8,7 +8,6 @@ coverage branch { true: BlockMarkerId(0), false: BlockMarkerId(1) } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0) + coverage body span: $DIR/instrument_coverage_cleanup.rs:13:11: 15:2 (#0) -+ coverage ExpressionId(0) => Expression { lhs: Counter(0), op: Subtract, rhs: Counter(1) }; + coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0); + coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0); + coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0); @@ -16,7 +15,7 @@ + coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0); + bb0: { -+ Coverage::CounterIncrement(0); ++ Coverage::VirtualCounter(bcb0); Coverage::SpanMarker; StorageLive(_1); _1 = std::hint::black_box::(const true) -> [return: bb1, unwind: bb5]; @@ -27,20 +26,21 @@ } bb2: { -+ Coverage::CounterIncrement(1); ++ Coverage::VirtualCounter(bcb1); Coverage::BlockMarker(1); _0 = const (); goto -> bb4; } bb3: { -+ Coverage::ExpressionUsed(0); ++ Coverage::VirtualCounter(bcb3); Coverage::BlockMarker(0); _0 = const (); goto -> bb4; } bb4: { ++ Coverage::VirtualCounter(bcb2); StorageDead(_1); return; } diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.rs b/tests/mir-opt/coverage/instrument_coverage_cleanup.rs index acc544a28af..e8af4d6174f 100644 --- a/tests/mir-opt/coverage/instrument_coverage_cleanup.rs +++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.rs @@ -3,7 +3,7 @@ // but leaves the statements that were added by InstrumentCoverage. // // Removed statement kinds: BlockMarker, SpanMarker -// Retained statement kinds: CounterIncrement, ExpressionUsed +// Retained statement kinds: VirtualCounter //@ test-mir-pass: InstrumentCoverage //@ compile-flags: -Cinstrument-coverage -Zcoverage-options=branch -Zno-profiler-runtime @@ -16,6 +16,6 @@ fn main() { // CHECK-NOT: Coverage::BlockMarker // CHECK-NOT: Coverage::SpanMarker -// CHECK: Coverage::CounterIncrement +// CHECK: Coverage::VirtualCounter // CHECK-NOT: Coverage::BlockMarker // CHECK-NOT: Coverage::SpanMarker -- cgit 1.4.1-3-g733a5 From bd855b6c9efc25a9f1eba3febd3d04f1b23a1ec5 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Sat, 25 Jan 2025 18:41:35 +1100 Subject: coverage: Remove the old code for simplifying counters after MIR opts --- .../src/coverageinfo/mapgen/covfun.rs | 25 ++-- .../rustc_codegen_llvm/src/coverageinfo/mod.rs | 4 +- compiler/rustc_middle/src/mir/coverage.rs | 32 +---- .../rustc_mir_transform/src/coverage/counters.rs | 2 +- compiler/rustc_mir_transform/src/coverage/query.rs | 137 +-------------------- 5 files changed, 17 insertions(+), 183 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index 1290c6efb09..c53ea6d4666 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -54,7 +54,7 @@ pub(crate) fn prepare_covfun_record<'tcx>( let fn_cov_info = tcx.instance_mir(instance.def).function_coverage_info.as_deref()?; let ids_info = tcx.coverage_ids_info(instance.def)?; - let expressions = prepare_expressions(ids_info, is_used); + let expressions = prepare_expressions(ids_info); let mut covfun = CovfunRecord { mangled_function_name: tcx.symbol_name(instance).name, @@ -76,16 +76,8 @@ pub(crate) fn prepare_covfun_record<'tcx>( } /// Convert the function's coverage-counter expressions into a form suitable for FFI. -fn prepare_expressions(ids_info: &CoverageIdsInfo, is_used: bool) -> Vec { - // If any counters or expressions were removed by MIR opts, replace their - // terms with zero. - let counter_for_term = |term| { - if !is_used || ids_info.is_zero_term(term) { - ffi::Counter::ZERO - } else { - ffi::Counter::from_term(term) - } - }; +fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec { + let counter_for_term = ffi::Counter::from_term; // We know that LLVM will optimize out any unused expressions before // producing the final coverage map, so there's no need to do the same @@ -133,13 +125,14 @@ fn fill_region_tables<'tcx>( // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - let is_zero_term = |term| !covfun.is_used || ids_info.is_zero_term(term); for &Mapping { ref kind, span } in &fn_cov_info.mappings { - // If the mapping refers to counters/expressions that were removed by - // MIR opts, replace those occurrences with zero. + // If this function is unused, replace all counters with zero. let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { - let term = ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term"); - let term = if is_zero_term(term) { CovTerm::Zero } else { term }; + let term = if covfun.is_used { + ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term") + } else { + CovTerm::Zero + }; ffi::Counter::from_term(term) }; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index e404db5e196..ea7f581a3cb 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -163,11 +163,9 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::VirtualCounter { bcb } if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) => { - let num_counters = ids_info.num_counters_after_mir_opts(); - let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); - let num_counters = bx.const_u32(num_counters); + let num_counters = bx.const_u32(ids_info.num_counters); let index = bx.const_u32(id.as_u32()); debug!( "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index 31b9eeb715b..8c6b11a681e 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -3,7 +3,6 @@ use std::fmt::{self, Debug, Formatter}; use rustc_data_structures::fx::FxIndexMap; -use rustc_index::bit_set::DenseBitSet; use rustc_index::{Idx, IndexVec}; use rustc_macros::{HashStable, TyDecodable, TyEncodable}; use rustc_span::Span; @@ -277,41 +276,12 @@ pub struct MCDCDecisionSpan { /// Returned by the `coverage_ids_info` query. #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable)] pub struct CoverageIdsInfo { - pub counters_seen: DenseBitSet, - pub zero_expressions: DenseBitSet, - + pub num_counters: u32, pub phys_counter_for_node: FxIndexMap, pub term_for_bcb: IndexVec>, pub expressions: IndexVec, } -impl CoverageIdsInfo { - /// Coverage codegen needs to know how many coverage counters are ever - /// incremented within a function, so that it can set the `num-counters` - /// argument of the `llvm.instrprof.increment` intrinsic. - /// - /// This may be less than the highest counter ID emitted by the - /// InstrumentCoverage MIR pass, if the highest-numbered counter increments - /// were removed by MIR optimizations. - pub fn num_counters_after_mir_opts(&self) -> u32 { - // FIXME(Zalathar): Currently this treats an unused counter as "used" - // if its ID is less than that of the highest counter that really is - // used. Fixing this would require adding a renumbering step somewhere. - self.counters_seen.last_set_in(..).map_or(0, |max| max.as_u32() + 1) - } - - /// Returns `true` if the given term is known to have a value of zero, taking - /// into account knowledge of which counters are unused and which expressions - /// are always zero. - pub fn is_zero_term(&self, term: CovTerm) -> bool { - match term { - CovTerm::Zero => true, - CovTerm::Counter(id) => !self.counters_seen.contains(id), - CovTerm::Expression(id) => self.zero_expressions.contains(id), - } - } -} - rustc_index::newtype_index! { /// During the `InstrumentCoverage` MIR pass, a BCB is a node in the /// "coverage graph", which is a refinement of the MIR control-flow graph diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index ac0039fd007..6f9984d5d0a 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -135,7 +135,7 @@ pub(super) struct CoverageCounters { /// List of places where a counter-increment statement should be injected /// into MIR, each with its corresponding counter ID. pub(crate) phys_counter_for_node: FxIndexMap, - next_counter_id: CounterId, + pub(crate) next_counter_id: CounterId, /// Coverage counters/expressions that are associated with individual BCBs. pub(crate) node_counters: IndexVec>, diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 995fa91e0a0..cd89fbe772d 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -1,11 +1,7 @@ use rustc_data_structures::captures::Captures; -use rustc_index::IndexSlice; use rustc_index::bit_set::DenseBitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::coverage::{ - BasicCoverageBlock, CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, - ExpressionId, MappingKind, Op, -}; +use rustc_middle::mir::coverage::{BasicCoverageBlock, CoverageIdsInfo, CoverageKind, MappingKind}; use rustc_middle::mir::{Body, Statement, StatementKind}; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::util::Providers; @@ -134,44 +130,12 @@ fn coverage_ids_info<'tcx>( let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &fn_cov_info.priority_list); let coverage_counters = transcribe_counters(&node_counters, &bcb_needs_counter, &bcbs_seen); - let mut counters_seen = DenseBitSet::new_empty(coverage_counters.node_counters.len()); - let mut expressions_seen = DenseBitSet::new_filled(coverage_counters.expressions.len()); - - // For each expression ID that is directly used by one or more mappings, - // mark it as not-yet-seen. This indicates that we expect to see a - // corresponding `VirtualCounter` statement during MIR traversal. - for mapping in fn_cov_info.mappings.iter() { - // Currently we only worry about ordinary code mappings. - // For branch and MC/DC mappings, expressions might not correspond - // to any particular point in the control-flow graph. - if let MappingKind::Code { bcb } = mapping.kind - && let Some(CovTerm::Expression(id)) = coverage_counters.node_counters[bcb] - { - expressions_seen.remove(id); - } - } - - for bcb in bcbs_seen.iter() { - if let Some(&id) = coverage_counters.phys_counter_for_node.get(&bcb) { - counters_seen.insert(id); - } - if let Some(CovTerm::Expression(id)) = coverage_counters.node_counters[bcb] { - expressions_seen.insert(id); - } - } - - let zero_expressions = identify_zero_expressions( - &coverage_counters.expressions, - &counters_seen, - &expressions_seen, - ); - - let CoverageCounters { phys_counter_for_node, node_counters, expressions, .. } = - coverage_counters; + let CoverageCounters { + phys_counter_for_node, next_counter_id, node_counters, expressions, .. + } = coverage_counters; Some(CoverageIdsInfo { - counters_seen, - zero_expressions, + num_counters: next_counter_id.as_u32(), phys_counter_for_node, term_for_bcb: node_counters, expressions, @@ -193,94 +157,3 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool { let scope_data = &body.source_scopes[statement.source_info.scope]; scope_data.inlined.is_some() || scope_data.inlined_parent_scope.is_some() } - -/// Identify expressions that will always have a value of zero, and note their -/// IDs in a `DenseBitSet`. Mappings that refer to a zero expression can instead -/// become mappings to a constant zero value. -/// -/// This function mainly exists to preserve the simplifications that were -/// already being performed by the Rust-side expression renumbering, so that -/// the resulting coverage mappings don't get worse. -fn identify_zero_expressions( - expressions: &IndexSlice, - counters_seen: &DenseBitSet, - expressions_seen: &DenseBitSet, -) -> DenseBitSet { - // The set of expressions that either were optimized out entirely, or - // have zero as both of their operands, and will therefore always have - // a value of zero. Other expressions that refer to these as operands - // can have those operands replaced with `CovTerm::Zero`. - let mut zero_expressions = DenseBitSet::new_empty(expressions.len()); - - // Simplify a copy of each expression based on lower-numbered expressions, - // and then update the set of always-zero expressions if necessary. - // (By construction, expressions can only refer to other expressions - // that have lower IDs, so one pass is sufficient.) - for (id, expression) in expressions.iter_enumerated() { - if !expressions_seen.contains(id) { - // If an expression was not seen, it must have been optimized away, - // so any operand that refers to it can be replaced with zero. - zero_expressions.insert(id); - continue; - } - - // We don't need to simplify the actual expression data in the - // expressions list; we can just simplify a temporary copy and then - // use that to update the set of always-zero expressions. - let Expression { mut lhs, op, mut rhs } = *expression; - - // If an expression has an operand that is also an expression, the - // operand's ID must be strictly lower. This is what lets us find - // all zero expressions in one pass. - let assert_operand_expression_is_lower = |operand_id: ExpressionId| { - assert!( - operand_id < id, - "Operand {operand_id:?} should be less than {id:?} in {expression:?}", - ) - }; - - // If an operand refers to a counter or expression that is always - // zero, then that operand can be replaced with `CovTerm::Zero`. - let maybe_set_operand_to_zero = |operand: &mut CovTerm| { - if let CovTerm::Expression(id) = *operand { - assert_operand_expression_is_lower(id); - } - - if is_zero_term(&counters_seen, &zero_expressions, *operand) { - *operand = CovTerm::Zero; - } - }; - maybe_set_operand_to_zero(&mut lhs); - maybe_set_operand_to_zero(&mut rhs); - - // Coverage counter values cannot be negative, so if an expression - // involves subtraction from zero, assume that its RHS must also be zero. - // (Do this after simplifications that could set the LHS to zero.) - if lhs == CovTerm::Zero && op == Op::Subtract { - rhs = CovTerm::Zero; - } - - // After the above simplifications, if both operands are zero, then - // we know that this expression is always zero too. - if lhs == CovTerm::Zero && rhs == CovTerm::Zero { - zero_expressions.insert(id); - } - } - - zero_expressions -} - -/// Returns `true` if the given term is known to have a value of zero, taking -/// into account knowledge of which counters are unused and which expressions -/// are always zero. -fn is_zero_term( - counters_seen: &DenseBitSet, - zero_expressions: &DenseBitSet, - term: CovTerm, -) -> bool { - match term { - CovTerm::Zero => true, - CovTerm::Counter(id) => !counters_seen.contains(id), - CovTerm::Expression(id) => zero_expressions.contains(id), - } -} -- cgit 1.4.1-3-g733a5 From 53f9852224e6e54141e55bed86dbdb966e215d22 Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Fri, 7 Feb 2025 18:08:19 +0000 Subject: rustc_target: Add the fp16 target feature for AArch32 --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 1 + compiler/rustc_target/src/target_features.rs | 1 + 2 files changed, 2 insertions(+) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 53611c746a7..ee8c59015e6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -271,6 +271,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option Some(LLVMFeature::new("fullfp16")), // Filter out features that are not supported by the current LLVM version ("aarch64", "fpmr") if get_version().0 != 18 => None, + ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single // feature called `fast-unaligned-access`. In LLVM 19, it was split back out. ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs index eb2417e0a20..e1efbe9cba5 100644 --- a/compiler/rustc_target/src/target_features.rs +++ b/compiler/rustc_target/src/target_features.rs @@ -145,6 +145,7 @@ const ARM_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ ("dotprod", Unstable(sym::arm_target_feature), &["neon"]), ("dsp", Unstable(sym::arm_target_feature), &[]), ("fp-armv8", Unstable(sym::arm_target_feature), &["vfp4"]), + ("fp16", Unstable(sym::arm_target_feature), &["neon"]), ("fpregs", Unstable(sym::arm_target_feature), &[]), ("i8mm", Unstable(sym::arm_target_feature), &["neon"]), ("mclass", Unstable(sym::arm_target_feature), &[]), -- cgit 1.4.1-3-g733a5 From 21d096184e3c7e05646ee756a2ff6a99e90aff77 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Feb 2025 22:27:46 -0500 Subject: fix non-enzyme builds --- compiler/rustc_codegen_llvm/src/back/write.rs | 4 +++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 17 ++++++++++------- 3 files changed, 14 insertions(+), 8 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 4706744f353..155c07ff635 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -551,11 +551,12 @@ pub(crate) unsafe fn llvm_optimize( let vectorize_slp; let vectorize_loop; + let run_enzyme = cfg!(llvm_enzyme); // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing // optimizations until after differentiation. FIXME(ZuseZ4): Before shipping on nightly, // we should make this more granular, or at least check that the user has at least one autodiff // call in their code, to justify altering the compilation pipeline. - if skip_size_increasing_opts && cfg!(llvm_enzyme) { + if skip_size_increasing_opts && run_enzyme { unroll_loops = false; vectorize_slp = false; vectorize_loop = false; @@ -633,6 +634,7 @@ pub(crate) unsafe fn llvm_optimize( vectorize_loop, config.no_builtins, config.emit_lifetime_markers, + run_enzyme, sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index cc7c5231aca..f2af400ce38 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2346,6 +2346,7 @@ unsafe extern "C" { LoopVectorize: bool, DisableSimplifyLibCalls: bool, EmitLifetimeMarkers: bool, + RunEnzyme: bool, SanitizerOptions: Option<&SanitizerOptions>, PGOGenPath: *const c_char, PGOUsePath: *const c_char, diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 0063f262c9b..0e591786d91 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -688,7 +688,8 @@ struct LLVMRustSanitizerOptions { bool SanitizeKernelAddressRecover; }; -extern "C" void registerEnzyme(llvm::PassBuilder &PB); +// This symbol won't be available or used when Enzyme is not enabled +extern "C" void registerEnzyme(llvm::PassBuilder &PB) __attribute__((weak)); extern "C" LLVMRustResult LLVMRustOptimize( LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, @@ -696,7 +697,7 @@ extern "C" LLVMRustResult LLVMRustOptimize( bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, bool LintIR, bool UseThinLTOBuffers, bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, - bool EmitLifetimeMarkers, LLVMRustSanitizerOptions *SanitizerOptions, + bool EmitLifetimeMarkers, bool RunEnzyme, LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, const char *PGOUsePath, bool InstrumentCoverage, const char *InstrProfileOutput, const char *PGOSampleUsePath, bool DebugInfoForProfiling, void *LlvmSelfProfiler, @@ -1013,11 +1014,13 @@ extern "C" LLVMRustResult LLVMRustOptimize( } // now load "-enzyme" pass: - registerEnzyme(PB); - if (auto Err = PB.parsePassPipeline(MPM, "enzyme")) { - std::string ErrMsg = toString(std::move(Err)); - LLVMRustSetLastError(ErrMsg.c_str()); - return LLVMRustResult::Failure; + if (RunEnzyme) { + registerEnzyme(PB); + if (auto Err = PB.parsePassPipeline(MPM, "enzyme")) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } } // Upgrade all calls to old intrinsics first. -- cgit 1.4.1-3-g733a5 From 1221cff55149d2dbbf8761345799ef06f8099b97 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 10 Feb 2025 01:35:22 -0500 Subject: move second opt run to lto phase and cleanup code --- compiler/rustc_codegen_llvm/src/back/lto.rs | 25 +++++++++++-- compiler/rustc_codegen_llvm/src/back/write.rs | 41 +++++++++++++--------- .../rustc_codegen_llvm/src/builder/autodiff.rs | 35 ++++-------------- compiler/rustc_llvm/build.rs | 4 +++ compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 11 ++++-- src/bootstrap/src/core/build_steps/compile.rs | 9 +++-- tests/codegen/autodiff.rs | 4 +-- 7 files changed, 75 insertions(+), 54 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 78c759bbe8c..8bad437eeb7 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -606,10 +606,31 @@ pub(crate) fn run_pass_manager( // If this rustc version was build with enzyme/autodiff enabled, and if users applied the // `#[autodiff]` macro at least once, then we will later call llvm_optimize a second time. - let first_run = true; debug!("running llvm pm opt pipeline"); unsafe { - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, first_run)?; + write::llvm_optimize( + cgcx, + dcx, + module, + config, + opt_level, + opt_stage, + write::AutodiffStage::DuringAD, + )?; + } + // FIXME(ZuseZ4): Make this more granular + if cfg!(llvm_enzyme) && !thin { + unsafe { + write::llvm_optimize( + cgcx, + dcx, + module, + config, + opt_level, + llvm::OptStage::FatLTO, + write::AutodiffStage::PostAD, + )?; + } } debug!("lto done"); Ok(()) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 155c07ff635..ae4c4d5876e 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -530,6 +530,16 @@ fn get_instr_profile_output_path(config: &ModuleConfig) -> Option { config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned()) } +// PreAD will run llvm opts but disable size increasing opts (vectorization, loop unrolling) +// DuringAD is the same as above, but also runs the enzyme opt and autodiff passes. +// PostAD will run all opts, including size increasing opts. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum AutodiffStage { + PreAD, + DuringAD, + PostAD, +} + pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext, dcx: DiagCtxtHandle<'_>, @@ -537,7 +547,7 @@ pub(crate) unsafe fn llvm_optimize( config: &ModuleConfig, opt_level: config::OptLevel, opt_stage: llvm::OptStage, - skip_size_increasing_opts: bool, + autodiff_stage: AutodiffStage, ) -> Result<(), FatalError> { // Enzyme: // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized @@ -550,13 +560,16 @@ pub(crate) unsafe fn llvm_optimize( let unroll_loops; let vectorize_slp; let vectorize_loop; + let run_enzyme = cfg!(llvm_enzyme) && autodiff_stage == AutodiffStage::DuringAD; - let run_enzyme = cfg!(llvm_enzyme); // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing - // optimizations until after differentiation. FIXME(ZuseZ4): Before shipping on nightly, + // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt). + // We therefore have two calls to llvm_optimize, if autodiff is used. + // + // FIXME(ZuseZ4): Before shipping on nightly, // we should make this more granular, or at least check that the user has at least one autodiff // call in their code, to justify altering the compilation pipeline. - if skip_size_increasing_opts && run_enzyme { + if cfg!(llvm_enzyme) && autodiff_stage != AutodiffStage::PostAD { unroll_loops = false; vectorize_slp = false; vectorize_loop = false; @@ -566,7 +579,7 @@ pub(crate) unsafe fn llvm_optimize( vectorize_slp = config.vectorize_slp; vectorize_loop = config.vectorize_loop; } - trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop); + trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme); let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); @@ -686,18 +699,14 @@ pub(crate) unsafe fn optimize( _ => llvm::OptStage::PreLinkNoLTO, }; - // If we know that we will later run AD, then we disable vectorization and loop unrolling - let skip_size_increasing_opts = cfg!(llvm_enzyme); + // If we know that we will later run AD, then we disable vectorization and loop unrolling. + // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). + // FIXME(ZuseZ4): Make this more granular, only set PreAD if we actually have autodiff + // usages, not just if we build rustc with autodiff support. + let autodiff_stage = + if cfg!(llvm_enzyme) { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; return unsafe { - llvm_optimize( - cgcx, - dcx, - module, - config, - opt_level, - opt_stage, - skip_size_increasing_opts, - ) + llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage) }; } Ok(()) diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 474b0940203..f82449479a8 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -4,10 +4,9 @@ use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivit use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::back::write::ModuleConfig; use rustc_errors::FatalError; -use rustc_session::config::Lto; use tracing::{debug, trace}; -use crate::back::write::{llvm_err, llvm_optimize}; +use crate::back::write::llvm_err; use crate::builder::SBuilder; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; @@ -153,7 +152,7 @@ fn generate_enzyme_call<'ll>( _ => {} } - trace!("matching autodiff arguments"); + debug!("matching autodiff arguments"); // We now handle the issue that Rust level arguments not always match the llvm-ir level // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on // llvm-ir level. The number of activities matches the number of Rust level arguments, so we @@ -222,7 +221,10 @@ fn generate_enzyme_call<'ll>( // A duplicated pointer will have the following two outer_fn arguments: // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call: // (..., metadata! enzyme_dup, ptr, ptr, ...). - if matches!(diff_activity, DiffActivity::Duplicated | DiffActivity::DuplicatedOnly) { + if matches!( + diff_activity, + DiffActivity::Duplicated | DiffActivity::DuplicatedOnly + ) { assert!( llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer ); @@ -282,7 +284,7 @@ pub(crate) fn differentiate<'ll>( module: &'ll ModuleCodegen, cgcx: &CodegenContext, diff_items: Vec, - config: &ModuleConfig, + _config: &ModuleConfig, ) -> Result<(), FatalError> { for item in &diff_items { trace!("{}", item); @@ -317,29 +319,6 @@ pub(crate) fn differentiate<'ll>( // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts - if let Some(opt_level) = config.opt_level { - let opt_stage = match cgcx.lto { - Lto::Fat => llvm::OptStage::PreLinkFatLTO, - Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, - _ => llvm::OptStage::PreLinkNoLTO, - }; - // This is our second opt call, so now we run all opts, - // to make sure we get the best performance. - let skip_size_increasing_opts = false; - trace!("running Module Optimization after differentiation"); - unsafe { - llvm_optimize( - cgcx, - diag_handler.handle(), - module, - config, - opt_level, - opt_stage, - skip_size_increasing_opts, - )? - }; - } trace!("done with differentiate()"); Ok(()) diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index d9d28299413..48806888b43 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -193,6 +193,10 @@ fn main() { cfg.define(&flag, None); } + if tracked_env_var_os("LLVM_ENZYME").is_some() { + cfg.define("ENZYME", None); + } + if tracked_env_var_os("LLVM_RUSTLLVM").is_some() { cfg.define("LLVM_RUSTLLVM", None); } diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 0e591786d91..a6b2384f2d7 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -689,7 +689,9 @@ struct LLVMRustSanitizerOptions { }; // This symbol won't be available or used when Enzyme is not enabled -extern "C" void registerEnzyme(llvm::PassBuilder &PB) __attribute__((weak)); +#ifdef ENZYME +extern "C" void registerEnzyme(llvm::PassBuilder &PB); +#endif extern "C" LLVMRustResult LLVMRustOptimize( LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, @@ -697,8 +699,9 @@ extern "C" LLVMRustResult LLVMRustOptimize( bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, bool LintIR, bool UseThinLTOBuffers, bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, - bool EmitLifetimeMarkers, bool RunEnzyme, LLVMRustSanitizerOptions *SanitizerOptions, - const char *PGOGenPath, const char *PGOUsePath, bool InstrumentCoverage, + bool EmitLifetimeMarkers, bool RunEnzyme, + LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, + const char *PGOUsePath, bool InstrumentCoverage, const char *InstrProfileOutput, const char *PGOSampleUsePath, bool DebugInfoForProfiling, void *LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, @@ -1014,6 +1017,7 @@ extern "C" LLVMRustResult LLVMRustOptimize( } // now load "-enzyme" pass: +#ifdef ENZYME if (RunEnzyme) { registerEnzyme(PB); if (auto Err = PB.parsePassPipeline(MPM, "enzyme")) { @@ -1022,6 +1026,7 @@ extern "C" LLVMRustResult LLVMRustOptimize( return LLVMRustResult::Failure; } } +#endif // Upgrade all calls to old intrinsics first. for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 4e360e94fd6..308d4723d98 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1049,9 +1049,9 @@ pub fn rustc_cargo( // . cargo.rustflag("-Zon-broken-pipe=kill"); - // We temporarily disable linking here as part of some refactoring. - // This way, people can manually use -Z llvm-plugins and -C passes=enzyme for now. - // In a follow-up PR, we will re-enable linking here and load the pass for them. + // We want to link against registerEnzyme and in the future we want to use additional + // functionality from Enzyme core. For that we need to link against Enzyme. + // FIXME(ZuseZ4): Get the LLVM version number automatically instead of hardcoding it. if builder.config.llvm_enzyme { cargo.rustflag("-l").rustflag("Enzyme-19"); } @@ -1234,6 +1234,9 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect if builder.is_rust_llvm(target) { cargo.env("LLVM_RUSTLLVM", "1"); } + if builder.config.llvm_enzyme { + cargo.env("LLVM_ENZYME", "1"); + } let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); cargo.env("LLVM_CONFIG", &llvm_config); diff --git a/tests/codegen/autodiff.rs b/tests/codegen/autodiff.rs index efc5ac77f1b..abf7fcf3e4b 100644 --- a/tests/codegen/autodiff.rs +++ b/tests/codegen/autodiff.rs @@ -15,9 +15,9 @@ fn square(x: &f64) -> f64 { // CHECK-NEXT:invertstart: // CHECK-NEXT: %_0 = fmul double %x.0.val, %x.0.val // CHECK-NEXT: %0 = fadd fast double %x.0.val, %x.0.val -// CHECK-NEXT: %1 = load double, ptr %"x'", align 8, !alias.scope !17816, !noalias !17819 +// CHECK-NEXT: %1 = load double, ptr %"x'", align 8 // CHECK-NEXT: %2 = fadd fast double %1, %0 -// CHECK-NEXT: store double %2, ptr %"x'", align 8, !alias.scope !17816, !noalias !17819 +// CHECK-NEXT: store double %2, ptr %"x'", align 8 // CHECK-NEXT: ret double %_0 // CHECK-NEXT:} -- cgit 1.4.1-3-g733a5 From 061abbc36928cce784c54463c266f4d43d14d419 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 10 Feb 2025 01:35:53 -0500 Subject: remove outdated *First autodiff variants for higher-order ad --- compiler/rustc_ast/src/expand/autodiff_attrs.rs | 24 ++++++---------------- .../rustc_codegen_llvm/src/builder/autodiff.rs | 2 -- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 2 -- 3 files changed, 6 insertions(+), 22 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_ast/src/expand/autodiff_attrs.rs b/compiler/rustc_ast/src/expand/autodiff_attrs.rs index ecc522ec39d..70222f4acab 100644 --- a/compiler/rustc_ast/src/expand/autodiff_attrs.rs +++ b/compiler/rustc_ast/src/expand/autodiff_attrs.rs @@ -30,14 +30,6 @@ pub enum DiffMode { Forward, /// The target function, to be created using reverse mode AD. Reverse, - /// The target function, to be created using forward mode AD. - /// This target function will also be used as a source for higher order derivatives, - /// so compute it before all Forward/Reverse targets and optimize it through llvm. - ForwardFirst, - /// The target function, to be created using reverse mode AD. - /// This target function will also be used as a source for higher order derivatives, - /// so compute it before all Forward/Reverse targets and optimize it through llvm. - ReverseFirst, } /// Dual and Duplicated (and their Only variants) are getting lowered to the same Enzyme Activity. @@ -92,10 +84,10 @@ pub struct AutoDiffAttrs { impl DiffMode { pub fn is_rev(&self) -> bool { - matches!(self, DiffMode::Reverse | DiffMode::ReverseFirst) + matches!(self, DiffMode::Reverse) } pub fn is_fwd(&self) -> bool { - matches!(self, DiffMode::Forward | DiffMode::ForwardFirst) + matches!(self, DiffMode::Forward) } } @@ -106,8 +98,6 @@ impl Display for DiffMode { DiffMode::Source => write!(f, "Source"), DiffMode::Forward => write!(f, "Forward"), DiffMode::Reverse => write!(f, "Reverse"), - DiffMode::ForwardFirst => write!(f, "ForwardFirst"), - DiffMode::ReverseFirst => write!(f, "ReverseFirst"), } } } @@ -125,12 +115,12 @@ pub fn valid_ret_activity(mode: DiffMode, activity: DiffActivity) -> bool { match mode { DiffMode::Error => false, DiffMode::Source => false, - DiffMode::Forward | DiffMode::ForwardFirst => { + DiffMode::Forward => { activity == DiffActivity::Dual || activity == DiffActivity::DualOnly || activity == DiffActivity::Const } - DiffMode::Reverse | DiffMode::ReverseFirst => { + DiffMode::Reverse => { activity == DiffActivity::Const || activity == DiffActivity::Active || activity == DiffActivity::ActiveOnly @@ -166,10 +156,10 @@ pub fn valid_input_activity(mode: DiffMode, activity: DiffActivity) -> bool { return match mode { DiffMode::Error => false, DiffMode::Source => false, - DiffMode::Forward | DiffMode::ForwardFirst => { + DiffMode::Forward => { matches!(activity, Dual | DualOnly | Const) } - DiffMode::Reverse | DiffMode::ReverseFirst => { + DiffMode::Reverse => { matches!(activity, Active | ActiveOnly | Duplicated | DuplicatedOnly | Const) } }; @@ -200,8 +190,6 @@ impl FromStr for DiffMode { "Source" => Ok(DiffMode::Source), "Forward" => Ok(DiffMode::Forward), "Reverse" => Ok(DiffMode::Reverse), - "ForwardFirst" => Ok(DiffMode::ForwardFirst), - "ReverseFirst" => Ok(DiffMode::ReverseFirst), _ => Err(()), } } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index f82449479a8..6e5185faedc 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -52,8 +52,6 @@ fn generate_enzyme_call<'ll>( let mut ad_name: String = match attrs.mode { DiffMode::Forward => "__enzyme_fwddiff", DiffMode::Reverse => "__enzyme_autodiff", - DiffMode::ForwardFirst => "__enzyme_fwddiff", - DiffMode::ReverseFirst => "__enzyme_autodiff", _ => panic!("logic bug in autodiff, unrecognized mode"), } .to_string(); diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 4166387dad0..ad4dcbb919c 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -916,8 +916,6 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option { let mode = match mode.as_str() { "Forward" => DiffMode::Forward, "Reverse" => DiffMode::Reverse, - "ForwardFirst" => DiffMode::ForwardFirst, - "ReverseFirst" => DiffMode::ReverseFirst, _ => { span_bug!(mode.span, "rustc_autodiff attribute contains invalid mode"); } -- cgit 1.4.1-3-g733a5 From e11e2b4d091635fad78d25c9ba444627ca094677 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Sun, 9 Feb 2025 23:05:51 -0800 Subject: compiler: internally merge `Conv::PtxKernel` into `GpuKernel` It is speculated that these two can be conceptually merged, and it can start by ripping out rustc's notion of the PtxKernel call convention. Leave the ExternAbi for now, but the nvptx target now should see it as just a different way to spell Conv::GpuKernel. --- compiler/rustc_codegen_llvm/src/abi.rs | 1 - compiler/rustc_smir/src/rustc_smir/convert/abi.rs | 1 - compiler/rustc_target/src/callconv/mod.rs | 6 ++---- compiler/rustc_target/src/json.rs | 1 - compiler/rustc_ty_utils/src/abi.rs | 2 +- 5 files changed, 3 insertions(+), 8 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 685b2f37c9c..e575e9830ac 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -687,7 +687,6 @@ impl llvm::CallConv { Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, Conv::ArmAapcs => llvm::ArmAapcsCallConv, Conv::Msp430Intr => llvm::Msp430Intr, - Conv::PtxKernel => llvm::PtxKernel, Conv::X86Fastcall => llvm::X86FastcallCallConv, Conv::X86Intr => llvm::X86_Intr, Conv::X86Stdcall => llvm::X86StdcallCallConv, diff --git a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs index 4a03ff4beae..fb2e838cdc9 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs @@ -105,7 +105,6 @@ impl<'tcx> Stable<'tcx> for callconv::Conv { Conv::CCmseNonSecureCall => CallConvention::CCmseNonSecureCall, Conv::CCmseNonSecureEntry => CallConvention::CCmseNonSecureEntry, Conv::Msp430Intr => CallConvention::Msp430Intr, - Conv::PtxKernel => CallConvention::PtxKernel, Conv::X86Fastcall => CallConvention::X86Fastcall, Conv::X86Intr => CallConvention::X86Intr, Conv::X86Stdcall => CallConvention::X86Stdcall, diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 50ac6c8fcde..ffc35f9a3be 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -543,8 +543,6 @@ pub enum Conv { Msp430Intr, - PtxKernel, - GpuKernel, X86Fastcall, @@ -701,7 +699,8 @@ impl<'a, Ty> FnAbi<'a, Ty> { "sparc" => sparc::compute_abi_info(cx, self), "sparc64" => sparc64::compute_abi_info(cx, self), "nvptx64" => { - if cx.target_spec().adjust_abi(abi, self.c_variadic) == ExternAbi::PtxKernel { + let abi = cx.target_spec().adjust_abi(abi, self.c_variadic); + if abi == ExternAbi::PtxKernel || abi == ExternAbi::GpuKernel { nvptx64::compute_ptx_kernel_abi_info(cx, self) } else { nvptx64::compute_abi_info(self) @@ -860,7 +859,6 @@ impl FromStr for Conv { "CCmseNonSecureCall" => Ok(Conv::CCmseNonSecureCall), "CCmseNonSecureEntry" => Ok(Conv::CCmseNonSecureEntry), "Msp430Intr" => Ok(Conv::Msp430Intr), - "PtxKernel" => Ok(Conv::PtxKernel), "X86Fastcall" => Ok(Conv::X86Fastcall), "X86Intr" => Ok(Conv::X86Intr), "X86Stdcall" => Ok(Conv::X86Stdcall), diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs index 15cf7e195db..8d6f8f4c6f6 100644 --- a/compiler/rustc_target/src/json.rs +++ b/compiler/rustc_target/src/json.rs @@ -105,7 +105,6 @@ impl ToJson for crate::callconv::Conv { Self::CCmseNonSecureCall => "CCmseNonSecureCall", Self::CCmseNonSecureEntry => "CCmseNonSecureEntry", Self::Msp430Intr => "Msp430Intr", - Self::PtxKernel => "PtxKernel", Self::X86Fastcall => "X86Fastcall", Self::X86Intr => "X86Intr", Self::X86Stdcall => "X86Stdcall", diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 169f3a78c26..25926c871dd 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -290,7 +290,7 @@ fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: ExternAbi, c_variadic: bool) -> Conv Aapcs { .. } => Conv::ArmAapcs, CCmseNonSecureCall => Conv::CCmseNonSecureCall, CCmseNonSecureEntry => Conv::CCmseNonSecureEntry, - PtxKernel => Conv::PtxKernel, + PtxKernel => Conv::GpuKernel, Msp430Interrupt => Conv::Msp430Intr, X86Interrupt => Conv::X86Intr, GpuKernel => Conv::GpuKernel, -- cgit 1.4.1-3-g733a5 From 5f29273921f2a15a440e373c640d5525756fdf41 Mon Sep 17 00:00:00 2001 From: Daniel Paoliello Date: Fri, 7 Feb 2025 16:42:55 -0800 Subject: rustc_codegen_llvm: Mark items as pub(crate) outside of the llvm module --- compiler/rustc_codegen_llvm/src/abi.rs | 2 +- compiler/rustc_codegen_llvm/src/back/lto.rs | 4 ++-- compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 2 +- compiler/rustc_codegen_llvm/src/lib.rs | 2 +- compiler/rustc_codegen_llvm/src/llvm_util.rs | 2 +- compiler/rustc_codegen_llvm/src/type_.rs | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 31ee0eeca11..bde80124a4b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -657,7 +657,7 @@ impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } impl llvm::CallConv { - pub fn from_conv(conv: Conv, arch: &str) -> Self { + pub(crate) fn from_conv(conv: Conv, arch: &str) -> Self { match conv { Conv::C | Conv::Rust diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 78c759bbe8c..98d04a54e91 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -621,7 +621,7 @@ unsafe impl Send for ModuleBuffer {} unsafe impl Sync for ModuleBuffer {} impl ModuleBuffer { - pub fn new(m: &llvm::Module) -> ModuleBuffer { + pub(crate) fn new(m: &llvm::Module) -> ModuleBuffer { ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) } } @@ -663,7 +663,7 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { + pub(crate) fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { unsafe { let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); ThinBuffer(buffer) diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 4cbd49aa44d..f075f332462 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -17,7 +17,7 @@ pub struct OwnedTargetMachine { } impl OwnedTargetMachine { - pub fn new( + pub(crate) fn new( triple: &CStr, cpu: &CStr, features: &CStr, diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 14346795fda..57a5bd43446 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -29,7 +29,7 @@ use std::mem::ManuallyDrop; use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; use errors::{AutoDiffWithoutLTO, ParseTargetMachineConfig}; -pub use llvm_util::target_features_cfg; +pub(crate) use llvm_util::target_features_cfg; use rustc_ast::expand::allocator::AllocatorKind; use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 53611c746a7..e72f8bdff84 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -303,7 +303,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option Vec { +pub(crate) fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec { let mut features: FxHashSet = Default::default(); // Add base features for the target. diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index c56ad886120..13dc2df3b0d 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -237,11 +237,11 @@ impl<'ll, 'tcx> BaseTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { impl Type { /// Creates an integer type with the given number of bits, e.g., i24 - pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { + pub(crate) fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } } - pub fn ptr_llcx(llcx: &llvm::Context) -> &Type { + pub(crate) fn ptr_llcx(llcx: &llvm::Context) -> &Type { unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::DATA.0) } } } -- cgit 1.4.1-3-g733a5 From cde7e805ad48e8766a07c95c2046693c6a6e236c Mon Sep 17 00:00:00 2001 From: Flakebi Date: Mon, 10 Feb 2025 21:38:44 +0100 Subject: Cast allocas to default address space Pointers for variables all need to be in the same address space for correct compilation. Therefore ensure that even if an `alloca` is created in a different address space, it is casted to the default address space before its value is used. This is necessary for the amdgpu target and others where the default address space for `alloca`s is not 0. For example the following code compiles incorrectly when not casting the address space to the default one: ```rust fn f(p: *const i8 /* addrspace(0) */) -> *const i8 /* addrspace(0) */ { let local = 0i8; /* addrspace(5) */ let res = if cond { p } else { &raw const local }; res } ``` results in ```llvm %local = alloca addrspace(5) i8 %res = alloca addrspace(5) ptr if: ; Store 64-bit flat pointer store ptr %p, ptr addrspace(5) %res else: ; Store 32-bit scratch pointer store ptr addrspace(5) %local, ptr addrspace(5) %res ret: ; Load and return 64-bit flat pointer %res.load = load ptr, ptr addrspace(5) %res ret ptr %res.load ``` For amdgpu, `addrspace(0)` are 64-bit pointers, `addrspace(5)` are 32-bit pointers. The above code may store a 32-bit pointer and read it back as a 64-bit pointer, which is obviously wrong and cannot work. Instead, we need to `addrspacecast %local to ptr addrspace(0)`, then we store and load the correct type. --- compiler/rustc_codegen_llvm/src/builder.rs | 6 ++++-- tests/codegen/amdgpu-addrspacecast.rs | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/codegen/amdgpu-addrspacecast.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index ab0e43e60a4..fbeedef314d 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -421,7 +421,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - alloca + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) } } @@ -430,7 +431,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - alloca + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) } } diff --git a/tests/codegen/amdgpu-addrspacecast.rs b/tests/codegen/amdgpu-addrspacecast.rs new file mode 100644 index 00000000000..7fe630a7efa --- /dev/null +++ b/tests/codegen/amdgpu-addrspacecast.rs @@ -0,0 +1,18 @@ +// Check that pointers are casted to addrspace(0) before they are used + +//@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 +//@ needs-llvm-components: amdgpu +//@ add-core-stubs +#![feature(no_core)] +#![no_core] + +extern crate minicore; + +// CHECK-LABEL: @ref_of_local +// CHECK: [[alloca:%[0-9]]] = alloca +// CHECK: %i = addrspacecast ptr addrspace(5) [[alloca]] to ptr +#[no_mangle] +pub fn ref_of_local(f: fn(&i32)) { + let i = 0; + f(&i); +} -- cgit 1.4.1-3-g733a5 From 51f49d84649d27af91e9fc8f6aac319be100cecd Mon Sep 17 00:00:00 2001 From: Askar Safin Date: Tue, 11 Feb 2025 09:58:53 +0300 Subject: compiler/rustc_codegen_llvm/src/lib.rs: remove "unsafe impl Send/Sync" --- compiler/rustc_codegen_llvm/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index f0d04b2b644..51380d82943 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -245,9 +245,6 @@ impl WriteBackendMethods for LlvmCodegenBackend { } } -unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis -unsafe impl Sync for LlvmCodegenBackend {} - impl LlvmCodegenBackend { pub fn new() -> Box { Box::new(LlvmCodegenBackend(())) -- cgit 1.4.1-3-g733a5 From b2cd1b8ead95530e61ad108a3113c588e590187a Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Tue, 14 Jan 2025 12:25:16 +0000 Subject: Remove an unsafe closure invariant by inlining the closure wrapper into the called function --- compiler/rustc_codegen_llvm/src/back/write.rs | 130 ++++++++++---------------- 1 file changed, 50 insertions(+), 80 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index ae4c4d5876e..6a7b6996347 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -40,7 +40,7 @@ use crate::errors::{ WithLlvmError, WriteBytecode, }; use crate::llvm::diagnostic::OptimizationDiagnosticKind::*; -use crate::llvm::{self, DiagnosticInfo, PassManager}; +use crate::llvm::{self, DiagnosticInfo}; use crate::type_::Type; use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util}; @@ -54,7 +54,7 @@ pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> Fatal fn write_output_file<'ll>( dcx: DiagCtxtHandle<'_>, target: &'ll llvm::TargetMachine, - pm: &llvm::PassManager<'ll>, + no_builtins: bool, m: &'ll llvm::Module, output: &Path, dwo_output: Option<&Path>, @@ -63,16 +63,19 @@ fn write_output_file<'ll>( verify_llvm_ir: bool, ) -> Result<(), FatalError> { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); - unsafe { - let output_c = path_to_c_string(output); - let dwo_output_c; - let dwo_output_ptr = if let Some(dwo_output) = dwo_output { - dwo_output_c = path_to_c_string(dwo_output); - dwo_output_c.as_ptr() - } else { - std::ptr::null() - }; - let result = llvm::LLVMRustWriteOutputFile( + let output_c = path_to_c_string(output); + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() + } else { + std::ptr::null() + }; + let result = unsafe { + let pm = llvm::LLVMCreatePassManager(); + llvm::LLVMAddAnalysisPasses(target, pm); + llvm::LLVMRustAddLibraryInfo(pm, m, no_builtins); + llvm::LLVMRustWriteOutputFile( target, pm, m, @@ -80,22 +83,22 @@ fn write_output_file<'ll>( dwo_output_ptr, file_type, verify_llvm_ir, - ); + ) + }; - // Record artifact sizes for self-profiling - if result == llvm::LLVMRustResult::Success { - let artifact_kind = match file_type { - llvm::FileType::ObjectFile => "object_file", - llvm::FileType::AssemblyFile => "assembly_file", - }; - record_artifact_size(self_profiler_ref, artifact_kind, output); - if let Some(dwo_file) = dwo_output { - record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); - } + // Record artifact sizes for self-profiling + if result == llvm::LLVMRustResult::Success { + let artifact_kind = match file_type { + llvm::FileType::ObjectFile => "object_file", + llvm::FileType::AssemblyFile => "assembly_file", + }; + record_artifact_size(self_profiler_ref, artifact_kind, output); + if let Some(dwo_file) = dwo_output { + record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); } - - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } + + result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } pub(crate) fn create_informational_target_machine( @@ -755,31 +758,6 @@ pub(crate) unsafe fn codegen( create_msvc_imps(cgcx, llcx, llmod); } - // A codegen-specific pass manager is used to generate object - // files for an LLVM module. - // - // Apparently each of these pass managers is a one-shot kind of - // thing, so we create a new one for each type of output. The - // pass manager passed to the closure should be ensured to not - // escape the closure itself, and the manager should only be - // used once. - unsafe fn with_codegen<'ll, F, R>( - tm: &'ll llvm::TargetMachine, - llmod: &'ll llvm::Module, - no_builtins: bool, - f: F, - ) -> R - where - F: FnOnce(&'ll mut PassManager<'ll>) -> R, - { - unsafe { - let cpm = llvm::LLVMCreatePassManager(); - llvm::LLVMAddAnalysisPasses(tm, cpm); - llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); - f(cpm) - } - } - // Note that if object files are just LLVM bitcode we write bitcode, // copy it to the .o file, and delete the bitcode if it wasn't // otherwise requested. @@ -898,21 +876,17 @@ pub(crate) unsafe fn codegen( } else { llmod }; - unsafe { - with_codegen(tm, llmod, config.no_builtins, |cpm| { - write_output_file( - dcx, - tm, - cpm, - llmod, - &path, - None, - llvm::FileType::AssemblyFile, - &cgcx.prof, - config.verify_llvm_ir, - ) - })?; - } + write_output_file( + dcx, + tm, + config.no_builtins, + llmod, + &path, + None, + llvm::FileType::AssemblyFile, + &cgcx.prof, + config.verify_llvm_ir, + )?; } match config.emit_obj { @@ -936,21 +910,17 @@ pub(crate) unsafe fn codegen( (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; - unsafe { - with_codegen(tm, llmod, config.no_builtins, |cpm| { - write_output_file( - dcx, - tm, - cpm, - llmod, - &obj_out, - dwo_out, - llvm::FileType::ObjectFile, - &cgcx.prof, - config.verify_llvm_ir, - ) - })?; - } + write_output_file( + dcx, + tm, + config.no_builtins, + llmod, + &obj_out, + dwo_out, + llvm::FileType::ObjectFile, + &cgcx.prof, + config.verify_llvm_ir, + )?; } EmitObj::Bitcode => { -- cgit 1.4.1-3-g733a5 From 4b83038d637c1664df94895c82cb8b26716bad1d Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Tue, 14 Jan 2025 12:25:16 +0000 Subject: Add a safe wrapper for `WriteBitcodeToFile` --- compiler/rustc_codegen_llvm/src/back/write.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 6a7b6996347..39aa3a033dc 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -328,13 +328,17 @@ pub(crate) fn save_temp_bitcode( if !cgcx.save_temps { return; } + let ext = format!("{name}.bc"); + let cgu = Some(&module.name[..]); + let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); + write_bitcode_to_file(module, &path) +} + +fn write_bitcode_to_file(module: &ModuleCodegen, path: &Path) { unsafe { - let ext = format!("{name}.bc"); - let cgu = Some(&module.name[..]); - let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); - let cstr = path_to_c_string(&path); + let path = path_to_c_string(&path); let llmod = module.module_llvm.llmod(); - llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); + llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr()); } } @@ -679,7 +683,6 @@ pub(crate) unsafe fn optimize( ) -> Result<(), FatalError> { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); - let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt); @@ -688,8 +691,7 @@ pub(crate) unsafe fn optimize( if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); - let out = path_to_c_string(&out); - unsafe { llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()) }; + write_bitcode_to_file(module, &out) } // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts -- cgit 1.4.1-3-g733a5 From dcf1e4d72b4dbda592050dcde7d7972414b05d3c Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Tue, 14 Jan 2025 12:25:16 +0000 Subject: Document some safety constraints and use more safe wrappers --- compiler/rustc_codegen_llvm/src/allocator.rs | 4 +- compiler/rustc_codegen_llvm/src/back/archive.rs | 11 ++-- compiler/rustc_codegen_llvm/src/back/write.rs | 64 +++++++++++------------- compiler/rustc_codegen_llvm/src/common.rs | 2 +- compiler/rustc_codegen_llvm/src/consts.rs | 6 +-- compiler/rustc_codegen_llvm/src/context.rs | 10 ++-- compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs | 2 +- compiler/rustc_codegen_llvm/src/declare.rs | 2 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 +- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 4 ++ 11 files changed, 50 insertions(+), 59 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 149ded28356..66723cbf882 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -81,13 +81,13 @@ pub(crate) unsafe fn codegen( llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); - llvm::LLVMSetInitializer(ll_g, llval); + llvm::set_initializer(ll_g, llval); let name = NO_ALLOC_SHIM_IS_UNSTABLE; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let llval = llvm::LLVMConstInt(i8, 0, False); - llvm::LLVMSetInitializer(ll_g, llval); + llvm::set_initializer(ll_g, llval); } if tcx.sess.opts.debuginfo != DebugInfo::None { diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 33a956e552f..93553f3f364 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -11,7 +11,7 @@ use rustc_codegen_ssa::back::archive::{ use rustc_session::Session; use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind}; +use crate::llvm::{self, ArchiveKind, last_error}; /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] @@ -169,6 +169,8 @@ impl<'a> LlvmArchiveBuilder<'a> { .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); let mut additions = mem::take(&mut self.additions); + // Values in the `members` list below will contain pointers to the strings allocated here. + // So they need to get dropped after all elements of `members` get freed. let mut strings = Vec::new(); let mut members = Vec::new(); @@ -229,12 +231,7 @@ impl<'a> LlvmArchiveBuilder<'a> { self.sess.target.arch == "arm64ec", ); let ret = if r.into_result().is_err() { - let err = llvm::LLVMRustGetLastError(); - let msg = if err.is_null() { - "failed to write archive".into() - } else { - String::from_utf8_lossy(CStr::from_ptr(err).to_bytes()) - }; + let msg = last_error().unwrap_or_else(|| "failed to write archive".into()); Err(io::Error::new(io::ErrorKind::Other, msg)) } else { Ok(!members.is_empty()) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 39aa3a033dc..58933a77e53 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1049,24 +1049,18 @@ unsafe fn embed_bitcode( { // We don't need custom section flags, create LLVM globals. let llconst = common::bytes_in_context(llcx, bitcode); - let llglobal = llvm::LLVMAddGlobal( - llmod, - common::val_ty(llconst), - c"rustc.embedded.module".as_ptr(), - ); - llvm::LLVMSetInitializer(llglobal, llconst); + let llglobal = + llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module"); + llvm::set_initializer(llglobal, llconst); llvm::set_section(llglobal, bitcode_section_name(cgcx)); llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); - let llglobal = llvm::LLVMAddGlobal( - llmod, - common::val_ty(llconst), - c"rustc.embedded.cmdline".as_ptr(), - ); - llvm::LLVMSetInitializer(llglobal, llconst); + let llglobal = + llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); + llvm::set_initializer(llglobal, llconst); let section = if cgcx.target_is_like_osx { c"__LLVM,__cmdline" } else if cgcx.target_is_like_aix { @@ -1106,31 +1100,29 @@ fn create_msvc_imps( // underscores added in front). let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; - unsafe { - let ptr_ty = Type::ptr_llcx(llcx); - let globals = base::iter_globals(llmod) - .filter(|&val| { - llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage - && llvm::LLVMIsDeclaration(val) == 0 - }) - .filter_map(|val| { - // Exclude some symbols that we know are not Rust symbols. - let name = llvm::get_value_name(val); - if ignored(name) { None } else { Some((val, name)) } - }) - .map(move |(val, name)| { - let mut imp_name = prefix.as_bytes().to_vec(); - imp_name.extend(name); - let imp_name = CString::new(imp_name).unwrap(); - (imp_name, val) - }) - .collect::>(); + let ptr_ty = Type::ptr_llcx(llcx); + let globals = base::iter_globals(llmod) + .filter(|&val| { + llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val) + }) + .filter_map(|val| { + // Exclude some symbols that we know are not Rust symbols. + let name = llvm::get_value_name(val); + if ignored(name) { None } else { Some((val, name)) } + }) + .map(move |(val, name)| { + let mut imp_name = prefix.as_bytes().to_vec(); + imp_name.extend(name); + let imp_name = CString::new(imp_name).unwrap(); + (imp_name, val) + }) + .collect::>(); - for (imp_name, val) in globals { - let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr()); - llvm::LLVMSetInitializer(imp, val); - llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); - } + for (imp_name, val) in globals { + let imp = llvm::add_global(llmod, ptr_ty, &imp_name); + + llvm::set_initializer(imp, val); + llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); } // Use this function to exclude certain symbols from `__imp` generation. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8c94a46ebf3..78b3a7f8541 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -219,8 +219,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { bug!("symbol `{}` is already defined", sym); }); + llvm::set_initializer(g, sc); unsafe { - llvm::LLVMSetInitializer(g, sc); llvm::LLVMSetGlobalConstant(g, True); llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index c6855dd42e5..4a5491ec7a1 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -191,7 +191,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( }) }); llvm::set_linkage(g2, llvm::Linkage::InternalLinkage); - unsafe { llvm::LLVMSetInitializer(g2, g1) }; + llvm::set_initializer(g2, g1); g2 } else if cx.tcx.sess.target.arch == "x86" && common::is_mingw_gnu_toolchain(&cx.tcx.sess.target) @@ -235,7 +235,7 @@ impl<'ll> CodegenCx<'ll, '_> { } _ => self.define_private_global(self.val_ty(cv)), }; - unsafe { llvm::LLVMSetInitializer(gv, cv) }; + llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); gv @@ -458,7 +458,7 @@ impl<'ll> CodegenCx<'ll, '_> { new_g }; set_global_alignment(self, g, alloc.align); - llvm::LLVMSetInitializer(g, v); + llvm::set_initializer(g, v); if self.should_assume_dso_local(g, true) { llvm::LLVMRustSetDSOLocal(g, true); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ba4fd75fb94..7fe527a4c07 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -616,12 +616,10 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let array = self.const_array(self.type_ptr(), values); - unsafe { - let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); - llvm::LLVMSetInitializer(g, array); - llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); - llvm::set_section(g, c"llvm.metadata"); - } + let g = llvm::add_global(self.llmod, self.val_ty(array), name); + llvm::set_initializer(g, array); + llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); + llvm::set_section(g, c"llvm.metadata"); } } impl<'ll> SimpleCx<'ll> { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 2c9f1cda13a..54c5d445f66 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -73,7 +73,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); llvm::set_section(section_var, c".debug_gdb_scripts"); - llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); + llvm::set_initializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index bdc83267cca..cebceef1b93 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -235,7 +235,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// name. pub(crate) fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { self.get_declared_value(name).and_then(|val| { - let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; + let declaration = llvm::is_declaration(val); if !declaration { Some(val) } else { None } }) } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 3200c94d977..8b976885904 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -824,7 +824,7 @@ fn codegen_msvc_try<'ll>( if bx.cx.tcx.sess.target.supports_comdat() { llvm::SetUniqueComdat(bx.llmod, tydesc); } - unsafe { llvm::LLVMSetInitializer(tydesc, type_info) }; + llvm::set_initializer(tydesc, type_info); // The flag value of 8 indicates that we are catching the exception by // reference instead of by value. We can't use catch by value because diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4d6a76b23ea..441d144ce50 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2359,7 +2359,7 @@ unsafe extern "C" { ); pub fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, - PM: &PassManager<'a>, + PM: *mut PassManager<'a>, M: &'a Module, Output: *const c_char, DwoOutput: *const c_char, diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 707aeba22cc..7becba4ccd4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -241,6 +241,10 @@ pub fn set_linkage(llglobal: &Value, linkage: Linkage) { } } +pub fn is_declaration(llglobal: &Value) -> bool { + unsafe { LLVMIsDeclaration(llglobal) == ffi::True } +} + pub fn get_visibility(llglobal: &Value) -> Visibility { unsafe { LLVMGetVisibility(llglobal) }.to_rust() } -- cgit 1.4.1-3-g733a5 From d82219a4fa06bfa47fc5aac64844c461905ae77d Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Wed, 12 Feb 2025 00:37:33 +0000 Subject: debuginfo: Set bitwidth appropriately in enum variant tags Previously, we unconditionally set the bitwidth to 128-bits, the largest an discrimnator would possibly be. Then, LLVM would cut down the constant by chopping off leading zeroes before emitting the DWARF. LLVM only supported 64-bit descriminators, so this would also have occasionally resulted in truncated data (or an assert) if more than 64-bits were used. LLVM added support for 128-bit enumerators in llvm/llvm-project#125578 That patchset also trusts the constant to describe how wide the variant tag is. As a result, we went from emitting tags that looked like: DW_AT_discr_value (0xfe) (`form1`) to emitting tags that looked like: DW_AT_discr_value (<0x10> fe ff ff ff 00 00 00 00 00 00 00 00 00 00 00 00 ) This makes the `DW_AT_discr_value` encode at the bitwidth of the tag, which: 1. Is probably closer to our intentions in terms of describing the data. 2. Doesn't invoke the 128-bit support which may not be supported by all debuggers / downstream tools. 3. Will result in smaller debug information. --- .../rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs | 8 +++++++- tests/codegen/enum/enum-debug-niche-2.rs | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 11824398f24..187d97c54c8 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -437,6 +437,12 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( .source_info .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + let discr = discr_value.opt_single_val().map(|value| { + let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); + let size = cx.size_of(tag_base_type); + cx.const_uint_big(cx.type_ix(size.bits()), value) + }); + unsafe { llvm::LLVMRustDIBuilderCreateVariantMemberType( DIB(cx), @@ -448,7 +454,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.opt_single_val().map(|value| cx.const_u128(value)), + discr, DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) diff --git a/tests/codegen/enum/enum-debug-niche-2.rs b/tests/codegen/enum/enum-debug-niche-2.rs index 58f43fe3ec6..80a4081f15b 100644 --- a/tests/codegen/enum/enum-debug-niche-2.rs +++ b/tests/codegen/enum/enum-debug-niche-2.rs @@ -5,8 +5,8 @@ //@ ignore-msvc // // CHECK: {{.*}}DICompositeType{{.*}}tag: DW_TAG_variant_part,{{.*}}size: 32,{{.*}} -// CHECK: {{.*}}DIDerivedType{{.*}}tag: DW_TAG_member,{{.*}}name: "Placeholder",{{.*}}extraData: i128 4294967295{{[,)].*}} -// CHECK: {{.*}}DIDerivedType{{.*}}tag: DW_TAG_member,{{.*}}name: "Error",{{.*}}extraData: i128 0{{[,)].*}} +// CHECK: {{.*}}DIDerivedType{{.*}}tag: DW_TAG_member,{{.*}}name: "Placeholder",{{.*}}extraData: i32 -1{{[,)].*}} +// CHECK: {{.*}}DIDerivedType{{.*}}tag: DW_TAG_member,{{.*}}name: "Error",{{.*}}extraData: i32 0{{[,)].*}} #![feature(never_type)] #[derive(Copy, Clone)] -- cgit 1.4.1-3-g733a5 From 659e20fa7524f8fd217476daf5ecbbe366b2ae61 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Sat, 8 Feb 2025 14:25:14 +1100 Subject: Remove `LLVMGetModuleContext` This was unused after the removal of `-Zprofile` in #131829. --- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 - 1 file changed, 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 441d144ce50..e0beaeab8a5 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -972,7 +972,6 @@ unsafe extern "C" { // Create modules. pub fn LLVMModuleCreateWithNameInContext(ModuleID: *const c_char, C: &Context) -> &Module; - pub fn LLVMGetModuleContext(M: &Module) -> &Context; pub fn LLVMCloneModule(M: &Module) -> &Module; /// Data layout. See Module::getDataLayout. -- cgit 1.4.1-3-g733a5 From e7cef26a3d76728b43cdf0686fb9a378b4980e30 Mon Sep 17 00:00:00 2001 From: Daniel Paoliello Date: Tue, 11 Feb 2025 10:33:45 -0800 Subject: cg_llvm: Reduce visibility of all functions in the llvm module --- compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs | 6 +- compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs | 27 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 778 +++++++++++---------- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 88 +-- 4 files changed, 479 insertions(+), 420 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 4dabde55e98..63b2b15c514 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -26,7 +26,7 @@ impl ArchiveRO { /// /// If this archive is used with a mutable method, then an error will be /// raised. - pub fn open(dst: &Path) -> Result { + pub(crate) fn open(dst: &Path) -> Result { unsafe { let s = path_to_c_string(dst); let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { @@ -36,7 +36,7 @@ impl ArchiveRO { } } - pub fn iter(&self) -> Iter<'_> { + pub(crate) fn iter(&self) -> Iter<'_> { unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } } } @@ -71,7 +71,7 @@ impl<'a> Drop for Iter<'a> { } impl<'a> Child<'a> { - pub fn name(&self) -> Option<&'a str> { + pub(crate) fn name(&self) -> Option<&'a str> { unsafe { let mut name_len = 0; let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 92b0ce8ffe1..39bac13a968 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -1,4 +1,5 @@ #![allow(non_camel_case_types)] +#![expect(dead_code)] use libc::{c_char, c_uint}; @@ -8,23 +9,23 @@ use crate::llvm::Bool; #[link(name = "llvm-wrapper", kind = "static")] extern "C" { // Enzyme - pub fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool; - pub fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value); - pub fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>; - pub fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>; - pub fn LLVMRustEraseInstFromParent(V: &Value); - pub fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value; - pub fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; + pub(crate) fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool; + pub(crate) fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value); + pub(crate) fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>; + pub(crate) fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>; + pub(crate) fn LLVMRustEraseInstFromParent(V: &Value); + pub(crate) fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value; + pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; } extern "C" { // Enzyme - pub fn LLVMDumpModule(M: &Module); - pub fn LLVMDumpValue(V: &Value); - pub fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; - pub fn LLVMGetReturnType(T: &Type) -> &Type; - pub fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); - pub fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>; + pub(crate) fn LLVMDumpModule(M: &Module); + pub(crate) fn LLVMDumpValue(V: &Value); + pub(crate) fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; + pub(crate) fn LLVMGetReturnType(T: &Type) -> &Type; + pub(crate) fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); + pub(crate) fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>; } #[repr(C)] diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e0beaeab8a5..618fdb09d4d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -160,7 +160,7 @@ pub enum Visibility { } impl Visibility { - pub fn from_generic(visibility: SymbolVisibility) -> Self { + pub(crate) fn from_generic(visibility: SymbolVisibility) -> Self { match visibility { SymbolVisibility::Hidden => Visibility::Hidden, SymbolVisibility::Protected => Visibility::Protected, @@ -255,7 +255,7 @@ pub enum IntPredicate { } impl IntPredicate { - pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { + pub(crate) fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { use rustc_codegen_ssa::common::IntPredicate as Common; match intpre { Common::IntEQ => Self::IntEQ, @@ -295,7 +295,7 @@ pub enum RealPredicate { } impl RealPredicate { - pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { + pub(crate) fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { use rustc_codegen_ssa::common::RealPredicate as Common; match realp { Common::RealPredicateFalse => Self::RealPredicateFalse, @@ -344,7 +344,7 @@ pub enum TypeKind { } impl TypeKind { - pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { + pub(crate) fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { use rustc_codegen_ssa::common::TypeKind as Common; match self { Self::Void => Common::Void, @@ -388,7 +388,7 @@ pub enum AtomicRmwBinOp { } impl AtomicRmwBinOp { - pub fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { + pub(crate) fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { use rustc_codegen_ssa::common::AtomicRmwBinOp as Common; match op { Common::AtomicXchg => Self::AtomicXchg, @@ -422,7 +422,7 @@ pub enum AtomicOrdering { } impl AtomicOrdering { - pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { + pub(crate) fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { use rustc_codegen_ssa::common::AtomicOrdering as Common; match ao { Common::Unordered => Self::Unordered, @@ -908,7 +908,7 @@ pub mod debuginfo { } impl DebugEmissionKind { - pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + pub(crate) fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { // We should be setting LLVM's emission kind to `LineTablesOnly` if // we are compiling with "limited" debuginfo. However, some of the // existing tools relied on slightly more debuginfo being generated than @@ -967,48 +967,55 @@ pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c unsafe extern "C" { // Create and destroy contexts. - pub fn LLVMContextDispose(C: &'static mut Context); - pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint; + pub(crate) fn LLVMContextDispose(C: &'static mut Context); + pub(crate) fn LLVMGetMDKindIDInContext( + C: &Context, + Name: *const c_char, + SLen: c_uint, + ) -> c_uint; // Create modules. - pub fn LLVMModuleCreateWithNameInContext(ModuleID: *const c_char, C: &Context) -> &Module; - pub fn LLVMCloneModule(M: &Module) -> &Module; + pub(crate) fn LLVMModuleCreateWithNameInContext( + ModuleID: *const c_char, + C: &Context, + ) -> &Module; + pub(crate) fn LLVMCloneModule(M: &Module) -> &Module; /// Data layout. See Module::getDataLayout. - pub fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; - pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); + pub(crate) fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; + pub(crate) fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); + pub(crate) fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); // Operations on integer types - pub fn LLVMInt1TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt8TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt16TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt32TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt64TypeInContext(C: &Context) -> &Type; - pub fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; + pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt16TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt32TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt64TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; - pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; + pub(crate) fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; // Operations on real types - pub fn LLVMHalfTypeInContext(C: &Context) -> &Type; - pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; - pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; - pub fn LLVMFP128TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMHalfTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; // Operations on function types - pub fn LLVMFunctionType<'a>( + pub(crate) fn LLVMFunctionType<'a>( ReturnType: &'a Type, ParamTypes: *const &'a Type, ParamCount: c_uint, IsVarArg: Bool, ) -> &'a Type; - pub fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; - pub fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; + pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); // Operations on struct types - pub fn LLVMStructTypeInContext<'a>( + pub(crate) fn LLVMStructTypeInContext<'a>( C: &'a Context, ElementTypes: *const &'a Type, ElementCount: c_uint, @@ -1016,111 +1023,123 @@ unsafe extern "C" { ) -> &'a Type; // Operations on array, pointer, and vector types (sequence types) - pub fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; - pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; + pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; + pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; - pub fn LLVMGetElementType(Ty: &Type) -> &Type; - pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; + pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type; + pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; // Operations on other types - pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMTokenTypeInContext(C: &Context) -> &Type; - pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMTokenTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values - pub fn LLVMIsUndef(Val: &Value) -> Bool; - pub fn LLVMTypeOf(Val: &Value) -> &Type; - pub fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; - pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); - pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); - pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); - pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + pub(crate) fn LLVMIsUndef(Val: &Value) -> Bool; + pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type; + pub(crate) fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; + pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); + pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); + pub(crate) fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); + pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub(crate) fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type - pub fn LLVMConstNull(Ty: &Type) -> &Value; - pub fn LLVMGetUndef(Ty: &Type) -> &Value; - pub fn LLVMGetPoison(Ty: &Type) -> &Value; + pub(crate) fn LLVMConstNull(Ty: &Type) -> &Value; + pub(crate) fn LLVMGetUndef(Ty: &Type) -> &Value; + pub(crate) fn LLVMGetPoison(Ty: &Type) -> &Value; // Operations on metadata - pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata; - pub fn LLVMMDNodeInContext2<'a>( + pub(crate) fn LLVMMDStringInContext2( + C: &Context, + Str: *const c_char, + SLen: size_t, + ) -> &Metadata; + pub(crate) fn LLVMMDNodeInContext2<'a>( C: &'a Context, Vals: *const &'a Metadata, Count: size_t, ) -> &'a Metadata; - pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); + pub(crate) fn LLVMAddNamedMetadataOperand<'a>( + M: &'a Module, + Name: *const c_char, + Val: &'a Value, + ); // Operations on scalar constants - pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; - pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; - pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; + pub(crate) fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; + pub(crate) fn LLVMConstIntOfArbitraryPrecision( + IntTy: &Type, + Wn: c_uint, + Ws: *const u64, + ) -> &Value; + pub(crate) fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; // Operations on composite constants - pub fn LLVMConstArray2<'a>( + pub(crate) fn LLVMConstArray2<'a>( ElementTy: &'a Type, ConstantVals: *const &'a Value, Length: u64, ) -> &'a Value; - pub fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; - pub fn LLVMConstStringInContext2( + pub(crate) fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; + pub(crate) fn LLVMConstStringInContext2( C: &Context, Str: *const c_char, Length: size_t, DontNullTerminate: Bool, ) -> &Value; - pub fn LLVMConstStructInContext<'a>( + pub(crate) fn LLVMConstStructInContext<'a>( C: &'a Context, ConstantVals: *const &'a Value, Count: c_uint, Packed: Bool, ) -> &'a Value; - pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; + pub(crate) fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions - pub fn LLVMConstInBoundsGEP2<'a>( + pub(crate) fn LLVMConstInBoundsGEP2<'a>( ty: &'a Type, ConstantVal: &'a Value, ConstantIndices: *const &'a Value, NumIndices: c_uint, ) -> &'a Value; - pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; - pub fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; - pub fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; + pub(crate) fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; + pub(crate) fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMIsDeclaration(Global: &Value) -> Bool; - pub fn LLVMGetLinkage(Global: &Value) -> RawEnum; - pub fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); - pub fn LLVMSetSection(Global: &Value, Section: *const c_char); - pub fn LLVMGetVisibility(Global: &Value) -> RawEnum; - pub fn LLVMSetVisibility(Global: &Value, Viz: Visibility); - pub fn LLVMGetAlignment(Global: &Value) -> c_uint; - pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); - pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); - pub fn LLVMGlobalGetValueType(Global: &Value) -> &Type; + pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub(crate) fn LLVMGetLinkage(Global: &Value) -> RawEnum; + pub(crate) fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); + pub(crate) fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub(crate) fn LLVMGetVisibility(Global: &Value) -> RawEnum; + pub(crate) fn LLVMSetVisibility(Global: &Value, Viz: Visibility); + pub(crate) fn LLVMGetAlignment(Global: &Value) -> c_uint; + pub(crate) fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); + pub(crate) fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); + pub(crate) fn LLVMGlobalGetValueType(Global: &Value) -> &Type; // Operations on global variables - pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; - pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; - pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMDeleteGlobal(GlobalVar: &Value); - pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); - pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; - pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); - pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; - pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); - pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub(crate) fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub(crate) fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; + pub(crate) fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; + pub(crate) fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMDeleteGlobal(GlobalVar: &Value); + pub(crate) fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); + pub(crate) fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; + pub(crate) fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); + pub(crate) fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; + pub(crate) fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); + pub(crate) fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); // Operations on attributes - pub fn LLVMCreateStringAttribute( + pub(crate) fn LLVMCreateStringAttribute( C: &Context, Name: *const c_char, NameLen: c_uint, @@ -1129,34 +1148,34 @@ unsafe extern "C" { ) -> &Attribute; // Operations on functions - pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); // Operations on parameters - pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; - pub fn LLVMCountParams(Fn: &Value) -> c_uint; - pub fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; + pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMCountParams(Fn: &Value) -> c_uint; + pub(crate) fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; // Operations on basic blocks - pub fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; - pub fn LLVMAppendBasicBlockInContext<'a>( + pub(crate) fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; + pub(crate) fn LLVMAppendBasicBlockInContext<'a>( C: &'a Context, Fn: &'a Value, Name: *const c_char, ) -> &'a BasicBlock; // Operations on instructions - pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; - pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; - pub fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; + pub(crate) fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; + pub(crate) fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; // Operations on call sites - pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); + pub(crate) fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); // Operations on load/store instructions (only) - pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); + pub(crate) fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); // Operations on phi nodes - pub fn LLVMAddIncoming<'a>( + pub(crate) fn LLVMAddIncoming<'a>( PhiNode: &'a Value, IncomingValues: *const &'a Value, IncomingBlocks: *const &'a BasicBlock, @@ -1164,266 +1183,276 @@ unsafe extern "C" { ); // Instruction builders - pub fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; - pub fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); - pub fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; - pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); + pub(crate) fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; + pub(crate) fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); + pub(crate) fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; + pub(crate) fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); - pub fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>; + pub(crate) fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); + pub(crate) fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>; // Terminators - pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; - pub fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; - pub fn LLVMBuildCondBr<'a>( + pub(crate) fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; + pub(crate) fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; + pub(crate) fn LLVMBuildCondBr<'a>( B: &Builder<'a>, If: &'a Value, Then: &'a BasicBlock, Else: &'a BasicBlock, ) -> &'a Value; - pub fn LLVMBuildSwitch<'a>( + pub(crate) fn LLVMBuildSwitch<'a>( B: &Builder<'a>, V: &'a Value, Else: &'a BasicBlock, NumCases: c_uint, ) -> &'a Value; - pub fn LLVMBuildLandingPad<'a>( + pub(crate) fn LLVMBuildLandingPad<'a>( B: &Builder<'a>, Ty: &'a Type, PersFn: Option<&'a Value>, NumClauses: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; - pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; + pub(crate) fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMBuildCleanupPad<'a>( + pub(crate) fn LLVMBuildCleanupPad<'a>( B: &Builder<'a>, ParentPad: Option<&'a Value>, Args: *const &'a Value, NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMBuildCleanupRet<'a>( + pub(crate) fn LLVMBuildCleanupRet<'a>( B: &Builder<'a>, CleanupPad: &'a Value, BB: Option<&'a BasicBlock>, ) -> Option<&'a Value>; - pub fn LLVMBuildCatchPad<'a>( + pub(crate) fn LLVMBuildCatchPad<'a>( B: &Builder<'a>, ParentPad: &'a Value, Args: *const &'a Value, NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMBuildCatchRet<'a>( + pub(crate) fn LLVMBuildCatchRet<'a>( B: &Builder<'a>, CatchPad: &'a Value, BB: &'a BasicBlock, ) -> Option<&'a Value>; - pub fn LLVMBuildCatchSwitch<'a>( + pub(crate) fn LLVMBuildCatchSwitch<'a>( Builder: &Builder<'a>, ParentPad: Option<&'a Value>, UnwindBB: Option<&'a BasicBlock>, NumHandlers: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); - pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); + pub(crate) fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); + pub(crate) fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); // Add a case to the switch instruction - pub fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); + pub(crate) fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); // Add a clause to the landing pad instruction - pub fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); + pub(crate) fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); // Set the cleanup on a landing pad instruction - pub fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); + pub(crate) fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); // Arithmetic - pub fn LLVMBuildAdd<'a>( + pub(crate) fn LLVMBuildAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFAdd<'a>( + pub(crate) fn LLVMBuildFAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSub<'a>( + pub(crate) fn LLVMBuildSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFSub<'a>( + pub(crate) fn LLVMBuildFSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildMul<'a>( + pub(crate) fn LLVMBuildMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFMul<'a>( + pub(crate) fn LLVMBuildFMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildUDiv<'a>( + pub(crate) fn LLVMBuildUDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExactUDiv<'a>( + pub(crate) fn LLVMBuildExactUDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSDiv<'a>( + pub(crate) fn LLVMBuildSDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExactSDiv<'a>( + pub(crate) fn LLVMBuildExactSDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFDiv<'a>( + pub(crate) fn LLVMBuildFDiv<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildURem<'a>( + pub(crate) fn LLVMBuildURem<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSRem<'a>( + pub(crate) fn LLVMBuildSRem<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFRem<'a>( + pub(crate) fn LLVMBuildFRem<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildShl<'a>( + pub(crate) fn LLVMBuildShl<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildLShr<'a>( + pub(crate) fn LLVMBuildLShr<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildAShr<'a>( + pub(crate) fn LLVMBuildAShr<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNSWAdd<'a>( + pub(crate) fn LLVMBuildNSWAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNUWAdd<'a>( + pub(crate) fn LLVMBuildNUWAdd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNSWSub<'a>( + pub(crate) fn LLVMBuildNSWSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNUWSub<'a>( + pub(crate) fn LLVMBuildNUWSub<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNSWMul<'a>( + pub(crate) fn LLVMBuildNSWMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNUWMul<'a>( + pub(crate) fn LLVMBuildNUWMul<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildAnd<'a>( + pub(crate) fn LLVMBuildAnd<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildOr<'a>( + pub(crate) fn LLVMBuildOr<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildXor<'a>( + pub(crate) fn LLVMBuildXor<'a>( B: &Builder<'a>, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub(crate) fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMBuildFNeg<'a>( + B: &Builder<'a>, + V: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) + -> &'a Value; // Extra flags on arithmetic - pub fn LLVMSetIsDisjoint(Instr: &Value, IsDisjoint: Bool); + pub(crate) fn LLVMSetIsDisjoint(Instr: &Value, IsDisjoint: Bool); // Memory - pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildArrayAlloca<'a>( + pub(crate) fn LLVMBuildAlloca<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildArrayAlloca<'a>( B: &Builder<'a>, Ty: &'a Type, Val: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildLoad2<'a>( + pub(crate) fn LLVMBuildLoad2<'a>( B: &Builder<'a>, Ty: &'a Type, PointerVal: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; - pub fn LLVMBuildGEP2<'a>( + pub(crate) fn LLVMBuildGEP2<'a>( B: &Builder<'a>, Ty: &'a Type, Pointer: &'a Value, @@ -1431,7 +1460,7 @@ unsafe extern "C" { NumIndices: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildInBoundsGEP2<'a>( + pub(crate) fn LLVMBuildInBoundsGEP2<'a>( B: &Builder<'a>, Ty: &'a Type, Pointer: &'a Value, @@ -1441,85 +1470,85 @@ unsafe extern "C" { ) -> &'a Value; // Casts - pub fn LLVMBuildTrunc<'a>( + pub(crate) fn LLVMBuildTrunc<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildZExt<'a>( + pub(crate) fn LLVMBuildZExt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSExt<'a>( + pub(crate) fn LLVMBuildSExt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPToUI<'a>( + pub(crate) fn LLVMBuildFPToUI<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPToSI<'a>( + pub(crate) fn LLVMBuildFPToSI<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildUIToFP<'a>( + pub(crate) fn LLVMBuildUIToFP<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildSIToFP<'a>( + pub(crate) fn LLVMBuildSIToFP<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPTrunc<'a>( + pub(crate) fn LLVMBuildFPTrunc<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFPExt<'a>( + pub(crate) fn LLVMBuildFPExt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildPtrToInt<'a>( + pub(crate) fn LLVMBuildPtrToInt<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildIntToPtr<'a>( + pub(crate) fn LLVMBuildIntToPtr<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildBitCast<'a>( + pub(crate) fn LLVMBuildBitCast<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildPointerCast<'a>( + pub(crate) fn LLVMBuildPointerCast<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildIntCast2<'a>( + pub(crate) fn LLVMBuildIntCast2<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, @@ -1528,14 +1557,14 @@ unsafe extern "C" { ) -> &'a Value; // Comparisons - pub fn LLVMBuildICmp<'a>( + pub(crate) fn LLVMBuildICmp<'a>( B: &Builder<'a>, Op: c_uint, LHS: &'a Value, RHS: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildFCmp<'a>( + pub(crate) fn LLVMBuildFCmp<'a>( B: &Builder<'a>, Op: c_uint, LHS: &'a Value, @@ -1544,47 +1573,48 @@ unsafe extern "C" { ) -> &'a Value; // Miscellaneous instructions - pub fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildSelect<'a>( + pub(crate) fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMBuildSelect<'a>( B: &Builder<'a>, If: &'a Value, Then: &'a Value, Else: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildVAArg<'a>( + pub(crate) fn LLVMBuildVAArg<'a>( B: &Builder<'a>, list: &'a Value, Ty: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExtractElement<'a>( + pub(crate) fn LLVMBuildExtractElement<'a>( B: &Builder<'a>, VecVal: &'a Value, Index: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildInsertElement<'a>( + pub(crate) fn LLVMBuildInsertElement<'a>( B: &Builder<'a>, VecVal: &'a Value, EltVal: &'a Value, Index: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildShuffleVector<'a>( + pub(crate) fn LLVMBuildShuffleVector<'a>( B: &Builder<'a>, V1: &'a Value, V2: &'a Value, Mask: &'a Value, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildExtractValue<'a>( + pub(crate) fn LLVMBuildExtractValue<'a>( B: &Builder<'a>, AggVal: &'a Value, Index: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildInsertValue<'a>( + pub(crate) fn LLVMBuildInsertValue<'a>( B: &Builder<'a>, AggVal: &'a Value, EltVal: &'a Value, @@ -1593,7 +1623,7 @@ unsafe extern "C" { ) -> &'a Value; // Atomic Operations - pub fn LLVMBuildAtomicCmpXchg<'a>( + pub(crate) fn LLVMBuildAtomicCmpXchg<'a>( B: &Builder<'a>, LHS: &'a Value, CMP: &'a Value, @@ -1603,9 +1633,9 @@ unsafe extern "C" { SingleThreaded: Bool, ) -> &'a Value; - pub fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool); + pub(crate) fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool); - pub fn LLVMBuildAtomicRMW<'a>( + pub(crate) fn LLVMBuildAtomicRMW<'a>( B: &Builder<'a>, Op: AtomicRmwBinOp, LHS: &'a Value, @@ -1614,7 +1644,7 @@ unsafe extern "C" { SingleThreaded: Bool, ) -> &'a Value; - pub fn LLVMBuildFence<'a>( + pub(crate) fn LLVMBuildFence<'a>( B: &Builder<'a>, Order: AtomicOrdering, SingleThreaded: Bool, @@ -1622,36 +1652,36 @@ unsafe extern "C" { ) -> &'a Value; /// Writes a module to the specified path. Returns 0 on success. - pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; + pub(crate) fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; /// Creates a legacy pass manager -- only used for final codegen. - pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; + pub(crate) fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; - pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); + pub(crate) fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - pub fn LLVMGetHostCPUFeatures() -> *mut c_char; + pub(crate) fn LLVMGetHostCPUFeatures() -> *mut c_char; - pub fn LLVMDisposeMessage(message: *mut c_char); + pub(crate) fn LLVMDisposeMessage(message: *mut c_char); - pub fn LLVMIsMultithreaded() -> Bool; + pub(crate) fn LLVMIsMultithreaded() -> Bool; - pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; + pub(crate) fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; - pub fn LLVMStructSetBody<'a>( + pub(crate) fn LLVMStructSetBody<'a>( StructTy: &'a Type, ElementTypes: *const &'a Type, ElementCount: c_uint, Packed: Bool, ); - pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + pub(crate) fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; - pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); + pub(crate) fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); - pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + pub(crate) fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - pub fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; - pub fn LLVMSetComdat(V: &Value, C: &Comdat); + pub(crate) fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; + pub(crate) fn LLVMSetComdat(V: &Value, C: &Comdat); pub(crate) fn LLVMCreateOperandBundle( Tag: *const c_char, @@ -1744,22 +1774,26 @@ unsafe extern "C" { #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { - pub fn LLVMRustInstallErrorHandlers(); - pub fn LLVMRustDisableSystemDialogsOnCrash(); + pub(crate) fn LLVMRustInstallErrorHandlers(); + pub(crate) fn LLVMRustDisableSystemDialogsOnCrash(); // Create and destroy contexts. - pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; + pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; /// See llvm::LLVMTypeKind::getTypeID. - pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; + pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; // Operations on all values - pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); - pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; + pub(crate) fn LLVMRustGlobalAddMetadata<'a>( + Val: &'a Value, + KindID: c_uint, + Metadata: &'a Metadata, + ); + pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; // Operations on scalar constants - pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; - pub fn LLVMRustConstInt128Get( + pub(crate) fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; + pub(crate) fn LLVMRustConstInt128Get( ConstantVal: &ConstantInt, SExt: bool, high: &mut u64, @@ -1767,35 +1801,38 @@ unsafe extern "C" { ) -> bool; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); + pub(crate) fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); // Operations on global variables - pub fn LLVMRustGetOrInsertGlobal<'a>( + pub(crate) fn LLVMRustGetOrInsertGlobal<'a>( M: &'a Module, Name: *const c_char, NameLen: size_t, T: &'a Type, ) -> &'a Value; - pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; - pub fn LLVMRustGetNamedValue( + pub(crate) fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; + pub(crate) fn LLVMRustGetNamedValue( M: &Module, Name: *const c_char, NameLen: size_t, ) -> Option<&Value>; // Operations on attributes - pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; - pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; - pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; - pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; - pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute; - pub fn LLVMRustCreateRangeAttribute( + pub(crate) fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; + pub(crate) fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub(crate) fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub(crate) fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; + pub(crate) fn LLVMRustCreateMemoryEffectsAttr( + C: &Context, + effects: MemoryEffects, + ) -> &Attribute; + pub(crate) fn LLVMRustCreateRangeAttribute( C: &Context, num_bits: c_uint, lower_words: *const u64, @@ -1803,13 +1840,13 @@ unsafe extern "C" { ) -> &Attribute; // Operations on functions - pub fn LLVMRustGetOrInsertFunction<'a>( + pub(crate) fn LLVMRustGetOrInsertFunction<'a>( M: &'a Module, Name: *const c_char, NameLen: size_t, FunctionTy: &'a Type, ) -> &'a Value; - pub fn LLVMRustAddFunctionAttributes<'a>( + pub(crate) fn LLVMRustAddFunctionAttributes<'a>( Fn: &'a Value, index: c_uint, Attrs: *const &'a Attribute, @@ -1817,19 +1854,19 @@ unsafe extern "C" { ); // Operations on call sites - pub fn LLVMRustAddCallSiteAttributes<'a>( + pub(crate) fn LLVMRustAddCallSiteAttributes<'a>( Instr: &'a Value, index: c_uint, Attrs: *const &'a Attribute, AttrsLen: size_t, ); - pub fn LLVMRustSetFastMath(Instr: &Value); - pub fn LLVMRustSetAlgebraicMath(Instr: &Value); - pub fn LLVMRustSetAllowReassoc(Instr: &Value); + pub(crate) fn LLVMRustSetFastMath(Instr: &Value); + pub(crate) fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub(crate) fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustBuildMemCpy<'a>( + pub(crate) fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, @@ -1838,7 +1875,7 @@ unsafe extern "C" { Size: &'a Value, IsVolatile: bool, ) -> &'a Value; - pub fn LLVMRustBuildMemMove<'a>( + pub(crate) fn LLVMRustBuildMemMove<'a>( B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, @@ -1847,7 +1884,7 @@ unsafe extern "C" { Size: &'a Value, IsVolatile: bool, ) -> &'a Value; - pub fn LLVMRustBuildMemSet<'a>( + pub(crate) fn LLVMRustBuildMemSet<'a>( B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, @@ -1856,47 +1893,55 @@ unsafe extern "C" { IsVolatile: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFAdd<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFAdd<'a>( B: &Builder<'a>, Acc: &'a Value, Src: &'a Value, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMul<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFMul<'a>( B: &Builder<'a>, Acc: &'a Value, Src: &'a Value, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMin<'a>( + pub(crate) fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMin<'a>( B: &Builder<'a>, Src: &'a Value, IsSigned: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMax<'a>( + pub(crate) fn LLVMRustBuildVectorReduceMax<'a>( B: &Builder<'a>, Src: &'a Value, IsSigned: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMin<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFMin<'a>( B: &Builder<'a>, Src: &'a Value, IsNaN: bool, ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMax<'a>( + pub(crate) fn LLVMRustBuildVectorReduceFMax<'a>( B: &Builder<'a>, Src: &'a Value, IsNaN: bool, ) -> &'a Value; - pub fn LLVMRustBuildMinNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; - pub fn LLVMRustBuildMaxNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildMinNum<'a>( + B: &Builder<'a>, + LHS: &'a Value, + LHS: &'a Value, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildMaxNum<'a>( + B: &Builder<'a>, + LHS: &'a Value, + LHS: &'a Value, + ) -> &'a Value; // Atomic Operations - pub fn LLVMRustBuildAtomicLoad<'a>( + pub(crate) fn LLVMRustBuildAtomicLoad<'a>( B: &Builder<'a>, ElementType: &'a Type, PointerVal: &'a Value, @@ -1904,21 +1949,21 @@ unsafe extern "C" { Order: AtomicOrdering, ) -> &'a Value; - pub fn LLVMRustBuildAtomicStore<'a>( + pub(crate) fn LLVMRustBuildAtomicStore<'a>( B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value, Order: AtomicOrdering, ) -> &'a Value; - pub fn LLVMRustTimeTraceProfilerInitialize(); + pub(crate) fn LLVMRustTimeTraceProfilerInitialize(); - pub fn LLVMRustTimeTraceProfilerFinishThread(); + pub(crate) fn LLVMRustTimeTraceProfilerFinishThread(); - pub fn LLVMRustTimeTraceProfilerFinish(FileName: *const c_char); + pub(crate) fn LLVMRustTimeTraceProfilerFinish(FileName: *const c_char); /// Returns a string describing the last error caused by an LLVMRust* call. - pub fn LLVMRustGetLastError() -> *const c_char; + pub(crate) fn LLVMRustGetLastError() -> *const c_char; /// Prints the timing information collected by `-Ztime-llvm-passes`. pub(crate) fn LLVMRustPrintPassTimings(OutStr: &RustString); @@ -1927,7 +1972,7 @@ unsafe extern "C" { pub(crate) fn LLVMRustPrintStatistics(OutStr: &RustString); /// Prepares inline assembly. - pub fn LLVMRustInlineAsm( + pub(crate) fn LLVMRustInlineAsm( Ty: &Type, AsmString: *const c_char, AsmStringLen: size_t, @@ -1938,7 +1983,7 @@ unsafe extern "C" { Dialect: AsmDialect, CanThrow: Bool, ) -> &Value; - pub fn LLVMRustInlineAsmVerify( + pub(crate) fn LLVMRustInlineAsmVerify( Ty: &Type, Constraints: *const c_char, ConstraintsLen: size_t, @@ -1982,16 +2027,16 @@ unsafe extern "C" { pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; - pub fn LLVMRustDebugMetadataVersion() -> u32; - pub fn LLVMRustVersionMajor() -> u32; - pub fn LLVMRustVersionMinor() -> u32; - pub fn LLVMRustVersionPatch() -> u32; + pub(crate) fn LLVMRustDebugMetadataVersion() -> u32; + pub(crate) fn LLVMRustVersionMajor() -> u32; + pub(crate) fn LLVMRustVersionMinor() -> u32; + pub(crate) fn LLVMRustVersionPatch() -> u32; /// Add LLVM module flags. /// /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What /// "compatible" means depends on the merge behaviors involved. - pub fn LLVMRustAddModuleFlagU32( + pub(crate) fn LLVMRustAddModuleFlagU32( M: &Module, MergeBehavior: ModuleFlagMergeBehavior, Name: *const c_char, @@ -1999,7 +2044,7 @@ unsafe extern "C" { Value: u32, ); - pub fn LLVMRustAddModuleFlagString( + pub(crate) fn LLVMRustAddModuleFlagString( M: &Module, MergeBehavior: ModuleFlagMergeBehavior, Name: *const c_char, @@ -2008,7 +2053,7 @@ unsafe extern "C" { ValueLen: size_t, ); - pub fn LLVMRustDIBuilderCreateCompileUnit<'a>( + pub(crate) fn LLVMRustDIBuilderCreateCompileUnit<'a>( Builder: &DIBuilder<'a>, Lang: c_uint, File: &'a DIFile, @@ -2025,7 +2070,7 @@ unsafe extern "C" { DebugNameTableKind: DebugNameTableKind, ) -> &'a DIDescriptor; - pub fn LLVMRustDIBuilderCreateFile<'a>( + pub(crate) fn LLVMRustDIBuilderCreateFile<'a>( Builder: &DIBuilder<'a>, Filename: *const c_char, FilenameLen: size_t, @@ -2038,12 +2083,12 @@ unsafe extern "C" { SourceLen: size_t, ) -> &'a DIFile; - pub fn LLVMRustDIBuilderCreateSubroutineType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>( Builder: &DIBuilder<'a>, ParameterTypes: &'a DIArray, ) -> &'a DICompositeType; - pub fn LLVMRustDIBuilderCreateFunction<'a>( + pub(crate) fn LLVMRustDIBuilderCreateFunction<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2061,7 +2106,7 @@ unsafe extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; - pub fn LLVMRustDIBuilderCreateMethod<'a>( + pub(crate) fn LLVMRustDIBuilderCreateMethod<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2076,7 +2121,7 @@ unsafe extern "C" { TParam: &'a DIArray, ) -> &'a DISubprogram; - pub fn LLVMRustDIBuilderCreateBasicType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, NameLen: size_t, @@ -2084,7 +2129,7 @@ unsafe extern "C" { Encoding: c_uint, ) -> &'a DIBasicType; - pub fn LLVMRustDIBuilderCreateTypedef<'a>( + pub(crate) fn LLVMRustDIBuilderCreateTypedef<'a>( Builder: &DIBuilder<'a>, Type: &'a DIBasicType, Name: *const c_char, @@ -2094,7 +2139,7 @@ unsafe extern "C" { Scope: Option<&'a DIScope>, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreatePointerType<'a>( + pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>( Builder: &DIBuilder<'a>, PointeeTy: &'a DIType, SizeInBits: u64, @@ -2104,7 +2149,7 @@ unsafe extern "C" { NameLen: size_t, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateStructType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateStructType<'a>( Builder: &DIBuilder<'a>, Scope: Option<&'a DIDescriptor>, Name: *const c_char, @@ -2122,7 +2167,7 @@ unsafe extern "C" { UniqueIdLen: size_t, ) -> &'a DICompositeType; - pub fn LLVMRustDIBuilderCreateMemberType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2136,7 +2181,7 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateVariantMemberType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, Name: *const c_char, @@ -2151,7 +2196,7 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIType; - pub fn LLVMRustDIBuilderCreateStaticMemberType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateStaticMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, Name: *const c_char, @@ -2164,13 +2209,13 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateQualifiedType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateQualifiedType<'a>( Builder: &DIBuilder<'a>, Tag: c_uint, Type: &'a DIType, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateStaticVariable<'a>( + pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( Builder: &DIBuilder<'a>, Context: Option<&'a DIScope>, Name: *const c_char, @@ -2186,7 +2231,7 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub fn LLVMRustDIBuilderCreateVariable<'a>( + pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( Builder: &DIBuilder<'a>, Tag: c_uint, Scope: &'a DIDescriptor, @@ -2201,7 +2246,7 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIVariable; - pub fn LLVMRustDIBuilderCreateArrayType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateArrayType<'a>( Builder: &DIBuilder<'a>, Size: u64, AlignInBits: u32, @@ -2209,19 +2254,19 @@ unsafe extern "C" { Subscripts: &'a DIArray, ) -> &'a DIType; - pub fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( + pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( Builder: &DIBuilder<'a>, Lo: i64, Count: i64, ) -> &'a DISubrange; - pub fn LLVMRustDIBuilderGetOrCreateArray<'a>( + pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( Builder: &DIBuilder<'a>, Ptr: *const Option<&'a DIDescriptor>, Count: c_uint, ) -> &'a DIArray; - pub fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( + pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( Builder: &DIBuilder<'a>, Val: &'a Value, VarInfo: &'a DIVariable, @@ -2231,7 +2276,7 @@ unsafe extern "C" { InsertAtEnd: &'a BasicBlock, ); - pub fn LLVMRustDIBuilderCreateEnumerator<'a>( + pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, NameLen: size_t, @@ -2240,7 +2285,7 @@ unsafe extern "C" { IsUnsigned: bool, ) -> &'a DIEnumerator; - pub fn LLVMRustDIBuilderCreateEnumerationType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateEnumerationType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, Name: *const c_char, @@ -2254,7 +2299,7 @@ unsafe extern "C" { IsScoped: bool, ) -> &'a DIType; - pub fn LLVMRustDIBuilderCreateUnionType<'a>( + pub(crate) fn LLVMRustDIBuilderCreateUnionType<'a>( Builder: &DIBuilder<'a>, Scope: Option<&'a DIScope>, Name: *const c_char, @@ -2270,7 +2315,7 @@ unsafe extern "C" { UniqueIdLen: size_t, ) -> &'a DIType; - pub fn LLVMRustDIBuilderCreateVariantPart<'a>( + pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, Name: *const c_char, @@ -2286,7 +2331,7 @@ unsafe extern "C" { UniqueIdLen: size_t, ) -> &'a DIDerivedType; - pub fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( + pub(crate) fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( Builder: &DIBuilder<'a>, Scope: Option<&'a DIScope>, Name: *const c_char, @@ -2294,37 +2339,37 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DITemplateTypeParameter; - pub fn LLVMRustDICompositeTypeReplaceArrays<'a>( + pub(crate) fn LLVMRustDICompositeTypeReplaceArrays<'a>( Builder: &DIBuilder<'a>, CompositeType: &'a DIType, Elements: Option<&'a DIArray>, Params: Option<&'a DIArray>, ); - pub fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( + pub(crate) fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( Location: &'a DILocation, BD: c_uint, ) -> Option<&'a DILocation>; - pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); - pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); + pub(crate) fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); + pub(crate) fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); - pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; + pub(crate) fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); - pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; - pub fn LLVMRustGetTargetFeature( + pub(crate) fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; + pub(crate) fn LLVMRustGetTargetFeature( T: &TargetMachine, Index: size_t, Feature: &mut *const c_char, Desc: &mut *const c_char, ); - pub fn LLVMRustGetHostCPUName(LenOut: &mut size_t) -> *const u8; + pub(crate) fn LLVMRustGetHostCPUName(LenOut: &mut size_t) -> *const u8; // This function makes copies of pointed to data, so the data's lifetime may end after this // function returns. - pub fn LLVMRustCreateTargetMachine( + pub(crate) fn LLVMRustCreateTargetMachine( Triple: *const c_char, CPU: *const c_char, Features: *const c_char, @@ -2350,13 +2395,13 @@ unsafe extern "C" { ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; - pub fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); - pub fn LLVMRustAddLibraryInfo<'a>( + pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); + pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, DisableSimplifyLibCalls: bool, ); - pub fn LLVMRustWriteOutputFile<'a>( + pub(crate) fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, PM: *mut PassManager<'a>, M: &'a Module, @@ -2365,7 +2410,7 @@ unsafe extern "C" { FileType: FileType, VerifyIR: bool, ) -> LLVMRustResult; - pub fn LLVMRustOptimize<'a>( + pub(crate) fn LLVMRustOptimize<'a>( M: &'a Module, TM: &'a TargetMachine, OptLevel: PassBuilderOptLevel, @@ -2397,29 +2442,32 @@ unsafe extern "C" { LLVMPlugins: *const c_char, LLVMPluginsLen: size_t, ) -> LLVMRustResult; - pub fn LLVMRustPrintModule( + pub(crate) fn LLVMRustPrintModule( M: &Module, Output: *const c_char, Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t, ) -> LLVMRustResult; - pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); - pub fn LLVMRustPrintPasses(); - pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); - pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - - pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub fn LLVMRustArchiveIteratorNext<'a>( + pub(crate) fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); + pub(crate) fn LLVMRustPrintPasses(); + pub(crate) fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); + pub(crate) fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); + + pub(crate) fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; + pub(crate) fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; + pub(crate) fn LLVMRustArchiveIteratorNext<'a>( AIR: &ArchiveIterator<'a>, ) -> Option<&'a mut ArchiveChild<'a>>; - pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); + pub(crate) fn LLVMRustArchiveChildName( + ACR: &ArchiveChild<'_>, + size: &mut size_t, + ) -> *const c_char; + pub(crate) fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); + pub(crate) fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); + pub(crate) fn LLVMRustDestroyArchive(AR: &'static mut Archive); - pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); + pub(crate) fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); - pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( + pub(crate) fn LLVMRustUnpackOptimizationDiagnostic<'a>( DI: &'a DiagnosticInfo, pass_name_out: &RustString, function_out: &mut Option<&'a Value>, @@ -2429,22 +2477,22 @@ unsafe extern "C" { message_out: &RustString, ); - pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( + pub(crate) fn LLVMRustUnpackInlineAsmDiagnostic<'a>( DI: &'a DiagnosticInfo, level_out: &mut DiagnosticLevel, cookie_out: &mut u64, message_out: &mut Option<&'a Twine>, ); - pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); - pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; + pub(crate) fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); + pub(crate) fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; - pub fn LLVMRustGetSMDiagnostic<'a>( + pub(crate) fn LLVMRustGetSMDiagnostic<'a>( DI: &'a DiagnosticInfo, cookie_out: &mut u64, ) -> &'a SMDiagnostic; - pub fn LLVMRustUnpackSMDiagnostic( + pub(crate) fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, message_out: &RustString, buffer_out: &RustString, @@ -2454,7 +2502,7 @@ unsafe extern "C" { num_ranges: &mut usize, ) -> bool; - pub fn LLVMRustWriteArchive( + pub(crate) fn LLVMRustWriteArchive( Dst: *const c_char, NumMembers: size_t, Members: *const &RustArchiveMember<'_>, @@ -2462,63 +2510,63 @@ unsafe extern "C" { Kind: ArchiveKind, isEC: bool, ) -> LLVMRustResult; - pub fn LLVMRustArchiveMemberNew<'a>( + pub(crate) fn LLVMRustArchiveMemberNew<'a>( Filename: *const c_char, Name: *const c_char, Child: Option<&ArchiveChild<'a>>, ) -> &'a mut RustArchiveMember<'a>; - pub fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); + pub(crate) fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); + pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); + pub(crate) fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); - pub fn LLVMRustSetModulePICLevel(M: &Module); - pub fn LLVMRustSetModulePIELevel(M: &Module); - pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); - pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; - pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; - pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; - pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); - pub fn LLVMRustModuleCost(M: &Module) -> u64; - pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustSetModulePICLevel(M: &Module); + pub(crate) fn LLVMRustSetModulePIELevel(M: &Module); + pub(crate) fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); + pub(crate) fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; + pub(crate) fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; + pub(crate) fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; + pub(crate) fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); + pub(crate) fn LLVMRustModuleCost(M: &Module) -> u64; + pub(crate) fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); - pub fn LLVMRustThinLTOBufferCreate( + pub(crate) fn LLVMRustThinLTOBufferCreate( M: &Module, is_thin: bool, emit_summary: bool, ) -> &'static mut ThinLTOBuffer; - pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); - pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; - pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; - pub fn LLVMRustThinLTOBufferThinLinkDataPtr(M: &ThinLTOBuffer) -> *const c_char; - pub fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; - pub fn LLVMRustCreateThinLTOData( + pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); + pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; + pub(crate) fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; + pub(crate) fn LLVMRustThinLTOBufferThinLinkDataPtr(M: &ThinLTOBuffer) -> *const c_char; + pub(crate) fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; + pub(crate) fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, NumModules: size_t, PreservedSymbols: *const *const c_char, PreservedSymbolsLen: size_t, ) -> Option<&'static mut ThinLTOData>; - pub fn LLVMRustPrepareThinLTORename( + pub(crate) fn LLVMRustPrepareThinLTORename( Data: &ThinLTOData, Module: &Module, Target: &TargetMachine, ); - pub fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; - pub fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; - pub fn LLVMRustPrepareThinLTOImport( + pub(crate) fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; + pub(crate) fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; + pub(crate) fn LLVMRustPrepareThinLTOImport( Data: &ThinLTOData, Module: &Module, Target: &TargetMachine, ) -> bool; - pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); - pub fn LLVMRustParseBitcodeForLTO( + pub(crate) fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); + pub(crate) fn LLVMRustParseBitcodeForLTO( Context: &Context, Data: *const u8, len: usize, Identifier: *const c_char, ) -> Option<&Module>; - pub fn LLVMRustGetSliceFromObjectDataByName( + pub(crate) fn LLVMRustGetSliceFromObjectDataByName( data: *const u8, len: usize, name: *const u8, @@ -2526,25 +2574,27 @@ unsafe extern "C" { out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; - pub fn LLVMRustLinkerAdd( + pub(crate) fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; + pub(crate) fn LLVMRustLinkerAdd( linker: &Linker<'_>, bytecode: *const c_char, bytecode_len: usize, ) -> bool; - pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); - pub fn LLVMRustComputeLTOCacheKey( + pub(crate) fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); + pub(crate) fn LLVMRustComputeLTOCacheKey( key_out: &RustString, mod_id: *const c_char, data: &ThinLTOData, ); - pub fn LLVMRustContextGetDiagnosticHandler(Context: &Context) -> Option<&DiagnosticHandler>; - pub fn LLVMRustContextSetDiagnosticHandler( + pub(crate) fn LLVMRustContextGetDiagnosticHandler( + Context: &Context, + ) -> Option<&DiagnosticHandler>; + pub(crate) fn LLVMRustContextSetDiagnosticHandler( context: &Context, diagnostic_handler: Option<&DiagnosticHandler>, ); - pub fn LLVMRustContextConfigureDiagnosticHandler( + pub(crate) fn LLVMRustContextConfigureDiagnosticHandler( context: &Context, diagnostic_handler_callback: DiagnosticHandlerTy, diagnostic_handler_context: *mut c_void, @@ -2555,15 +2605,15 @@ unsafe extern "C" { pgo_available: bool, ); - pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); + pub(crate) fn LLVMRustGetMangledName(V: &Value, out: &RustString); - pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; + pub(crate) fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; - pub fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; + pub(crate) fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; - pub fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; + pub(crate) fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; - pub fn LLVMRustGetSymbols( + pub(crate) fn LLVMRustGetSymbols( buf_ptr: *const u8, buf_len: usize, state: *mut c_void, @@ -2571,10 +2621,10 @@ unsafe extern "C" { error_callback: GetSymbolsErrorCallback, ) -> *mut c_void; - pub fn LLVMRustIs64BitSymbolicFile(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIs64BitSymbolicFile(buf_ptr: *const u8, buf_len: usize) -> bool; - pub fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; - pub fn LLVMRustSetNoSanitizeAddress(Global: &Value); - pub fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); + pub(crate) fn LLVMRustSetNoSanitizeAddress(Global: &Value); + pub(crate) fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 7becba4ccd4..efc9cf2ef69 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -24,7 +24,7 @@ mod ffi; pub use self::enzyme_ffi::*; impl LLVMRustResult { - pub fn into_result(self) -> Result<(), ()> { + pub(crate) fn into_result(self) -> Result<(), ()> { match self { LLVMRustResult::Success => Ok(()), LLVMRustResult::Failure => Err(()), @@ -32,13 +32,17 @@ impl LLVMRustResult { } } -pub fn AddFunctionAttributes<'ll>(llfn: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute]) { +pub(crate) fn AddFunctionAttributes<'ll>( + llfn: &'ll Value, + idx: AttributePlace, + attrs: &[&'ll Attribute], +) { unsafe { LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len()); } } -pub fn AddCallSiteAttributes<'ll>( +pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute], @@ -48,7 +52,11 @@ pub fn AddCallSiteAttributes<'ll>( } } -pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) -> &'ll Attribute { +pub(crate) fn CreateAttrStringValue<'ll>( + llcx: &'ll Context, + attr: &str, + value: &str, +) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, @@ -60,7 +68,7 @@ pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) - } } -pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { +pub(crate) fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, @@ -72,39 +80,39 @@ pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { } } -pub fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { +pub(crate) fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) } } -pub fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { +pub(crate) fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) } } -pub fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { +pub(crate) fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) } } -pub fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { +pub(crate) fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { unsafe { LLVMRustCreateByValAttr(llcx, ty) } } -pub fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { +pub(crate) fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } } -pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { +pub(crate) fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } } -pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { +pub(crate) fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } } -pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { +pub(crate) fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } } -pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { +pub(crate) fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { let lower = range.start; let upper = range.end.wrapping_add(1); let lower_words = [lower as u64, (lower >> 64) as u64]; @@ -127,7 +135,7 @@ pub enum AttributePlace { } impl AttributePlace { - pub fn as_uint(self) -> c_uint { + pub(crate) fn as_uint(self) -> c_uint { match self { AttributePlace::ReturnValue => 0, AttributePlace::Argument(i) => 1 + i, @@ -159,12 +167,12 @@ impl FromStr for ArchiveKind { } } -pub fn SetInstructionCallConv(instr: &Value, cc: CallConv) { +pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); } } -pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { +pub(crate) fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { unsafe { LLVMSetFunctionCallConv(fn_, cc as c_uint); } @@ -176,20 +184,20 @@ pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { // value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the // function. // For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 -pub fn SetUniqueComdat(llmod: &Module, val: &Value) { +pub(crate) fn SetUniqueComdat(llmod: &Module, val: &Value) { let name_buf = get_value_name(val).to_vec(); let name = CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap(); set_comdat(llmod, val, &name); } -pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { +pub(crate) fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { unsafe { LLVMSetUnnamedAddress(global, unnamed); } } -pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { +pub(crate) fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { unsafe { LLVMSetThreadLocalMode(global, mode); } @@ -197,65 +205,65 @@ pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { impl AttributeKind { /// Create an LLVM Attribute with no associated value. - pub fn create_attr(self, llcx: &Context) -> &Attribute { + pub(crate) fn create_attr(self, llcx: &Context) -> &Attribute { unsafe { LLVMRustCreateAttrNoValue(llcx, self) } } } impl MemoryEffects { /// Create an LLVM Attribute with these memory effects. - pub fn create_attr(self, llcx: &Context) -> &Attribute { + pub(crate) fn create_attr(self, llcx: &Context) -> &Attribute { unsafe { LLVMRustCreateMemoryEffectsAttr(llcx, self) } } } -pub fn set_section(llglobal: &Value, section_name: &CStr) { +pub(crate) fn set_section(llglobal: &Value, section_name: &CStr) { unsafe { LLVMSetSection(llglobal, section_name.as_ptr()); } } -pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { +pub(crate) fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } } -pub fn set_initializer(llglobal: &Value, constant_val: &Value) { +pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) { unsafe { LLVMSetInitializer(llglobal, constant_val); } } -pub fn set_global_constant(llglobal: &Value, is_constant: bool) { +pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) { unsafe { LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); } } -pub fn get_linkage(llglobal: &Value) -> Linkage { +pub(crate) fn get_linkage(llglobal: &Value) -> Linkage { unsafe { LLVMGetLinkage(llglobal) }.to_rust() } -pub fn set_linkage(llglobal: &Value, linkage: Linkage) { +pub(crate) fn set_linkage(llglobal: &Value, linkage: Linkage) { unsafe { LLVMSetLinkage(llglobal, linkage); } } -pub fn is_declaration(llglobal: &Value) -> bool { +pub(crate) fn is_declaration(llglobal: &Value) -> bool { unsafe { LLVMIsDeclaration(llglobal) == ffi::True } } -pub fn get_visibility(llglobal: &Value) -> Visibility { +pub(crate) fn get_visibility(llglobal: &Value) -> Visibility { unsafe { LLVMGetVisibility(llglobal) }.to_rust() } -pub fn set_visibility(llglobal: &Value, visibility: Visibility) { +pub(crate) fn set_visibility(llglobal: &Value, visibility: Visibility) { unsafe { LLVMSetVisibility(llglobal, visibility); } } -pub fn set_alignment(llglobal: &Value, align: Align) { +pub(crate) fn set_alignment(llglobal: &Value, align: Align) { unsafe { ffi::LLVMSetAlignment(llglobal, align.bytes() as c_uint); } @@ -265,7 +273,7 @@ pub fn set_alignment(llglobal: &Value, align: Align) { /// /// Inserts the comdat into `llmod` if it does not exist. /// It is an error to call this if the target does not support comdat. -pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { +pub(crate) fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { unsafe { let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr()); LLVMSetComdat(llglobal, comdat); @@ -273,7 +281,7 @@ pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { } /// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. -pub fn get_param(llfn: &Value, index: c_uint) -> &Value { +pub(crate) fn get_param(llfn: &Value, index: c_uint) -> &Value { unsafe { assert!( index < LLVMCountParams(llfn), @@ -286,7 +294,7 @@ pub fn get_param(llfn: &Value, index: c_uint) -> &Value { } /// Safe wrapper for `LLVMGetValueName2` into a byte slice -pub fn get_value_name(value: &Value) -> &[u8] { +pub(crate) fn get_value_name(value: &Value) -> &[u8] { unsafe { let mut len = 0; let data = LLVMGetValueName2(value, &mut len); @@ -295,28 +303,28 @@ pub fn get_value_name(value: &Value) -> &[u8] { } /// Safe wrapper for `LLVMSetValueName2` from a byte slice -pub fn set_value_name(value: &Value, name: &[u8]) { +pub(crate) fn set_value_name(value: &Value, name: &[u8]) { unsafe { let data = name.as_c_char_ptr(); LLVMSetValueName2(value, data, name.len()); } } -pub fn build_string(f: impl FnOnce(&RustString)) -> Result { +pub(crate) fn build_string(f: impl FnOnce(&RustString)) -> Result { String::from_utf8(RustString::build_byte_buffer(f)) } -pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec { +pub(crate) fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec { RustString::build_byte_buffer(f) } -pub fn twine_to_string(tr: &Twine) -> String { +pub(crate) fn twine_to_string(tr: &Twine) -> String { unsafe { build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM") } } -pub fn last_error() -> Option { +pub(crate) fn last_error() -> Option { unsafe { let cstr = LLVMRustGetLastError(); if cstr.is_null() { -- cgit 1.4.1-3-g733a5 From 296625613386532e8a6963f821716db119c2ab62 Mon Sep 17 00:00:00 2001 From: clubby789 Date: Mon, 13 Jan 2025 15:52:08 +0000 Subject: Make `-O` mean `-C opt-level=3` --- compiler/rustc_codegen_cranelift/src/lib.rs | 2 +- compiler/rustc_codegen_gcc/src/lib.rs | 2 +- compiler/rustc_codegen_llvm/src/back/write.rs | 4 ++-- compiler/rustc_codegen_ssa/src/back/linker.rs | 16 ++++++------- compiler/rustc_codegen_ssa/src/back/write.rs | 4 ++-- compiler/rustc_codegen_ssa/src/base.rs | 6 ++--- compiler/rustc_mir_transform/src/inline.rs | 3 +-- compiler/rustc_session/src/config.rs | 26 +++++++++++++--------- src/doc/rustc/src/codegen-options/index.md | 2 +- src/doc/rustc/src/command-line-arguments.md | 2 +- .../run-make-support/src/external_deps/rustc.rs | 2 +- tests/run-make/rustc-help/help-v.stdout | 2 +- tests/run-make/rustc-help/help.stdout | 2 +- 13 files changed, 39 insertions(+), 34 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index b0153419903..a3f43744875 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -289,7 +289,7 @@ fn build_isa(sess: &Session) -> Arc { flags_builder.set("opt_level", "none").unwrap(); } OptLevel::Less - | OptLevel::Default + | OptLevel::More | OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => { diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index ce88ac39021..6455bcec685 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -476,7 +476,7 @@ fn to_gcc_opt_level(optlevel: Option) -> OptimizationLevel { Some(level) => match level { OptLevel::No => OptimizationLevel::None, OptLevel::Less => OptimizationLevel::Limited, - OptLevel::Default => OptimizationLevel::Standard, + OptLevel::More => OptimizationLevel::Standard, OptLevel::Aggressive => OptimizationLevel::Aggressive, OptLevel::Size | OptLevel::SizeMin => OptimizationLevel::Limited, }, diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 58933a77e53..9fa10e96068 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -141,7 +141,7 @@ fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm:: match cfg { No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), - Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), + More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), @@ -153,7 +153,7 @@ fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel match cfg { No => llvm::PassBuilderOptLevel::O0, Less => llvm::PassBuilderOptLevel::O1, - Default => llvm::PassBuilderOptLevel::O2, + More => llvm::PassBuilderOptLevel::O2, Aggressive => llvm::PassBuilderOptLevel::O3, Size => llvm::PassBuilderOptLevel::Os, SizeMin => llvm::PassBuilderOptLevel::Oz, diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 8fb831471a9..05d6ff35751 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -410,7 +410,7 @@ impl<'a> GccLinker<'a> { let opt_level = match self.sess.opts.optimize { config::OptLevel::No => "O0", config::OptLevel::Less => "O1", - config::OptLevel::Default | config::OptLevel::Size | config::OptLevel::SizeMin => "O2", + config::OptLevel::More | config::OptLevel::Size | config::OptLevel::SizeMin => "O2", config::OptLevel::Aggressive => "O3", }; @@ -685,7 +685,7 @@ impl<'a> Linker for GccLinker<'a> { // GNU-style linkers support optimization with -O. GNU ld doesn't // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::Default + if self.sess.opts.optimize == config::OptLevel::More || self.sess.opts.optimize == config::OptLevel::Aggressive { self.link_arg("-O1"); @@ -1213,7 +1213,7 @@ impl<'a> Linker for EmLinker<'a> { self.cc_arg(match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", OptLevel::Size => "-Os", OptLevel::SizeMin => "-Oz", @@ -1384,7 +1384,7 @@ impl<'a> Linker for WasmLd<'a> { self.link_arg(match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", // Currently LLD doesn't support `Os` and `Oz`, so pass through `O2` // instead. @@ -1451,7 +1451,7 @@ impl<'a> WasmLd<'a> { let opt_level = match self.sess.opts.optimize { config::OptLevel::No => "O0", config::OptLevel::Less => "O1", - config::OptLevel::Default => "O2", + config::OptLevel::More => "O2", config::OptLevel::Aggressive => "O3", // wasm-ld only handles integer LTO opt levels. Use O2 config::OptLevel::Size | config::OptLevel::SizeMin => "O2", @@ -1525,7 +1525,7 @@ impl<'a> Linker for L4Bender<'a> { fn optimize(&mut self) { // GNU-style linkers support optimization with -O. GNU ld doesn't // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::Default + if self.sess.opts.optimize == config::OptLevel::More || self.sess.opts.optimize == config::OptLevel::Aggressive { self.link_arg("-O1"); @@ -1929,7 +1929,7 @@ impl<'a> Linker for LlbcLinker<'a> { match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", OptLevel::Size => "-Os", OptLevel::SizeMin => "-Oz", @@ -2006,7 +2006,7 @@ impl<'a> Linker for BpfLinker<'a> { self.link_arg(match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", OptLevel::Size => "-Os", OptLevel::SizeMin => "-Oz", diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index ce2161a07eb..f029c08a808 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -236,7 +236,7 @@ impl ModuleConfig { // Copy what clang does by turning on loop vectorization at O2 and // slp vectorization at O3. vectorize_loop: !sess.opts.cg.no_vectorize_loops - && (sess.opts.optimize == config::OptLevel::Default + && (sess.opts.optimize == config::OptLevel::More || sess.opts.optimize == config::OptLevel::Aggressive), vectorize_slp: !sess.opts.cg.no_vectorize_slp && sess.opts.optimize == config::OptLevel::Aggressive, @@ -260,7 +260,7 @@ impl ModuleConfig { MergeFunctions::Trampolines | MergeFunctions::Aliases => { use config::OptLevel::*; match sess.opts.optimize { - Aggressive | Default | SizeMin | Size => true, + Aggressive | More | SizeMin | Size => true, Less | No => false, } } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index d9fbf539fd3..e800492dad0 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -1054,12 +1054,12 @@ pub(crate) fn provide(providers: &mut Providers) { config::OptLevel::No => return config::OptLevel::No, // If globally optimise-speed is already specified, just use that level. config::OptLevel::Less => return config::OptLevel::Less, - config::OptLevel::Default => return config::OptLevel::Default, + config::OptLevel::More => return config::OptLevel::More, config::OptLevel::Aggressive => return config::OptLevel::Aggressive, // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size) // are present). - config::OptLevel::Size => config::OptLevel::Default, - config::OptLevel::SizeMin => config::OptLevel::Default, + config::OptLevel::Size => config::OptLevel::More, + config::OptLevel::SizeMin => config::OptLevel::More, }; let defids = tcx.collect_and_partition_mono_items(cratenum).all_mono_items; diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index ab617e2ce6f..1f8392b2118 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -49,8 +49,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { match sess.mir_opt_level() { 0 | 1 => false, 2 => { - (sess.opts.optimize == OptLevel::Default - || sess.opts.optimize == OptLevel::Aggressive) + (sess.opts.optimize == OptLevel::More || sess.opts.optimize == OptLevel::Aggressive) && sess.opts.incremental == None } _ => true, diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 2fb4b27b889..7d473b86ff5 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -86,12 +86,18 @@ pub enum CFProtection { #[derive(Clone, Copy, Debug, PartialEq, Hash, HashStable_Generic)] pub enum OptLevel { - No, // -O0 - Less, // -O1 - Default, // -O2 - Aggressive, // -O3 - Size, // -Os - SizeMin, // -Oz + /// `-Copt-level=0` + No, + /// `-Copt-level=1` + Less, + /// `-Copt-level=2` + More, + /// `-Copt-level=3` / `-O` + Aggressive, + /// `-Copt-level=s` + Size, + /// `-Copt-level=z` + SizeMin, } /// This is what the `LtoCli` values get mapped to after resolving defaults and @@ -1253,7 +1259,7 @@ impl Options { Some(setting) => setting, None => match self.optimize { OptLevel::No | OptLevel::Less | OptLevel::Size | OptLevel::SizeMin => true, - OptLevel::Default | OptLevel::Aggressive => false, + OptLevel::More | OptLevel::Aggressive => false, }, } } @@ -1572,7 +1578,7 @@ pub fn rustc_optgroups() -> Vec { stack-protector-strategies|link-args|deployment-target]", ), opt(Stable, FlagMulti, "g", "", "Equivalent to -C debuginfo=2", ""), - opt(Stable, FlagMulti, "O", "", "Equivalent to -C opt-level=2", ""), + opt(Stable, FlagMulti, "O", "", "Equivalent to -C opt-level=3", ""), opt(Stable, Opt, "o", "", "Write output to ", "FILENAME"), opt(Stable, Opt, "", "out-dir", "Write output to compiler-chosen filename in ", "DIR"), opt( @@ -2127,12 +2133,12 @@ fn parse_opt_level( }) .max(); if max_o > max_c { - OptLevel::Default + OptLevel::Aggressive } else { match cg.opt_level.as_ref() { "0" => OptLevel::No, "1" => OptLevel::Less, - "2" => OptLevel::Default, + "2" => OptLevel::More, "3" => OptLevel::Aggressive, "s" => OptLevel::Size, "z" => OptLevel::SizeMin, diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index f45217c69ff..8c1769a8c77 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -368,7 +368,7 @@ This flag controls the optimization level. * `s`: optimize for binary size. * `z`: optimize for binary size, but also turn off loop vectorization. -Note: The [`-O` flag][option-o-optimize] is an alias for `-C opt-level=2`. +Note: The [`-O` flag][option-o-optimize] is an alias for `-C opt-level=3`. The default is `0`. diff --git a/src/doc/rustc/src/command-line-arguments.md b/src/doc/rustc/src/command-line-arguments.md index 0942b5ebfee..9dd2e7de1b3 100644 --- a/src/doc/rustc/src/command-line-arguments.md +++ b/src/doc/rustc/src/command-line-arguments.md @@ -308,7 +308,7 @@ A synonym for [`-C debuginfo=2`](codegen-options/index.md#debuginfo). ## `-O`: optimize your code -A synonym for [`-C opt-level=2`](codegen-options/index.md#opt-level). +A synonym for [`-C opt-level=3`](codegen-options/index.md#opt-level). ## `-o`: filename of the output diff --git a/src/tools/run-make-support/src/external_deps/rustc.rs b/src/tools/run-make-support/src/external_deps/rustc.rs index 33dc57cbc07..fd4a20278ad 100644 --- a/src/tools/run-make-support/src/external_deps/rustc.rs +++ b/src/tools/run-make-support/src/external_deps/rustc.rs @@ -84,7 +84,7 @@ impl Rustc { self } - /// Specify default optimization level `-O` (alias for `-C opt-level=2`). + /// Specify default optimization level `-O` (alias for `-C opt-level=3`). pub fn opt(&mut self) -> &mut Self { self.cmd.arg("-O"); self diff --git a/tests/run-make/rustc-help/help-v.stdout b/tests/run-make/rustc-help/help-v.stdout index 8f6fde69c29..382b1c96682 100644 --- a/tests/run-make/rustc-help/help-v.stdout +++ b/tests/run-make/rustc-help/help-v.stdout @@ -32,7 +32,7 @@ Options: --print [crate-name|file-names|sysroot|target-libdir|cfg|check-cfg|calling-conventions|target-list|target-cpus|target-features|relocation-models|code-models|tls-models|target-spec-json|all-target-specs-json|native-static-libs|stack-protector-strategies|link-args|deployment-target] Compiler information to print on stdout -g Equivalent to -C debuginfo=2 - -O Equivalent to -C opt-level=2 + -O Equivalent to -C opt-level=3 -o FILENAME Write output to --out-dir DIR Write output to compiler-chosen filename in --explain OPT Provide a detailed explanation of an error message diff --git a/tests/run-make/rustc-help/help.stdout b/tests/run-make/rustc-help/help.stdout index 131efa93282..08bb7f85098 100644 --- a/tests/run-make/rustc-help/help.stdout +++ b/tests/run-make/rustc-help/help.stdout @@ -32,7 +32,7 @@ Options: --print [crate-name|file-names|sysroot|target-libdir|cfg|check-cfg|calling-conventions|target-list|target-cpus|target-features|relocation-models|code-models|tls-models|target-spec-json|all-target-specs-json|native-static-libs|stack-protector-strategies|link-args|deployment-target] Compiler information to print on stdout -g Equivalent to -C debuginfo=2 - -O Equivalent to -C opt-level=2 + -O Equivalent to -C opt-level=3 -o FILENAME Write output to --out-dir DIR Write output to compiler-chosen filename in --explain OPT Provide a detailed explanation of an error message -- cgit 1.4.1-3-g733a5 From bfdc96114c4b591025dd6ec18b4282f0b21e641c Mon Sep 17 00:00:00 2001 From: Daniel Paoliello Date: Thu, 13 Feb 2025 15:04:39 -0800 Subject: [cg_llvm] Remove dead error message --- compiler/rustc_codegen_llvm/messages.ftl | 3 --- compiler/rustc_codegen_llvm/src/errors.rs | 5 ----- 2 files changed, 8 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index 9585848cbf0..399d7ffea8e 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -38,9 +38,6 @@ codegen_llvm_lto_proc_macro = lto cannot be used for `proc-macro` crate type wit codegen_llvm_mismatch_data_layout = data-layout for target `{$rustc_target}`, `{$rustc_layout}`, differs from LLVM target's `{$llvm_target}` default layout, `{$llvm_layout}` -codegen_llvm_multiple_source_dicompileunit = multiple source DICompileUnits found -codegen_llvm_multiple_source_dicompileunit_with_llvm_err = multiple source DICompileUnits found: {$llvm_err} - codegen_llvm_parse_bitcode = failed to parse bitcode for LTO module codegen_llvm_parse_bitcode_with_llvm_err = failed to parse bitcode for LTO module: {$llvm_err} diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index f4c9491f758..97f49256165 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -131,8 +131,6 @@ pub enum LlvmError<'a> { LoadBitcode { name: CString }, #[diag(codegen_llvm_write_thinlto_key)] WriteThinLtoKey { err: std::io::Error }, - #[diag(codegen_llvm_multiple_source_dicompileunit)] - MultipleSourceDiCompileUnit, #[diag(codegen_llvm_prepare_thin_lto_module)] PrepareThinLtoModule, #[diag(codegen_llvm_parse_bitcode)] @@ -155,9 +153,6 @@ impl Diagnostic<'_, G> for WithLlvmError<'_> { PrepareThinLtoContext => fluent::codegen_llvm_prepare_thin_lto_context_with_llvm_err, LoadBitcode { .. } => fluent::codegen_llvm_load_bitcode_with_llvm_err, WriteThinLtoKey { .. } => fluent::codegen_llvm_write_thinlto_key_with_llvm_err, - MultipleSourceDiCompileUnit => { - fluent::codegen_llvm_multiple_source_dicompileunit_with_llvm_err - } PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, PrepareAutoDiff { .. } => fluent::codegen_llvm_prepare_autodiff_with_llvm_err, -- cgit 1.4.1-3-g733a5 From 9ad6839f7a64d7a5aa8799661d6974a9d7938646 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Wed, 5 Feb 2025 03:43:54 -0800 Subject: Set both `nuw` and `nsw` in slice size calculation There's an old note in the code to do this, and now that LLVM-C has an API for it, we might as well. --- compiler/rustc_codegen_gcc/src/builder.rs | 26 +--------------- compiler/rustc_codegen_llvm/src/builder.rs | 31 +++++++++++++++++++ compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 ++ compiler/rustc_codegen_ssa/src/size_of_val.rs | 11 ++----- compiler/rustc_codegen_ssa/src/traits/builder.rs | 35 ++++++++++++++++++---- .../issues/issue-96497-slice-size-nowrap.rs | 12 +++++++- tests/codegen/slice-ref-equality.rs | 8 ++--- 7 files changed, 81 insertions(+), 44 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 1c3d6cc899a..b8e37b60480 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -665,6 +665,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { a + b } + // TODO(antoyo): should we also override the `unchecked_` versions? fn sub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { self.gcc_sub(a, b) } @@ -832,31 +833,6 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { set_rvalue_location(self, self.gcc_not(a)) } - fn unchecked_sadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - set_rvalue_location(self, self.gcc_add(a, b)) - } - - fn unchecked_uadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - set_rvalue_location(self, self.gcc_add(a, b)) - } - - fn unchecked_ssub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - set_rvalue_location(self, self.gcc_sub(a, b)) - } - - fn unchecked_usub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - // TODO(antoyo): should generate poison value? - set_rvalue_location(self, self.gcc_sub(a, b)) - } - - fn unchecked_smul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - set_rvalue_location(self, self.gcc_mul(a, b)) - } - - fn unchecked_umul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - set_rvalue_location(self, self.gcc_mul(a, b)) - } - fn fadd_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> { // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC. set_rvalue_location(self, lhs + rhs) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index acae0b444c0..57f09b1fa2f 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -421,6 +421,37 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unchecked_umul(x, y) => LLVMBuildNUWMul, } + fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(add).is_some() { + llvm::LLVMSetNUW(add, True); + llvm::LLVMSetNSW(add, True); + } + add + } + } + fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(sub).is_some() { + llvm::LLVMSetNUW(sub, True); + llvm::LLVMSetNSW(sub, True); + } + sub + } + } + fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED); + if llvm::LLVMIsAInstruction(mul).is_some() { + llvm::LLVMSetNUW(mul, True); + llvm::LLVMSetNSW(mul, True); + } + mul + } + } + fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { unsafe { let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 618fdb09d4d..3b0187b9d37 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1430,6 +1430,8 @@ unsafe extern "C" { // Extra flags on arithmetic pub(crate) fn LLVMSetIsDisjoint(Instr: &Value, IsDisjoint: Bool); + pub(crate) fn LLVMSetNUW(ArithInst: &Value, HasNUW: Bool); + pub(crate) fn LLVMSetNSW(ArithInst: &Value, HasNSW: Bool); // Memory pub(crate) fn LLVMBuildAlloca<'a>( diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 71a2f916db5..ac2366340fb 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -45,14 +45,9 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // The info in this case is the length of the str, so the size is that // times the unit size. ( - // All slice sizes must fit into `isize`, so this multiplication cannot (signed) - // wrap. - // NOTE: ideally, we want the effects of both `unchecked_smul` and `unchecked_umul` - // (resulting in `mul nsw nuw` in LLVM IR), since we know that the multiplication - // cannot signed wrap, and that both operands are non-negative. But at the time of - // writing, the `LLVM-C` binding can't do this, and it doesn't seem to enable any - // further optimizations. - bx.unchecked_smul(info.unwrap(), bx.const_usize(unit.size.bytes())), + // All slice sizes must fit into `isize`, so this multiplication cannot + // wrap -- neither signed nor unsigned. + bx.unchecked_sumul(info.unwrap(), bx.const_usize(unit.size.bytes())), bx.const_usize(unit.align.abi.bytes()), ) } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index b7dcf16fa2b..871bf8b0d1b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -159,12 +159,35 @@ pub trait BuilderMethods<'a, 'tcx>: /// must be interpreted as unsigned and can be assumed to be less than the size of the left /// operand. fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.add(lhs, rhs) + } + fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.add(lhs, rhs) + } + fn unchecked_suadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.unchecked_sadd(lhs, rhs) + } + fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.sub(lhs, rhs) + } + fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.sub(lhs, rhs) + } + fn unchecked_susub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.unchecked_ssub(lhs, rhs) + } + fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.mul(lhs, rhs) + } + fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.mul(lhs, rhs) + } + fn unchecked_sumul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + // Which to default to is a fairly arbitrary choice, + // but this is what slice layout was using before. + self.unchecked_smul(lhs, rhs) + } fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends diff --git a/tests/codegen/issues/issue-96497-slice-size-nowrap.rs b/tests/codegen/issues/issue-96497-slice-size-nowrap.rs index dce156dd425..7b3a20a295e 100644 --- a/tests/codegen/issues/issue-96497-slice-size-nowrap.rs +++ b/tests/codegen/issues/issue-96497-slice-size-nowrap.rs @@ -11,7 +11,7 @@ pub fn simple_size_of_nowrap(x: &[u32]) -> usize { // Make sure the shift used to compute the size has a nowrap flag. - // CHECK: [[A:%.*]] = shl nsw {{.*}}, 2 + // CHECK: [[A:%.*]] = shl nuw nsw {{.*}}, 2 // CHECK-NEXT: ret {{.*}} [[A]] core::mem::size_of_val(x) } @@ -26,3 +26,13 @@ pub fn drop_write(mut x: Box<[u32]>) { // CHECK-NOT: store i32 42 x[1] = 42; } + +// CHECK-LABEL: @slice_size_plus_2 +#[no_mangle] +pub fn slice_size_plus_2(x: &[u16]) -> usize { + // Before #136575 this didn't get the `nuw` in the add. + + // CHECK: [[BYTES:%.+]] = shl nuw nsw {{i16|i32|i64}} %x.1, 1 + // CHECK: = add nuw {{i16|i32|i64}} [[BYTES]], 2 + core::mem::size_of_val(x) + 2 +} diff --git a/tests/codegen/slice-ref-equality.rs b/tests/codegen/slice-ref-equality.rs index a5046a75944..2940378da3c 100644 --- a/tests/codegen/slice-ref-equality.rs +++ b/tests/codegen/slice-ref-equality.rs @@ -47,7 +47,7 @@ pub fn is_zero_array(data: &[u8; 4]) -> bool { #[no_mangle] fn eq_slice_of_nested_u8(x: &[[u8; 3]], y: &[[u8; 3]]) -> bool { // CHECK: icmp eq [[USIZE]] %x.1, %y.1 - // CHECK: %[[BYTES:.+]] = mul nsw [[USIZE]] {{%x.1|%y.1}}, 3 + // CHECK: %[[BYTES:.+]] = mul nuw nsw [[USIZE]] {{%x.1|%y.1}}, 3 // CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr // CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]]) x == y @@ -59,7 +59,7 @@ fn eq_slice_of_nested_u8(x: &[[u8; 3]], y: &[[u8; 3]]) -> bool { #[no_mangle] fn eq_slice_of_i32(x: &[i32], y: &[i32]) -> bool { // CHECK: icmp eq [[USIZE]] %x.1, %y.1 - // CHECK: %[[BYTES:.+]] = shl nsw [[USIZE]] {{%x.1|%y.1}}, 2 + // CHECK: %[[BYTES:.+]] = shl nuw nsw [[USIZE]] {{%x.1|%y.1}}, 2 // CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr // CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]]) x == y @@ -71,7 +71,7 @@ fn eq_slice_of_i32(x: &[i32], y: &[i32]) -> bool { #[no_mangle] fn eq_slice_of_nonzero(x: &[NonZero], y: &[NonZero]) -> bool { // CHECK: icmp eq [[USIZE]] %x.1, %y.1 - // CHECK: %[[BYTES:.+]] = shl nsw [[USIZE]] {{%x.1|%y.1}}, 2 + // CHECK: %[[BYTES:.+]] = shl nuw nsw [[USIZE]] {{%x.1|%y.1}}, 2 // CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr // CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]]) x == y @@ -83,7 +83,7 @@ fn eq_slice_of_nonzero(x: &[NonZero], y: &[NonZero]) -> bool { #[no_mangle] fn eq_slice_of_option_of_nonzero(x: &[Option>], y: &[Option>]) -> bool { // CHECK: icmp eq [[USIZE]] %x.1, %y.1 - // CHECK: %[[BYTES:.+]] = shl nsw [[USIZE]] {{%x.1|%y.1}}, 1 + // CHECK: %[[BYTES:.+]] = shl nuw nsw [[USIZE]] {{%x.1|%y.1}}, 1 // CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr // CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]]) x == y -- cgit 1.4.1-3-g733a5