diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src/back')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 87 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 15 |
2 files changed, 68 insertions, 34 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 7262fce4911..99906ea7bce 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -362,8 +362,8 @@ fn fat_lto( ptr as *const *const libc::c_char, symbols_below_threshold.len() as libc::size_t, ); - save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } + save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } Ok(LtoModuleCodegen::Fat(module)) @@ -586,6 +586,42 @@ fn thin_lto( } } +fn enable_autodiff_settings(ad: &[config::AutoDiff], module: &mut ModuleCodegen<ModuleLlvm>) { + for &val in ad { + match val { + config::AutoDiff::PrintModBefore => { + unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) }; + } + config::AutoDiff::PrintPerf => { + llvm::set_print_perf(true); + } + config::AutoDiff::PrintAA => { + llvm::set_print_activity(true); + } + config::AutoDiff::PrintTA => { + llvm::set_print_type(true); + } + config::AutoDiff::Inline => { + llvm::set_inline(true); + } + config::AutoDiff::LooseTypes => { + llvm::set_loose_types(false); + } + config::AutoDiff::PrintSteps => { + llvm::set_print(true); + } + // We handle this below + config::AutoDiff::PrintModAfter => {} + // This is required and already checked + config::AutoDiff::Enable => {} + } + } + // This helps with handling enums for now. + llvm::set_strict_aliasing(false); + // FIXME(ZuseZ4): Test this, since it was added a long time ago. + llvm::set_rust_rules(true); +} + pub(crate) fn run_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, dcx: DiagCtxtHandle<'_>, @@ -604,34 +640,37 @@ pub(crate) fn run_pass_manager( let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - // If this rustc version was build with enzyme/autodiff enabled, and if users applied the - // `#[autodiff]` macro at least once, then we will later call llvm_optimize a second time. - debug!("running llvm pm opt pipeline"); + // The PostAD behavior is the same that we would have if no autodiff was used. + // It will run the default optimization pipeline. If AD is enabled we select + // the DuringAD stage, which will disable vectorization and loop unrolling, and + // schedule two autodiff optimization + differentiation passes. + // We then run the llvm_optimize function a second time, to optimize the code which we generated + // in the enzyme differentiation pass. + let enable_ad = config.autodiff.contains(&config::AutoDiff::Enable); + let stage = + if enable_ad { write::AutodiffStage::DuringAD } else { write::AutodiffStage::PostAD }; + + if enable_ad { + enable_autodiff_settings(&config.autodiff, module); + } + unsafe { - write::llvm_optimize( - cgcx, - dcx, - module, - config, - opt_level, - opt_stage, - write::AutodiffStage::DuringAD, - )?; + write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?; } - // FIXME(ZuseZ4): Make this more granular - if cfg!(llvm_enzyme) && !thin { + + if cfg!(llvm_enzyme) && enable_ad { + let opt_stage = llvm::OptStage::FatLTO; + let stage = write::AutodiffStage::PostAD; unsafe { - write::llvm_optimize( - cgcx, - dcx, - module, - config, - opt_level, - llvm::OptStage::FatLTO, - write::AutodiffStage::PostAD, - )?; + write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?; + } + + // This is the final IR, so people should be able to inspect the optimized autodiff output. + if config.autodiff.contains(&config::AutoDiff::PrintModAfter) { + unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) }; } } + debug!("lto done"); Ok(()) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 9fa10e96068..b67890c0465 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -564,19 +564,16 @@ pub(crate) unsafe fn llvm_optimize( // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting // differentiated. + let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let run_enzyme = autodiff_stage == AutodiffStage::DuringAD; let unroll_loops; let vectorize_slp; let vectorize_loop; - let run_enzyme = cfg!(llvm_enzyme) && autodiff_stage == AutodiffStage::DuringAD; // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt). // We therefore have two calls to llvm_optimize, if autodiff is used. - // - // FIXME(ZuseZ4): Before shipping on nightly, - // we should make this more granular, or at least check that the user has at least one autodiff - // call in their code, to justify altering the compilation pipeline. - if cfg!(llvm_enzyme) && autodiff_stage != AutodiffStage::PostAD { + if consider_ad && autodiff_stage != AutodiffStage::PostAD { unroll_loops = false; vectorize_slp = false; vectorize_loop = false; @@ -706,10 +703,8 @@ pub(crate) unsafe fn optimize( // If we know that we will later run AD, then we disable vectorization and loop unrolling. // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). - // FIXME(ZuseZ4): Make this more granular, only set PreAD if we actually have autodiff - // usages, not just if we build rustc with autodiff support. - let autodiff_stage = - if cfg!(llvm_enzyme) { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; + let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; return unsafe { llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage) }; |