diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
4 files changed, 84 insertions, 6 deletions
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 35134e9f5a0..27f7f95f100 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -123,7 +123,7 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> { /// Empty string, to be used where LLVM expects an instruction name, indicating /// that the instruction is to be left unnamed (i.e. numbered, in textual IR). // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. -const UNNAMED: *const c_char = c"".as_ptr(); +pub(crate) const UNNAMED: *const c_char = c"".as_ptr(); impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> { type Value = <GenericCx<'ll, CX> as BackendTypes>::Value; diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 5e7ef27143b..e7c071d05aa 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -10,7 +10,7 @@ use rustc_middle::bug; use tracing::{debug, trace}; use crate::back::write::llvm_err; -use crate::builder::SBuilder; +use crate::builder::{SBuilder, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; use crate::errors::{AutoDiffWithoutEnable, LlvmError}; @@ -51,6 +51,7 @@ fn has_sret(fnc: &Value) -> bool { // using iterators and peek()? fn match_args_from_caller_to_enzyme<'ll>( cx: &SimpleCx<'ll>, + builder: &SBuilder<'ll, 'll>, width: u32, args: &mut Vec<&'ll llvm::Value>, inputs: &[DiffActivity], @@ -78,7 +79,9 @@ fn match_args_from_caller_to_enzyme<'ll>( let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap(); let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap(); let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap(); + let enzyme_dupv = cx.create_metadata("enzyme_dupv".to_string()).unwrap(); let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap(); + let enzyme_dupnoneedv = cx.create_metadata("enzyme_dupnoneedv".to_string()).unwrap(); while activity_pos < inputs.len() { let diff_activity = inputs[activity_pos as usize]; @@ -90,13 +93,34 @@ fn match_args_from_caller_to_enzyme<'ll>( DiffActivity::Active => (enzyme_out, false), DiffActivity::ActiveOnly => (enzyme_out, false), DiffActivity::Dual => (enzyme_dup, true), + DiffActivity::Dualv => (enzyme_dupv, true), DiffActivity::DualOnly => (enzyme_dupnoneed, true), + DiffActivity::DualvOnly => (enzyme_dupnoneedv, true), DiffActivity::Duplicated => (enzyme_dup, true), DiffActivity::DuplicatedOnly => (enzyme_dupnoneed, true), - DiffActivity::FakeActivitySize => (enzyme_const, false), + DiffActivity::FakeActivitySize(_) => (enzyme_const, false), }; let outer_arg = outer_args[outer_pos]; args.push(cx.get_metadata_value(activity)); + if matches!(diff_activity, DiffActivity::Dualv) { + let next_outer_arg = outer_args[outer_pos + 1]; + let elem_bytes_size: u64 = match inputs[activity_pos + 1] { + DiffActivity::FakeActivitySize(Some(s)) => s.into(), + _ => bug!("incorrect Dualv handling recognized."), + }; + // stride: sizeof(T) * n_elems. + // n_elems is the next integer. + // Now we multiply `4 * next_outer_arg` to get the stride. + let mul = unsafe { + llvm::LLVMBuildMul( + builder.llbuilder, + cx.get_const_i64(elem_bytes_size), + next_outer_arg, + UNNAMED, + ) + }; + args.push(mul); + } args.push(outer_arg); if duplicated { // We know that duplicated args by construction have a following argument, @@ -114,7 +138,7 @@ fn match_args_from_caller_to_enzyme<'ll>( } else { let next_activity = inputs[activity_pos + 1]; // We analyze the MIR types and add this dummy activity if we visit a slice. - next_activity == DiffActivity::FakeActivitySize + matches!(next_activity, DiffActivity::FakeActivitySize(_)) } }; if slice { @@ -125,7 +149,10 @@ fn match_args_from_caller_to_enzyme<'ll>( // int2 >= int1, which means the shadow vector is large enough to store the gradient. assert_eq!(cx.type_kind(next_outer_ty), TypeKind::Integer); - for i in 0..(width as usize) { + let iterations = + if matches!(diff_activity, DiffActivity::Dualv) { 1 } else { width as usize }; + + for i in 0..iterations { let next_outer_arg2 = outer_args[outer_pos + 2 * (i + 1)]; let next_outer_ty2 = cx.val_ty(next_outer_arg2); assert_eq!(cx.type_kind(next_outer_ty2), TypeKind::Pointer); @@ -136,7 +163,7 @@ fn match_args_from_caller_to_enzyme<'ll>( } args.push(cx.get_metadata_value(enzyme_const)); args.push(next_outer_arg); - outer_pos += 2 + 2 * width as usize; + outer_pos += 2 + 2 * iterations; activity_pos += 2; } else { // A duplicated pointer will have the following two outer_fn arguments: @@ -360,6 +387,7 @@ fn generate_enzyme_call<'ll>( let outer_args: Vec<&llvm::Value> = get_params(outer_fn); match_args_from_caller_to_enzyme( &cx, + &builder, attrs.width, &mut args, &attrs.input_activity, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index ae2ab32ef53..56fb12d3c22 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -247,6 +247,16 @@ pub(super) fn stub<'ll, 'tcx>( StubInfo { metadata, unique_type_id } } +struct AdtStackPopGuard<'ll, 'tcx, 'a> { + cx: &'a CodegenCx<'ll, 'tcx>, +} + +impl<'ll, 'tcx, 'a> Drop for AdtStackPopGuard<'ll, 'tcx, 'a> { + fn drop(&mut self) { + debug_context(self.cx).adt_stack.borrow_mut().pop(); + } +} + /// This function enables creating debuginfo nodes that can recursively refer to themselves. /// It will first insert the given stub into the type map and only then execute the `members` /// and `generics` closures passed in. These closures have access to the stub so they can @@ -261,6 +271,44 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( ) -> DINodeCreationResult<'ll> { assert_eq!(debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), None); + let mut _adt_stack_pop_guard = None; + if let UniqueTypeId::Ty(ty, ..) = stub_info.unique_type_id + && let ty::Adt(adt_def, args) = ty.kind() + { + let def_id = adt_def.did(); + // If any sub type reference the original type definition and the sub type has a type + // parameter that strictly contains the original parameter, the original type is a recursive + // type that can expanding indefinitely. Example, + // ``` + // enum Recursive<T> { + // Recurse(*const Recursive<Wrap<T>>), + // Item(T), + // } + // ``` + let is_expanding_recursive = + debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| { + if def_id == *parent_def_id { + args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| { + if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type()) + { + arg != parent_arg && arg.contains(parent_arg) + } else { + false + } + }) + } else { + false + } + }); + if is_expanding_recursive { + // FIXME: indicate that this is an expanding recursive type in stub metadata? + return DINodeCreationResult::new(stub_info.metadata, false); + } else { + debug_context(cx).adt_stack.borrow_mut().push((def_id, args)); + _adt_stack_pop_guard = Some(AdtStackPopGuard { cx }); + } + } + debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata); let members: SmallVec<_> = diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 0f94a1dbb0d..c5085927923 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -66,6 +66,7 @@ pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, + adt_stack: RefCell<Vec<(DefId, GenericArgsRef<'tcx>)>>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, recursion_marker_type: OnceCell<&'ll DIType>, } @@ -80,6 +81,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { builder, created_files: Default::default(), type_map: Default::default(), + adt_stack: Default::default(), namespace_map: RefCell::new(Default::default()), recursion_marker_type: OnceCell::new(), } |