diff --git a/crates/cuda_builder/src/lib.rs b/crates/cuda_builder/src/lib.rs index 6cc3a7f6..7e33ab13 100644 --- a/crates/cuda_builder/src/lib.rs +++ b/crates/cuda_builder/src/lib.rs @@ -699,10 +699,12 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { let mut rustflags = vec![ format!("-Zcodegen-backend={}", rustc_codegen_nvvm.display()), + "-Zunstable-options".into(), "-Zcrate-attr=feature(register_tool)".into(), "-Zcrate-attr=register_tool(nvvm_internal)".into(), "-Zcrate-attr=no_std".into(), "-Zsaturating_float_casts=false".into(), + "-Cpanic=immediate-abort".into(), ]; if let Some(emit) = &builder.emit { @@ -775,12 +777,7 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { cargo.arg("--release"); } - // TODO(RDambrosio016): Remove this once we can get meaningful error messages in panic to work. - // for now we enable it to remove some useless indirect calls in the ptx. - cargo.arg("-Zbuild-std-features=panic_immediate_abort"); - if builder.optix { - cargo.arg("-Zbuild-std-features=panic_immediate_abort"); cargo.arg("-Zunstable-options"); cargo.arg("--config"); cargo.arg("optix=\"1\""); @@ -838,7 +835,7 @@ struct RustcOutput { } fn get_last_artifact(out: &str) -> Option { - let artifacts = + let mut artifacts = out.lines() .filter_map(|line| match serde_json::from_str::(line) { Ok(line) => Some(line), @@ -849,9 +846,7 @@ fn get_last_artifact(out: &str) -> Option { } }); - let last = artifacts - .filter(|line| line.reason == "compiler-artifact") - .next_back()?; + let last = artifacts.rfind(|line| line.reason == "compiler-artifact")?; let mut filenames = last .filenames diff --git a/crates/cuda_std/src/thread.rs b/crates/cuda_std/src/thread.rs index 8df6f1b8..01f25706 100644 --- a/crates/cuda_std/src/thread.rs +++ b/crates/cuda_std/src/thread.rs @@ -236,7 +236,7 @@ pub fn index() -> u32 { #[inline(always)] pub fn index_1d() -> u32 { - thread_idx_x() as u32 + block_idx_x() as u32 * block_dim_x() as u32 + thread_idx_x() + block_idx_x() * block_dim_x() } #[inline(always)] diff --git a/crates/gpu_rand/src/lib.rs b/crates/gpu_rand/src/lib.rs index 51383463..263486e6 100644 --- a/crates/gpu_rand/src/lib.rs +++ b/crates/gpu_rand/src/lib.rs @@ -40,8 +40,6 @@ #![deny(missing_docs)] #![deny(missing_debug_implementations)] #![allow(clippy::unreadable_literal)] -#![feature(doc_cfg)] - pub mod xoroshiro; mod default; diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp index 1c105a75..8d90dc9e 100644 --- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp +++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp @@ -1500,6 +1500,18 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn, unwrap(Fn), makeArrayRef(unwrap(Args), NumArgs), Bundles, Name)); } +extern "C" LLVMValueRef LLVMRustBuildCall2(LLVMBuilderRef B, LLVMTypeRef FnTy, + LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, + OperandBundleDef *Bundle, + const char *Name) +{ + assert(Bundle == nullptr && "LLVM 7 lacks CreateCall(FunctionType, ..., Bundles)"); + return wrap(unwrap(B)->CreateCall( + unwrap(FnTy), unwrap(Fn), + makeArrayRef(unwrap(Args), NumArgs), Name)); +} + extern "C" LLVMValueRef LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, @@ -1925,4 +1937,4 @@ extern "C" LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty, unwrap(B)->CreateInBoundsGEP(unwrap(Ty), unwrap(Pointer), IdxList, Name)); } -#endif \ No newline at end of file +#endif diff --git a/crates/rustc_codegen_nvvm/src/abi.rs b/crates/rustc_codegen_nvvm/src/abi.rs index 996723ca..bb438795 100644 --- a/crates/rustc_codegen_nvvm/src/abi.rs +++ b/crates/rustc_codegen_nvvm/src/abi.rs @@ -115,7 +115,10 @@ impl ArgAttributeExt for ArgAttribute { where F: FnMut(llvm::Attribute), { - for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, InReg) + for_each_kind!(self, f, NoAlias, NonNull, ReadOnly, InReg); + if self.contains(ArgAttribute::CapturesNone) { + f(llvm::Attribute::NoCapture); + } } } @@ -672,6 +675,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); diff --git a/crates/rustc_codegen_nvvm/src/allocator.rs b/crates/rustc_codegen_nvvm/src/allocator.rs index 1403fde3..c7a34973 100644 --- a/crates/rustc_codegen_nvvm/src/allocator.rs +++ b/crates/rustc_codegen_nvvm/src/allocator.rs @@ -3,155 +3,128 @@ use crate::llvm::{self, False, True}; use crate::target; use libc::c_uint; use rustc_ast::expand::allocator::{ - ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, alloc_error_handler_name, default_fn_name, + AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; use rustc_middle::ty::TyCtxt; +use rustc_symbol_mangling::mangle_internal_symbol; // adapted from rustc_codegen_llvm -pub(crate) unsafe fn codegen( - _tcx: TyCtxt<'_>, +pub(crate) fn codegen( + tcx: TyCtxt<'_>, mods: &mut LlvmMod, _module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) { - let llcx = &*mods.llcx; - let llmod = unsafe { mods.llmod.as_ref().unwrap() }; - let usize = unsafe { target::usize_ty(llcx) }; - let i8 = unsafe { llvm::LLVMInt8TypeInContext(llcx) }; - let i8p = unsafe { llvm::LLVMPointerType(i8, 0) }; - let void = unsafe { llvm::LLVMVoidTypeInContext(llcx) }; + unsafe { + let llcx = &*mods.llcx; + let llmod = mods.llmod.as_ref().unwrap(); + let usize = target::usize_ty(llcx); + let i8 = llvm::LLVMInt8TypeInContext(llcx); + let i8p = llvm::LLVMPointerType(i8, 0); + let void = llvm::LLVMVoidTypeInContext(llcx); - let mut used = Vec::new(); + let mut used = Vec::new(); - if kind == AllocatorKind::Default { - for method in ALLOCATOR_METHODS { + for method in methods { let mut args = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match ty.ty { + for input in method.inputs { + match input.ty { AllocatorTy::Layout => { - args.push(usize); // size - args.push(usize); // align + args.push(usize); + args.push(usize); } AllocatorTy::Ptr => args.push(i8p), AllocatorTy::Usize => args.push(usize), - - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + AllocatorTy::Never | AllocatorTy::ResultPtr | AllocatorTy::Unit => { + panic!("invalid allocator arg") + } } } + + let no_return = matches!(method.output, AllocatorTy::Never); let output = match method.output { AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, - + AllocatorTy::Never | AllocatorTy::Unit => None, AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { panic!("invalid allocator output") } }; - let ty = unsafe { - llvm::LLVMFunctionType( - output.unwrap_or(void), - args.as_ptr(), - args.len() as c_uint, - False, - ) - }; - let name = format!("__rust_{}", method.name); - let llfn = unsafe { - llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty) - }; - + let ty = llvm::LLVMFunctionType( + output.unwrap_or(void), + args.as_ptr(), + args.len() as c_uint, + False, + ); + let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); + let llfn = llvm::LLVMRustGetOrInsertFunction( + llmod, + from_name.as_ptr().cast(), + from_name.len(), + ty, + ); used.push(llfn); - // nvvm doesnt support uwtable so dont try to generate it - - let callee = default_fn_name(method.name); - let callee = unsafe { - llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty) - }; - unsafe { llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden) }; + if no_return { + llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); + } - let llbb = unsafe { - llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()) - }; + let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); + let callee = llvm::LLVMRustGetOrInsertFunction( + llmod, + to_name.as_ptr().cast(), + to_name.len(), + ty, + ); + used.push(callee); + if no_return { + llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee); + } + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(llcx) }; - unsafe { llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb) }; + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); let args = args .iter() .enumerate() - .map(|(i, _)| unsafe { llvm::LLVMGetParam(llfn, i as c_uint) }) + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::>(); - let ret = unsafe { - llvm::LLVMRustBuildCall( - llbuilder, - callee, - args.as_ptr(), - args.len() as c_uint, - None, - ) - }; - unsafe { llvm::LLVMSetTailCall(ret, True) }; + let ret = llvm::LLVMRustBuildCall( + llbuilder, + callee, + args.as_ptr(), + args.len() as c_uint, + None, + ); + llvm::LLVMSetTailCall(ret, True); if output.is_some() { - unsafe { llvm::LLVMBuildRet(llbuilder, ret) }; + llvm::LLVMBuildRet(llbuilder, ret); } else { - unsafe { llvm::LLVMBuildRetVoid(llbuilder) }; + llvm::LLVMBuildRetVoid(llbuilder); } - unsafe { llvm::LLVMDisposeBuilder(llbuilder) }; + llvm::LLVMDisposeBuilder(llbuilder); } - } - - // rust alloc error handler - let args = [usize, usize]; // size, align - - let ty = unsafe { llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False) }; - let name = "__rust_alloc_error_handler".to_string(); - let llfn = - unsafe { llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty) }; - - used.push(llfn); - - // -> ! DIFlagNoReturn - llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); - - let callee = alloc_error_handler_name(alloc_error_handler_kind); - let callee = unsafe { - llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty) - }; - used.push(callee); - - // -> ! DIFlagNoReturn - llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee); - unsafe { llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden) }; - - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()) }; - - let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(llcx) }; - unsafe { llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb) }; - let args = args - .iter() - .enumerate() - .map(|(i, _)| unsafe { llvm::LLVMGetParam(llfn, i as c_uint) }) - .collect::>(); - let ret = unsafe { - llvm::LLVMRustBuildCall(llbuilder, callee, args.as_ptr(), args.len() as c_uint, None) - }; - unsafe { llvm::LLVMSetTailCall(ret, True) }; - unsafe { llvm::LLVMBuildRetVoid(llbuilder) }; - unsafe { llvm::LLVMDisposeBuilder(llbuilder) }; - - let ptr_ty = unsafe { llvm::LLVMPointerType(llvm::LLVMInt8TypeInContext(llcx), 0) }; + let shim_ty = llvm::LLVMFunctionType(void, std::ptr::null(), 0, False); + let shim_name = mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE); + let shim = llvm::LLVMRustGetOrInsertFunction( + llmod, + shim_name.as_ptr().cast(), + shim_name.len(), + shim_ty, + ); + used.push(shim); + + let ptr_ty = llvm::LLVMPointerType(llvm::LLVMInt8TypeInContext(llcx), 0); + for used in &mut used { + *used = llvm::LLVMConstBitCast(used, ptr_ty); + } - for used in &mut used { - *used = unsafe { llvm::LLVMConstBitCast(used, ptr_ty) }; + let section = c"llvm.metadata"; + let array = llvm::LLVMConstArray(ptr_ty, used.as_ptr(), used.len() as u32); + let g = llvm::LLVMAddGlobal(llmod, llvm::LLVMTypeOf(array), c"llvm.used".as_ptr().cast()); + llvm::LLVMSetInitializer(g, array); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); + llvm::LLVMSetSection(g, section.as_ptr()); } - - let section = c"llvm.metadata"; - let array = unsafe { llvm::LLVMConstArray(ptr_ty, used.as_ptr(), used.len() as u32) }; - let g = unsafe { - llvm::LLVMAddGlobal(llmod, llvm::LLVMTypeOf(array), c"llvm.used".as_ptr().cast()) - }; - unsafe { llvm::LLVMSetInitializer(g, array) }; - unsafe { llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage) }; - unsafe { llvm::LLVMSetSection(g, section.as_ptr()) }; } diff --git a/crates/rustc_codegen_nvvm/src/asm.rs b/crates/rustc_codegen_nvvm/src/asm.rs index b9d05fbc..411c9a1f 100644 --- a/crates/rustc_codegen_nvvm/src/asm.rs +++ b/crates/rustc_codegen_nvvm/src/asm.rs @@ -13,7 +13,7 @@ use rustc_codegen_ssa::{ ConstCodegenMethods, GlobalAsmOperandRef, InlineAsmOperandRef, }, }; -use rustc_hash::FxHashMap; +use rustc_data_structures::fx::FxHashMap; use rustc_middle::{span_bug, ty::Instance}; use rustc_span::{Pos, Span}; use rustc_target::asm::{InlineAsmRegClass, InlineAsmRegOrRegClass, NvptxInlineAsmRegClass}; diff --git a/crates/rustc_codegen_nvvm/src/back.rs b/crates/rustc_codegen_nvvm/src/back.rs index 102097a1..c3112164 100644 --- a/crates/rustc_codegen_nvvm/src/back.rs +++ b/crates/rustc_codegen_nvvm/src/back.rs @@ -4,15 +4,16 @@ use std::sync::Arc; use libc::{c_char, size_t}; use rustc_codegen_ssa::back::write::{TargetMachineFactoryConfig, TargetMachineFactoryFn}; -use rustc_codegen_ssa::traits::{DebugInfoCodegenMethods, MiscCodegenMethods}; +use rustc_codegen_ssa::traits::{DebugInfoCodegenMethods, MiscCodegenMethods, ModuleBufferMethods}; use rustc_codegen_ssa::{ CompiledModule, ModuleCodegen, - back::write::{CodegenContext, ModuleConfig}, + back::write::{CodegenContext, ModuleConfig, SharedEmitter}, base::maybe_create_entry_wrapper, mono_item::MonoItemExt, - traits::{BaseTypeCodegenMethods, ThinBufferMethods}, + traits::BaseTypeCodegenMethods, }; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_errors::{DiagCtxt, DiagCtxtHandle, FatalError}; use rustc_fs_util::path_to_c_string; use rustc_middle::bug; use rustc_middle::mir::mono::{MonoItem, MonoItemData}; @@ -25,7 +26,7 @@ use rustc_target::spec::{CodeModel, RelocModel}; use crate::common::AsCCharPtr; use crate::llvm::{self}; use crate::override_fns::define_or_override_fn; -use crate::{LlvmMod, NvvmCodegenBackend, builder::Builder, context::CodegenCx, lto::ThinBuffer}; +use crate::{LlvmMod, NvvmCodegenBackend, builder::Builder, context::CodegenCx, lto::ModuleBuffer}; pub fn llvm_err(handle: DiagCtxtHandle, msg: &str) -> FatalError { match llvm::last_error() { @@ -81,7 +82,7 @@ pub fn target_machine_factory( let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); - let use_softfp = sess.opts.cg.soft_float; + let use_softfp = false; let ffunction_sections = sess .opts @@ -101,7 +102,7 @@ pub fn target_machine_factory( .trap_unreachable .unwrap_or(sess.target.trap_unreachable); - Arc::new(move |_config: TargetMachineFactoryConfig| { + Arc::new(move |dcx, _config: TargetMachineFactoryConfig| { let tm = unsafe { llvm::LLVMRustCreateTargetMachine( triple.as_c_char_ptr(), @@ -121,7 +122,11 @@ pub fn target_machine_factory( false, ) }; - tm.ok_or_else(|| format!("Could not create LLVM TargetMachine for triple: {triple}")) + tm.unwrap_or_else(|| { + dcx.fatal(format!( + "Could not create LLVM TargetMachine for triple: {triple}" + )) + }) }) } @@ -156,11 +161,13 @@ pub extern "C" fn demangle_callback( /// Compile a single module (in an nvvm context this means getting the llvm bitcode out of it) pub(crate) unsafe fn codegen( - cgcx: &CodegenContext, + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> Result { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); // For NVVM, all the codegen we need to do is turn the llvm modules @@ -172,28 +179,29 @@ pub(crate) unsafe fn codegen( // we also implement emit_ir so we can dump the IR fed to nvvm in case we // feed it anything it doesnt like - let _timer = cgcx - .prof - .generic_activity_with_arg("NVVM_module_codegen", &module.name[..]); + let _timer = prof.generic_activity_with_arg("NVVM_module_codegen", &module.name[..]); let llmod = unsafe { module.module_llvm.llmod.as_ref().unwrap() }; let mod_name = module.name.clone(); let module_name = &mod_name[..]; - let out = cgcx - .output_filenames - .temp_path_for_cgu(OutputType::Object, module_name, None); + let out = cgcx.output_filenames.temp_path_for_cgu( + OutputType::Object, + module_name, + cgcx.invocation_temp.as_deref(), + ); // nvvm ir *is* llvm ir so emit_ir fits the expectation of llvm ir which is why we // implement this. this is copy and pasted straight from rustc_codegen_llvm // because im too lazy to make it seem like i rewrote this when its the same logic if config.emit_ir { - let _timer = cgcx - .prof - .generic_activity_with_arg("NVVM_module_codegen_emit_ir", &module.name[..]); - let out = - cgcx.output_filenames - .temp_path_for_cgu(OutputType::LlvmAssembly, module_name, None); + let _timer = + prof.generic_activity_with_arg("NVVM_module_codegen_emit_ir", &module.name[..]); + let out = cgcx.output_filenames.temp_path_for_cgu( + OutputType::LlvmAssembly, + module_name, + cgcx.invocation_temp.as_deref(), + ); let out = out.to_str().unwrap(); let result = unsafe { @@ -206,17 +214,15 @@ pub(crate) unsafe fn codegen( })?; } - let _bc_timer = cgcx - .prof - .generic_activity_with_arg("NVVM_module_codegen_make_bitcode", &module.name[..]); + let _bc_timer = + prof.generic_activity_with_arg("NVVM_module_codegen_make_bitcode", &module.name[..]); - let thin = ThinBuffer::new(llmod); + let thin = ModuleBuffer::new(llmod, false); let data = thin.data(); - let _bc_emit_timer = cgcx - .prof - .generic_activity_with_arg("NVVM_module_codegen_emit_bitcode", &module.name[..]); + let _bc_emit_timer = + prof.generic_activity_with_arg("NVVM_module_codegen_emit_bitcode", &module.name[..]); if let Err(e) = std::fs::write(&out, data) { let msg = format!("failed to write bytecode to {}: {}", out.display(), e); @@ -330,30 +336,24 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen // are incompatible with llvm 7 nowadays. Although we should probably consult a rustc dev on whether // any big things were discovered in that timespan that we should modify. pub(crate) unsafe fn optimize( - cgcx: &CodegenContext, - diag_handler: DiagCtxtHandle<'_>, - module: &ModuleCodegen, + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &SharedEmitter, + module: &mut ModuleCodegen, config: &ModuleConfig, ) -> Result<(), FatalError> { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_module_optimize", &module.name[..]); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + let diag_handler = dcx.handle(); + let _timer = prof.generic_activity_with_arg("LLVM_module_optimize", &module.name[..]); let llmod = unsafe { &*module.module_llvm.llmod }; - if config.emit_no_opt_bc { + if config.emit_pre_lto_bc { let out = cgcx.output_filenames.with_extension("no-opt.bc"); let out = path_to_c_string(&out); unsafe { llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()) }; } - let tm_factory_config = TargetMachineFactoryConfig { - split_dwarf_file: None, - output_obj_file: None, - }; - - let tm = (cgcx.tm_factory)(tm_factory_config).expect("failed to create target machine"); - if config.opt_level.is_some() { unsafe { let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); @@ -379,8 +379,6 @@ pub(crate) unsafe fn optimize( }; if !config.no_prepopulate_passes { - llvm::LLVMRustAddAnalysisPasses(tm, fpm, llmod); - llvm::LLVMRustAddAnalysisPasses(tm, mpm, llmod); let opt_level = config .opt_level .map_or(llvm::CodeGenOptLevel::None, |x| to_llvm_opt_settings(x).0); @@ -422,7 +420,7 @@ unsafe fn with_llvm_pmb( let builder = llvm::LLVMPassManagerBuilderCreate(); let opt_size = config - .opt_size + .opt_level .map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); llvm::LLVMRustConfigurePassManagerBuilder( diff --git a/crates/rustc_codegen_nvvm/src/builder.rs b/crates/rustc_codegen_nvvm/src/builder.rs index b51709d8..7997504d 100644 --- a/crates/rustc_codegen_nvvm/src/builder.rs +++ b/crates/rustc_codegen_nvvm/src/builder.rs @@ -67,13 +67,12 @@ impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { type Function = as BackendTypes>::Function; type BasicBlock = as BackendTypes>::BasicBlock; type Type = as BackendTypes>::Type; + type FunctionSignature = as BackendTypes>::FunctionSignature; type Funclet = as BackendTypes>::Funclet; type DIScope = as BackendTypes>::DIScope; type DILocation = as BackendTypes>::DILocation; type DIVariable = as BackendTypes>::DIVariable; - - type Metadata = as BackendTypes>::Metadata; } impl HasDataLayout for Builder<'_, '_, '_> { @@ -499,6 +498,10 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> &'ll Value { + self.unsupported("scalable vector allocas"); + } + fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { trace!("Load {ty:?} {:?}", ptr); let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty)); @@ -656,6 +659,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandRef { val, layout: place.layout, + move_annotation: None, } } @@ -984,6 +988,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + _tt: Option, ) { assert!( !flags.contains(MemFlags::NONTEMPORAL), @@ -1139,6 +1144,10 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.unsupported("catch switches"); } + fn get_funclet_cleanuppad(&self, _funclet: &()) -> &'ll Value { + self.unsupported("funclet cleanuppads"); + } + fn set_personality_fn(&mut self, _personality: &'ll Value) {} // Atomic Operations @@ -1346,10 +1355,17 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { trace!("Calling fn {:?} with args {:?}", llfn, args); self.cx.last_call_llfn.set(None); let args = self.check_call("call", llty, llfn, args); + let llfn_ptr_ty = unsafe { llvm::LLVMPointerType(llty, 0) }; + let llfn = if self.val_ty(llfn) == llfn_ptr_ty { + llfn + } else { + self.pointercast(llfn, llfn_ptr_ty) + }; let mut call = unsafe { - llvm::LLVMRustBuildCall( + llvm::LLVMRustBuildCall2( self.llbuilder, + llty, llfn, args.as_ptr(), args.len() as c_uint, @@ -1452,7 +1468,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - fn check_call<'b>( + pub(crate) fn check_call<'b>( &mut self, typ: &str, fn_ty: &'ll Type, diff --git a/crates/rustc_codegen_nvvm/src/const_ty.rs b/crates/rustc_codegen_nvvm/src/const_ty.rs index 10b28191..634da3a4 100644 --- a/crates/rustc_codegen_nvvm/src/const_ty.rs +++ b/crates/rustc_codegen_nvvm/src/const_ty.rs @@ -10,15 +10,11 @@ use rustc_codegen_ssa::traits::*; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_hashes::Hash128; use rustc_middle::bug; -use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; +use rustc_middle::mir::interpret::{GlobalAlloc, Scalar}; use rustc_middle::ty::layout::LayoutOf; use tracing::trace; impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { - fn const_data_from_alloc(&self, alloc: ConstAllocation) -> &'ll Value { - const_alloc_to_llvm(self, alloc, /*static*/ false) - } - fn const_null(&self, t: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMConstNull(t) } } @@ -168,16 +164,18 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { }; } else { let init = const_alloc_to_llvm(self, alloc, /*static*/ false); - let alloc = alloc.inner(); - let value = match alloc.mutability { - Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), - _ => self.static_addr_of(init, alloc.align, None), + let inner_alloc = alloc.inner(); + let value = match inner_alloc.mutability { + Mutability::Mut => { + self.static_addr_of_mut(init, inner_alloc.align, None) + } + _ => self.static_addr_of(alloc, None), }; if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() { let hash = self.tcx.with_stable_hashing_context(|mut hcx| { let mut hasher = StableHasher::new(); - alloc.hash_stable(&mut hcx, &mut hasher); + inner_alloc.hash_stable(&mut hcx, &mut hasher); hasher.finish::() }); llvm::set_value_name( @@ -202,8 +200,8 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { }), ))) .unwrap_memory(); - let init = const_alloc_to_llvm(self, alloc, /*static*/ false); - let value = self.static_addr_of(init, alloc.inner().align, None); + let _ = const_alloc_to_llvm(self, alloc, /*static*/ false); + let value = self.static_addr_of(alloc, None); (value, AddressSpace::ZERO) } GlobalAlloc::Static(def_id) => { diff --git a/crates/rustc_codegen_nvvm/src/consts.rs b/crates/rustc_codegen_nvvm/src/consts.rs index 1d904f68..d6a7987c 100644 --- a/crates/rustc_codegen_nvvm/src/consts.rs +++ b/crates/rustc_codegen_nvvm/src/consts.rs @@ -5,6 +5,7 @@ use crate::llvm::{self, True, Type, Value}; use libc::{c_char, c_uint}; use rustc_abi::{AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::Linkage; use rustc_hir::def_id::DefId; use rustc_middle::mir::interpret::{ Allocation, ConstAllocation, ErrorHandled, Pointer, read_target_uint, @@ -16,7 +17,7 @@ use rustc_middle::{ middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}, mir::{ interpret::{InitChunk, Scalar as InterpScalar}, - mono::{Linkage, MonoItem}, + mono::MonoItem, }, span_bug, }; @@ -179,10 +180,15 @@ fn check_and_apply_linkage<'ll, 'tcx>( let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.linkage { // https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#linkage-types-nvvm - use Linkage::*; match linkage { - External | Internal | Common | AvailableExternally | LinkOnceAny | LinkOnceODR - | WeakAny | WeakODR => {} + Linkage::External + | Linkage::Internal + | Linkage::Common + | Linkage::AvailableExternally + | Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR => {} _ => cx .sess() .dcx() @@ -315,24 +321,13 @@ impl<'ll> CodegenCx<'ll, '_> { } impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> { - fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { - if let Some(&gv) = self.const_globals.borrow().get(&cv) { - unsafe { - // Upgrade the alignment in cases where the same constant is used with different - // alignment requirements - let llalign = align.bytes() as u32; - if llalign > llvm::LLVMGetAlignment(gv) { - llvm::LLVMSetAlignment(gv, llalign); - } - } - return gv; - } - let gv = self.static_addr_of_mut(cv, align, kind); + fn static_addr_of(&self, alloc: ConstAllocation<'_>, kind: Option<&str>) -> &'ll Value { + let cv = const_alloc_to_llvm(self, alloc, /*static*/ false); + let gv = self.static_addr_of_mut(cv, alloc.inner().align, kind); unsafe { llvm::LLVMSetGlobalConstant(gv, True); } - self.const_globals.borrow_mut().insert(cv, gv); - gv + self.const_ptrcast(gv, self.type_ptr()) } fn codegen_static(&mut self, def_id: DefId) { diff --git a/crates/rustc_codegen_nvvm/src/context.rs b/crates/rustc_codegen_nvvm/src/context.rs index e5ed5be0..41481808 100644 --- a/crates/rustc_codegen_nvvm/src/context.rs +++ b/crates/rustc_codegen_nvvm/src/context.rs @@ -18,9 +18,8 @@ use rustc_codegen_ssa::traits::{ MiscCodegenMethods, }; use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN}; +use rustc_data_structures::fx::FxHashMap; use rustc_errors::DiagMessage; -use rustc_hash::FxHashMap; -use rustc_middle::dep_graph::DepContext; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOf, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOf, }; @@ -33,8 +32,7 @@ use rustc_middle::{ }; use rustc_session::Session; use rustc_session::config::DebugInfo; -use rustc_span::source_map::Spanned; -use rustc_span::{Span, Symbol}; +use rustc_span::{Span, Spanned, Symbol}; use rustc_target::callconv::FnAbi; use rustc_target::spec::{HasTargetSpec, Target}; @@ -66,9 +64,6 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { pub remapped_integer_args: RefCell, Vec<(usize, &'ll Type)>)>>, - /// Cache of emitted const globals (value -> global) - pub const_globals: RefCell>, - /// List of globals for static variables which need to be passed to the /// LLVM function ReplaceAllUsesWith (RAUW) when codegen is complete. /// (We have to make sure we don't invalidate any Values referring @@ -150,7 +145,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { vtables: Default::default(), const_cstr_cache: Default::default(), remapped_integer_args: Default::default(), - const_globals: Default::default(), statics_to_rauw: RefCell::new(Vec::new()), used_statics: RefCell::new(Vec::new()), compiler_used_statics: RefCell::new(Vec::new()), @@ -178,7 +172,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { addrspace: Symbol::intern("addrspace"), }, dbg_cx, - codegen_args: CodegenArgs::from_session(tcx.sess()), + codegen_args: CodegenArgs::from_session(tcx.sess), last_call_llfn: Cell::new(None), constant_memory_usage: Cell::new(0), }; @@ -267,7 +261,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub fn static_addrspace(&self, instance: Instance<'tcx>) -> AddressSpace { let ty = instance.ty(self.tcx, self.typing_env()); let is_mutable = self.tcx().is_mutable_static(instance.def_id()); - let attrs = self.tcx.get_all_attrs(instance.def_id()); // TODO: replace with get_attrs + let attrs = if let Some(def_id) = instance.def_id().as_local() { + self.tcx.hir_attrs(self.tcx.local_def_id_to_hir_id(def_id)) + } else { + self.tcx.attrs_for_def(instance.def_id()) + }; let nvvm_attrs = NvvmAttributes::parse(self, attrs); if let Some(addr) = nvvm_attrs.addrspace { @@ -728,9 +726,9 @@ impl CodegenArgs { impl<'ll> BackendTypes for CodegenCx<'ll, '_> { type Value = &'ll Value; type Function = &'ll Value; - type BasicBlock = &'ll BasicBlock; type Type = &'ll Type; + type FunctionSignature = &'ll Type; // not applicable to nvvm, unwinding/exception handling // doesnt exist on the gpu type Funclet = (); @@ -738,8 +736,6 @@ impl<'ll> BackendTypes for CodegenCx<'ll, '_> { type DIScope = &'ll llvm::DIScope; type DILocation = &'ll llvm::DILocation; type DIVariable = &'ll llvm::DIVariable; - - type Metadata = &'ll llvm::Metadata; } impl HasDataLayout for CodegenCx<'_, '_> { @@ -769,16 +765,9 @@ impl<'tcx> HasTypingEnv<'tcx> for CodegenCx<'_, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { - self.tcx.dcx().emit_fatal(Spanned { - span, - node: err.into_diagnostic(), - }) - } else { - self.tcx - .dcx() - .emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) - } + self.tcx + .dcx() + .emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) } } diff --git a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs b/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs index d0464212..553b1195 100644 --- a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs +++ b/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs @@ -30,8 +30,10 @@ impl<'ll> CodegenCx<'ll, '_> { let t_i16 = self.type_i16(); let t_i32 = self.type_i32(); let t_i64 = self.type_i64(); + let t_f16 = self.type_f16(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_f128 = self.type_f128(); let t_isize = self.type_isize(); let t_i8_i1 = self.type_struct(&[t_i8, i1], false); @@ -141,6 +143,31 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!(map, "llvm.expect.i1", fn(i1, i1) -> i1); ifn!(map, "llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + ifn!(map, "llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16); + ifn!(map, "llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128); + ifn!(map, "llvm.fabs.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.fabs.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.sqrt.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.sqrt.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.floor.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.floor.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.ceil.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.ceil.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.trunc.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.trunc.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.round.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.round.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.rint.f16", fn(t_f16) -> t_f16); + ifn!(map, "llvm.rint.f128", fn(t_f128) -> t_f128); + ifn!(map, "llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16); + ifn!(map, "llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!(map, "llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16); + ifn!(map, "llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!(map, "llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16); + ifn!(map, "llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!(map, "llvm.powi.f16.i32", fn(t_f16, t_i32) -> t_f16); + ifn!(map, "llvm.powi.f128.i32", fn(t_f128, t_i32) -> t_f128); + // This isn't an "LLVM intrinsic", but LLVM's optimization passes // recognize it like one and we assume it exists in `core::slice::cmp` ifn!(map, "memcmp", fn(i8p, i8p, t_isize) -> t_i32); diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs b/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs index 7d148d2a..549d5102 100644 --- a/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs +++ b/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs @@ -16,7 +16,7 @@ use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtKind, CoroutineArgsExt, Instance, Ty, TyCtxt, Visibility}; use rustc_session::config::{self, DebugInfo}; use rustc_span::symbol::Symbol; -use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, hygiene}; +use rustc_span::{DUMMY_SP, FileName, RemapPathScopeComponents, SourceFile, hygiene}; use smallvec::smallvec; use tracing::debug; @@ -556,82 +556,29 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi ) -> &'ll DIFile { debug!(?source_file.name); - let filename_display_preference = cx - .sess() - .filename_display_preference(RemapPathScopeComponents::DEBUGINFO); - - use rustc_session::config::RemapPathScopeComponents; let (directory, file_name) = match &source_file.name { FileName::Real(filename) => { - let working_directory = &cx.sess().opts.working_dir; - debug!(?working_directory); - - if filename_display_preference == FileNameDisplayPreference::Remapped { - let filename = cx - .sess() - .source_map() - .path_mapping() - .to_embeddable_absolute_path(filename.clone(), working_directory); - - // Construct the absolute path of the file - let abs_path = filename.remapped_path_if_available(); - debug!(?abs_path); - - if let Ok(rel_path) = - abs_path.strip_prefix(working_directory.remapped_path_if_available()) - { - // If the compiler's working directory (which also is the DW_AT_comp_dir of - // the compilation unit) is a prefix of the path we are about to emit, then - // only emit the part relative to the working directory. Because of path - // remapping we sometimes see strange things here: `abs_path` might - // actually look like a relative path (e.g. - // `/src/lib.rs`), so if we emit it without taking - // the working directory into account, downstream tooling will interpret it - // as `//src/lib.rs`, which - // makes no sense. Usually in such cases the working directory will also be - // remapped to `` or some other prefix of the path - // we are remapping, so we end up with - // `//src/lib.rs`. - // By moving the working directory portion into the `directory` part of the - // DIFile, we allow LLVM to emit just the relative path for DWARF, while - // still emitting the correct absolute path for CodeView. - ( - working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), - rel_path.to_string_lossy().into_owned(), - ) - } else { - ("".into(), abs_path.to_string_lossy().into_owned()) - } - } else { - let working_directory = working_directory.local_path_if_available(); - let filename = filename.local_path_if_available(); + let (working_directory, embeddable_name) = + filename.embeddable_name(RemapPathScopeComponents::DEBUGINFO); - debug!(?working_directory, ?filename); - - let abs_path: Cow<'_, _> = if filename.is_absolute() { - filename.into() - } else { - let mut p = PathBuf::new(); - p.push(working_directory); - p.push(filename); - p.into() - }; + debug!(?working_directory, ?embeddable_name); - if let Ok(rel_path) = abs_path.strip_prefix(working_directory) { - ( - working_directory.to_string_lossy(), - rel_path.to_string_lossy().into_owned(), - ) - } else { - ("".into(), abs_path.to_string_lossy().into_owned()) - } + if let Ok(rel_path) = embeddable_name.strip_prefix(working_directory) { + ( + working_directory.to_string_lossy(), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), embeddable_name.to_string_lossy().into_owned()) } } other => { debug!(?other); ( "".into(), - other.display(filename_display_preference).to_string(), + other + .display(RemapPathScopeComponents::DEBUGINFO) + .to_string(), ) } }; @@ -763,15 +710,10 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( codegen_unit_name: &str, debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, ) -> &'ll DIDescriptor { - use rustc_session::RemapFileNameExt; - use rustc_session::config::RemapPathScopeComponents; let mut name_in_debuginfo = tcx .sess .local_crate_source_file() - .map(|src| { - src.for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO) - .to_path_buf() - }) + .map(|src| src.path(RemapPathScopeComponents::DEBUGINFO).to_path_buf()) .unwrap_or_else(|| PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str())); // To avoid breaking split DWARF, we need to ensure that each codegen unit @@ -807,7 +749,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( .sess .opts .working_dir - .for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO) + .path(RemapPathScopeComponents::DEBUGINFO) .to_string_lossy(); let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() @@ -818,14 +760,19 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( None, ) { // We get a path relative to the working directory from split_dwarf_path - Some(tcx.sess.source_map().path_mapping().to_real_filename(f)) + Some( + tcx.sess + .source_map() + .path_mapping() + .to_real_filename(&tcx.sess.opts.working_dir, f), + ) } else { None }; let split_name = split_name .as_ref() .map(|f| { - f.for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO) + f.path(RemapPathScopeComponents::DEBUGINFO) .to_string_lossy() }) .unwrap_or_default(); @@ -1317,7 +1264,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); - let trait_ref = tcx.erase_regions(trait_ref); + let trait_ref = tcx.erase_and_anonymize_regions(trait_ref); tcx.vtable_entries(trait_ref) } else { diff --git a/crates/rustc_codegen_nvvm/src/debug_info/mod.rs b/crates/rustc_codegen_nvvm/src/debug_info/mod.rs index 15331473..f9e7ade1 100644 --- a/crates/rustc_codegen_nvvm/src/debug_info/mod.rs +++ b/crates/rustc_codegen_nvvm/src/debug_info/mod.rs @@ -105,7 +105,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { } } -impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { +impl<'ll, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn dbg_var_addr( &mut self, dbg_var: &'ll DIVariable, @@ -113,7 +113,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { variable_alloca: &'ll Value, direct_offset: Size, indirect_offsets: &[Size], - fragment: Option>, + fragment: &Option>, ) { use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst}; @@ -159,6 +159,23 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { } } + fn dbg_var_value( + &mut self, + _dbg_var: &'ll DIVariable, + _dbg_loc: &'ll DILocation, + _value: &'ll Value, + _direct_offset: Size, + _indirect_offsets: &[Size], + _fragment: &Option>, + ) { + } + + fn clear_dbg_loc(&mut self) { + unsafe { + llvm::LLVMSetCurrentDebugLocation(self.llbuilder, None); + } + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { // do nothing } @@ -184,12 +201,6 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { llvm::set_value_name(value, name.as_bytes()); } } - - fn clear_dbg_loc(&mut self) { - unsafe { - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, None); - } - } } /// A source code location used to generate debug information. @@ -402,31 +413,24 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - if let Some(impl_def_id) = cx.tcx.impl_of_assoc(instance.def_id()) { - // If the method does *not* belong to a trait, proceed - if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( - instance.args, - cx.typing_env(), - cx.tcx.type_of(impl_def_id), - ); - - // Only "class" methods are generally understood by LLVM, - // so avoid methods on other types (e.g., `<*mut T>::null`). - if let ty::Adt(def, ..) = impl_self_ty.kind() - && !def.is_box() - { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() - { - return (type_di_node(cx, impl_self_ty), true); - } else { - return (namespace::item_namespace(cx, def.did()), false); - } + if let Some(impl_def_id) = cx.tcx.inherent_impl_of_assoc(instance.def_id()) { + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + cx.tcx.type_of(impl_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - } else { - // For trait method impls we still use the "parallel namespace" - // strategy } } let scope = namespace::item_namespace( diff --git a/crates/rustc_codegen_nvvm/src/intrinsic.rs b/crates/rustc_codegen_nvvm/src/intrinsic.rs index 31e1efd4..f6e9d161 100644 --- a/crates/rustc_codegen_nvvm/src/intrinsic.rs +++ b/crates/rustc_codegen_nvvm/src/intrinsic.rs @@ -1,12 +1,12 @@ use rustc_abi as abi; -use rustc_abi::{self, BackendRepr, Float, HasDataLayout, Primitive}; +use rustc_abi::{self, BackendRepr, Float, HasDataLayout, Primitive, WrappingRange}; use rustc_codegen_ssa::errors::InvalidMonomorphization; use rustc_codegen_ssa::mir::operand::OperandValue; use rustc_codegen_ssa::mir::place::PlaceValue; use rustc_codegen_ssa::mir::{operand::OperandRef, place::PlaceRef}; use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, IntrinsicCallBuilderMethods, - OverflowOp, + LayoutTypeCodegenMethods, OverflowOp, }; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Ty}; @@ -187,12 +187,6 @@ fn get_simple_intrinsic<'ll>( sym::log2f64 => "__nv_log2", sym::fmaf32 => "__nv_fmaf", sym::fmaf64 => "__nv_fma", - sym::fabsf32 => "__nv_fabsf", - sym::fabsf64 => "__nv_fabs", - sym::minnumf32 => "__nv_fminf", - sym::minnumf64 => "__nv_fmin", - sym::maxnumf32 => "__nv_fmaxf", - sym::maxnumf64 => "__nv_fmax", sym::copysignf32 => "__nv_copysignf", sym::copysignf64 => "__nv_copysign", sym::floorf32 => "__nv_floorf", @@ -211,6 +205,34 @@ fn get_simple_intrinsic<'ll>( Some(cx.get_intrinsic(llvm_name)) } +fn get_llvm_float_intrinsic<'ll>( + cx: &CodegenCx<'ll, '_>, + intrinsic: &str, + width: u32, +) -> (&'ll Type, &'ll Value) { + let suffix = match width { + 16 => "f16", + 32 => "f32", + 64 => "f64", + 128 => "f128", + _ => bug!("unsupported float width {width} for intrinsic {intrinsic}"), + }; + let name = format!("{intrinsic}.{suffix}"); + cx.get_intrinsic(&name) +} + +fn get_llvm_powi_intrinsic<'ll>(cx: &CodegenCx<'ll, '_>, width: u32) -> (&'ll Type, &'ll Value) { + let suffix = match width { + 16 => "f16", + 32 => "f32", + 64 => "f64", + 128 => "f128", + _ => bug!("unsupported float width {width} for llvm.powi"), + }; + let name = format!("llvm.powi.{suffix}.i32"); + cx.get_intrinsic(&name) +} + impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_intrinsic_call( &mut self, @@ -230,8 +252,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig); let arg_tys = sig.inputs(); let ret_ty = sig.output(); - let name = tcx.item_name(def_id); - let name_str = name.as_str(); + let intrinsic = tcx.intrinsic(def_id).unwrap(); + let name = intrinsic.name; + let name_str: &str = name.as_str(); trace!( "Beginning intrinsic call: `{:?}`, args: `{:?}`, ret: `{:?}`", @@ -257,6 +280,132 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Some(instance), ) } + n if matches!( + n, + sym::fabs | sym::minimumf32 | sym::minimumf64 | sym::maximumf32 | sym::maximumf64 + ) || matches!( + name_str, + "sqrtf16" + | "sqrtf128" + | "powif16" + | "powif128" + | "fmaf16" + | "fmaf128" + | "copysignf16" + | "copysignf128" + | "floorf16" + | "floorf128" + | "ceilf16" + | "ceilf128" + | "truncf16" + | "truncf128" + | "roundf16" + | "roundf128" + | "round_ties_even_f16" + | "round_ties_even_f128" + | "minimumf16" + | "minimumf128" + | "maximumf16" + | "maximumf128" + | "minimum_number_nsz_f16" + | "minimum_number_nsz_f32" + | "minimum_number_nsz_f64" + | "minimum_number_nsz_f128" + | "maximum_number_nsz_f16" + | "maximum_number_nsz_f32" + | "maximum_number_nsz_f64" + | "maximum_number_nsz_f128" + ) => + { + let ty = args[0].layout.ty; + let width = match ty.kind() { + ty::Float(ty::FloatTy::F16) => 16, + ty::Float(ty::FloatTy::F32) => 32, + ty::Float(ty::FloatTy::F64) => 64, + ty::Float(ty::FloatTy::F128) => 128, + _ => span_bug!( + span, + "unsupported float intrinsic {name:?} for argument type {ty:?}" + ), + }; + let (simple_ty, simple_fn) = match name { + sym::fabs if width == 32 => self.cx.get_intrinsic("__nv_fabsf"), + sym::fabs if width == 64 => self.cx.get_intrinsic("__nv_fabs"), + sym::fabs => get_llvm_float_intrinsic(self.cx, "llvm.fabs", width), + sym::minimumf32 if width == 32 => self.cx.get_intrinsic("__nv_fminf"), + sym::minimumf64 if width == 64 => self.cx.get_intrinsic("__nv_fmin"), + sym::maximumf32 if width == 32 => self.cx.get_intrinsic("__nv_fmaxf"), + sym::maximumf64 if width == 64 => self.cx.get_intrinsic("__nv_fmax"), + _ => match name_str { + "sqrtf16" | "sqrtf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.sqrt", width) + } + "powif16" | "powif128" => get_llvm_powi_intrinsic(self.cx, width), + "fmaf16" | "fmaf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.fma", width) + } + "copysignf16" | "copysignf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.copysign", width) + } + "floorf16" | "floorf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.floor", width) + } + "ceilf16" | "ceilf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.ceil", width) + } + "truncf16" | "truncf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.trunc", width) + } + "roundf16" | "roundf128" => { + get_llvm_float_intrinsic(self.cx, "llvm.round", width) + } + "round_ties_even_f16" | "round_ties_even_f128" => { + get_llvm_float_intrinsic(self.cx, "llvm.rint", width) + } + "minimumf16" + | "minimumf128" + | "minimum_number_nsz_f16" + | "minimum_number_nsz_f32" + | "minimum_number_nsz_f64" + | "minimum_number_nsz_f128" => { + if width == 32 { + self.cx.get_intrinsic("__nv_fminf") + } else if width == 64 { + self.cx.get_intrinsic("__nv_fmin") + } else { + get_llvm_float_intrinsic(self.cx, "llvm.minnum", width) + } + } + "maximumf16" + | "maximumf128" + | "maximum_number_nsz_f16" + | "maximum_number_nsz_f32" + | "maximum_number_nsz_f64" + | "maximum_number_nsz_f128" => { + if width == 32 { + self.cx.get_intrinsic("__nv_fmaxf") + } else if width == 64 { + self.cx.get_intrinsic("__nv_fmax") + } else { + get_llvm_float_intrinsic(self.cx, "llvm.maxnum", width) + } + } + _ => span_bug!( + span, + "unsupported float intrinsic {name:?} for argument type {ty:?}" + ), + }, + }; + self.call( + simple_ty, + None, + None, + simple_fn, + &args.iter().map(|arg| arg.immediate()).collect::>(), + None, + Some(instance), + ) + } sym::is_val_statically_known => { // LLVM 7 does not support this intrinsic, so always assume false. self.const_bool(false) @@ -287,7 +436,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"), } } - sym::likely => self.call_intrinsic( + _ if name_str == "likely" => self.call_intrinsic( "llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)], ), @@ -300,7 +449,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let data = args[1].immediate(); self.call(self.type_i1(), None, None, try_func, &[data], None, None); - let ret_align = self.data_layout().i32_align.abi; + let ret_align = self.data_layout().i32_align; self.store(self.const_i32(0), result.val.llval, ret_align) } sym::breakpoint => { @@ -533,6 +682,15 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let use_integer_compare = match layout.backend_repr() { BackendRepr::Scalar(_) | BackendRepr::ScalarPair(_, _) => true, BackendRepr::SimdVector { .. } => false, + BackendRepr::SimdScalableVector { .. } => { + tcx.dcx() + .emit_err(InvalidMonomorphization::NonScalableType { + span, + name: sym::raw_eq, + ty: tp_ty, + }); + return Ok(()); + } BackendRepr::Memory { .. } => { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), @@ -610,7 +768,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Fall back to a fallback intrinsic implementation, if possible _ => { // This piece of code was adapted from `rustc_codegen_cranelift`. - let intrinsic = self.tcx.intrinsic(instance.def_id()).unwrap(); if intrinsic.must_be_overridden { span_bug!( span, @@ -639,6 +796,97 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + is_cleanup: bool, + ) -> &'ll Value { + let tcx = self.tcx; + + let fn_ty = instance.ty(tcx, self.typing_env()); + let fn_sig = match *fn_ty.kind() { + ty::FnDef(def_id, args) => { + tcx.instantiate_bound_regions_with_erased(tcx.fn_sig(def_id).instantiate(tcx, args)) + } + _ => unreachable!(), + }; + assert!(!fn_sig.c_variadic); + + let ret_layout = self.layout_of(fn_sig.output()); + let llreturn_ty = if ret_layout.is_zst() { + self.type_void() + } else { + ret_layout.immediate_llvm_type(self) + }; + + let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len()); + for &arg in fn_sig.inputs() { + let arg_layout = self.layout_of(arg); + if arg_layout.is_zst() { + continue; + } + llargument_tys.push(arg_layout.immediate_llvm_type(self)); + } + + let fn_ty = self.type_func(&llargument_tys, llreturn_ty); + let sym = tcx.symbol_name(instance).name; + let fn_ptr = self + .cx + .get_declared_value(sym) + .unwrap_or_else(|| self.cx.declare_fn(sym, fn_ty, None)); + let fn_ptr_ty = unsafe { llvm::LLVMPointerType(fn_ty, 0) }; + let fn_ptr = if self.val_ty(fn_ptr) == fn_ptr_ty { + fn_ptr + } else { + self.pointercast(fn_ptr, fn_ptr_ty) + }; + + let mut llargs = Vec::with_capacity(args.len()); + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + trace!( + "call llvm intrinsic {:?} with args ({:?})", + instance, llargs + ); + let args = self.check_call("call", fn_ty, fn_ptr, &llargs); + let llret = unsafe { + llvm::LLVMRustBuildCall2( + self.llbuilder, + fn_ty, + fn_ptr, + args.as_ptr(), + args.len() as _, + None, + ) + }; + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { trace!("Generate abort call"); self.call_intrinsic("llvm.trap", &[]); @@ -658,7 +906,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &mut self, _llvtable: Self::Value, _vtable_byte_offset: u64, - _typeid: Self::Metadata, + _typeid: &[u8], ) -> Self::Value { // LLVM CFI doesnt make sense on the GPU self.const_i32(0) diff --git a/crates/rustc_codegen_nvvm/src/lib.rs b/crates/rustc_codegen_nvvm/src/lib.rs index 851724b6..61fe4894 100644 --- a/crates/rustc_codegen_nvvm/src/lib.rs +++ b/crates/rustc_codegen_nvvm/src/lib.rs @@ -3,7 +3,6 @@ // make our lives a lot easier for llvm ffi with this. And since rustc's core infra // relies on it its almost guaranteed to not be removed/broken #![feature(extern_types)] -#![feature(slice_as_array)] extern crate rustc_abi; extern crate rustc_arena; @@ -14,7 +13,6 @@ extern crate rustc_data_structures; extern crate rustc_driver; extern crate rustc_errors; extern crate rustc_fs_util; -extern crate rustc_hash; extern crate rustc_hashes; extern crate rustc_hir; extern crate rustc_index; @@ -22,7 +20,6 @@ extern crate rustc_interface; extern crate rustc_macros; extern crate rustc_metadata; extern crate rustc_middle; -extern crate rustc_query_system; extern crate rustc_session; extern crate rustc_span; extern crate rustc_symbol_mangling; @@ -56,11 +53,9 @@ mod ty; use abi::readjust_fn_abi; use back::target_machine_factory; -use lto::ThinBuffer; -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; +use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::{ - CodegenResults, CompiledModule, ModuleCodegen, TargetConfig, + CompiledModule, CompiledModules, CrateInfo, ModuleCodegen, TargetConfig, back::{ lto::{SerializedModule, ThinModule}, write::{CodegenContext, FatLtoInput, ModuleConfig, OngoingCodegen}, @@ -68,7 +63,8 @@ use rustc_codegen_ssa::{ traits::{CodegenBackend, ExtraBackendMethods, WriteBackendMethods}, }; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_errors::DiagCtxtHandle; use rustc_metadata::creader::MetadataLoaderDyn; use rustc_middle::util::Providers; use rustc_middle::{ @@ -107,8 +103,8 @@ unsafe impl Send for NvvmCodegenBackend {} unsafe impl Sync for NvvmCodegenBackend {} impl CodegenBackend for NvvmCodegenBackend { - fn locale_resource(&self) -> &'static str { - "" + fn name(&self) -> &'static str { + "nvvm" } fn init(&self, sess: &Session) { @@ -136,7 +132,7 @@ impl CodegenBackend for NvvmCodegenBackend { // Following NVIDIA semantics, we enable "at least this capability" matching - for example, // when targeting compute_70, we also enable compute_60, compute_50, and all lower capabilities. // This allows libraries to gate features based on minimum required compute capability. - providers.global_backend_features = |tcx, ()| { + providers.queries.global_backend_features = |tcx, ()| { let mut features = vec![]; // Parse CodegenArgs to get the architecture from llvm-args (e.g., "-arch=compute_70") @@ -158,27 +154,32 @@ impl CodegenBackend for NvvmCodegenBackend { features }; - providers.fn_abi_of_fn_ptr = |tcx, key| { - let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS.fn_abi_of_fn_ptr)(tcx, key); + providers.queries.fn_abi_of_fn_ptr = |tcx, key| { + let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS + .queries + .fn_abi_of_fn_ptr)(tcx, key); Ok(readjust_fn_abi(tcx, result?)) }; - providers.fn_abi_of_instance = |tcx, key| { - let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS.fn_abi_of_instance)(tcx, key); + providers.queries.fn_abi_of_instance_raw = |tcx, key| { + let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS + .queries + .fn_abi_of_instance_raw)(tcx, key); Ok(readjust_fn_abi(tcx, result?)) }; } - fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box { + fn target_cpu(&self, sess: &Session) -> String { + sess.opts + .cg + .target_cpu + .clone() + .unwrap_or_else(|| sess.target.cpu.to_string()) + } + + fn codegen_crate(&self, tcx: TyCtxt<'_>, crate_info: &CrateInfo) -> Box { debug!("Codegen crate"); Box::new(rustc_codegen_ssa::base::codegen_crate( - Self, - tcx, - tcx.sess - .opts - .cg - .target_cpu - .clone() - .unwrap_or_else(|| tcx.sess.target.cpu.to_string()), + Self, tcx, crate_info, )) } @@ -187,32 +188,25 @@ impl CodegenBackend for NvvmCodegenBackend { ongoing_codegen: Box, sess: &Session, _outputs: &OutputFilenames, - ) -> (CodegenResults, FxIndexMap) { + ) -> (CompiledModules, FxIndexMap) { debug!("Join codegen"); - let (codegen_results, work_products) = ongoing_codegen + let (compiled_modules, work_products) = ongoing_codegen .downcast::>() .expect("Expected OngoingCodegen, found Box") .join(sess); - // sess.compile_status(); - - (codegen_results, work_products) + (compiled_modules, work_products) } fn link( &self, sess: &rustc_session::Session, - codegen_results: rustc_codegen_ssa::CodegenResults, + compiled_modules: CompiledModules, + crate_info: CrateInfo, metadata: rustc_metadata::EncodedMetadata, outputs: &config::OutputFilenames, ) { - link::link( - sess, - &codegen_results, - outputs, - codegen_results.crate_info.local_crate_name.as_str(), - metadata, - ); + link::link(sess, compiled_modules, crate_info, metadata, outputs); } fn target_config(&self, sess: &Session) -> TargetConfig { @@ -262,85 +256,76 @@ impl WriteBackendMethods for NvvmCodegenBackend { type Module = LlvmMod; type ModuleBuffer = lto::ModuleBuffer; type TargetMachine = &'static mut llvm::TargetMachine; - type TargetMachineError = String; type ThinData = (); - type ThinBuffer = ThinBuffer; - fn run_and_optimize_fat_lto( - _cgcx: &CodegenContext, + fn target_machine_factory( + &self, + sess: &Session, + opt_level: config::OptLevel, + _target_features: &[String], + ) -> rustc_codegen_ssa::back::write::TargetMachineFactoryFn { + target_machine_factory(sess, opt_level) + } + + fn optimize_and_codegen_fat_lto( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _shared_emitter: &rustc_codegen_ssa::back::write::SharedEmitter, + _tm_factory: rustc_codegen_ssa::back::write::TargetMachineFactoryFn, _exported_symbols_for_lto: &[String], _each_linked_rlib_for_lto: &[PathBuf], _modules: Vec>, - _diff_fncs: Vec, - ) -> Result, FatalError> { + ) -> CompiledModule { todo!() } fn run_thin_lto( - cgcx: &CodegenContext, - // FIXME: Limit LTO exports to these symbols + cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _dcx: DiagCtxtHandle<'_>, _exported_symbols_for_lto: &[String], - // FIXME: handle these? but only relevant for non-thin LTO? _each_linked_rlib_for_lto: &[PathBuf], - modules: Vec<(String, Self::ThinBuffer)>, + modules: Vec<(String, Self::ModuleBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError> { + ) -> (Vec>, Vec) { lto::run_thin(cgcx, modules, cached_modules) } - fn print_pass_timings(&self) { - // Not applicable, nvvm doesnt expose pass timing info, maybe we could print llvm pass stuff here. - } - - fn print_statistics(&self) { - // Not applicable, nvvm doesnt expose pass timing info, maybe we could print llvm pass stuff here. - } - fn optimize( - cgcx: &CodegenContext, - diag_handler: DiagCtxtHandle<'_>, + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &rustc_codegen_ssa::back::write::SharedEmitter, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError> { - unsafe { back::optimize(cgcx, diag_handler, module, config) } + ) { + unsafe { back::optimize(cgcx, prof, shared_emitter, module, config) } + .unwrap_or_else(|err| err.raise()) } - fn optimize_thin( - cgcx: &CodegenContext, - thin_module: ThinModule, - ) -> Result, FatalError> { - unsafe { lto::optimize_thin(cgcx, thin_module) } + fn optimize_and_codegen_thin( + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &rustc_codegen_ssa::back::write::SharedEmitter, + tm_factory: rustc_codegen_ssa::back::write::TargetMachineFactoryFn, + thin: ThinModule, + ) -> CompiledModule { + unsafe { lto::optimize_and_codegen_thin(cgcx, prof, shared_emitter, tm_factory, thin) } } fn codegen( - cgcx: &CodegenContext, + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &rustc_codegen_ssa::back::write::SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result { - unsafe { back::codegen(cgcx, module, config) } + ) -> CompiledModule { + unsafe { back::codegen(cgcx, prof, shared_emitter, module, config) } + .unwrap_or_else(|err| err.raise()) } - fn prepare_thin( - module: ModuleCodegen, - _want_summary: bool, - ) -> (String, Self::ThinBuffer) { - debug!("Prepare thin"); - unsafe { - ( - module.name, - lto::ThinBuffer::new(module.module_llvm.llmod.as_ref().unwrap()), - ) - } - } - - fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { + fn serialize_module(module: Self::Module, is_thin: bool) -> Self::ModuleBuffer { debug!("Serializing module"); - unsafe { - ( - module.name, - lto::ModuleBuffer::new(module.module_llvm.llmod.as_ref().unwrap()), - ) - } + unsafe { lto::ModuleBuffer::new(module.llmod.as_ref().unwrap(), is_thin) } } } @@ -349,19 +334,10 @@ impl ExtraBackendMethods for NvvmCodegenBackend { &self, tcx: TyCtxt<'_>, module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) -> LlvmMod { let mut module_llvm = LlvmMod::new(module_name); - unsafe { - allocator::codegen( - tcx, - &mut module_llvm, - module_name, - kind, - alloc_error_handler_kind, - ); - } + allocator::codegen(tcx, &mut module_llvm, module_name, methods); module_llvm } @@ -372,15 +348,6 @@ impl ExtraBackendMethods for NvvmCodegenBackend { ) -> (rustc_codegen_ssa::ModuleCodegen, u64) { back::compile_codegen_unit(tcx, cgu_name) } - - fn target_machine_factory( - &self, - sess: &Session, - opt_level: config::OptLevel, - _target_features: &[String], - ) -> rustc_codegen_ssa::back::write::TargetMachineFactoryFn { - target_machine_factory(sess, opt_level) - } } /// Create the LLVM module for the rest of the compilation, this houses diff --git a/crates/rustc_codegen_nvvm/src/link.rs b/crates/rustc_codegen_nvvm/src/link.rs index b84812c2..ad9c2816 100644 --- a/crates/rustc_codegen_nvvm/src/link.rs +++ b/crates/rustc_codegen_nvvm/src/link.rs @@ -1,11 +1,9 @@ use object::{Object, ObjectSection}; -use rustc_ast::CRATE_NODE_ID; -use rustc_codegen_ssa::CodegenResults; -use rustc_codegen_ssa::CompiledModule; -use rustc_codegen_ssa::NativeLib; +use rustc_codegen_ssa::{CompiledModule, CompiledModules, CrateInfo, NativeLib}; +use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owned_slice::{OwnedSlice, slice_owned, try_slice_owned}; -use rustc_hash::FxHashSet; +use rustc_hir::attrs::NativeLibKind; use rustc_metadata::creader::MetadataLoader; use rustc_middle::bug; use rustc_middle::middle::dependency_format::Linkage; @@ -14,7 +12,6 @@ use rustc_session::{ Session, config::{CrateType, OutputFilenames, OutputType}, output::check_file_is_writeable, - utils::NativeLibKind, }; use rustc_span::Symbol; use rustc_target::spec::Target; @@ -108,11 +105,12 @@ fn search_for_section<'a>(path: &Path, bytes: &'a [u8], section: &str) -> Result pub fn link( sess: &Session, - codegen_results: &CodegenResults, - outputs: &OutputFilenames, - crate_name: &str, + compiled_modules: CompiledModules, + crate_info: CrateInfo, metadata: rustc_metadata::EncodedMetadata, + outputs: &OutputFilenames, ) { + let crate_name = crate_info.local_crate_name.as_str(); debug!("Linking crate `{}`", crate_name); // largely inspired by rust-gpu let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); @@ -124,7 +122,7 @@ pub fn link( continue; } - for obj in codegen_results + for obj in compiled_modules .modules .iter() .filter_map(|m| m.object.as_ref()) @@ -140,18 +138,20 @@ pub fn link( CrateType::Rlib => { link_rlib( sess, - codegen_results, + &compiled_modules, + &crate_info, &out_filename_file_for_writing, &metadata, ); } CrateType::Executable | CrateType::Cdylib | CrateType::Dylib => { let _ = link_exe( - &codegen_results.allocator_module, + &compiled_modules.allocator_module, sess, crate_type, &out_filename_file_for_writing, - codegen_results, + &compiled_modules, + &crate_info, ); } other => sess.dcx().fatal(format!("Invalid crate type: {other:?}")), @@ -162,14 +162,15 @@ pub fn link( fn link_rlib( sess: &Session, - codegen_results: &CodegenResults, + compiled_modules: &CompiledModules, + crate_info: &CrateInfo, out_filename: &Path, metadata: &rustc_metadata::EncodedMetadata, ) { debug!("Linking rlib `{:?}`", out_filename); let mut file_list = Vec::<&Path>::new(); - for obj in codegen_results + for obj in compiled_modules .modules .iter() .filter_map(|m| m.object.as_ref()) @@ -177,7 +178,7 @@ fn link_rlib( file_list.push(obj); } - for lib in codegen_results.crate_info.used_libraries.iter() { + for lib in crate_info.used_libraries.iter() { // native libraries in cuda doesnt make much sense, extern functions // do exist in nvvm for stuff like cuda syscalls and cuda provided functions // but including libraries doesnt make sense because nvvm would have to translate @@ -204,11 +205,12 @@ fn link_exe( sess: &Session, crate_type: CrateType, out_filename: &Path, - codegen_results: &CodegenResults, + compiled_modules: &CompiledModules, + crate_info: &CrateInfo, ) -> io::Result<()> { let mut objects = Vec::new(); let mut rlibs = Vec::new(); - for obj in codegen_results + for obj in compiled_modules .modules .iter() .filter_map(|m| m.object.as_ref()) @@ -216,12 +218,7 @@ fn link_exe( objects.push(obj.clone()); } - link_local_crate_native_libs_and_dependent_crate_libs( - &mut rlibs, - sess, - crate_type, - codegen_results, - ); + link_local_crate_native_libs_and_dependent_crate_libs(&mut rlibs, sess, crate_type, crate_info); let mut root_file_name = out_filename.file_name().unwrap().to_owned(); root_file_name.push(".dir"); @@ -370,20 +367,19 @@ fn link_local_crate_native_libs_and_dependent_crate_libs( rlibs: &mut Vec, sess: &Session, crate_type: CrateType, - codegen_results: &CodegenResults, + crate_info: &CrateInfo, ) { if sess.opts.unstable_opts.link_native_libraries { - add_local_native_libraries(sess, codegen_results); + add_local_native_libraries(sess, crate_info); } - add_upstream_rust_crates(sess, rlibs, codegen_results, crate_type); + add_upstream_rust_crates(sess, rlibs, crate_info, crate_type); if sess.opts.unstable_opts.link_native_libraries { - add_upstream_native_libraries(sess, codegen_results, crate_type); + add_upstream_native_libraries(sess, crate_info, crate_type); } } -fn add_local_native_libraries(sess: &Session, codegen_results: &CodegenResults) { - let relevant_libs = codegen_results - .crate_info +fn add_local_native_libraries(sess: &Session, crate_info: &CrateInfo) { + let relevant_libs = crate_info .used_libraries .iter() .filter(|l| relevant_lib(sess, l)); @@ -393,21 +389,20 @@ fn add_local_native_libraries(sess: &Session, codegen_results: &CodegenResults) fn add_upstream_rust_crates( sess: &Session, rlibs: &mut Vec, - codegen_results: &CodegenResults, + crate_info: &CrateInfo, crate_type: CrateType, ) { - let (_, data) = codegen_results - .crate_info + let (_, data) = crate_info .dependency_formats .iter() .find(|(ty, _)| **ty == crate_type) .expect("failed to find crate type in dependency format list"); - let deps = &codegen_results.crate_info.used_crates; + let deps = &crate_info.used_crates; for cnum in deps.iter() { - let src = &codegen_results.crate_info.used_crate_source[cnum]; + let src = &crate_info.used_crate_source[cnum]; match data[*cnum] { Linkage::NotLinked => {} - Linkage::Static => rlibs.push(src.rlib.as_ref().unwrap().0.clone()), + Linkage::Static => rlibs.push(src.rlib.as_ref().unwrap().clone()), // should we just ignore includedFromDylib? Linkage::Dynamic | Linkage::IncludedFromDylib => { sess.dcx().fatal("Dynamic Linking is not supported in CUDA") @@ -416,14 +411,10 @@ fn add_upstream_rust_crates( } } -fn add_upstream_native_libraries( - sess: &Session, - codegen_results: &CodegenResults, - _crate_type: CrateType, -) { - let crates = &codegen_results.crate_info.used_crates; +fn add_upstream_native_libraries(sess: &Session, crate_info: &CrateInfo, _crate_type: CrateType) { + let crates = &crate_info.used_crates; for cnum in crates { - for lib in codegen_results.crate_info.native_libraries[cnum].iter() { + for lib in crate_info.native_libraries[cnum].iter() { if !relevant_lib(sess, lib) { continue; } @@ -435,7 +426,7 @@ fn add_upstream_native_libraries( fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool { match &lib.cfg { - Some(cfg) => rustc_attr_parsing::cfg_matches(cfg, sess, CRATE_NODE_ID, None), + Some(cfg) => rustc_attr_parsing::eval_config_entry(sess, cfg).as_bool(), None => true, } } diff --git a/crates/rustc_codegen_nvvm/src/llvm.rs b/crates/rustc_codegen_nvvm/src/llvm.rs index 37c14ee8..a9cbcad3 100644 --- a/crates/rustc_codegen_nvvm/src/llvm.rs +++ b/crates/rustc_codegen_nvvm/src/llvm.rs @@ -619,6 +619,17 @@ pub(crate) unsafe fn LLVMRustBuildCall<'a>( unsafe { __LLVMRustBuildCall(B, Fn, Args, NumArgs, Bundle, unnamed()) } } +pub(crate) unsafe fn LLVMRustBuildCall2<'a>( + B: &Builder<'a>, + FnTy: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Bundle: Option<&OperandBundleDef<'a>>, +) -> &'a Value { + unsafe { __LLVMRustBuildCall2(B, FnTy, Fn, Args, NumArgs, Bundle, unnamed()) } +} + /// LLVMRustCodeGenOptLevel #[derive(Copy, Clone, PartialEq)] #[repr(C)] @@ -667,6 +678,17 @@ unsafe extern "C" { Name: *const c_char, ) -> &'a Value; + #[link_name = "LLVMRustBuildCall2"] + pub(crate) fn __LLVMRustBuildCall2<'a>( + B: &Builder<'a>, + FnTy: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Bundle: Option<&OperandBundleDef<'a>>, + Name: *const c_char, + ) -> &'a Value; + // see comment on function before this extern block #[link_name = "LLVMRustGetOrInsertFunction"] fn __LLVMRustGetOrInsertFunction<'a>( diff --git a/crates/rustc_codegen_nvvm/src/lto.rs b/crates/rustc_codegen_nvvm/src/lto.rs index 197812da..bed126a8 100644 --- a/crates/rustc_codegen_nvvm/src/lto.rs +++ b/crates/rustc_codegen_nvvm/src/lto.rs @@ -2,14 +2,15 @@ use std::ffi::CString; use std::sync::Arc; use rustc_codegen_ssa::{ - ModuleCodegen, + CompiledModule, ModuleCodegen, back::{ lto::{SerializedModule, ThinModule, ThinShared}, - write::CodegenContext, + write::{CodegenContext, SharedEmitter, TargetMachineFactoryFn}, }, - traits::{ModuleBufferMethods, ThinBufferMethods}, + traits::ModuleBufferMethods, }; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_errors::{DiagCtxt, DiagCtxtHandle, FatalError}; use rustc_middle::dep_graph::WorkProduct; use tracing::{debug, trace}; @@ -23,7 +24,7 @@ unsafe impl Send for ModuleBuffer {} unsafe impl Sync for ModuleBuffer {} impl ModuleBuffer { - pub(crate) fn new(m: &llvm::Module) -> ModuleBuffer { + pub(crate) fn new(m: &llvm::Module, _is_thin: bool) -> ModuleBuffer { ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) } } @@ -47,46 +48,6 @@ impl Drop for ModuleBuffer { } } -pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); - -unsafe impl Send for ThinBuffer {} -unsafe impl Sync for ThinBuffer {} - -impl ThinBuffer { - pub(crate) fn new(m: &llvm::Module) -> ThinBuffer { - unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m); - - ThinBuffer(buffer) - } - } -} - -impl ThinBufferMethods for ThinBuffer { - fn data(&self) -> &[u8] { - unsafe { - trace!("Retrieving data in thin buffer"); - let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; - - let len = llvm::LLVMRustThinLTOBufferLen(self.0); - - std::slice::from_raw_parts(ptr, len) - } - } - - fn thin_link_data(&self) -> &[u8] { - todo!() - } -} - -impl Drop for ThinBuffer { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); - } - } -} - #[allow(dead_code)] pub struct ThinData(&'static mut llvm::ThinLTOData); @@ -101,34 +62,23 @@ impl Drop for ThinData { } } -// essentially does nothing for now. pub(crate) fn run_thin( - _cgcx: &CodegenContext, - modules: Vec<(String, ThinBuffer)>, + _cgcx: &CodegenContext, + modules: Vec<(String, ModuleBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { debug!("Running thin LTO"); let mut thin_buffers = Vec::with_capacity(modules.len()); let mut module_names = Vec::with_capacity(modules.len() + cached_modules.len()); - // let thin_modules = Vec::with_capacity(modules.len() + cached_modules.len()); for (name, buf) in modules { - let cname = CString::new(name.clone()).unwrap(); - // thin_modules.push( - // llvm::ThinLTOModule { - // identifier: cname.as_ptr(), - // data: buf.data().as_ptr(), - // len: buf.data().len() - // } - // ); thin_buffers.push(buf); - module_names.push(cname); + module_names.push(CString::new(name).unwrap()); } let mut serialized_modules = Vec::with_capacity(cached_modules.len()); - for (sm, wp) in cached_modules { - let _slice_u8 = sm.data(); + let _ = sm.data(); serialized_modules.push(sm); module_names.push(CString::new(wp.cgu_name).unwrap()); } @@ -140,34 +90,39 @@ pub(crate) fn run_thin( module_names, }); - let mut opt_jobs = vec![]; - for (module_index, _) in shared.module_names.iter().enumerate() { + let mut opt_jobs = Vec::with_capacity(shared.module_names.len()); + for module_index in 0..shared.module_names.len() { opt_jobs.push(ThinModule { shared: shared.clone(), idx: module_index, }); } - Ok((opt_jobs, vec![])) + (opt_jobs, vec![]) } -pub(crate) unsafe fn optimize_thin( - cgcx: &CodegenContext, +pub(crate) unsafe fn optimize_and_codegen_thin( + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &SharedEmitter, + _tm_factory: TargetMachineFactoryFn, thin_module: ThinModule, -) -> Result, FatalError> { - // essentially does nothing - let dcx = cgcx.create_dcx(); - let dcx = dcx.handle(); - +) -> CompiledModule { let module_name = &thin_module.shared.module_names[thin_module.idx]; - + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let llcx = unsafe { llvm::LLVMRustContextCreate(cgcx.fewer_names) }; - let llmod = - parse_module(llcx, module_name.to_str().unwrap(), thin_module.data(), dcx)? as *const _; + let llmod = parse_module( + llcx, + module_name.to_str().unwrap(), + thin_module.data(), + dcx.handle(), + ) + .unwrap_or_else(|err| err.raise()) as *const _; let module = ModuleCodegen::new_regular(thin_module.name().to_string(), LlvmMod { llcx, llmod }); - Ok(module) + unsafe { crate::back::codegen(cgcx, prof, shared_emitter, module, &cgcx.module_config) } + .unwrap_or_else(|err| err.raise()) } pub(crate) fn parse_module<'a>( diff --git a/crates/rustc_codegen_nvvm/src/mono_item.rs b/crates/rustc_codegen_nvvm/src/mono_item.rs index efb01b68..d0997dea 100644 --- a/crates/rustc_codegen_nvvm/src/mono_item.rs +++ b/crates/rustc_codegen_nvvm/src/mono_item.rs @@ -6,8 +6,9 @@ use crate::context::CodegenCx; use crate::llvm; use crate::ty::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::Linkage; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance}; @@ -90,7 +91,11 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { attributes::from_fn_attrs(self, lldecl, instance); let def_id = instance.def_id(); - let attrs = self.tcx.get_all_attrs(def_id); // TODO: Replace with get_attrs + let attrs = if let Some(def_id) = def_id.as_local() { + self.tcx.hir_attrs(self.tcx.local_def_id_to_hir_id(def_id)) + } else { + self.tcx.attrs_for_def(def_id) + }; let nvvm_attrs = NvvmAttributes::parse(self, attrs); unsafe { diff --git a/crates/rustc_codegen_nvvm/src/nvvm.rs b/crates/rustc_codegen_nvvm/src/nvvm.rs index e165225f..735a2973 100644 --- a/crates/rustc_codegen_nvvm/src/nvvm.rs +++ b/crates/rustc_codegen_nvvm/src/nvvm.rs @@ -5,9 +5,9 @@ use crate::builder::unnamed; use crate::common::AsCCharPtr; use crate::context::CodegenArgs; use crate::llvm::*; -use crate::lto::ThinBuffer; +use crate::lto::ModuleBuffer; use nvvm::*; -use rustc_codegen_ssa::traits::ThinBufferMethods; +use rustc_codegen_ssa::traits::ModuleBufferMethods; use rustc_session::{Session, config::DebugInfo}; use std::fmt::Display; use std::marker::PhantomData; @@ -101,7 +101,7 @@ pub fn codegen_bitcode_modules( } } - let buf = ThinBuffer::new(module); + let buf = ModuleBuffer::new(module, false); prog.add_module(buf.data(), "merged".to_string())?; prog.add_lazy_module(LIBDEVICE_BITCODE, "libdevice".to_string())?; diff --git a/crates/rustc_codegen_nvvm/src/override_fns.rs b/crates/rustc_codegen_nvvm/src/override_fns.rs index e4c0ee23..8534d790 100644 --- a/crates/rustc_codegen_nvvm/src/override_fns.rs +++ b/crates/rustc_codegen_nvvm/src/override_fns.rs @@ -8,8 +8,9 @@ use crate::context::CodegenCx; use crate::llvm; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_hir::attrs::Linkage; use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility}; +use rustc_middle::mir::mono::{MonoItem, MonoItemData, Visibility}; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::{self, Instance}; diff --git a/crates/rustc_codegen_nvvm/src/target.rs b/crates/rustc_codegen_nvvm/src/target.rs index 4438925f..48249ed8 100644 --- a/crates/rustc_codegen_nvvm/src/target.rs +++ b/crates/rustc_codegen_nvvm/src/target.rs @@ -1,7 +1,5 @@ use crate::llvm::{self, Type}; -use rustc_target::spec::{ - LinkerFlavor, MergeFunctions, PanicStrategy, Target, TargetMetadata, TargetOptions, -}; +use rustc_target::spec::{Target, TargetTuple}; // This data layout must match `datalayout` in `crates/rustc_codegen_nvvm/libintrinsics.ll`. pub const DATA_LAYOUT: &str = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"; @@ -14,43 +12,8 @@ pub(crate) unsafe fn usize_ty(llcx: &'_ llvm::Context) -> &'_ Type { } pub fn target() -> Target { - let mut options = TargetOptions::default(); - - options.os = "cuda".into(); - options.vendor = "nvidia".into(); - options.linker_flavor = LinkerFlavor::Ptx; - // nvvm does all the linking for us, but technically its not a linker - options.linker = None; - options.cpu = "sm_30".into(); - options.max_atomic_width = Some(64); - // Unwinding on CUDA is neither feasible nor useful. - options.panic_strategy = PanicStrategy::Abort; - // Needed to use `dylib` and `bin` crate types and the linker. - options.dynamic_linking = true; - options.executables = true; - options.only_cdylib = true; - - // nvvm does all the work of turning the bitcode into ptx - options.obj_is_bitcode = true; - - options.dll_prefix = "".into(); - options.dll_suffix = ".ptx".into(); - options.exe_suffix = ".ptx".into(); - - // Disable MergeFunctions LLVM optimisation pass because it can - // produce kernel functions that call other kernel functions. - // This behavior is not supported by PTX ISA. - options.merge_functions = MergeFunctions::Disabled; - - Target { - arch: "nvptx".into(), - data_layout: DATA_LAYOUT.into(), - llvm_target: "nvptx64-nvidia-cuda".into(), - pointer_width: POINTER_WIDTH, - options, - metadata: TargetMetadata { - description: Some("NVIDIA CUDA".into()), - ..Default::default() - }, - } + let mut target = Target::expect_builtin(&TargetTuple::TargetTuple(TARGET_TRIPLE.into())); + target.data_layout = DATA_LAYOUT.into(); + target.pointer_width = POINTER_WIDTH as u16; + target } diff --git a/crates/rustc_codegen_nvvm/src/ty.rs b/crates/rustc_codegen_nvvm/src/ty.rs index b88a2dea..5633c71b 100644 --- a/crates/rustc_codegen_nvvm/src/ty.rs +++ b/crates/rustc_codegen_nvvm/src/ty.rs @@ -228,10 +228,12 @@ impl<'ll, 'tcx> BaseTypeCodegenMethods for CodegenCx<'ll, 'tcx> { fn float_width(&self, ty: &'ll Type) -> usize { match self.type_kind(ty) { + TypeKind::Half => 16, TypeKind::Float => 32, TypeKind::Double => 64, TypeKind::X86_FP80 => 80, TypeKind::FP128 | TypeKind::PPC_FP128 => 128, + TypeKind::Vector | TypeKind::ScalableVector => self.float_width(self.element_type(ty)), _ => bug!("llvm_float_width called on a non-float type"), } } @@ -305,6 +307,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true, BackendRepr::ScalarPair(..) => false, BackendRepr::Memory { .. } => self.is_zst(), + BackendRepr::SimdScalableVector { .. } => bug!("scalable vectors are unsupported"), } } @@ -314,6 +317,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => false, + BackendRepr::SimdScalableVector { .. } => false, } } @@ -372,7 +376,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // Make sure lifetimes are erased, to avoid generating distinct LLVM // types for Rust types that only differ in the choice of lifetimes. - let normal_ty = cx.tcx.erase_regions(self.ty); + let normal_ty = cx.tcx.erase_and_anonymize_regions(self.ty); let mut defer = None; let llty = if self.ty != normal_ty { @@ -399,11 +403,10 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { match self.backend_repr { - BackendRepr::Scalar(ref scalar) => { - if scalar.is_bool() { - return cx.type_i1(); - } + BackendRepr::Scalar(ref scalar) if scalar.is_bool() => { + return cx.type_i1(); } + BackendRepr::Scalar(..) => {} BackendRepr::ScalarPair(..) => { // An immediate pair always contains just the two elements, without any padding // filler, as it should never be stored to memory. @@ -513,6 +516,7 @@ fn uncached_llvm_type<'a, 'tcx>( let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } + BackendRepr::SimdScalableVector { .. } => bug!("scalable vectors are unsupported"), BackendRepr::ScalarPair(..) => { return cx.type_struct( &[ @@ -655,10 +659,6 @@ impl<'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { fn set_type_metadata(&self, _function: Self::Function, _typeid: &[u8]) {} - fn typeid_metadata(&self, _typeid: &[u8]) -> Option { - None - } - fn add_kcfi_type_metadata(&self, _function: Self::Function, _typeid: u32) {} fn set_kcfi_type_metadata(&self, _function: Self::Function, _typeid: u32) {} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 74829085..2885bd1b 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2025-08-11" +channel = "nightly-2026-04-02" components = ["clippy", "llvm-tools-preview", "rust-src", "rustc-dev", "rustfmt", "rust-analyzer"] diff --git a/tests/compiletests/src/main.rs b/tests/compiletests/src/main.rs index 8672854b..3f551e81 100644 --- a/tests/compiletests/src/main.rs +++ b/tests/compiletests/src/main.rs @@ -210,6 +210,7 @@ fn build_deps( // Build compiletests-deps-helper using the same approach as cuda_builder let rustflags = vec![ format!("-Zcodegen-backend={}", codegen_backend_path.display()), + "-Zunstable-options".into(), "-Zcrate-attr=feature(register_tool)".into(), "-Zcrate-attr=register_tool(nvvm_internal)".into(), "-Zcrate-attr=no_std".into(), @@ -219,7 +220,7 @@ fn build_deps( "-Cdebuginfo=0".into(), "-Coverflow-checks=off".into(), "-Copt-level=3".into(), - "-Cpanic=abort".into(), + "-Cpanic=immediate-abort".into(), "-Cno-redzone=yes".into(), format!("-Cllvm-args=-arch={} --override-libm", arch), format!("-Ctarget-feature=+{}", arch), @@ -235,7 +236,6 @@ fn build_deps( "compiletests-deps-helper", "--release", "-Zbuild-std=core,alloc", - "-Zbuild-std-features=panic_immediate_abort", &*format!("--target={target}"), ]) .arg("--target-dir") @@ -403,7 +403,7 @@ fn rust_flags(codegen_backend_path: &Path, target_arch: &str) -> String { "-Cdebuginfo=2", "-Cembed-bitcode=no", &format!("-Ctarget-feature=+{target_arch}"), - "-Cpanic=abort", + "-Cpanic=immediate-abort", "-Cno-redzone=yes", &format!("-Cllvm-args=-arch={target_arch}"), "-Cllvm-args=--override-libm", diff --git a/tests/compiletests/ui/lang/compile_fail.stderr b/tests/compiletests/ui/lang/compile_fail.stderr index 3d4332bb..b1e3483c 100644 --- a/tests/compiletests/ui/lang/compile_fail.stderr +++ b/tests/compiletests/ui/lang/compile_fail.stderr @@ -1,4 +1,4 @@ -error[E0433]: failed to resolve: use of unresolved module or unlinked crate `std` +error[E0433]: cannot find module or crate `std` in this scope --> $DIR/compile_fail.rs:5:14 | LL | let _s = std::fs::File::create("foo.txt");