diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index f4e08e08ef8db..89b58b32a2527 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -263,6 +263,12 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { bx.lifetime_end(tmp, size); } fx.store_return(bx, ret_dest, &fn_abi.ret, invokeret); + + // If the return value was retagged as it was stored, + // then we might be in a different basic block now. + // Update the cached block for `target` to point to this new + // block, where codegen will continue. + fx.cached_llbbs[target] = CachedLlbb::Some(bx.llbb()); } MergingSucc::False } else { @@ -2148,19 +2154,27 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { llval: Bx::Value, ) { use self::ReturnDest::*; - + let retags_enabled = bx.tcx().sess.opts.unstable_opts.codegen_emit_retag.is_some(); match dest { Nothing => (), - Store(dst) => bx.store_arg(ret_abi, llval, dst), + Store(dst) => { + bx.store_arg(ret_abi, llval, dst); + if retags_enabled { + self.codegen_retag_place(bx, dst, false); + } + } IndirectOperand(tmp, index) => { - let op = bx.load_operand(tmp); + let mut op = bx.load_operand(tmp); tmp.storage_dead(bx); + if retags_enabled { + op = self.codegen_retag_operand(bx, op, false); + } self.overwrite_local(index, LocalRef::Operand(op)); self.debug_introduce_local(bx, index); } DirectOperand(index) => { // If there is a cast, we have to store and reload. - let op = if let PassMode::Cast { .. } = ret_abi.mode { + let mut op = if let PassMode::Cast { .. } = ret_abi.mode { let tmp = PlaceRef::alloca(bx, ret_abi.layout); tmp.storage_live(bx); bx.store_arg(ret_abi, llval, tmp); @@ -2170,6 +2184,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout) }; + if retags_enabled { + op = self.codegen_retag_operand(bx, op, false); + } self.overwrite_local(index, LocalRef::Operand(op)); self.debug_introduce_local(bx, index); } diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 84013a00d79df..f8c651c38a617 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -23,6 +23,7 @@ mod locals; pub mod naked_asm; pub mod operand; pub mod place; +mod retag; mod rvalue; mod statement; @@ -401,7 +402,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( return vec![]; } - let args = mir + let mut args = mir .args_iter() .enumerate() .map(|(arg_index, local)| { @@ -535,6 +536,32 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } }) .collect::>(); + if bx.tcx().sess.opts.unstable_opts.codegen_emit_retag.is_some() { + args = args + .iter() + .map(|arg| match arg { + &LocalRef::Place(place_ref) => { + fx.codegen_retag_place(bx, place_ref, true); + LocalRef::Place(place_ref) + } + &LocalRef::UnsizedPlace(place_ref) => { + let operand = bx.load_operand(place_ref); + let retagged = fx.codegen_retag_operand(bx, operand, true); + assert!(matches!(retagged.val, OperandValue::Pair(_, _))); + retagged.val.store(bx, place_ref); + LocalRef::UnsizedPlace(place_ref) + } + &LocalRef::Operand(operand_ref) => { + let retagged = fx.codegen_retag_operand(bx, operand_ref, true); + LocalRef::Operand(retagged) + } + LocalRef::PendingOperand => LocalRef::PendingOperand, + }) + .collect::>(); + // If we branched during retagging, then we need to update the + // start block to the new location. + fx.cached_llbbs[mir::START_BLOCK] = CachedLlbb::Some(bx.llbb()); + } if fx.instance.def.requires_caller_location(bx.tcx()) { let mir_args = if let Some(num_untupled) = num_untupled { diff --git a/compiler/rustc_codegen_ssa/src/mir/retag.rs b/compiler/rustc_codegen_ssa/src/mir/retag.rs new file mode 100644 index 0000000000000..a08a434326bb5 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/retag.rs @@ -0,0 +1,34 @@ +//! Experimental support for emitting retags as function calls in generated code. + +use rustc_middle::mir::{Rvalue, WithRetag}; + +use crate::mir::FunctionCx; +use crate::mir::operand::OperandRef; +use crate::mir::place::PlaceRef; +use crate::traits::BuilderMethods; + +pub(crate) fn rvalue_needs_retag(rvalue: &Rvalue<'_>) -> bool { + // `Ref` has its own internal retagging + !matches!(rvalue, Rvalue::Ref(..)) && !matches!(rvalue, Rvalue::Use(.., WithRetag::No)) +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + /// Retags the pointers within an [`OperandRef`]. + pub(crate) fn codegen_retag_operand( + &mut self, + _bx: &mut Bx, + operand: OperandRef<'tcx, Bx::Value>, + _is_fn_entry: bool, + ) -> OperandRef<'tcx, Bx::Value> { + operand + } + + /// Retags the pointers within a [`PlaceRef`]. + pub(crate) fn codegen_retag_place( + &mut self, + _bx: &mut Bx, + _place_ref: PlaceRef<'tcx, Bx::Value>, + _is_fn_entry: bool, + ) { + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 90ac8c89ba9ad..469d6af12923e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -522,7 +522,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { Ty::new_ref(tcx, tcx.lifetimes.re_erased, ty, bk.to_mutbl_lossy()) }; - self.codegen_place_to_pointer(bx, place, mk_ref) + let op = self.codegen_place_to_pointer(bx, place, mk_ref); + if self.cx.tcx().sess.opts.unstable_opts.codegen_emit_retag.is_some() { + self.codegen_retag_operand(bx, op, false) + } else { + op + } } // Note: Exclusive reborrowing is always equal to a memcpy, as the types do not change. diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index afbef86bd089e..433964218e43b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -3,6 +3,7 @@ use rustc_middle::{bug, span_bug, ty}; use tracing::instrument; use super::{FunctionCx, LocalRef}; +use crate::mir::retag; use crate::traits::*; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { @@ -12,9 +13,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.set_debug_loc(bx, statement.source_info); match statement.kind { mir::StatementKind::Assign((ref place, ref rvalue)) => { + let needs_retag = bx.tcx().sess.opts.unstable_opts.codegen_emit_retag.is_some() + && retag::rvalue_needs_retag(rvalue); + if let Some(index) = place.as_local() { match self.locals[index] { - LocalRef::Place(cg_dest) => self.codegen_rvalue(bx, cg_dest, rvalue), + LocalRef::Place(cg_dest) => { + self.codegen_rvalue(bx, cg_dest, rvalue); + if needs_retag { + self.codegen_retag_place(bx, cg_dest, false); + } + } LocalRef::UnsizedPlace(cg_indirect_dest) => { let ty = cg_indirect_dest.layout.ty; span_bug!( @@ -24,7 +33,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ); } LocalRef::PendingOperand => { - let operand = self.codegen_rvalue_operand(bx, rvalue); + let mut operand = self.codegen_rvalue_operand(bx, rvalue); + if needs_retag { + operand = self.codegen_retag_operand(bx, operand, false); + } self.overwrite_local(index, LocalRef::Operand(operand)); self.debug_introduce_local(bx, index); } @@ -39,12 +51,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // If the type is zero-sized, it's already been set here, // but we still need to make sure we codegen the operand - self.codegen_rvalue_operand(bx, rvalue); + // and emit a retag. + let operand = self.codegen_rvalue_operand(bx, rvalue); + if needs_retag { + self.codegen_retag_operand(bx, operand, false); + } } } } else { let cg_dest = self.codegen_place(bx, place.as_ref()); self.codegen_rvalue(bx, cg_dest, rvalue); + if needs_retag { + self.codegen_retag_place(bx, cg_dest, false); + } } } mir::StatementKind::SetDiscriminant { ref place, variant_index } => { diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 0837e7767605c..906b444cce605 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -10,9 +10,9 @@ use rustc_errors::ColorConfig; use rustc_errors::emitter::HumanReadableErrorType; use rustc_hir::attrs::{CollapseMacroDebuginfo, NativeLibKind}; use rustc_session::config::{ - AnnotateMoves, AutoDiff, BranchProtection, CFGuard, Cfg, CoverageLevel, CoverageOptions, - DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, - FmtDebug, FunctionReturn, IncrementalStateAssertion, InliningThreshold, Input, + AnnotateMoves, AutoDiff, BranchProtection, CFGuard, Cfg, CodegenRetagOptions, CoverageLevel, + CoverageOptions, DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, + Externs, FmtDebug, FunctionReturn, IncrementalStateAssertion, InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli, MirIncludeSpans, NextSolverConfig, Offload, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet, Passes, PatchableFunctionEntry, Polonius, ProcMacroExecutionStrategy, Strip, @@ -772,6 +772,7 @@ fn test_unstable_options_tracking_hash() { }) ); tracked!(codegen_backend, Some("abc".to_string())); + tracked!(codegen_emit_retag, Some(CodegenRetagOptions::default())); tracked!( coverage_options, CoverageOptions { diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 2040cd48ec44f..83f70ae7032d1 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -200,6 +200,15 @@ pub enum Offload { Test, } +/// The different settings that the `-Z codegen-emit-retag` flag can have. +#[derive(Copy, Clone, Debug, Default, PartialEq, Hash, Encodable, Decodable)] +pub struct CodegenRetagOptions { + /// Track interior mutable data on the level of references, instead of on the byte level. + pub no_precise_im: bool, + /// Track `UnsafePinned` data on the level of references, instead of on the byte level. + pub no_precise_pin: bool, +} + /// The different settings that the `-Z autodiff` flag can have. #[derive(Clone, PartialEq, Hash, Debug, Encodable, Decodable)] pub enum AutoDiff { @@ -3048,12 +3057,13 @@ pub(crate) mod dep_tracking { }; use super::{ - AnnotateMoves, AutoDiff, BranchProtection, CFGuard, CFProtection, CoverageOptions, - CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, - InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, - LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OptLevel, OutFileName, OutputType, - OutputTypes, PatchableFunctionEntry, Polonius, ResolveDocLinks, SourceFileHashAlgorithm, - SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, + AnnotateMoves, AutoDiff, BranchProtection, CFGuard, CFProtection, CodegenRetagOptions, + CoverageOptions, CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, + FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, + LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OptLevel, + OutFileName, OutputType, OutputTypes, PatchableFunctionEntry, Polonius, ResolveDocLinks, + SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, + WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3157,6 +3167,7 @@ pub(crate) mod dep_tracking { InliningThreshold, FunctionReturn, Align, + CodegenRetagOptions ); impl DepTrackingHash for (T1, T2) diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index aa9331ee8f659..d9b8deec6bc49 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -783,6 +783,8 @@ mod desc { pub(crate) const parse_dump_mono_stats: &str = "`markdown` (default) or `json`"; pub(crate) const parse_instrument_coverage: &str = parse_bool; pub(crate) const parse_coverage_options: &str = "`block` | `branch` | `condition`"; + pub(crate) const parse_codegen_retag_options: &str = + "either no value or a comma-separated list of settings: `no-precise-im`, `no-precise-pin`"; pub(crate) const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`"; pub(crate) const parse_unpretty: &str = "`string` or `string=string`"; pub(crate) const parse_treat_err_as_bug: &str = "either no value or a non-negative number"; @@ -1523,6 +1525,29 @@ pub mod parse { true } + pub(crate) fn parse_codegen_retag_options( + slot: &mut Option, + v: Option<&str>, + ) -> bool { + let mut no_precise_im = false; + let mut no_precise_pin = false; + if let Some(opt_list) = v.map(|s| s.split(',')) { + for opt in opt_list { + match opt { + "no-precise-im" => { + no_precise_im = true; + } + "no-precise-pin" => { + no_precise_pin = true; + } + _ => return false, + } + } + } + *slot = Some(CodegenRetagOptions { no_precise_im, no_precise_pin }); + true + } + pub(crate) fn parse_coverage_options(slot: &mut CoverageOptions, v: Option<&str>) -> bool { let Some(v) = v else { return true }; @@ -2242,6 +2267,8 @@ options! { "hash algorithm of source files used to check freshness in cargo (`blake3` or `sha256`)"), codegen_backend: Option = (None, parse_opt_string, [TRACKED], "the backend to use"), + codegen_emit_retag: Option = (None, parse_codegen_retag_options, [TRACKED], + "emit retag function calls in generated code"), codegen_source_order: bool = (false, parse_bool, [UNTRACKED], "emit mono items in the order of spans in source files (default: no)"), contract_checks: Option = (None, parse_opt_bool, [TRACKED], diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 003164e8f9054..1db0a259e7155 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -550,6 +550,8 @@ impl Session { // HWAddressSanitizer and KernelHWAddressSanitizer will use lifetimes to detect use after // scope bugs in the future. || self.sanitizers().intersects(SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS | SanitizerSet::MEMORY | SanitizerSet::HWADDRESS | SanitizerSet::KERNELHWADDRESS) + // Lifetimes are necessary for retagging semantics. + || self.opts.unstable_opts.codegen_emit_retag.is_some() } pub fn diagnostic_width(&self) -> usize {