Pointer authentication config and user facing options

compiler/rustc_codegen_gcc/src/common.rs

@@ -7,6 +7,7 @@ use rustc_codegen_ssa::traits::{
 use rustc_middle::mir::Mutability;
 use rustc_middle::mir::interpret::{GlobalAlloc, PointerArithmetic, Scalar};
 use rustc_middle::ty::layout::LayoutOf;
+use rustc_session::PointerAuthSchema;
 
 use crate::consts::const_alloc_to_gcc;
 use crate::context::{CodegenCx, new_array_type};
@@ -229,7 +230,13 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> {
         None
     }
 
-    fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, ty: Type<'gcc>) -> RValue<'gcc> {
+    fn scalar_to_backend_with_pac(
+        &self,
+        cv: Scalar,
+        layout: abi::Scalar,
+        ty: Type<'gcc>,
+        _schema: Option<&PointerAuthSchema>,
+    ) -> RValue<'gcc> {
         let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
         match cv {
             Scalar::Int(int) => {
@@ -278,7 +285,7 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> {
                         }
                         value
                     }
-                    GlobalAlloc::Function { instance, .. } => self.get_fn_addr(instance),
+                    GlobalAlloc::Function { instance, .. } => self.get_fn_addr(instance, None),
                     GlobalAlloc::VTable(ty, dyn_ty) => {
                         let alloc = self
                             .tcx

compiler/rustc_codegen_gcc/src/context.rs

@@ -16,9 +16,9 @@ use rustc_middle::ty::layout::{
     LayoutOfHelpers,
 };
 use rustc_middle::ty::{self, ExistentialTraitRef, Instance, Ty, TyCtxt};
-use rustc_session::Session;
 #[cfg(feature = "master")]
 use rustc_session::config::DebugInfo;
+use rustc_session::{PointerAuthSchema, Session};
 use rustc_span::{DUMMY_SP, Span, Symbol, respan};
 use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, TlsModel, X86Abi};
 
@@ -398,7 +398,11 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         get_fn(self, instance)
     }
 
-    fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> {
+    fn get_fn_addr(
+        &self,
+        instance: Instance<'tcx>,
+        _pointer_auth_schema: Option<&PointerAuthSchema>,
+    ) -> RValue<'gcc> {
         let func_name = self.tcx.symbol_name(instance).name;
 
         let func = if let Some(variable) = self.get_declared_value(func_name) {

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -608,6 +608,13 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         }
     }
 
+    if sess.pointer_authentication() {
+        let cfg = sess.pointer_auth_config.as_ref().unwrap();
+        for ptrauth_attr in cfg.fn_attrs() {
+            to_add.push(llvm::CreateAttrString(cx.llcx, ptrauth_attr));
+        }
+    }
+
     to_add.extend(target_features_attr(cx, tcx, function_features));
 
     attributes::apply_to_llfn(llfn, Function, &to_add);

compiler/rustc_codegen_llvm/src/base.rs

@@ -123,7 +123,19 @@ pub(crate) fn compile_codegen_unit(
             if let Some(entry) =
                 maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit)
             {
-                let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerFnAttrs::default());
+                let mut attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerFnAttrs::default());
+                // When pointer authentication is enabled, ensure that the ptrauth-* attributes are
+                // also attached to the entry wrapper.
+                //
+                // FIXME(jchlanda) If it ever becomes necessary to ensure that all compiler
+                // generated functions receive the ptrauth-* attributes, `declare_fn` or
+                // `declare_raw_fn` could be used to provide those.
+                if cx.sess().pointer_authentication() {
+                    let cfg = cx.sess().pointer_auth_config.as_ref().unwrap();
+                    for ptrauth_attr in cfg.fn_attrs() {
+                        attrs.push(llvm::CreateAttrString(cx.llcx, ptrauth_attr));
+                    }
+                }
                 attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
             }
 
@@ -140,6 +152,24 @@ pub(crate) fn compile_codegen_unit(
                 cx.add_objc_module_flags();
             }
 
+            if cx.sess().pointer_authentication() {
+                let cfg = cx.sess().pointer_auth_config.as_ref().unwrap();
+
+                let aarch64_elf_pauthabi_version =
+                    cfg.calculate_pauth_abi_version(&cx.sess().target);
+                if aarch64_elf_pauthabi_version != 0 {
+                    cx.add_ptrauth_pauthabi_version_and_platform_flags(
+                        aarch64_elf_pauthabi_version,
+                    );
+                }
+                if cfg.elf_got {
+                    cx.add_ptrauth_elf_got_flag();
+                }
+                if cx.sess().pointer_authentication_functions() {
+                    cx.add_ptrauth_sign_personality_flag();
+                }
+            }
+
             // Finalize code coverage by injecting the coverage map. Note, the coverage map will
             // also be added to the `llvm.compiler.used` variable, created next.
             if cx.sess().instrument_coverage() {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -7,7 +7,7 @@ pub(crate) mod autodiff;
 pub(crate) mod gpu_offload;
 
 use libc::{c_char, c_uint};
-use rustc_abi::{self as abi, Align, Size, WrappingRange};
+use rustc_abi::{self as abi, Align, CanonAbi, Size, WrappingRange};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
@@ -429,6 +429,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             bundles.push(kcfi_bundle);
         }
 
+        let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
+        if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
+            bundles.push(p);
+        }
+
         let invoke = unsafe {
             llvm::LLVMBuildInvokeWithOperandBundles(
                 self.llbuilder,
@@ -1406,6 +1411,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             bundles.push(kcfi_bundle);
         }
 
+        let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
+        if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
+            bundles.push(p);
+        }
+
         let call = unsafe {
             llvm::LLVMBuildCallWithOperandBundles(
                 self.llbuilder,
@@ -1836,6 +1846,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             bundles.push(kcfi_bundle);
         }
 
+        let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
+        if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
+            bundles.push(p);
+        }
+
         let callbr = unsafe {
             llvm::LLVMBuildCallBr(
                 self.llbuilder,
@@ -1955,6 +1970,40 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         kcfi_bundle
     }
 
+    // Emits pauth operand bundle.
+    fn ptrauth_operand_bundle(
+        &mut self,
+        llfn: &'ll Value,
+        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
+    ) -> Option<llvm::OperandBundleBox<'ll>> {
+        if !self.sess().pointer_authentication_functions() {
+            return None;
+        }
+        // Pointer authentication support is currently limited to extern "C" calls; filter out other
+        // ABIs.
+        if fn_abi?.conv != CanonAbi::C {
+            return None;
+        }
+        // Filter out LLVM intrinsics.
+        if llvm::get_value_name(llfn).starts_with(b"llvm.") {
+            return None;
+        }
+
+        // FIXME(jchlanda) Operand bundles should only be attached to indirect function calls.
+        // However, function pointer signing is currently performed in `get_fn_addr`, which causes
+        // the logic to be applied too broadly, including to function values (not just pointers).
+        // As a result, direct calls using signed function values must also receive operand
+        // bundles.
+        // Once this is resolved, we should analyze each call and skip direct calls. See the
+        // discussion in the rust-lang issue: <https://github.com/rust-lang/rust/issues/152532>
+        let key: u32 = 0;
+        let discriminator: u64 = 0;
+        Some(llvm::OperandBundleBox::new(
+            "ptrauth",
+            &[self.const_u32(key), self.const_u64(discriminator)],
+        ))
+    }
+
     /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn instrprof_increment(

compiler/rustc_codegen_llvm/src/common.rs

@@ -4,23 +4,67 @@ use std::borrow::Borrow;
 
 use libc::{c_char, c_uint};
 use rustc_abi::Primitive::Pointer;
-use rustc_abi::{self as abi, HasDataLayout as _};
+use rustc_abi::{self as abi, ExternAbi, HasDataLayout as _};
 use rustc_ast::Mutability;
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::stable_hash::{StableHash, StableHasher};
 use rustc_hashes::Hash128;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::{GlobalAlloc, PointerArithmetic, Scalar};
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{Instance, TyCtxt};
 use rustc_session::cstore::DllImport;
+use rustc_session::{PointerAuthAddressDiscriminator, PointerAuthSchema};
 use tracing::debug;
 
-use crate::consts::const_alloc_to_llvm;
+use crate::consts::{IsInitOrFini, IsStatic, const_alloc_to_llvm};
 pub(crate) use crate::context::CodegenCx;
 use crate::context::{GenericCx, SCx};
-use crate::llvm::{self, BasicBlock, ConstantInt, FALSE, TRUE, ToLlvmBool, Type, Value};
+use crate::llvm::{
+    self, BasicBlock, ConstantInt, FALSE, TRUE, ToLlvmBool, Type, Value, const_ptr_auth,
+};
+
+pub(crate) fn maybe_sign_fn_ptr<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, '_>,
+    instance: Instance<'tcx>,
+    llfn: &'ll llvm::Value,
+    schema: &PointerAuthSchema,
+) -> &'ll llvm::Value {
+    if !cx.tcx.sess.pointer_authentication_functions() {
+        return llfn;
+    }
+
+    // Only free functions or methods
+    let def_id = instance.def_id();
+    if !matches!(cx.tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) {
+        return llfn;
+    }
+    // Only C ABI
+    let abi = cx.tcx.fn_sig(def_id).skip_binder().abi();
+    if !matches!(abi, ExternAbi::C { .. } | ExternAbi::System { .. }) {
+        return llfn;
+    }
+    // Ignore LLVM intrinsics
+    if llvm::get_value_name(llfn).starts_with(b"llvm.") {
+        return llfn;
+    }
+    if Some(def_id) == cx.tcx.lang_items().eh_personality() {
+        return llfn;
+    }
+
+    let addr_diversity = match schema.is_address_discriminated {
+        PointerAuthAddressDiscriminator::HardwareAddress(true) => Some(llfn),
+        PointerAuthAddressDiscriminator::HardwareAddress(false) => None,
+        PointerAuthAddressDiscriminator::Synthetic(val) => {
+            let llval = cx.const_u64(val);
+            let llty = cx.val_ty(llfn);
+            Some(unsafe { llvm::LLVMConstIntToPtr(llval, llty) })
+        }
+    };
+    const_ptr_auth(llfn, schema.key as u32, schema.constant_discriminator as u64, addr_diversity)
+}
 
 /*
 * A note on nomenclature of linking: "extern", "foreign", and "upcall".
@@ -268,7 +312,13 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
         })
     }
 
-    fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value {
+    fn scalar_to_backend_with_pac(
+        &self,
+        cv: Scalar,
+        layout: abi::Scalar,
+        llty: &'ll Type,
+        schema: Option<&PointerAuthSchema>,
+    ) -> &'ll Value {
         let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
         match cv {
             Scalar::Int(int) => {
@@ -297,8 +347,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                                 self.const_bitcast(llval, llty)
                             };
                         } else {
-                            let init =
-                                const_alloc_to_llvm(self, alloc.inner(), /*static*/ false);
+                            let init = const_alloc_to_llvm(
+                                self,
+                                alloc.inner(),
+                                IsStatic::No,
+                                IsInitOrFini::No,
+                            );
                             let alloc = alloc.inner();
                             let value = match alloc.mutability {
                                 Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
@@ -319,7 +373,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                             value
                         }
                     }
-                    GlobalAlloc::Function { instance, .. } => self.get_fn_addr(instance),
+                    GlobalAlloc::Function { instance, .. } => self.get_fn_addr(instance, schema),
                     GlobalAlloc::VTable(ty, dyn_ty) => {
                         let alloc = self
                             .tcx
@@ -330,7 +384,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                                 }),
                             )))
                             .unwrap_memory();
-                        let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false);
+                        let init = const_alloc_to_llvm(
+                            self,
+                            alloc.inner(),
+                            IsStatic::No,
+                            IsInitOrFini::No,
+                        );
                         self.static_addr_of_impl(init, alloc.inner().align, None)
                     }
                     GlobalAlloc::Static(def_id) => {