Add LLVM KCFI support to the Rust compiler

Cargo.lock

@@ -4269,6 +4269,7 @@ dependencies = [
  "rustc_span",
  "rustc_target",
  "tracing",
+ "twox-hash",
 ]
 
 [[package]]
@@ -5197,6 +5198,17 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "twox-hash"
+version = "1.6.3"
+source = "registry+https:/rust-lang/crates.io-index"
+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
+dependencies = [
+ "cfg-if 1.0.0",
+ "rand 0.8.5",
+ "static_assertions",
+]
+
 [[package]]
 name = "type-map"
 version = "0.4.0"

compiler/rustc_codegen_gcc/src/type_.rs

@@ -300,4 +300,8 @@ impl<'gcc, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
  // Unsupported.
  self.context.new_rvalue_from_int(self.int_type, 0)
  }
+
+ fn set_kcfi_type_metadata(&self, _function: RValue<'gcc>, _kcfi_typeid: u32) {
+ // Unsupported.
+ }
 }

compiler/rustc_codegen_llvm/src/allocator.rs

@@ -88,7 +88,8 @@ pub(crate) unsafe fn codegen(
  callee,
  args.as_ptr(),
  args.len() as c_uint,
- None,
+ [].as_ptr(),
+ 0 as c_uint,
  );
  llvm::LLVMSetTailCall(ret, True);
  if output.is_some() {
@@ -132,8 +133,15 @@ pub(crate) unsafe fn codegen(
  .enumerate()
  .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint))
  .collect::<Vec<_>>();
- let ret =
- llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None);
+ let ret = llvm::LLVMRustBuildCall(
+ llbuilder,
+ ty,
+ callee,
+ args.as_ptr(),
+ args.len() as c_uint,
+ [].as_ptr(),
+ 0 as c_uint,
+ );
  llvm::LLVMSetTailCall(ret, True);
  llvm::LLVMBuildRetVoid(llbuilder);
  llvm::LLVMDisposeBuilder(llbuilder);

compiler/rustc_codegen_llvm/src/builder.rs

@@ -20,6 +20,7 @@ use rustc_middle::ty::layout::{
 };
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::Span;
+use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi;
 use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
 use rustc_target::spec::{HasTargetSpec, Target};
 use std::borrow::Cow;
@@ -225,9 +226,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
  debug!("invoke {:?} with args ({:?})", llfn, args);
 
  let args = self.check_call("invoke", llty, llfn, args);
- let bundle = funclet.map(|funclet| funclet.bundle());
- let bundle = bundle.as_ref().map(|b| &*b.raw);
+ let funclet_bundle = funclet.map(|funclet| funclet.bundle());
+ let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
+ let mut bundles = vec![funclet_bundle];
+
+ // Set KCFI operand bundle
+ let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
+ let kcfi_bundle =
+ if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
+ let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
+ Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
+ } else {
+ None
+ };
+ if kcfi_bundle.is_some() {
+ let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
+ bundles.push(kcfi_bundle);
+ }
 
+ bundles.retain(|bundle| bundle.is_some());
  let invoke = unsafe {
  llvm::LLVMRustBuildInvoke(
  self.llbuilder,
@@ -237,7 +254,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
  args.len() as c_uint,
  then,
  catch,
- bundle,
+ bundles.as_ptr(),
+ bundles.len() as c_uint,
  UNNAMED,
  )
  };
@@ -1143,7 +1161,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
  llfn,
  args.as_ptr() as *const &llvm::Value,
  args.len() as c_uint,
- None,
+ [].as_ptr(),
+ 0 as c_uint,
  );
  }
  }
@@ -1159,17 +1178,34 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
  debug!("call {:?} with args ({:?})", llfn, args);
 
  let args = self.check_call("call", llty, llfn, args);
- let bundle = funclet.map(|funclet| funclet.bundle());
- let bundle = bundle.as_ref().map(|b| &*b.raw);
+ let funclet_bundle = funclet.map(|funclet| funclet.bundle());
+ let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
+ let mut bundles = vec![funclet_bundle];
+
+ // Set KCFI operand bundle
+ let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
+ let kcfi_bundle =
+ if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
+ let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
+ Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
+ } else {
+ None
+ };
+ if kcfi_bundle.is_some() {
+ let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
+ bundles.push(kcfi_bundle);
+ }
 
+ bundles.retain(|bundle| bundle.is_some());
  let call = unsafe {
  llvm::LLVMRustBuildCall(
  self.llbuilder,
  llty,
  llfn,
  args.as_ptr() as *const &llvm::Value,
  args.len() as c_uint,
- bundle,
+ bundles.as_ptr(),
+ bundles.len() as c_uint,
  )
  };
  if let Some(fn_abi) = fn_abi {

compiler/rustc_codegen_llvm/src/context.rs

@@ -250,6 +250,11 @@ pub unsafe fn create_module<'ll>(
  );
  }
 
+ if sess.is_sanitizer_kcfi_enabled() {
+ let kcfi = "kcfi\0".as_ptr().cast();
+ llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1);
+ }
+
  // Control Flow Guard is currently only supported by the MSVC linker on Windows.
  if sess.target.is_like_msvc {
  match sess.opts.cg.control_flow_guard {

compiler/rustc_codegen_llvm/src/declare.rs

@@ -20,7 +20,7 @@ use crate::type_::Type;
 use crate::value::Value;
 use rustc_codegen_ssa::traits::TypeMembershipMethods;
 use rustc_middle::ty::Ty;
-use rustc_symbol_mangling::typeid::typeid_for_fnabi;
+use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi};
 use smallvec::SmallVec;
 
 /// Declare a function.
@@ -136,6 +136,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
  self.set_type_metadata(llfn, typeid);
  }
 
+ if self.tcx.sess.is_sanitizer_kcfi_enabled() {
+ let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
+ self.set_kcfi_type_metadata(llfn, kcfi_typeid);
+ }
+
  llfn
  }
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -427,6 +427,7 @@ pub enum MetadataType {
  MD_type = 19,
  MD_vcall_visibility = 28,
  MD_noundef = 29,
+ MD_kcfi_type = 36,
 }
 
 /// LLVMRustAsmDialect
@@ -1060,6 +1061,7 @@ extern "C" {
  pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
  pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
  pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;
+ pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>;
 
  // Operations on constants of any type
  pub fn LLVMConstNull(Ty: &Type) -> &Value;
@@ -1270,7 +1272,8 @@ extern "C" {
  NumArgs: c_uint,
  Then: &'a BasicBlock,
  Catch: &'a BasicBlock,
- Bundle: Option<&OperandBundleDef<'a>>,
+ OpBundles: *const Option<&OperandBundleDef<'a>>,
+ NumOpBundles: c_uint,
  Name: *const c_char,
  ) -> &'a Value;
  pub fn LLVMBuildLandingPad<'a>(
@@ -1640,7 +1643,8 @@ extern "C" {
  Fn: &'a Value,
  Args: *const &'a Value,
  NumArgs: c_uint,
- Bundle: Option<&OperandBundleDef<'a>>,
+ OpBundles: *const Option<&OperandBundleDef<'a>>,
+ NumOpBundles: c_uint,
  ) -> &'a Value;
  pub fn LLVMRustBuildMemCpy<'a>(
  B: &Builder<'a>,

compiler/rustc_codegen_llvm/src/type_.rs

@@ -316,4 +316,19 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> {
  )
  }
  }
+
+ fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) {
+ let kcfi_type_metadata = self.const_u32(kcfi_typeid);
+ unsafe {
+ llvm::LLVMGlobalSetMetadata(
+ function,
+ llvm::MD_kcfi_type as c_uint,
+ llvm::LLVMMDNodeInContext2(
+ self.llcx,
+ &llvm::LLVMValueAsMetadata(kcfi_type_metadata),
+ 1,
+ ),
+ )
+ }
+ }
 }

compiler/rustc_codegen_ssa/src/traits/type_.rs

@@ -122,6 +122,7 @@ pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> {
 pub trait TypeMembershipMethods<'tcx>: Backend<'tcx> {
  fn set_type_metadata(&self, function: Self::Function, typeid: String);
  fn typeid_metadata(&self, typeid: String) -> Self::Value;
+ fn set_kcfi_type_metadata(&self, function: Self::Function, typeid: u32);
 }
 
 pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> {

compiler/rustc_feature/src/builtin_attrs.rs

@@ -394,7 +394,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
  ungated!(instruction_set, Normal, template!(List: "set"), ErrorPreceding),
  gated!(
  no_sanitize, Normal,
- template!(List: "address, memory, thread"), DuplicatesOk,
+ template!(List: "address, kcfi, memory, thread"), DuplicatesOk,
  experimental!(no_sanitize)
  ),
  gated!(no_coverage, Normal, template!(Word), WarnFollowing, experimental!(no_coverage)),

compiler/rustc_hir_analysis/src/collect.rs

@@ -1859,6 +1859,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
  codegen_fn_attrs.no_sanitize |= SanitizerSet::ADDRESS;
  } else if item.has_name(sym::cfi) {
  codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI;
+ } else if item.has_name(sym::kcfi) {
+ codegen_fn_attrs.no_sanitize |= SanitizerSet::KCFI;
  } else if item.has_name(sym::memory) {
  codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMORY;
  } else if item.has_name(sym::memtag) {
@@ -1872,7 +1874,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
  } else {
  tcx.sess
  .struct_span_err(item.span(), "invalid argument for `no_sanitize`")
- .note("expected one of: `address`, `cfi`, `hwaddress`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
+ .note("expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
  .emit();
  }
  }

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1460,13 +1460,13 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) {
 
 extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
  LLVMValueRef *Args, unsigned NumArgs,
- OperandBundleDef *Bundle) {
+ OperandBundleDef **OpBundles,
+ unsigned NumOpBundles) {
  Value *Callee = unwrap(Fn);
  FunctionType *FTy = unwrap<FunctionType>(Ty);
- unsigned Len = Bundle ? 1 : 0;
- ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len);
  return wrap(unwrap(B)->CreateCall(
- FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles));
+ FTy, Callee, makeArrayRef(unwrap(Args), NumArgs),
+ makeArrayRef(*OpBundles, NumOpBundles)));
 }
 
 extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
@@ -1506,14 +1506,14 @@ extern "C" LLVMValueRef
 LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
  LLVMValueRef *Args, unsigned NumArgs,
  LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
- OperandBundleDef *Bundle, const char *Name) {
+ OperandBundleDef **OpBundles, unsigned NumOpBundles,
+ const char *Name) {
  Value *Callee = unwrap(Fn);
  FunctionType *FTy = unwrap<FunctionType>(Ty);
- unsigned Len = Bundle ? 1 : 0;
- ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len);
  return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch),
  makeArrayRef(unwrap(Args), NumArgs),
- Bundles, Name));
+ makeArrayRef(*OpBundles, NumOpBundles),
+ Name));
 }
 
 extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B,

compiler/rustc_session/src/options.rs

@@ -368,7 +368,7 @@ mod desc {
  pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
  pub const parse_oom_strategy: &str = "either `panic` or `abort`";
  pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
- pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
+ pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `kcfi`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
  pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2";
  pub const parse_cfguard: &str =
  "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
@@ -675,6 +675,7 @@ mod parse {
  *slot |= match s {
  "address" => SanitizerSet::ADDRESS,
  "cfi" => SanitizerSet::CFI,
+ "kcfi" => SanitizerSet::KCFI,
  "leak" => SanitizerSet::LEAK,
  "memory" => SanitizerSet::MEMORY,
  "memtag" => SanitizerSet::MEMTAG,

compiler/rustc_session/src/session.rs

@@ -683,6 +683,10 @@ impl Session {
  self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI)
  }
 
+ pub fn is_sanitizer_kcfi_enabled(&self) -> bool {
+ self.opts.unstable_opts.sanitizer.contains(SanitizerSet::KCFI)
+ }
+
  /// Check whether this compile session and crate type use static crt.
  pub fn crt_static(&self, crate_type: Option<CrateType>) -> bool {
  if !self.target.crt_static_respected {
@@ -1530,6 +1534,14 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
  }
  }
 
+ // LLVM CFI and KCFI are mutually exclusive
+ if sess.is_sanitizer_cfi_enabled() && sess.is_sanitizer_kcfi_enabled() {
+ sess.emit_err(CannotMixAndMatchSanitizers {
+ first: "cfi".to_string(),
+ second: "kcfi".to_string(),
+ });
+ }
+
  if sess.opts.unstable_opts.stack_protector != StackProtector::None {
  if !sess.target.options.supports_stack_protector {
  sess.emit_warning(StackProtectorNotSupportedForTarget {

compiler/rustc_span/src/symbol.rs

@@ -828,6 +828,7 @@ symbols! {
  item_like_imports,
  iter,
  iter_repeat,
+ kcfi,
  keyword,
  kind,
  kreg,

compiler/rustc_symbol_mangling/Cargo.toml

@@ -10,6 +10,7 @@ bitflags = "1.2.1"
 tracing = "0.1"
 punycode = "0.4.0"
 rustc-demangle = "0.1.21"
+twox-hash = "1.6.3"
 
 rustc_span = { path = "../rustc_span" }
 rustc_middle = { path = "../rustc_middle" }