Auto merge of #46093 - scottmcm:lower-128-mir, r=nagisa

Add a MIR pass to lower 128-bit operators to lang item calls

Runs only with `-Z lower_128bit_ops` since it's not hooked into targets yet.

This isn't really useful on its own, but the declarations for the lang items need to be in the compiler before compiler-builtins can be updated to define them, so this is part 1 of at least 3.

cc #45676 @est31 @nagisa

src/librustc/middle/lang_items.rs

@@ -310,6 +310,34 @@ language_item_table! {
  NonZeroItem, "non_zero", non_zero;
 
  DebugTraitLangItem, "debug_trait", debug_trait;
+
+ // A lang item for each of the 128-bit operators we can optionally lower.
+ I128AddFnLangItem, "i128_add", i128_add_fn;
+ U128AddFnLangItem, "u128_add", u128_add_fn;
+ I128SubFnLangItem, "i128_sub", i128_sub_fn;
+ U128SubFnLangItem, "u128_sub", u128_sub_fn;
+ I128MulFnLangItem, "i128_mul", i128_mul_fn;
+ U128MulFnLangItem, "u128_mul", u128_mul_fn;
+ I128DivFnLangItem, "i128_div", i128_div_fn;
+ U128DivFnLangItem, "u128_div", u128_div_fn;
+ I128RemFnLangItem, "i128_rem", i128_rem_fn;
+ U128RemFnLangItem, "u128_rem", u128_rem_fn;
+ I128ShlFnLangItem, "i128_shl", i128_shl_fn;
+ U128ShlFnLangItem, "u128_shl", u128_shl_fn;
+ I128ShrFnLangItem, "i128_shr", i128_shr_fn;
+ U128ShrFnLangItem, "u128_shr", u128_shr_fn;
+ // And overflow versions for the operators that are checkable.
+ // While MIR calls these Checked*, they return (T,bool), not Option<T>.
+ I128AddoFnLangItem, "i128_addo", i128_addo_fn;
+ U128AddoFnLangItem, "u128_addo", u128_addo_fn;
+ I128SuboFnLangItem, "i128_subo", i128_subo_fn;
+ U128SuboFnLangItem, "u128_subo", u128_subo_fn;
+ I128MuloFnLangItem, "i128_mulo", i128_mulo_fn;
+ U128MuloFnLangItem, "u128_mulo", u128_mulo_fn;
+ I128ShloFnLangItem, "i128_shlo", i128_shlo_fn;
+ U128ShloFnLangItem, "u128_shlo", u128_shlo_fn;
+ I128ShroFnLangItem, "i128_shro", i128_shro_fn;
+ U128ShroFnLangItem, "u128_shro", u128_shro_fn;
 }
 
 impl<'a, 'tcx, 'gcx> TyCtxt<'a, 'tcx, 'gcx> {

src/librustc/mir/mod.rs

@@ -170,6 +170,15 @@ impl<'tcx> Mir<'tcx> {
  &mut self.basic_blocks
  }
 
+ #[inline]
+ pub fn basic_blocks_and_local_decls_mut(&mut self) -> (
+ &mut IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+ &mut LocalDecls<'tcx>,
+ ) {
+ self.cache.invalidate();
+ (&mut self.basic_blocks, &mut self.local_decls)
+ }
+
  #[inline]
  pub fn predecessors(&self) -> Ref<IndexVec<BasicBlock, Vec<BasicBlock>>> {
  self.cache.predecessors(self)

src/librustc/session/config.rs

@@ -1144,6 +1144,9 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
  saturating_float_casts: bool = (false, parse_bool, [TRACKED],
  "make float->int casts UB-free: numbers outside the integer type's range are clipped to \
  the max/min integer respectively, and NaN is mapped to 0"),
+ lower_128bit_ops: bool = (false, parse_bool, [TRACKED],
+ "rewrite operators on i128 and u128 into lang item calls (typically provided \
+ by compiler-builtins) so translation doesn't need to support them"),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc_mir/transform/lower_128bit.rs

@@ -0,0 +1,240 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Replaces 128-bit operators with lang item calls
+
+use rustc::hir::def_id::DefId;
+use rustc::middle::lang_items::LangItem;
+use rustc::mir::*;
+use rustc::ty::{Slice, Ty, TyCtxt, TypeVariants};
+use rustc_data_structures::indexed_vec::{Idx};
+use transform::{MirPass, MirSource};
+use syntax;
+
+pub struct Lower128Bit;
+
+impl MirPass for Lower128Bit {
+ fn run_pass<'a, 'tcx>(&self,
+ tcx: TyCtxt<'a, 'tcx, 'tcx>,
+ _src: MirSource,
+ mir: &mut Mir<'tcx>) {
+ if !tcx.sess.opts.debugging_opts.lower_128bit_ops {
+ return
+ }
+
+ self.lower_128bit_ops(tcx, mir);
+ }
+}
+
+impl Lower128Bit {
+ fn lower_128bit_ops<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>, mir: &mut Mir<'tcx>) {
+ let mut new_blocks = Vec::new();
+ let cur_len = mir.basic_blocks().len();
+
+ let (basic_blocks, local_decls) = mir.basic_blocks_and_local_decls_mut();
+ for block in basic_blocks.iter_mut() {
+ for i in (0..block.statements.len()).rev() {
+ let (lang_item, rhs_kind) =
+ if let Some((lang_item, rhs_kind)) =
+ lower_to(&block.statements[i], local_decls, tcx)
+ {
+ (lang_item, rhs_kind)
+ } else {
+ continue;
+ };
+
+ let rhs_override_ty = rhs_kind.ty(tcx);
+ let cast_local =
+ match rhs_override_ty {
+ None => None,
+ Some(ty) => {
+ let local_decl = LocalDecl::new_internal(
+ ty, block.statements[i].source_info.span);
+ Some(local_decls.push(local_decl))
+ },
+ };
+
+ let storage_dead = cast_local.map(|local| {
+ Statement {
+ source_info: block.statements[i].source_info,
+ kind: StatementKind::StorageDead(local),
+ }
+ });
+ let after_call = BasicBlockData {
+ statements: storage_dead.into_iter()
+ .chain(block.statements.drain((i+1)..)).collect(),
+ is_cleanup: block.is_cleanup,
+ terminator: block.terminator.take(),
+ };
+
+ let bin_statement = block.statements.pop().unwrap();
+ let (source_info, lvalue, lhs, mut rhs) = match bin_statement {
+ Statement {
+ source_info,
+ kind: StatementKind::Assign(
+ lvalue,
+ Rvalue::BinaryOp(_, lhs, rhs))
+ } => (source_info, lvalue, lhs, rhs),
+ Statement {
+ source_info,
+ kind: StatementKind::Assign(
+ lvalue,
+ Rvalue::CheckedBinaryOp(_, lhs, rhs))
+ } => (source_info, lvalue, lhs, rhs),
+ _ => bug!("Statement doesn't match pattern any more?"),
+ };
+
+ if let Some(local) = cast_local {
+ block.statements.push(Statement {
+ source_info: source_info,
+ kind: StatementKind::StorageLive(local),
+ });
+ block.statements.push(Statement {
+ source_info: source_info,
+ kind: StatementKind::Assign(
+ Lvalue::Local(local),
+ Rvalue::Cast(
+ CastKind::Misc,
+ rhs,
+ rhs_override_ty.unwrap())),
+ });
+ rhs = Operand::Consume(Lvalue::Local(local));
+ }
+
+ let call_did = check_lang_item_type(
+ lang_item, &lvalue, &lhs, &rhs, local_decls, tcx);
+
+ let bb = BasicBlock::new(cur_len + new_blocks.len());
+ new_blocks.push(after_call);
+
+ block.terminator =
+ Some(Terminator {
+ source_info,
+ kind: TerminatorKind::Call {
+ func: Operand::function_handle(tcx, call_did,
+ Slice::empty(), source_info.span),
+ args: vec![lhs, rhs],
+ destination: Some((lvalue, bb)),
+ cleanup: None,
+ },
+ });
+ }
+ }
+
+ basic_blocks.extend(new_blocks);
+ }
+}
+
+fn check_lang_item_type<'a, 'tcx, D>(
+ lang_item: LangItem,
+ lvalue: &Lvalue<'tcx>,
+ lhs: &Operand<'tcx>,
+ rhs: &Operand<'tcx>,
+ local_decls: &D,
+ tcx: TyCtxt<'a, 'tcx, 'tcx>)
+-> DefId
+ where D: HasLocalDecls<'tcx>
+{
+ let did = tcx.require_lang_item(lang_item);
+ let poly_sig = tcx.fn_sig(did);
+ let sig = tcx.no_late_bound_regions(&poly_sig).unwrap();
+ let lhs_ty = lhs.ty(local_decls, tcx);
+ let rhs_ty = rhs.ty(local_decls, tcx);
+ let lvalue_ty = lvalue.ty(local_decls, tcx).to_ty(tcx);
+ let expected = [lhs_ty, rhs_ty, lvalue_ty];
+ assert_eq!(sig.inputs_and_output[..], expected,
+ "lang item {}", tcx.def_symbol_name(did));
+ did
+}
+
+fn lower_to<'a, 'tcx, D>(statement: &Statement<'tcx>, local_decls: &D, tcx: TyCtxt<'a, 'tcx, 'tcx>)
+ -> Option<(LangItem, RhsKind)>
+ where D: HasLocalDecls<'tcx>
+{
+ match statement.kind {
+ StatementKind::Assign(_, Rvalue::BinaryOp(bin_op, ref lhs, _)) => {
+ let ty = lhs.ty(local_decls, tcx);
+ if let Some(is_signed) = sign_of_128bit(ty) {
+ return item_for_op(bin_op, is_signed);
+ }
+ },
+ StatementKind::Assign(_, Rvalue::CheckedBinaryOp(bin_op, ref lhs, _)) => {
+ let ty = lhs.ty(local_decls, tcx);
+ if let Some(is_signed) = sign_of_128bit(ty) {
+ return item_for_checked_op(bin_op, is_signed);
+ }
+ },
+ _ => {},
+ }
+ None
+}
+
+#[derive(Copy, Clone)]
+enum RhsKind {
+ Unchanged,
+ ForceU128,
+ ForceU32,
+}
+
+impl RhsKind {
+ fn ty<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) -> Option<Ty<'tcx>> {
+ match *self {
+ RhsKind::Unchanged => None,
+ RhsKind::ForceU128 => Some(tcx.types.u128),
+ RhsKind::ForceU32 => Some(tcx.types.u32),
+ }
+ }
+}
+
+fn sign_of_128bit(ty: Ty) -> Option<bool> {
+ match ty.sty {
+ TypeVariants::TyInt(syntax::ast::IntTy::I128) => Some(true),
+ TypeVariants::TyUint(syntax::ast::UintTy::U128) => Some(false),
+ _ => None,
+ }
+}
+
+fn item_for_op(bin_op: BinOp, is_signed: bool) -> Option<(LangItem, RhsKind)> {
+ let i = match (bin_op, is_signed) {
+ (BinOp::Add, true) => (LangItem::I128AddFnLangItem, RhsKind::Unchanged),
+ (BinOp::Add, false) => (LangItem::U128AddFnLangItem, RhsKind::Unchanged),
+ (BinOp::Sub, true) => (LangItem::I128SubFnLangItem, RhsKind::Unchanged),
+ (BinOp::Sub, false) => (LangItem::U128SubFnLangItem, RhsKind::Unchanged),
+ (BinOp::Mul, true) => (LangItem::I128MulFnLangItem, RhsKind::Unchanged),
+ (BinOp::Mul, false) => (LangItem::U128MulFnLangItem, RhsKind::Unchanged),
+ (BinOp::Div, true) => (LangItem::I128DivFnLangItem, RhsKind::Unchanged),
+ (BinOp::Div, false) => (LangItem::U128DivFnLangItem, RhsKind::Unchanged),
+ (BinOp::Rem, true) => (LangItem::I128RemFnLangItem, RhsKind::Unchanged),
+ (BinOp::Rem, false) => (LangItem::U128RemFnLangItem, RhsKind::Unchanged),
+ (BinOp::Shl, true) => (LangItem::I128ShlFnLangItem, RhsKind::ForceU32),
+ (BinOp::Shl, false) => (LangItem::U128ShlFnLangItem, RhsKind::ForceU32),
+ (BinOp::Shr, true) => (LangItem::I128ShrFnLangItem, RhsKind::ForceU32),
+ (BinOp::Shr, false) => (LangItem::U128ShrFnLangItem, RhsKind::ForceU32),
+ _ => return None,
+ };
+ Some(i)
+}
+
+fn item_for_checked_op(bin_op: BinOp, is_signed: bool) -> Option<(LangItem, RhsKind)> {
+ let i = match (bin_op, is_signed) {
+ (BinOp::Add, true) => (LangItem::I128AddoFnLangItem, RhsKind::Unchanged),
+ (BinOp::Add, false) => (LangItem::U128AddoFnLangItem, RhsKind::Unchanged),
+ (BinOp::Sub, true) => (LangItem::I128SuboFnLangItem, RhsKind::Unchanged),
+ (BinOp::Sub, false) => (LangItem::U128SuboFnLangItem, RhsKind::Unchanged),
+ (BinOp::Mul, true) => (LangItem::I128MuloFnLangItem, RhsKind::Unchanged),
+ (BinOp::Mul, false) => (LangItem::U128MuloFnLangItem, RhsKind::Unchanged),
+ (BinOp::Shl, true) => (LangItem::I128ShloFnLangItem, RhsKind::ForceU128),
+ (BinOp::Shl, false) => (LangItem::U128ShloFnLangItem, RhsKind::ForceU128),
+ (BinOp::Shr, true) => (LangItem::I128ShroFnLangItem, RhsKind::ForceU128),
+ (BinOp::Shr, false) => (LangItem::U128ShroFnLangItem, RhsKind::ForceU128),
+ _ => bug!("That should be all the checked ones?"),
+ };
+ Some(i)
+}

src/librustc_mir/transform/mod.rs

@@ -42,6 +42,7 @@ pub mod copy_prop;
 pub mod generator;
 pub mod inline;
 pub mod nll;
+pub mod lower_128bit;
 
 pub(crate) fn provide(providers: &mut Providers) {
  self::qualify_consts::provide(providers);
@@ -241,6 +242,8 @@ fn optimized_mir<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> &'tcx
  // From here on out, regions are gone.
  erase_regions::EraseRegions,
 
+ lower_128bit::Lower128Bit,
+
  // Optimizations begin.
  inline::Inline,
  instcombine::InstCombine,