Trigger a panic on allocation error on nightly Rust

src/librustc_driver/lib.rs

@@ -12,6 +12,8 @@
 #![feature(set_stdio)]
 #![feature(no_debug)]
 #![feature(integer_atomics)]
+#![feature(alloc_error_hook)]
+#![feature(alloc_internals)]
 
 #![recursion_limit="256"]
 
@@ -149,6 +151,14 @@ pub fn run_compiler(
  file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
  emitter: Option<Box<dyn Write + Send>>
 ) -> interface::Result<()> {
+
+ // Install a custom alloc error hook on Nightly, to try
+ // so we can try to generate a backtrace on OOM in the compiler
+ // See https:/rust-lang/rust/issues/66342 for details
+ if UnstableFeatures::from_environment().is_nightly_build() {
+ std::alloc::set_alloc_error_hook(std::alloc::rustc_alloc_error_hook);
+ }
+
  let mut args = Vec::new();
  for arg in at_args {
  match args::arg_expand(arg.clone()) {

src/libstd/alloc.rs

@@ -198,6 +198,21 @@ pub fn take_alloc_error_hook() -> fn(Layout) {
  }
 }
 
+/// A custom error hook, used by librustc_driver.
+/// This lives in `libstd` so that we can call `dumb_print`.
+///
+/// This hook prints out a message like the default error hook,
+/// and then attempts to print a backtrace (without panicking).
+/// If we are completely out of memory, backtrace generation may fail:
+/// this is fine, since the backtrace is best-effort only. We
+/// are guaranteed to print the actual error message, though.
+#[doc(hidden)]
+#[unstable(feature = "alloc_internals", issue = "0")]
+pub fn rustc_alloc_error_hook(layout: Layout) {
+ dumb_print(format_args!("memory allocation of {} bytes failed. backtrace:", layout.size()));
+ dumb_print(format_args!("{:?}", crate::backtrace::Backtrace::capture()));
+}
+
 fn default_alloc_error_hook(layout: Layout) {
  dumb_print(format_args!("memory allocation of {} bytes failed", layout.size()));
 }