Implement sret

Cargo.lock

@@ -56,18 +56,18 @@ dependencies = [
 
 [[package]]
 name = "gccjit"
-version = "7.0.0"
+version = "7.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfdb84ba432f6679794790de652bfbed86201893a30c03dc669c2bfe7424f472"
+checksum = "21312c2cc52f15e394551018c6ee878b84b039f47ad00ab027cdd13fe8b5fb08"
 dependencies = [
  "gccjit_sys",
 ]
 
 [[package]]
 name = "gccjit_sys"
-version = "3.2.0"
+version = "4.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e081669728b490723537f9def7eb674b7c9acd8de0b92ad4f4abf5f5cc75ea4b"
+checksum = "3b85bc720f5c4194dcea10f95d8eb82b30b74d63255ee01285b990424f5a0a46"
 dependencies = [
  "libc",
 ]

Cargo.toml

@@ -20,7 +20,7 @@ default = ["master"]
 [dependencies]
 object = { version = "0.39.0", default-features = false, features = ["std", "read"] }
 tempfile = "3.20"
-gccjit = { version = "7.0.0", features = ["dlopen"] }
+gccjit = { version = "7.1.0", features = ["dlopen"] }
 #gccjit = { git = "https://github.com/rust-lang/gccjit.rs", branch = "error-dlopen", features = ["dlopen"] }
 
 # Local copy.

build_system/src/test.rs

@@ -935,7 +935,9 @@ fn test_libcore_doctests(env: &Env, args: &TestArg) -> Result<(), String> {
         // `core::io::ErrorKind`'s `Display` impl declares `#![feature(core_io)]` upstream: without
         // it, that doctest fails to compile with `E0658` on any backend.
         &"-Zforce-unstable-if-unmarked",
-        // FIXME: one test cannot compile due to an upstream bug in the new trait solver.
+        // FIXME: one test (mem::transmutability::Assume::alignment in core/src/mem/transmutability.rs)
+        // cannot compile due to an upstream bug in the new trait solver.
+        // See: https://github.com/rust-lang/rust/issues/161251
         &"-Znext-solver=coherence",
     ];
     for flag in &rustflags {

libgccjit.version

@@ -1 +1 @@
-badf78d09d16e66f4ca07971c51aa6a227558d4f
+c86ad8032df732309fe949ea68da1874fbb76f5c

src/abi.rs

@@ -20,8 +20,16 @@ use crate::context::CodegenCx;
 use crate::type_of::LayoutGccExt;
 
 impl AbiBuilderMethods for Builder<'_, '_, '_> {
-    fn get_param(&mut self, index: usize) -> Self::Value {
+    fn get_param(&mut self, mut index: usize) -> Self::Value {
         let func = self.current_func();
+        if let Some(&return_value) = self.functions_with_indirect_return.borrow().get(&func) {
+            // cg_ssa sees the return pointer as the first parameter, but in GCC it is a hidden
+            // parameter: hand out the address of the local holding the return value instead.
+            if index == 0 {
+                return return_value.get_address(self.location);
+            }
+            index -= 1;
+        }
         let param = func.get_param(index as i32);
         let on_stack = if let Some(on_stack_param_indices) =
             self.on_stack_function_params.borrow().get(&func)
@@ -104,6 +112,10 @@ pub struct FnAbiGcc<'gcc> {
     pub on_stack_param_indices: FxHashSet<usize>,
     #[cfg(feature = "master")]
     pub fn_attributes: Vec<FnAttribute<'gcc>>,
+    /// Whether the value is returned in memory, through a pointer that GCC passes as a hidden
+    /// parameter.
+    #[cfg(feature = "master")]
+    pub has_indirect_return: bool,
 }
 
 pub trait FnAbiGccExt<'gcc, 'tcx> {
@@ -117,6 +129,7 @@ pub trait FnAbiGccExt<'gcc, 'tcx> {
 impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
     fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> FnAbiGcc<'gcc> {
         let mut on_stack_param_indices = FxHashSet::default();
+        let has_indirect_return = cfg!(feature = "master") && self.ret.is_indirect();
 
         // This capacity calculation is approximate.
         let mut argument_tys = Vec::with_capacity(
@@ -127,6 +140,10 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_gcc_type(cx),
             PassMode::Cast { ref cast, .. } => cast.gcc_type(cx),
+            // Returned by value: the function (or function pointer type) is flagged as returning
+            // in memory, so GCC does the sret lowering itself, with the hidden pointer in the
+            // register the target ABI reserves for it.
+            PassMode::Indirect { .. } if has_indirect_return => self.ret.layout.gcc_type(cx),
             PassMode::Indirect { .. } => {
                 argument_tys.push(cx.type_ptr_to(self.ret.layout.gcc_type(cx)));
                 cx.type_void()
@@ -277,13 +294,30 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             on_stack_param_indices,
             #[cfg(feature = "master")]
             fn_attributes: fn_attrs,
+            #[cfg(feature = "master")]
+            has_indirect_return,
         }
     }
 
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
         // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
-        let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = self.gcc_type(cx);
-        cx.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)
+        let fn_abi_gcc = self.gcc_type(cx);
+        let pointer_type = cx.context.new_function_pointer_type(
+            None,
+            fn_abi_gcc.return_type,
+            &fn_abi_gcc.arguments_type,
+            fn_abi_gcc.is_c_variadic,
+        );
+        #[cfg(feature = "master")]
+        if fn_abi_gcc.has_indirect_return {
+            // Calls through this pointer must use the same convention as direct calls to a
+            // function declared with an indirect return.
+            pointer_type
+                .dyncast_function_ptr_type()
+                .expect("function pointer type")
+                .set_indirect_return();
+        }
+        pointer_type
     }
 
     #[cfg(feature = "master")]

src/builder.rs

@@ -358,25 +358,28 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         funclet: Option<&Funclet>,
         must_tail: bool,
     ) -> RValue<'gcc> {
-        // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport.
-        let args = match return_slot {
-            ReturnSlot::Direct => Cow::Borrowed(args),
+        // Without libgccjit's indirect return support, the return pointer is an explicit parameter.
+        #[cfg(not(feature = "master"))]
+        let (args, return_slot) = match return_slot {
+            ReturnSlot::Direct => (Cow::Borrowed(args), ReturnSlot::Direct),
             ReturnSlot::Indirect(sret_ptr) => {
                 let mut args = args.to_vec();
                 // Prepend the indirect return pointer
                 args.insert(0, sret_ptr);
-                Cow::Owned(args)
+                (Cow::Owned(args), ReturnSlot::Direct)
             }
         };
+        #[cfg(not(feature = "master"))]
+        let args: &[RValue<'gcc>] = &args;
         // FIXME(antoyo): remove when having a proper API.
         let gcc_func = unsafe { std::mem::transmute::<RValue<'gcc>, Function<'gcc>>(func) };
         let call = if self.functions.borrow().values().any(|value| *value == gcc_func) {
             // FIXME(antoyo): remove when the API supports a different type for functions.
             let func: Function<'gcc> = self.cx.rvalue_as_function(func);
-            self.function_call(func, &args, funclet, must_tail)
+            self.function_call(func, return_slot, args, funclet, must_tail)
         } else {
             // If it's a not function that was defined, it's a function pointer.
-            self.function_ptr_call(typ, fn_abi, func, &args, funclet, must_tail)
+            self.function_ptr_call(typ, fn_abi, func, return_slot, args, funclet, must_tail)
         };
         if let Some(_fn_abi) = fn_abi {
             // FIXME(bjorn3): Apply function attributes
@@ -387,6 +390,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     pub fn function_call(
         &mut self,
         func: Function<'gcc>,
+        return_slot: ReturnSlot<RValue<'gcc>>,
         args: &[RValue<'gcc>],
         _funclet: Option<&Funclet>,
         must_tail: bool,
@@ -403,23 +407,22 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         // That's why we assign the result to a local or call add_eval().
         let return_type = func.get_return_type();
         let void_type = self.context.new_type::<()>();
-        let current_func = self.block.get_function();
         if return_type != void_type {
-            let result = self.new_temp(current_func, self.location, return_type);
-            self.block.add_assignment(self.location, result, call);
-            result.to_rvalue()
+            self.store_call_result(return_slot, call)
         } else {
             self.block.add_eval(self.location, call);
             // Return dummy value when not having return value.
             self.context.new_rvalue_zero(self.isize_type)
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn function_ptr_call(
         &mut self,
         typ: Type<'gcc>,
         fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
         mut func_ptr: RValue<'gcc>,
+        return_slot: ReturnSlot<RValue<'gcc>>,
         args: &[RValue<'gcc>],
         _funclet: Option<&Funclet>,
         must_tail: bool,
@@ -434,6 +437,16 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             }
         };
         let gcc_func = func_ptr_type.dyncast_function_ptr_type().expect("function ptr");
+        // A function pointer type built without `ptr_to_gcc_type` would lack the indirect-return
+        // flag: GCC would then expect the result in registers while the callee returns in memory.
+        #[cfg(feature = "master")]
+        if let Some(fn_abi) = fn_abi {
+            assert_eq!(
+                gcc_func.is_indirect_return(),
+                fn_abi.ret.is_indirect(),
+                "function pointer type with a wrong indirect-return flag: {func_ptr:?}",
+            );
+        }
         let on_stack_param_indices = fn_abi
             .map(|fn_abi| fn_abi.gcc_type(self.cx).on_stack_param_indices)
             .unwrap_or_default();
@@ -457,7 +470,6 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         // That's why we assign the result to a local or call add_eval().
         let return_type = gcc_func.get_return_type();
         let void_type = self.context.new_type::<()>();
-        let current_func = self.block.get_function();
 
         if return_type != void_type {
             let return_value = self.cx.context.new_call_through_ptr(self.location, func_ptr, &args);
@@ -469,9 +481,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 args_adjusted,
                 orig_args,
             );
-            let result = self.new_temp(current_func, self.location, return_value.get_type());
-            self.block.add_assignment(self.location, result, return_value);
-            result.to_rvalue()
+            self.store_call_result(return_slot, return_value)
         } else {
             #[cfg(not(feature = "master"))]
             if gcc_func.get_param_count() == 0 {
@@ -615,6 +625,14 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn ret_void(&mut self) {
+        if let Some(&return_value) =
+            self.functions_with_indirect_return.borrow().get(&self.current_func())
+        {
+            // cg_ssa returns nothing for an indirect return, but the GCC function returns the
+            // value itself: GCC copies it to the caller's slot and returns the hidden pointer.
+            self.llbb().end_with_return(self.location, return_value.to_rvalue());
+            return;
+        }
         self.llbb().end_with_void_return(self.location)
     }
 
@@ -768,7 +786,14 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         } else {
             let trap = self.context.get_builtin_function("__builtin_trap");
             self.block.add_eval(self.location, self.context.new_call(self.location, trap, &[]));
-            let return_value = self.new_temp(self.current_func(), self.location, return_type);
+            // Reuse the local of an indirect return: a new temporary per unreachable block would
+            // add the whole return value to the stack frame each time.
+            let indirect_return_value =
+                self.functions_with_indirect_return.borrow().get(&self.current_func()).copied();
+            let return_value = match indirect_return_value {
+                Some(return_value) => return_value,
+                None => self.new_temp(self.current_func(), self.location, return_type),
+            };
             self.block.end_with_return(self.location, return_value)
         }
     }
@@ -2516,6 +2541,31 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         self.llbb().add_assignment(self.location, var, value);
         var.to_rvalue()
     }
+
+    /// Anchor a call in a statement, since an rvalue is evaluated at each use. An indirect result
+    /// goes to the caller-provided return slot and a dummy value is returned to cg_ssa.
+    fn store_call_result(
+        &self,
+        return_slot: ReturnSlot<RValue<'gcc>>,
+        call: RValue<'gcc>,
+    ) -> RValue<'gcc> {
+        match return_slot {
+            ReturnSlot::Direct => self.assign_to_var(call),
+            ReturnSlot::Indirect(sret_ptr) => {
+                // Cast the opaque return slot to the exact return type, so that GCC stores the
+                // result there directly, without a conversion or an intermediate copy.
+                let return_type = call.get_type();
+                let sret_ptr =
+                    self.context.new_cast(self.location, sret_ptr, return_type.make_pointer());
+                self.llbb().add_assignment(
+                    self.location,
+                    sret_ptr.dereference(self.location),
+                    call,
+                );
+                self.context.new_rvalue_zero(self.cx.type_u32())
+            }
+        }
+    }
 }
 
 fn difference_or_zero<'gcc>(

src/context.rs

@@ -117,6 +117,9 @@ pub struct CodegenCx<'gcc, 'tcx> {
     /// Mapping from function to indexes of on stack parameters.
     pub on_stack_function_params: RefCell<FxHashMap<Function<'gcc>, FxHashSet<usize>>>,
 
+    /// Local holding the return value of the defined functions returning it in memory.
+    pub functions_with_indirect_return: RefCell<FxHashMap<Function<'gcc>, LValue<'gcc>>>,
+
     /// Cache of emitted const globals (value -> global)
     pub const_globals: RefCell<FxHashMap<RValue<'gcc>, RValue<'gcc>>>,
 
@@ -308,6 +311,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             function_instances: Default::default(),
             intrinsic_instances: Default::default(),
             on_stack_function_params: Default::default(),
+            functions_with_indirect_return: Default::default(),
             vtables: Default::default(),
             const_globals: Default::default(),
             global_lvalues: Default::default(),

src/declare.rs

@@ -132,6 +132,18 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         );
         self.on_stack_function_params.borrow_mut().insert(func, fn_abi_gcc.on_stack_param_indices);
         #[cfg(feature = "master")]
+        if fn_abi_gcc.has_indirect_return {
+            // Return in memory even where the target ABI would use registers, as cg_llvm does for
+            // `PassMode::Indirect`.
+            func.set_indirect_return();
+            // `get_param(0)` hands out this local's address and `ret_void` returns it.
+            if self.linkage.get() != FunctionType::Extern {
+                self.functions_with_indirect_return.borrow_mut().entry(func).or_insert_with(|| {
+                    func.new_local(None, fn_abi_gcc.return_type, "indirectReturn")
+                });
+            }
+        }
+        #[cfg(feature = "master")]
         for fn_attr in fn_abi_gcc.fn_attributes {
             func.add_attribute(fn_attr);
         }

src/intrinsic/mod.rs

@@ -831,8 +831,11 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         idx: &mut usize,
         dst: PlaceRef<'tcx, RValue<'gcc>>,
     ) {
+        let func = bx.current_func();
+        // The return pointer counted by cg_ssa is a hidden parameter in GCC.
+        let hidden_params = bx.functions_with_indirect_return.borrow().contains_key(&func) as usize;
         let mut next = || {
-            let val = bx.current_func().get_param(*idx as i32);
+            let val = func.get_param((*idx - hidden_params) as i32);
             *idx += 1;
             val.to_rvalue()
         };

src/type_of.rs

@@ -15,7 +15,7 @@ use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt};
 use rustc_span::bug;
 use rustc_target::callconv::{CastTarget, FnAbi};
 
-use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType};
+use crate::abi::{FnAbiGccExt, GccType};
 use crate::context::CodegenCx;
 use crate::type_::{struct_attributes, struct_fields};
 
@@ -356,8 +356,7 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     }
 
     fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
-        // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
-        let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = fn_abi.gcc_type(self);
-        self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)
+        // Call sites cast the callee to this type, so it must carry the indirect-return flag too.
+        fn_abi.ptr_to_gcc_type(self)
     }
 }