diff options
Diffstat (limited to 'src/rt/rust_upcall.cpp')
| -rw-r--r-- | src/rt/rust_upcall.cpp | 408 |
1 files changed, 319 insertions, 89 deletions
diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp index 653db471fc0..e4b646e39ff 100644 --- a/src/rt/rust_upcall.cpp +++ b/src/rt/rust_upcall.cpp @@ -6,7 +6,31 @@ #include "rust_upcall.h" #include <stdint.h> -// Upcalls. +extern "C" void record_sp(void *limit); + +/** + * Switches to the C-stack and invokes |fn_ptr|, passing |args| as argument. + */ +extern "C" CDECL void +upcall_call_shim_on_c_stack(void *args, void *fn_ptr) { + rust_task *task = rust_scheduler::get_task(); + + // FIXME (1226) - The shim functions generated by rustc contain the + // morestack prologue, so we need to let them know they have enough + // stack. + record_sp(0); + + rust_scheduler *sched = task->sched; + try { + sched->c_context.call_shim_on_c_stack(args, fn_ptr); + } catch (...) { + task = rust_scheduler::get_task(); + task->record_stack_limit(); + throw; + } + task = rust_scheduler::get_task(); + task->record_stack_limit(); +} #if defined(__i386__) || defined(__x86_64__) || defined(_M_X64) void @@ -47,24 +71,34 @@ copy_elements(rust_task *task, type_desc *elem_t, } } +struct s_fail_args { + char const *expr; + char const *file; + size_t line; +}; + extern "C" CDECL void -upcall_fail(char const *expr, - char const *file, - size_t line) { +upcall_s_fail(s_fail_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); - LOG_ERR(task, upcall, "upcall fail '%s', %s:%" PRIdPTR, expr, file, line); + LOG_ERR(task, upcall, "upcall fail '%s', %s:%" PRIdPTR, + args->expr, args->file, args->line); task->fail(); } +struct s_malloc_args { + size_t nbytes; + type_desc *td; +}; + extern "C" CDECL uintptr_t -upcall_malloc(size_t nbytes, type_desc *td) { +upcall_s_malloc(s_malloc_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); LOG(task, mem, "upcall malloc(%" PRIdPTR ", 0x%" PRIxPTR ")", - nbytes, td); + args->nbytes, args->td); gc::maybe_gc(task); cc::maybe_cc(task); @@ -72,107 +106,136 @@ upcall_malloc(size_t nbytes, type_desc *td) { // TODO: Maybe use dladdr here to find a more useful name for the // type_desc. - void *p = task->malloc(nbytes, "tdesc", td); - memset(p, '\0', nbytes); + void *p = task->malloc(args->nbytes, "tdesc", args->td); + memset(p, '\0', args->nbytes); - task->local_allocs[p] = td; + task->local_allocs[p] = args->td; debug::maybe_track_origin(task, p); LOG(task, mem, "upcall malloc(%" PRIdPTR ", 0x%" PRIxPTR ") = 0x%" PRIxPTR, - nbytes, td, (uintptr_t)p); + args->nbytes, args->td, (uintptr_t)p); return (uintptr_t) p; } +struct s_free_args { + void *ptr; + uintptr_t is_gc; +}; + /** * Called whenever an object's ref count drops to zero. */ extern "C" CDECL void -upcall_free(void* ptr, uintptr_t is_gc) { +upcall_s_free(s_free_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); rust_scheduler *sched = task->sched; DLOG(sched, mem, "upcall free(0x%" PRIxPTR ", is_gc=%" PRIdPTR ")", - (uintptr_t)ptr, is_gc); + (uintptr_t)args->ptr, args->is_gc); - task->local_allocs.erase(ptr); - debug::maybe_untrack_origin(task, ptr); + task->local_allocs.erase(args->ptr); + debug::maybe_untrack_origin(task, args->ptr); - task->free(ptr, (bool) is_gc); + task->free(args->ptr, (bool) args->is_gc); } +struct s_shared_malloc_args { + size_t nbytes; + type_desc *td; +}; + extern "C" CDECL uintptr_t -upcall_shared_malloc(size_t nbytes, type_desc *td) { +upcall_s_shared_malloc(s_shared_malloc_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); LOG(task, mem, - "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR ")", - nbytes, td); - void *p = task->kernel->malloc(nbytes, "shared malloc"); - memset(p, '\0', nbytes); + "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR ")", + args->nbytes, args->td); + void *p = task->kernel->malloc(args->nbytes, "shared malloc"); + memset(p, '\0', args->nbytes); LOG(task, mem, - "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR - ") = 0x%" PRIxPTR, - nbytes, td, (uintptr_t)p); + "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR + ") = 0x%" PRIxPTR, + args->nbytes, args->td, (uintptr_t)p); return (uintptr_t) p; } +struct s_shared_free_args { + void *ptr; +}; + /** * Called whenever an object's ref count drops to zero. */ extern "C" CDECL void -upcall_shared_free(void* ptr) { +upcall_s_shared_free(s_shared_free_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); rust_scheduler *sched = task->sched; DLOG(sched, mem, "upcall shared_free(0x%" PRIxPTR")", - (uintptr_t)ptr); - task->kernel->free(ptr); + (uintptr_t)args->ptr); + task->kernel->free(args->ptr); } +struct s_get_type_desc_args { + size_t size; + size_t align; + size_t n_descs; + type_desc const **descs; + uintptr_t n_obj_params; +}; + extern "C" CDECL type_desc * -upcall_get_type_desc(void *curr_crate, // ignored, legacy compat. - size_t size, - size_t align, - size_t n_descs, - type_desc const **descs, - uintptr_t n_obj_params) { +upcall_s_get_type_desc(s_get_type_desc_args *args) { rust_task *task = rust_scheduler::get_task(); check_stack(task); LOG_UPCALL_ENTRY(task); LOG(task, cache, "upcall get_type_desc with size=%" PRIdPTR - ", align=%" PRIdPTR ", %" PRIdPTR " descs", size, align, - n_descs); + ", align=%" PRIdPTR ", %" PRIdPTR " descs", args->size, args->align, + args->n_descs); rust_crate_cache *cache = task->get_crate_cache(); - type_desc *td = cache->get_type_desc(size, align, n_descs, descs, - n_obj_params); + type_desc *td = cache->get_type_desc(args->size, args->align, args->n_descs, + args->descs, args->n_obj_params); LOG(task, cache, "returning tydesc 0x%" PRIxPTR, td); return td; } +struct s_vec_grow_args { + rust_vec** vp; + size_t new_sz; +}; + extern "C" CDECL void -upcall_vec_grow(rust_vec** vp, size_t new_sz) { +upcall_s_vec_grow(s_vec_grow_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); - reserve_vec(task, vp, new_sz); - (*vp)->fill = new_sz; + reserve_vec(task, args->vp, args->new_sz); + (*args->vp)->fill = args->new_sz; } +struct s_vec_push_args { + rust_vec** vp; + type_desc* elt_ty; + void* elt; +}; + extern "C" CDECL void -upcall_vec_push(rust_vec** vp, type_desc* elt_ty, void* elt) { +upcall_s_vec_push(s_vec_push_args *args) { rust_task *task = rust_scheduler::get_task(); LOG_UPCALL_ENTRY(task); - size_t new_sz = (*vp)->fill + elt_ty->size; - reserve_vec(task, vp, new_sz); - rust_vec* v = *vp; - copy_elements(task, elt_ty, &v->data[0] + v->fill, elt, elt_ty->size); - v->fill += elt_ty->size; + size_t new_sz = (*args->vp)->fill + args->elt_ty->size; + reserve_vec(task, args->vp, new_sz); + rust_vec* v = *args->vp; + copy_elements(task, args->elt_ty, &v->data[0] + v->fill, + args->elt, args->elt_ty->size); + v->fill += args->elt_ty->size; } /** @@ -180,60 +243,235 @@ upcall_vec_push(rust_vec** vp, type_desc* elt_ty, void* elt) { * space in the dynamic stack. */ extern "C" CDECL void * -upcall_dynastack_mark() { +upcall_s_dynastack_mark() { return rust_scheduler::get_task()->dynastack.mark(); } +struct s_dynastack_alloc_args { + size_t sz; +}; + /** * Allocates space in the dynamic stack and returns it. * * FIXME: Deprecated since dynamic stacks need to be self-describing for GC. */ extern "C" CDECL void * -upcall_dynastack_alloc(size_t sz) { +upcall_s_dynastack_alloc(s_dynastack_alloc_args *args) { + size_t sz = args->sz; return sz ? rust_scheduler::get_task()->dynastack.alloc(sz, NULL) : NULL; } +struct s_dynastack_alloc_2_args { + size_t sz; + type_desc *ty; +}; + /** * Allocates space associated with a type descriptor in the dynamic stack and * returns it. */ extern "C" CDECL void * -upcall_dynastack_alloc_2(size_t sz, type_desc *ty) { +upcall_s_dynastack_alloc_2(s_dynastack_alloc_2_args *args) { + size_t sz = args->sz; + type_desc *ty = args->ty; return sz ? rust_scheduler::get_task()->dynastack.alloc(sz, ty) : NULL; } +struct s_dynastack_free_args { + void *ptr; +}; + /** Frees space in the dynamic stack. */ extern "C" CDECL void -upcall_dynastack_free(void *ptr) { - return rust_scheduler::get_task()->dynastack.free(ptr); +upcall_s_dynastack_free(s_dynastack_free_args *args) { + return rust_scheduler::get_task()->dynastack.free(args->ptr); } -extern "C" void record_sp(void *limit); +extern "C" _Unwind_Reason_Code +__gxx_personality_v0(int version, + _Unwind_Action actions, + uint64_t exception_class, + _Unwind_Exception *ue_header, + _Unwind_Context *context); + +struct s_rust_personality_args { + int version; + _Unwind_Action actions; + uint64_t exception_class; + _Unwind_Exception *ue_header; + _Unwind_Context *context; +}; + +extern "C" _Unwind_Reason_Code +upcall_s_rust_personality(s_rust_personality_args *args) { + return __gxx_personality_v0(args->version, + args->actions, + args->exception_class, + args->ue_header, + args->context); +} + +extern "C" void +shape_cmp_type(int8_t *result, const type_desc *tydesc, + const type_desc **subtydescs, uint8_t *data_0, + uint8_t *data_1, uint8_t cmp_type); + +struct s_cmp_type_args { + int8_t *result; + const type_desc *tydesc; + const type_desc **subtydescs; + uint8_t *data_0; + uint8_t *data_1; + uint8_t cmp_type; +}; + +extern "C" void +upcall_s_cmp_type(s_cmp_type_args *args) { + shape_cmp_type(args->result, args->tydesc, args->subtydescs, + args->data_0, args->data_1, args->cmp_type); +} + +extern "C" void +shape_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level); + +struct s_log_type_args { + const type_desc *tydesc; + uint8_t *data; + uint32_t level; +}; + +extern "C" void +upcall_s_log_type(s_log_type_args *args) { + shape_log_type(args->tydesc, args->data, args->level); +} + + +// ______________________________________________________________________________ +// Upcalls in original format: deprecated and should be removed once snapshot +// transitions them away. + +extern "C" CDECL void +upcall_fail(char const *expr, + char const *file, + size_t line) { + s_fail_args args = {expr,file,line}; + upcall_s_fail(&args); +} + +extern "C" CDECL uintptr_t +upcall_malloc(size_t nbytes, type_desc *td) { + s_malloc_args args = {nbytes, td}; + return upcall_s_malloc(&args); +} /** - * Switch to the C stack and call the given function, passing a single pointer - * argument. + * Called whenever an object's ref count drops to zero. */ extern "C" CDECL void -upcall_call_shim_on_c_stack(void *args, void *fn_ptr) { - rust_task *task = rust_scheduler::get_task(); +upcall_free(void* ptr, uintptr_t is_gc) { + s_free_args args = {ptr, is_gc}; + upcall_s_free(&args); +} - // FIXME (1226) - The shim functions generated by rustc contain the - // morestack prologue, so we need to let them know they have enough - // stack. - record_sp(0); +extern "C" CDECL uintptr_t +upcall_shared_malloc(size_t nbytes, type_desc *td) { + s_shared_malloc_args args = {nbytes, td}; + return upcall_s_shared_malloc(&args); +} - rust_scheduler *sched = task->sched; - try { - sched->c_context.call_shim_on_c_stack(args, fn_ptr); - } catch (...) { - task = rust_scheduler::get_task(); - task->record_stack_limit(); - throw; - } - task = rust_scheduler::get_task(); - task->record_stack_limit(); +/** + * Called whenever an object's ref count drops to zero. + */ +extern "C" CDECL void +upcall_shared_free(void* ptr) { + s_shared_free_args args = {ptr}; + upcall_s_shared_free(&args); +} + +extern "C" CDECL type_desc * +upcall_get_type_desc(void *curr_crate, // ignored, legacy compat. + size_t size, + size_t align, + size_t n_descs, + type_desc const **descs, + uintptr_t n_obj_params) { + s_get_type_desc_args args = {size,align,n_descs,descs,n_obj_params}; + return upcall_s_get_type_desc(&args); +} + +extern "C" CDECL void +upcall_vec_grow(rust_vec** vp, size_t new_sz) { + s_vec_grow_args args = {vp, new_sz}; + upcall_s_vec_grow(&args); +} + +extern "C" CDECL void +upcall_vec_push(rust_vec** vp, type_desc* elt_ty, void* elt) { + s_vec_push_args args = {vp, elt_ty, elt}; + upcall_s_vec_push(&args); +} + +/** + * Returns a token that can be used to deallocate all of the allocated space + * space in the dynamic stack. + */ +extern "C" CDECL void * +upcall_dynastack_mark() { + return upcall_s_dynastack_mark(); +} + +/** + * Allocates space in the dynamic stack and returns it. + * + * FIXME: Deprecated since dynamic stacks need to be self-describing for GC. + */ +extern "C" CDECL void * +upcall_dynastack_alloc(size_t sz) { + s_dynastack_alloc_args args = {sz}; + return upcall_s_dynastack_alloc(&args); +} + +/** + * Allocates space associated with a type descriptor in the dynamic stack and + * returns it. + */ +extern "C" CDECL void * +upcall_dynastack_alloc_2(size_t sz, type_desc *ty) { + s_dynastack_alloc_2_args args = {sz, ty}; + return upcall_s_dynastack_alloc_2(&args); +} + +/** Frees space in the dynamic stack. */ +extern "C" CDECL void +upcall_dynastack_free(void *ptr) { + s_dynastack_free_args args = {ptr}; + return upcall_s_dynastack_free(&args); +} + +extern "C" _Unwind_Reason_Code +upcall_rust_personality(int version, + _Unwind_Action actions, + uint64_t exception_class, + _Unwind_Exception *ue_header, + _Unwind_Context *context) { + s_rust_personality_args args = {version, actions, exception_class, ue_header, + context}; + return upcall_s_rust_personality(&args); +} + +extern "C" void +upcall_cmp_type(int8_t *result, const type_desc *tydesc, + const type_desc **subtydescs, uint8_t *data_0, + uint8_t *data_1, uint8_t cmp_type) { + s_cmp_type_args args = {result, tydesc, subtydescs, data_0, data_1, cmp_type}; + upcall_s_cmp_type(&args); +} + +extern "C" void +upcall_log_type(const type_desc *tydesc, uint8_t *data, uint32_t level) { + s_log_type_args args = {tydesc, data, level}; + upcall_s_log_type(&args); } struct rust_new_stack2_args { @@ -245,6 +483,8 @@ struct rust_new_stack2_args { // A new stack function suitable for calling through // upcall_call_shim_on_c_stack +// FIXME: Convert this to the same arrangement as +// the other upcalls, simplify __morestack extern "C" CDECL void upcall_new_stack(struct rust_new_stack2_args *args) { rust_task *task = rust_scheduler::get_task(); @@ -253,30 +493,21 @@ upcall_new_stack(struct rust_new_stack2_args *args) { args->args_sz); } +// FIXME: As above extern "C" CDECL void upcall_del_stack() { rust_task *task = rust_scheduler::get_task(); task->del_stack(); } -extern "C" _Unwind_Reason_Code -__gxx_personality_v0(int version, - _Unwind_Action actions, - uint64_t exception_class, - _Unwind_Exception *ue_header, - _Unwind_Context *context); - -extern "C" _Unwind_Reason_Code -upcall_rust_personality(int version, - _Unwind_Action actions, - uint64_t exception_class, - _Unwind_Exception *ue_header, - _Unwind_Context *context) { - return __gxx_personality_v0(version, - actions, - exception_class, - ue_header, - context); +// Landing pads need to call this to insert the +// correct limit into TLS. +// NB: This must run on the Rust stack because it +// needs to acquire the value of the stack pointer +extern "C" CDECL void +upcall_reset_stack_limit() { + rust_task *task = rust_scheduler::get_task(); + task->reset_stack_limit(); } // @@ -286,6 +517,5 @@ upcall_rust_personality(int version, // indent-tabs-mode: nil // c-basic-offset: 4 // buffer-file-coding-system: utf-8-unix -// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'"; // End: // |