From 65cfde953dbb7eaff1338937a8d8a479713c32ed Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 10 Feb 2015 23:24:16 -0500 Subject: [PATCH 1/2] hide the julia type pointer from the struct jl_value_t definition this makes references to types allocated in Julia recursively forward compatible with C structs --- base/pointer.jl | 2 +- doc/devdocs/object.rst | 186 +++++++++++++++++++++++++++---------- src/alloc.c | 70 +++++++------- src/array.c | 42 +++++---- src/builtins.c | 4 +- src/ccall.cpp | 18 ++-- src/cgutils.cpp | 206 ++++++++++++++++++++--------------------- src/codegen.cpp | 131 ++++++++++++-------------- src/dump.c | 24 ++--- src/gc.c | 137 ++++++++++++++------------- src/gf.c | 4 +- src/intrinsics.cpp | 20 ++-- src/jlapi.c | 11 +++ src/jltypes.c | 4 +- src/julia.h | 122 +++++++++++++----------- src/julia_internal.h | 14 +-- src/module.c | 2 +- src/task.c | 4 +- ui/repl.c | 2 +- 19 files changed, 551 insertions(+), 452 deletions(-) diff --git a/base/pointer.jl b/base/pointer.jl index fde8982206df3..b148690f77589 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -51,7 +51,7 @@ unsafe_store!{T}(p::Ptr{T}, x) = pointerset(p, convert(T,x), 1) # convert a raw Ptr to an object reference, and vice-versa unsafe_pointer_to_objref(x::Ptr) = ccall(:jl_value_ptr, Any, (Ptr{Void},), x) pointer_from_objref(x::ANY) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x) -data_pointer_from_objref(x::ANY) = pointer_from_objref(x)::Ptr{Void}+Core.sizeof(Int) +data_pointer_from_objref(x::ANY) = pointer_from_objref(x)::Ptr{Void} eltype{T}(::Type{Ptr{T}}) = T diff --git a/doc/devdocs/object.rst b/doc/devdocs/object.rst index 6eda9816584be..c5fabd9e4a689 100644 --- a/doc/devdocs/object.rst +++ b/doc/devdocs/object.rst @@ -5,84 +5,170 @@ Memory layout of Julia Objects Object layout (jl_value_t) -------------------------- -.. sidebar:: `special case. `_ +The :code:`jl_value_t` struct is the name for a block of memory owned by the Julia Garbage Collector, +representing the data associated with a Julia object in memory. +Absent any type information, it is simply an opaque pointer:: - :code:`jl_tuple_type->type = jl_tuple_type` + typedef struct jl_value_t* jl_pvalue_t; -The :code:`jl_value_t` struct defines the minimal header for a Julia -object in memory. -The :code:`type` field points to a -`jl_datatype_t `_ object, -(the jl_typeof() macro should be used to query it):: +Each :code:`jl_value_t` struct is contained in a :code:`jl_typetag_t` struct that contains metadata information +about the Julia object, such as its type and garbage-collector (gc) reachability:: - typedef struct _jl_value_t { - struct _jl_value_t *type; - } jl_value_t; + typedef struct { + opaque metadata; + jl_value_t value; + } jl_typetag_t; - #define jl_typeof(v) (((jl_value_t*)(v))->type) +The type of any julia object is an instance of a leaf :func:`jl_datatype_t` object. +The :func:`jl_typeof` function can be used to query for it:: + jl_value_t *jl_typeof(jl_value_t *v); -The layout of the rest of the object is dependant on its type. +The layout of the object is dependant on its type. +Reflection methods can be used to inspect that layout. +A field can be accessed by calling one of the get-field methods:: -e.g. a :func:`Base.tuple` object has an array of pointers to the -objects contained by the tuple:: + jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i); + jl_value_t *jl_get_field(jl_value_t *o, char *fld); - typedef struct { - struct _jl_value_t *type; - size_t length; - jl_value_t *data[]; - } jl_tuple_t; +If the field types are known, a priori, to be all pointers, +the values can also be extracted directly as an array access:: + + jl_value_t *v = value->fieldptr[n]; -e.g. a "boxed" uint16_t (created by :func:`jl_box_uint16`) is stored as -follows (assuming machine is 64-bit):: +As an example, a "boxed" uint16_t is stored as follows:: struct { - struct _jl_value_t *type; -- 8 bytes - uint16_t data; -- 2 bytes - -- 6 bytes padding + oqaque metadata; + struct { + uint16_t data; -- 2 bytes + } jl_value_t; }; -Structs for the built-in types are `defined in julia.h `_. The corresponding global jl_datatype_t objects are created by `jl_init_types() `_. +This object is created by :func:`jl_box_uint16`. +Note that the ``jl_value_t*`` pointer references the data portion, +not the metadata at the top of the struct. +A value may be stored "unboxed" in many circumstances +(just the data, without the metadata, and possibly not even stored but just kept in registers), +so it is unsafe to assume that a the address of a box is a unique identifier. +The "egal" test (corresponding to the `is` function in Julia), +should instead be used to compare two unknown objects for equivalence:: -Garbage collector mark bit --------------------------- + int jl_egal(jl_value_t *a, jl_value_t *b); + +This optimization should be relatively transparant to the API, +since the object will be "boxed" on-demand, whenever a :code:`jl_value_t*` is needed. + +Note that modification of a jl_value_t* in memory is permitted only if the object is mutable. +Otherwise, modification of the value may corrupt the program and the result will be undefined. +The mutability property of a value can be queried for with:: + + int jl_is_mutable(jl_value_t *v); + +If the object being stored is a :code:`jl_value_t*`, the Julia garbage-collector must be notified also:: -The garbage collector uses the low bit of the :code:`jl_value_t.type` -pointer as a flag to mark reachable objects (see :code:`gcval_t`). -During each mark/sweep cycle, the gc sets the mark bit of each -reachable object, deallocates objects that are not marked, then -clears the mark bits. While the mark/sweep is in progress the -:code:`jl_value_t.type` pointer is altered by the mark bit. The gc -uses the :func:`gc_typeof` macro to retrieve the original type -pointer:: + void gc_wb(jl_value_t *parent, jl_value_t *ptr); - #define gc_typeof(v) ((jl_value_t*)(((uptrint_t)jl_typeof(v))&~1UL)) +However, the embedding section of the manual is also required reading at this point, +for covering other details of boxing and unboxing various types, +and understanding the gc-interactions. +Mirror structs for some of the built-in types are `defined in julia.h `_. +The corresponding global ``jl_datatype_t`` objects are created by `jl_init_types in jltypes.c `_. + +Garbage collector mark bits +--------------------------- + +The garbage collector uses several bits from the metadata portion of the :code:`jl_typetag_t` +to track each object in the system. +Further details about this algorithm can be found in the comments of the `garbage-collector implementation in gc.c +`_. Object allocation ----------------- -Storage for new objects is allocated by :func:`newobj` in julia_internal.h:: +Most new objects are allocated by :func:`jl_new_structv`:: - STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) - { - jl_value_t *jv = (jl_value_t*)allocobj((1+nfields) * sizeof(void*)); - jv->type = type; - return jv; - } + jl_value_t *jl_new_struct(jl_datatype_t *type, ...); + jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uint32_t na); -.. sidebar:: :ref:`man-singleton-types` +Although, `isbits` objects can be also constructed directly from memory:: + + jl_value_t *jl_new_bits(jl_value_t *bt, void *data) + +And some objects have special constructors that must be used instead of the above functions: + +Types:: + + jl_datatype_t *jl_apply_type(jl_datatype_t *tc, jl_tuple_t *params); + jl_datatype_t *jl_apply_array_type(jl_datatype_t *type, size_t dim); + jl_uniontype_t *jl_new_uniontype(jl_tuple_t *types); + +While these are the most commonly used options, there are more low-level constructors too, +which you can find declared in `julia.h `_. +These are used in :func:`jl_init_types` to create the initial types needed to bootstrap the creation of the Julia system image. + +Tuples:: + jl_tuple_t *jl_tuple(size_t n, ...); + jl_tuple_t *jl_tuplev(size_t n, jl_value_t **v); + jl_tuple_t *jl_alloc_tuple(size_t n); + +The representation of tuples is highly unique in the Julia object representation ecosystem. +In some cases, a :func:`Base.tuple` object may be an array of pointers to the +objects contained by the tuple equivalent to:: + + typedef struct { + size_t length; + jl_value_t *data[length]; + } jl_tuple_t; + +However, in other cases, the tuple may be converted to an anonymous :func:`isbits` type +and stored unboxed, or it may not stored at all (if it is not being used in a generic context as a :code:`jl_value_t*`). + +Symbols:: + + jl_sym_t *jl_symbol(const char *str); + +Functions and LambdaStaticData:: + + jl_function_t *jl_new_generic_function(jl_sym_t *name); + jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_tuple_t *sparams); + jl_function_t *jl_new_closure(jl_fptr_t proc, jl_value_t *env, jl_lambda_info_t *li); + +Arrays:: + + jl_array_t *jl_new_array(jl_value_t *atype, jl_tuple_t *dims); + jl_array_t *jl_new_arrayv(jl_value_t *atype, ...); + jl_array_t *jl_alloc_array_1d(jl_value_t *atype, size_t nr); + jl_array_t *jl_alloc_array_2d(jl_value_t *atype, size_t nr, size_t nc); + jl_array_t *jl_alloc_array_3d(jl_value_t *atype, size_t nr, size_t nc, size_t z); + jl_array_t *jl_alloc_cell_1d(size_t n); + +Note that many of these have alternative allocation functions for various special-purposes. +The list here reflects the more common usages, but a more complete list can be found by reading the `julia.h header file +`_. + +Internal to Julia, storage is typically allocated by :func:`newstruct` (or :func:`newobj` for the special types):: + + jl_value_t *newstruct(jl_value_t *type); + jl_value_t *newobj(jl_value_t *type, size_t nfields); + +And at the lowest level, memory is getting allocated by a call to the garbage collector (in gc.c), +then tagged with its type:: + + jl_value_t *allocobj(size_t nbytes); + void jl_set_typeof(jl_value_t *v, jl_datatype_t *type); + +.. sidebar:: :ref:`man-singleton-types` Singleton types have only one instance and no data fields. - Singleton instances use only 8 bytes. + Singleton instances have a size of 0 bytes, + and consist only of their metadata. e.g. :data:`nothing::Void`. See :ref:`man-singleton-types` and :ref:`man-nothing` -Note that all objects are allocated in multiples of 8 bytes, so the -smallest object size is 16 bytes (8 byte type pointer + 8 bytes -data). :func:`allocobj` in gc.c allocates memory for new objects. -Memory is allocated from a pool for objects up to 2048 bytes, or -by malloc() otherwise. +Note that all objects are allocated in multiples of 4 bytes and aligned to the platform pointer size. +Memory is allocated from a pool for smaller objects, or directly with malloc() for large objects. diff --git a/src/alloc.c b/src/alloc.c index 8a817d3eb826e..3d4a67ac84661 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -160,10 +160,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); - jl_value_t *v = - (jl_value_t*)allocobj((NWORDS(LLT_ALIGN(nb,sizeof(void*)))+1)* - sizeof(void*)); - v->type = (jl_value_t*)bt; + jl_value_t *v = (jl_value_t*)newobj((jl_value_t*)bt, NWORDS(nb)); switch (nb) { case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break; case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break; @@ -251,7 +248,7 @@ jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i) { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); assert(i < jl_tuple_len(st->names)); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { return *(jl_value_t**)((char*)v + offs); } @@ -263,7 +260,7 @@ jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i) jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); if (i >= jl_tuple_len(st->names)) jl_bounds_error_int(v, i+1); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { jl_value_t *fval = *(jl_value_t**)((char*)v + offs); if (fval == NULL) @@ -276,7 +273,7 @@ jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i) void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs) { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { *(jl_value_t**)((char*)v + offs) = rhs; if(rhs != NULL) gc_wb(v, rhs); @@ -289,7 +286,7 @@ void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs) int jl_field_isdefined(jl_value_t *v, size_t i) { jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v); - size_t offs = jl_field_offset(st,i) + sizeof(void*); + size_t offs = jl_field_offset(st,i); if (st->fields[i].isptr) { return *(jl_value_t**)((char*)v + offs) != NULL; } @@ -320,7 +317,7 @@ DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uin } for(size_t i=na; i < nf; i++) { if (type->fields[i].isptr) - *(jl_value_t**)((char*)jv+jl_field_offset(type,i)+sizeof(void*)) = NULL; + *(jl_value_t**)((char*)jl_data_ptr(jv)+jl_field_offset(type,i)) = NULL; } return jv; } @@ -330,7 +327,7 @@ DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) if (type->instance != NULL) return type->instance; jl_value_t *jv = newstruct(type); if (type->size > 0) - memset(&((void**)jv)[1], 0, type->size); + memset(jl_data_ptr(jv), 0, type->size); return jv; } @@ -359,11 +356,11 @@ DLLEXPORT jl_tuple_t *jl_tuplev(size_t n, jl_value_t **v) jl_tuple_t *jl_tuple1(void *a) { #ifdef OVERLAP_TUPLE_LEN - jl_tuple_t *t = (jl_tuple_t*)alloc_2w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_1w(); #else - jl_tuple_t *t = (jl_tuple_t*)alloc_3w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_2w(); #endif - t->type = (jl_value_t*)jl_tuple_type; + jl_set_typeof(t, jl_tuple_type); jl_tuple_set_len_unsafe(t, 1); jl_tupleset(t, 0, a); return t; @@ -372,11 +369,11 @@ jl_tuple_t *jl_tuple1(void *a) jl_tuple_t *jl_tuple2(void *a, void *b) { #ifdef OVERLAP_TUPLE_LEN - jl_tuple_t *t = (jl_tuple_t*)alloc_3w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_2w(); #else - jl_tuple_t *t = (jl_tuple_t*)alloc_4w(); + jl_tuple_t *t = (jl_tuple_t*)alloc_3w(); #endif - t->type = (jl_value_t*)jl_tuple_type; + jl_set_typeof(t, jl_tuple_type); jl_tuple_set_len_unsafe(t, 2); jl_tupleset(t, 0, a); jl_tupleset(t, 1, b); @@ -433,8 +430,8 @@ jl_tuple_t *jl_tuple_fill(size_t n, jl_value_t *v) DLLEXPORT jl_function_t *jl_new_closure(jl_fptr_t fptr, jl_value_t *env, jl_lambda_info_t *linfo) { - jl_function_t *f = (jl_function_t*)alloc_4w(); - f->type = (jl_value_t*)jl_function_type; + jl_function_t *f = (jl_function_t*)alloc_3w(); assert(NWORDS(sizeof(jl_function_t))==3); + jl_set_typeof(f, jl_function_type); f->fptr = (fptr!=NULL ? fptr : linfo->fptr); f->env = env; f->linfo = linfo; @@ -446,7 +443,7 @@ jl_lambda_info_t *jl_new_lambda_info(jl_value_t *ast, jl_tuple_t *sparams) { jl_lambda_info_t *li = (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - LAMBDA_INFO_NW); + NWORDS(sizeof(jl_lambda_info_t))); li->ast = ast; li->file = null_sym; li->line = 0; @@ -498,23 +495,23 @@ static jl_sym_t *mk_symbol(const char *str) #endif jl_sym_t *sym; size_t len = strlen(str); - size_t nb = (sizeof(jl_sym_t)+len+1+7)&-8; + size_t nb = (sizeof(jl_typetag_t)+sizeof(jl_sym_t)+len+1+7)&-8; if (nb >= SYM_POOL_SIZE) { jl_exceptionf(jl_argumenterror_type, "Symbol length exceeds maximum length"); } #ifdef MEMDEBUG - sym = (jl_sym_t*)malloc(nb); + sym = (jl_sym_t*)((jl_typetag_t*)malloc(nb))->value; #else if (sym_pool == NULL || pool_ptr+nb > sym_pool+SYM_POOL_SIZE) { sym_pool = (char*)malloc(SYM_POOL_SIZE); pool_ptr = sym_pool; } - sym = (jl_sym_t*)pool_ptr; + sym = (jl_sym_t*)((jl_typetag_t*)malloc(nb))->value; pool_ptr += nb; #endif - sym->type = (jl_value_t*)jl_sym_type; + jl_set_typeof(sym, jl_sym_type); sym->left = sym->right = NULL; sym->hash = hash_symbol(str, len); strcpy(&sym->name[0], str); @@ -524,7 +521,7 @@ static jl_sym_t *mk_symbol(const char *str) static void unmark_symbols_(jl_sym_t *root) { while (root != NULL) { - root->type = (jl_value_t*)(((uptrint_t)root->type)&~3UL); + jl_set_typeof(root, jl_sym_type); unmark_symbols_(root->left); root = root->right; } @@ -618,7 +615,7 @@ DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, int32_t len) jl_typename_t *jl_new_typename(jl_sym_t *name) { - jl_typename_t *tn=(jl_typename_t*)newobj((jl_value_t*)jl_typename_type, 6); + jl_typename_t *tn=(jl_typename_t*)newobj((jl_value_t*)jl_typename_type, NWORDS(sizeof(jl_typename_t))); tn->name = name; tn->module = jl_current_module; tn->primary = NULL; @@ -640,8 +637,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields) { return (jl_datatype_t*) newobj((jl_value_t*)jl_datatype_type, - NWORDS(sizeof(jl_datatype_t) - sizeof(void*) + - nfields*sizeof(jl_fielddesc_t))); + NWORDS(sizeof(jl_datatype_t) + nfields*sizeof(jl_fielddesc_t))); } void jl_compute_field_offsets(jl_datatype_t *st) @@ -765,7 +761,7 @@ jl_datatype_t *jl_new_bitstype(jl_value_t *name, jl_datatype_t *super, jl_uniontype_t *jl_new_uniontype(jl_tuple_t *types) { - jl_uniontype_t *t = (jl_uniontype_t*)newobj((jl_value_t*)jl_uniontype_type,1); + jl_uniontype_t *t = (jl_uniontype_t*)newobj((jl_value_t*)jl_uniontype_type,NWORDS(sizeof(jl_uniontype_t))); // don't make unions of 1 type; Union(T)==T assert(jl_tuple_len(types) != 1); t->types = types; @@ -776,7 +772,7 @@ jl_uniontype_t *jl_new_uniontype(jl_tuple_t *types) jl_typector_t *jl_new_type_ctor(jl_tuple_t *params, jl_value_t *body) { - jl_typector_t *tc = (jl_typector_t*)newobj((jl_value_t*)jl_typector_type,2); + jl_typector_t *tc = (jl_typector_t*)newobj((jl_value_t*)jl_typector_type,NWORDS(sizeof(jl_typector_t))); tc->parameters = params; tc->body = body; return (jl_typector_t*)tc; @@ -790,7 +786,7 @@ jl_value_t *jl_box##nb(jl_datatype_t *t, int##nb##_t x) \ assert(jl_isbits(t)); \ assert(jl_datatype_size(t) == sizeof(x)); \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)t; \ + jl_set_typeof(v, t); \ *(int##nb##_t*)jl_data_ptr(v) = x; \ return v; \ } @@ -828,7 +824,7 @@ UNBOX_FUNC(gensym, ssize_t) jl_value_t *pfx##_##typ(c_type x) \ { \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)jl_##typ##_type; \ + jl_set_typeof(v, jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -850,7 +846,7 @@ jl_value_t *jl_box_##typ(c_type x) \ if ((u##c_type)idx < (u##c_type)NBOX_C) \ return boxed_##typ##_cache[idx]; \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)jl_##typ##_type; \ + jl_set_typeof(v, jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -861,7 +857,7 @@ jl_value_t *jl_box_##typ(c_type x) \ if (x < NBOX_C) \ return boxed_##typ##_cache[x]; \ jl_value_t *v = (jl_value_t*)alloc_##nw##w(); \ - v->type = (jl_value_t*)jl_##typ##_type; \ + jl_set_typeof(v, jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -954,8 +950,8 @@ jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) { jl_array_t *ar = n==0 ? (jl_array_t*)jl_an_empty_cell : jl_alloc_cell_1d(n); JL_GC_PUSH1(&ar); - jl_expr_t *ex = (jl_expr_t*)alloc_4w(); - ex->type = (jl_value_t*)jl_expr_type; + jl_expr_t *ex = (jl_expr_t*)alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); + jl_set_typeof(ex, jl_expr_type); ex->head = head; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; @@ -971,8 +967,8 @@ JL_CALLABLE(jl_f_new_expr) JL_GC_PUSH1(&ar); for(size_t i=0; i < nargs-1; i++) jl_cellset(ar, i, args[i+1]); - jl_expr_t *ex = (jl_expr_t*)alloc_4w(); - ex->type = (jl_value_t*)jl_expr_type; + jl_expr_t *ex = (jl_expr_t*)alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); + jl_set_typeof(ex, jl_expr_type); ex->head = (jl_sym_t*)args[0]; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; diff --git a/src/array.c b/src/array.c index d6f80a720f3f6..fcce1ee085333 100644 --- a/src/array.c +++ b/src/array.c @@ -14,6 +14,9 @@ extern "C" { #endif +#define JL_ARRAY_ALIGN(jl_value, nbytes) (LLT_ALIGN((jl_value)+sizeof(jl_typetag_t), nbytes)-sizeof(jl_typetag_t)) + + // array constructors --------------------------------------------------------- static inline int store_unboxed(jl_value_t *el_type) @@ -69,16 +72,15 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, } int ndimwords = jl_array_ndimwords(ndims); - size_t tsz = sizeof(jl_array_t); - tsz += ndimwords*sizeof(size_t); + int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); if (tot <= ARRAY_INLINE_NBYTES) { if (isunboxed && elsz >= 4) - tsz = (tsz+15)&-16; // align data area 16 + tsz = JL_ARRAY_ALIGN(tsz, 16); // align data area 16 size_t doffs = tsz; tsz += tot; - tsz = (tsz+15)&-16; // align whole object 16 + tsz = JL_ARRAY_ALIGN(tsz, 16); // align whole object 16 a = (jl_array_t*)allocobj(tsz); - a->type = atype; + jl_set_typeof(a, atype); a->how = 0; data = (char*)a + doffs; if (tot > 0 && !isunboxed) { @@ -86,10 +88,10 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, } } else { - tsz = (tsz+15)&-16; // align whole object size 16 + tsz = JL_ARRAY_ALIGN(tsz, 16); // align whole object 16 a = (jl_array_t*)allocobj(tsz); JL_GC_PUSH1(&a); - a->type = atype; + jl_set_typeof(a, atype); // temporarily initialize to make gc-safe a->data = NULL; a->how = 2; @@ -99,7 +101,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, memset(data, 0, tot); JL_GC_POP(); } - a->pooled = tsz <= 2048; + a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; a->data = data; if (elsz == 1) ((char*)data)[tot-1] = '\0'; @@ -148,10 +150,10 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *di size_t ndims = jl_tuple_len(dims); int ndimwords = jl_array_ndimwords(ndims); - int tsz = (sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16; + int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), 16); a = (jl_array_t*)allocobj(tsz); - a->type = atype; - a->pooled = tsz <= 2048; + jl_set_typeof(a, atype); + a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; a->ndims = ndims; a->offset = 0; a->data = NULL; @@ -214,10 +216,12 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, elsz = jl_datatype_size(el_type); else elsz = sizeof(void*); - int tsz = (sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16; + + int ndimwords = jl_array_ndimwords(1); + int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); a = (jl_array_t*)allocobj(tsz); - a->pooled = tsz <= 2048; - a->type = atype; + jl_set_typeof(a, atype); + a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; @@ -265,10 +269,10 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims, elsz = sizeof(void*); int ndimwords = jl_array_ndimwords(ndims); - int tsz = (sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16; + int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); a = (jl_array_t*)allocobj(tsz); - a->pooled = tsz <= 2048; - a->type = atype; + jl_set_typeof(a, atype); + a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; @@ -340,8 +344,8 @@ jl_value_t *jl_array_to_string(jl_array_t *a) // TODO: check type of array? jl_datatype_t *string_type = u8_isvalid((char*)a->data, jl_array_len(a)) == 1 ? // ASCII jl_ascii_string_type : jl_utf8_string_type; - jl_value_t *s = (jl_value_t*)alloc_2w(); - s->type = (jl_value_t*)string_type; + jl_value_t *s = (jl_value_t*)alloc_1w(); + jl_set_typeof(s, string_type); jl_set_nth_field(s, 0, (jl_value_t*)a); return s; } diff --git a/src/builtins.c b/src/builtins.c index 836ccb2ea7d76..a6a1e591cb440 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1291,8 +1291,8 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) else if (jl_typeof(v) == NULL) { n += jl_printf(out, ""); } - else if ((uptrint_t)v->type < 4096U) { - n += jl_printf(out, "", (int)(uptrint_t)v->type); + else if (jl_typetagof(v)->type_bits < 4096U) { + n += jl_printf(out, "", (int)jl_typetagof(v)->type_bits); } else if (jl_is_lambda_info(v)) { jl_lambda_info_t *li = (jl_lambda_info_t*)v; diff --git a/src/ccall.cpp b/src/ccall.cpp index c37b607b3921e..a1bb5e4724521 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -328,7 +328,7 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv, } if (jl_is_mutable_datatype(ety)) { // no copy, just reference the data field - return builder.CreateBitCast(emit_nthptr_addr(jv, (size_t)1), ty); // skip type tag field + return builder.CreateBitCast(jv, ty); } else if (jl_is_immutable_datatype(ety) && jt != (jl_value_t*)jl_voidpointer_type) { // yes copy @@ -338,12 +338,12 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv, else nbytes = tbaa_decorate(tbaa_datatype, builder.CreateLoad( builder.CreateGEP(builder.CreatePointerCast(emit_typeof(jv), T_pint32), - ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/4)), + ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/sizeof(int32_t))), false)); *needStackRestore = true; AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes); ai->setAlignment(16); - builder.CreateMemCpy(ai, builder.CreateBitCast(emit_nthptr_addr(jv, (size_t)1), T_pint8), nbytes, 1); + builder.CreateMemCpy(ai, builder.CreateBitCast(jv, T_pint8), nbytes, 1); return builder.CreateBitCast(ai, ty); } // emit maybe copy @@ -360,16 +360,16 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv, T_int1); builder.CreateCondBr(ismutable, mutableBB, immutableBB); builder.SetInsertPoint(mutableBB); - Value *p1 = builder.CreatePointerCast(emit_nthptr_addr(jv, (size_t)1), ty); // skip type tag field + Value *p1 = builder.CreatePointerCast(jv, ty); builder.CreateBr(afterBB); builder.SetInsertPoint(immutableBB); Value *nbytes = tbaa_decorate(tbaa_datatype, builder.CreateLoad( builder.CreateGEP(builder.CreatePointerCast(jvt, T_pint32), - ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/4)), + ConstantInt::get(T_size, offsetof(jl_datatype_t,size)/sizeof(int32_t))), false)); AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes); ai->setAlignment(16); - builder.CreateMemCpy(ai, builder.CreatePointerCast(emit_nthptr_addr(jv, (size_t)1), T_pint8), nbytes, 1); + builder.CreateMemCpy(ai, builder.CreatePointerCast(jv, T_pint8), nbytes, 1); Value *p2 = builder.CreatePointerCast(ai, ty); builder.CreateBr(afterBB); builder.SetInsertPoint(afterBB); @@ -1158,7 +1158,7 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) ary = emit_unbox(largty, emit_unboxed(argi, ctx), jl_tupleref(tt, 0)); } JL_GC_POP(); - return mark_or_box_ccall_result(builder.CreateBitCast(emit_nthptr_addr(ary, addressOf?1:0), lrt), + return mark_or_box_ccall_result(builder.CreateBitCast(ary, lrt), args[2], rt, static_rt, ctx); } if (fptr == (void *) &jl_is_leaf_type || @@ -1217,7 +1217,7 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) argvals[0] = result; } else { - argvals[0] = builder.CreateBitCast(emit_nthptr_addr(result, (size_t)1), fargt_sig[0]); + argvals[0] = builder.CreateBitCast(result, fargt_sig[0]); } } @@ -1414,7 +1414,7 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) Value *newst = emit_new_struct(rt,1,NULL,ctx); assert(newst != NULL && "Type was not concrete"); if (newst->getType()->isPointerTy()) { - builder.CreateStore(result,builder.CreateBitCast(emit_nthptr_addr(newst, (size_t)1), prt->getPointerTo())); + builder.CreateStore(result,builder.CreateBitCast(newst, prt->getPointerTo())); result = newst; } else if (lrt != prt) { diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 3803916dc7698..a3e8ca8561907 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -785,32 +785,73 @@ static Value *mark_julia_type(Value *v, jl_value_t *jt) return v; } -// --- generating various error checks --- +// --- generating various field accessors --- -static jl_value_t *llvm_type_to_julia(Type *t, bool err=true); +static Value *emit_nthptr_addr(Value *v, ssize_t n) +{ + return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), + ConstantInt::get(T_size, (ssize_t)n)); +} + +static Value *emit_nthptr_addr(Value *v, Value *idx) +{ + return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), idx); +} + +static Value *emit_nthptr(Value *v, ssize_t n, MDNode *tbaa) +{ + // p = (jl_value_t**)v; p[n] + Value *vptr = emit_nthptr_addr(v, n); + return tbaa_decorate(tbaa,builder.CreateLoad(vptr, false)); +} + +static Value *emit_nthptr_recast(Value *v, ssize_t n, MDNode *tbaa, Type* ptype) +{ + // p = (jl_value_t**)v; *(ptype)&p[n] + Value *vptr = emit_nthptr_addr(v, n); + return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); +} + +static Value *emit_nthptr_recast(Value *v, Value *idx, MDNode *tbaa, Type *ptype) +{ + // p = (jl_value_t**)v; *(ptype)&p[n] + Value *vptr = emit_nthptr_addr(v, idx); + return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); +} + +static Value *emit_typeptr_addr(Value *p) +{ + ssize_t offset = offsetof(jl_typetag_t,type) - offsetof(jl_typetag_t,value); + offset /= (signed)sizeof(jl_value_t*); // important: division must be signed + return emit_nthptr_addr(p, offset); +} static Value *emit_typeof(Value *p) { // given p, a jl_value_t*, compute its type tag if (p->getType() == jl_pvalue_llvmt) { Value *tt = builder.CreateBitCast(p, jl_ppvalue_llvmt); - tt = builder. - CreateLoad(builder.CreateGEP(tt,ConstantInt::get(T_size,0)), - false); - tt = builder. - CreateIntToPtr(builder. CreateAnd(builder.CreatePtrToInt(tt, T_int64), ConstantInt::get(T_int64,~(uptrint_t)3)), jl_pvalue_llvmt); + tt = builder.CreateLoad(emit_typeptr_addr(tt), false); #ifdef OVERLAP_TUPLE_LEN - tt = builder. - CreateIntToPtr(builder. - CreateAnd(builder.CreatePtrToInt(tt, T_int64), - ConstantInt::get(T_int64,0x000ffffffffffffe)), - jl_pvalue_llvmt); + tt = builder.CreateIntToPtr(builder.CreateAnd( + builder.CreatePtrToInt(tt, T_int64), + ConstantInt::get(T_int64,0x000ffffffffffffe)), + jl_pvalue_llvmt); +#else + tt = builder.CreateIntToPtr(builder.CreateAnd( + builder.CreatePtrToInt(tt, T_size), + ConstantInt::get(T_size,~(uptrint_t)3)), + jl_pvalue_llvmt); #endif return tt; } return literal_pointer_val(julia_type_of(p)); } +// --- generating various error checks --- + +static jl_value_t *llvm_type_to_julia(Type *t, bool err=true); + static void just_emit_error(const std::string &txt, jl_codectx_t *ctx) { Value *zeros[2] = { ConstantInt::get(T_int32, 0), @@ -960,38 +1001,6 @@ static Value *emit_bounds_check(Value *a, jl_value_t *ty, Value *i, Value *len, // --- loading and storing --- -static Value *emit_nthptr_addr(Value *v, size_t n) -{ - return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), - ConstantInt::get(T_size, n)); -} - -static Value *emit_nthptr_addr(Value *v, Value *idx) -{ - return builder.CreateGEP(builder.CreateBitCast(v, jl_ppvalue_llvmt), idx); -} - -static Value *emit_nthptr(Value *v, size_t n, MDNode *tbaa) -{ - // p = (jl_value_t**)v; p[n] - Value *vptr = emit_nthptr_addr(v, n); - return tbaa_decorate(tbaa,builder.CreateLoad(vptr, false)); -} - -static Value *emit_nthptr_recast(Value *v, size_t n, MDNode *tbaa, Type* ptype) -{ - // p = (jl_value_t**)v; *(ptype)&p[n] - Value *vptr = emit_nthptr_addr(v, n); - return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); -} - -static Value *emit_nthptr_recast(Value *v, Value *idx, MDNode *tbaa, Type *ptype) -{ - // p = (jl_value_t**)v; *(ptype)&p[n] - Value *vptr = emit_nthptr_addr(v, idx); - return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); -} - static Value *ghostValue(jl_value_t *ty); static Value *typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, @@ -1139,11 +1148,11 @@ static Value *emit_tuplelen(Value *t,jl_value_t *jt) Type *ty = t->getType(); if (ty == jl_pvalue_llvmt) { //boxed #ifdef OVERLAP_TUPLE_LEN - Value *lenbits = emit_nthptr(t, (size_t)0); + Value *lenbits = builder.CreateLoad(emit_typeptr_addr(t)); return builder.CreateLShr(builder.CreatePtrToInt(lenbits, T_int64), ConstantInt::get(T_int32, 52)); #else - return emit_nthptr_recast(t, 1, tbaa_tuplelen, T_psize); + return emit_nthptr_recast(t, offsetof(jl_tuple_t,length)/sizeof(jl_value_t*), tbaa_tuplelen, T_psize); #endif } else { //unboxed @@ -1151,7 +1160,7 @@ static Value *emit_tuplelen(Value *t,jl_value_t *jt) } } -static Value *emit_tupleset(Value *tuple, Value *ival, Value *x, jl_value_t *jt, jl_codectx_t *ctx) +static Value *emit_tupleset(Value *tuple, Value *ival0, Value *x, jl_value_t *jt, jl_codectx_t *ctx) { if (tuple == NULL) { // A typecheck must have caught this one @@ -1162,17 +1171,17 @@ static Value *emit_tupleset(Value *tuple, Value *ival, Value *x, jl_value_t *jt, if (ty == jl_pvalue_llvmt) { //boxed #ifdef OVERLAP_TUPLE_LEN Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt), - ival); + ival0); #else Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt), - builder.CreateAdd(ConstantInt::get(T_size,1),ival)); + builder.CreateAdd(ConstantInt::get(T_size,1),ival0)); #endif builder.CreateStore(x,slot); return tuple; } - ConstantInt *idx = dyn_cast(ival); + ConstantInt *idx = dyn_cast(ival0); assert(idx != NULL && "tuplesets must use constant indices"); - unsigned ci = (unsigned)idx->getZExtValue()-1; + unsigned ci = (unsigned)idx->getZExtValue(); if (ty->isVectorTy()) { return mark_julia_type(builder.CreateInsertElement(tuple,x,ConstantInt::get(T_int32,ci)), jt); } @@ -1199,7 +1208,7 @@ static Value *allocate_box_dynamic(Value *jlty, Value *nb, Value *v); static void jl_add_linfo_root(jl_lambda_info_t *li, jl_value_t *val); // Julia semantics -static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codectx_t *ctx) +static Value *emit_tupleref(Value *tuple, Value *ival0, jl_value_t *jt, jl_codectx_t *ctx) { if (tuple == NULL) { // A typecheck must have caught this one @@ -1209,38 +1218,38 @@ static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codect Type *ty = tuple->getType(); if (ty == jl_pvalue_llvmt) { //boxed #ifdef OVERLAP_TUPLE_LEN - Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt),ival); + Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt),ival0); #else Value *slot = builder.CreateGEP(builder.CreateBitCast(tuple, jl_ppvalue_llvmt), - builder.CreateAdd(ConstantInt::get(T_size,1),ival)); + builder.CreateAdd(ConstantInt::get(T_size,1),ival0)); #endif return builder.CreateLoad(slot); } - ConstantInt *idx = dyn_cast(ival); - unsigned ci = idx ? (unsigned)idx->getZExtValue()-1 : (unsigned)-1; + ConstantInt *idx = dyn_cast(ival0); + unsigned ci = idx ? (unsigned)idx->getZExtValue() : (unsigned)-1; if (ty->isVectorTy()) { - Type *ity = ival->getType(); + Type *ity = ival0->getType(); assert(ity->isIntegerTy()); IntegerType *iity = dyn_cast(ity); // ExtractElement needs i32 *sigh* if (iity->getBitWidth() > 32) - ival = builder.CreateTrunc(ival,T_int32); + ival0 = builder.CreateTrunc(ival0,T_int32); else if (iity->getBitWidth() < 32) - ival = builder.CreateZExt(ival,T_int32); - Value *v = builder.CreateExtractElement(tuple,builder.CreateSub(ival,ConstantInt::get(T_int32,1))); + ival0 = builder.CreateZExt(ival0,T_int32); + Value *v = builder.CreateExtractElement(tuple,ival0); if (idx) { v = mark_julia_type(v,jl_tupleref(jt,ci)); } else { if (sizeof(void*) != 4) - ival = builder.CreateZExt(ival,T_size); + ival0 = builder.CreateZExt(ival0,T_size); if (is_tupletype_homogeneous((jl_tuple_t*)jt)) { v = mark_julia_type(v, jl_t0(jt)); } else { jl_add_linfo_root(ctx->linfo, jt); v = allocate_box_dynamic(emit_tupleref(literal_pointer_val(jt), - ival, jl_typeof(jt), ctx), + ival0, jl_typeof(jt), ctx), ConstantInt::get(T_size,ty->getScalarSizeInBits()/8), v); } } @@ -1277,14 +1286,14 @@ static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codect tbaa_decorate(tbaa_user, builder.CreateStore(tuple,tempSpace)); Value *idxs[2]; idxs[0] = ConstantInt::get(T_size,0); - idxs[1] = builder.CreateSub(ival,ConstantInt::get(T_size,1)); + idxs[1] = ival0; Value *v = builder.CreateGEP(tempSpace,ArrayRef(&idxs[0],2)); if (idx) { v = mark_julia_type(tbaa_decorate(tbaa_user, builder.CreateLoad(v)), jl_tupleref(jt,ci)); } else { jl_add_linfo_root(ctx->linfo, jt); - Value *lty = emit_tupleref(literal_pointer_val(jt), ival, jl_typeof(jt), ctx); + Value *lty = emit_tupleref(literal_pointer_val(jt), ival0, jl_typeof(jt), ctx); size_t i, l = jl_tuple_len(jt); if (is_tupletype_homogeneous((jl_tuple_t*)jt) && jl_isbits(jl_t0(jt))) { v = mark_julia_type(tbaa_decorate(tbaa_user, builder.CreateLoad(v)), jl_t0(jt)); @@ -1316,20 +1325,20 @@ static Value *emit_tupleref(Value *tuple, Value *ival, jl_value_t *jt, jl_codect BasicBlock *after = BasicBlock::Create(getGlobalContext(),"after_switch",ctx->f); BasicBlock *deflt = BasicBlock::Create(getGlobalContext(),"default_case",ctx->f); // Create the switch - SwitchInst *sw = builder.CreateSwitch(ival,deflt,n); + SwitchInst *sw = builder.CreateSwitch(ival0,deflt,n); // Anything else is a bounds error builder.SetInsertPoint(deflt); Value *tmp = builder.CreateAlloca(ty); builder.CreateStore(tuple, tmp); jl_add_linfo_root(ctx->linfo, jt); - builder.CreateCall3(prepare_call(jluboundserror_func), builder.CreatePointerCast(tmp, T_pint8), literal_pointer_val(jt), ival); + builder.CreateCall3(prepare_call(jluboundserror_func), builder.CreatePointerCast(tmp, T_pint8), literal_pointer_val(jt), ival0); builder.CreateUnreachable(); size_t ntuple = jl_tuple_len(jt); PHINode *ret = PHINode::Create(jl_pvalue_llvmt, ntuple); // Now for the cases for (size_t i = 0, j = 0; i < ntuple; ++i) { BasicBlock *blk = BasicBlock::Create(getGlobalContext(),"case",ctx->f); - sw->addCase(ConstantInt::get((IntegerType*)T_size,i+1),blk); + sw->addCase(ConstantInt::get((IntegerType*)T_size,i),blk); builder.SetInsertPoint(blk); jl_value_t *jltype = jl_tupleref(jt,i); Type *ty = julia_struct_to_llvm(jltype); @@ -1368,19 +1377,7 @@ static Value *emit_n_varargs(jl_codectx_t *ctx) static Value *emit_arraysize(Value *t, Value *dim) { -#ifdef STORE_ARRAY_LEN -#ifdef _P64 - int o = 3; -#else - int o = 4; -#endif -#else -#ifdef _P64 - int o = 2; -#else - int o = 3; -#endif -#endif + int o = offsetof(jl_array_t, nrows)/sizeof(void*) - 1; return emit_nthptr_recast(t, builder.CreateAdd(dim, ConstantInt::get(dim->getType(), o)), tbaa_arraysize, T_psize); @@ -1410,7 +1407,7 @@ static Value *emit_arraylen_prim(Value *t, jl_value_t *ty) { #ifdef STORE_ARRAY_LEN (void)ty; - Value *addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 2); + Value *addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 1); //index (not offset) of length field in jl_parray_llvmt return tbaa_decorate(tbaa_arraylen, builder.CreateLoad(addr, false)); #else jl_value_t *p1 = jl_tparam1(ty); @@ -1442,7 +1439,7 @@ static Value *emit_arraylen(Value *t, jl_value_t *ex, jl_codectx_t *ctx) static Value *emit_arrayptr(Value *t) { - Value* addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 1); + Value* addr = builder.CreateStructGEP(builder.CreateBitCast(t,jl_parray_llvmt), 0); //index (not offset) of data field in jl_parray_llvmt return tbaa_decorate(tbaa_arrayptr, builder.CreateLoad(addr, false)); } @@ -1474,8 +1471,7 @@ static void assign_arrayvar(jl_arrayvar_t &av, Value *ar) static Value *data_pointer(Value *x) { - return builder.CreateGEP(builder.CreateBitCast(x, jl_ppvalue_llvmt), - ConstantInt::get(T_size, 1)); + return builder.CreateBitCast(x, jl_ppvalue_llvmt); } static Value *emit_array_nd_index(Value *a, jl_value_t *ex, size_t nd, jl_value_t **args, @@ -1551,7 +1547,7 @@ static Value *tpropagate(Value *a, Value *b) static Value *init_bits_value(Value *newv, Value *jt, Type *t, Value *v) { - builder.CreateStore(jt, builder.CreateBitCast(newv, jl_ppvalue_llvmt)); + builder.CreateStore(jt, builder.CreateBitCast(emit_typeptr_addr(newv), jl_ppvalue_llvmt)); builder.CreateStore(v , builder.CreateBitCast(data_pointer(newv), PointerType::get(t,0))); return newv; @@ -1564,9 +1560,7 @@ static Value *allocate_box_dynamic(Value *jlty, Value *nb, Value *v) if (v->getType()->isPointerTy()) { v = builder.CreatePtrToInt(v, T_size); } - Value *newv = builder.CreateCall(prepare_call(jlallocobj_func), - builder.CreateAdd(nb, - ConstantInt::get(T_size, sizeof(void*)))); + Value *newv = builder.CreateCall(prepare_call(jlallocobj_func), nb); // TODO: make sure this is rooted. I think it is. return init_bits_value(newv, jlty, v->getType(), v); } @@ -1714,9 +1708,9 @@ static Value *boxed(Value *v, jl_codectx_t *ctx, jl_value_t *jt) make_gcroot(tpl,ctx); for (size_t i = 0; i < n; ++i) { jl_value_t *jti = jl_tupleref(jt,i); - Value *vi = emit_tupleref(v, ConstantInt::get(T_size,i+1), jt, ctx); + Value *vi = emit_tupleref(v, ConstantInt::get(T_size,i), jt, ctx); Value *boxedvi = boxed(vi, ctx, jti); - emit_tupleset(tpl, ConstantInt::get(T_size,i+1), boxedvi, jt, ctx); + emit_tupleset(tpl, ConstantInt::get(T_size,i), boxedvi, jt, ctx); } ctx->argDepth = last_depth; return tpl; @@ -1735,9 +1729,9 @@ static Value *boxed(Value *v, jl_codectx_t *ctx, jl_value_t *jt) if (jb == jl_float64_type) { // manually inline alloc & init of Float64 box. cheap, I know. #ifdef _P64 - Value *newv = builder.CreateCall(prepare_call(jlalloc2w_func)); + Value *newv = builder.CreateCall(prepare_call(jlalloc1w_func)); #else - Value *newv = builder.CreateCall(prepare_call(jlalloc3w_func)); + Value *newv = builder.CreateCall(prepare_call(jlalloc2w_func)); #endif return init_bits_value(newv, literal_pointer_val(jt), t, v); } @@ -1773,7 +1767,7 @@ static void emit_cpointercheck(Value *x, const std::string &msg, emit_typecheck(t, (jl_value_t*)jl_datatype_type, msg, ctx); Value *istype = - builder.CreateICmpEQ(emit_nthptr(t, offsetof(jl_datatype_t,name)/sizeof(char*), tbaa_datatype), + builder.CreateICmpEQ(emit_nthptr(t, (ssize_t)(offsetof(jl_datatype_t,name)/sizeof(char*)), tbaa_datatype), literal_pointer_val((jl_value_t*)jl_pointer_type->name)); BasicBlock *failBB = BasicBlock::Create(getGlobalContext(),"fail",ctx->f); BasicBlock *passBB = BasicBlock::Create(getGlobalContext(),"pass"); @@ -1790,12 +1784,12 @@ static void emit_cpointercheck(Value *x, const std::string &msg, // allocation for known size object static Value* emit_allocobj(size_t static_size) { - if (static_size == sizeof(void*)*2) + if (static_size == sizeof(void*)*1) + return builder.CreateCall(prepare_call(jlalloc1w_func)); + else if (static_size == sizeof(void*)*2) return builder.CreateCall(prepare_call(jlalloc2w_func)); else if (static_size == sizeof(void*)*3) return builder.CreateCall(prepare_call(jlalloc3w_func)); - else if (static_size == sizeof(void*)*4) - return builder.CreateCall(prepare_call(jlalloc4w_func)); else return builder.CreateCall(prepare_call(jlallocobj_func), ConstantInt::get(T_size, static_size)); @@ -1804,10 +1798,8 @@ static Value* emit_allocobj(size_t static_size) // if ptr is NULL this emits a write barrier _back_ static void emit_write_barrier(jl_codectx_t* ctx, Value *parent, Value *ptr) { - /* builder.CreateCall2(wbfunc, builder.CreateBitCast(parent, jl_pvalue_llvmt), builder.CreateBitCast(ptr, jl_pvalue_llvmt)); - return;*/ - parent = builder.CreateBitCast(parent, T_psize); - Value* parent_type = builder.CreateLoad(parent); + Value* parenttag = builder.CreateBitCast(emit_typeptr_addr(parent), T_psize); + Value* parent_type = builder.CreateLoad(parenttag); Value* parent_mark_bits = builder.CreateAnd(parent_type, 1); // the branch hint does not seem to make it to the generated code @@ -1820,7 +1812,7 @@ static void emit_write_barrier(jl_codectx_t* ctx, Value *parent, Value *ptr) builder.CreateCondBr(parent_marked, barrier_may_trigger, cont); builder.SetInsertPoint(barrier_may_trigger); - Value* ptr_mark_bit = builder.CreateAnd(builder.CreateLoad(builder.CreateBitCast(ptr, T_psize)), 1); + Value* ptr_mark_bit = builder.CreateAnd(builder.CreateLoad(builder.CreateBitCast(emit_typeptr_addr(ptr), T_psize)), 1); Value* ptr_not_marked = builder.CreateICmpEQ(ptr_mark_bit, ConstantInt::get(T_size, 0)); builder.CreateCondBr(ptr_not_marked, barrier_trigger, cont); builder.SetInsertPoint(barrier_trigger); @@ -1844,15 +1836,15 @@ static void emit_checked_write_barrier(jl_codectx_t *ctx, Value *parent, Value * builder.SetInsertPoint(cont); } -static void emit_setfield(jl_datatype_t *sty, Value *strct, size_t idx, +static void emit_setfield(jl_datatype_t *sty, Value *strct, size_t idx0, Value *rhs, jl_codectx_t *ctx, bool checked, bool wb) { if (sty->mutabl || !checked) { Value *addr = builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), - ConstantInt::get(T_size, sty->fields[idx].offset + sizeof(void*))); - jl_value_t *jfty = jl_tupleref(sty->types, idx); - if (sty->fields[idx].isptr) { + ConstantInt::get(T_size, sty->fields[idx0].offset)); + jl_value_t *jfty = jl_tupleref(sty->types, idx0); + if (sty->fields[idx0].isptr) { rhs = boxed(rhs, ctx); builder.CreateStore(rhs, builder.CreateBitCast(addr, jl_ppvalue_llvmt)); @@ -1915,9 +1907,9 @@ static Value *emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **args, j if (might_need_root(args[1]) || fval->getType() != jl_pvalue_llvmt) make_gcroot(f1, ctx); } - Value *strct = emit_allocobj(sizeof(void*)+sty->size); + Value *strct = emit_allocobj(sty->size); builder.CreateStore(literal_pointer_val((jl_value_t*)ty), - emit_nthptr_addr(strct, (size_t)0)); + emit_typeptr_addr(strct)); if (f1) { if (!jl_subtype(expr_type(args[1],ctx), jl_t0(sty->types), 0)) emit_typecheck(f1, jl_t0(sty->types), "new", ctx); diff --git a/src/codegen.cpp b/src/codegen.cpp index 552e3d70ba169..2161aa26998b5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -310,9 +310,9 @@ static Function *jlenter_func; static Function *jlleave_func; static Function *jlegal_func; static Function *jlallocobj_func; +static Function *jlalloc1w_func; static Function *jlalloc2w_func; static Function *jlalloc3w_func; -static Function *jlalloc4w_func; static Function *jl_alloc_tuple_func; static Function *jlsubtype_func; static Function *setjmp_func; @@ -349,7 +349,7 @@ static Function *show_execution_point_func; static std::vector two_pvalue_llvmt; static std::vector three_pvalue_llvmt; -extern "C" DLLEXPORT void gc_wb_slow(void* parent, void* ptr) +extern "C" DLLEXPORT void gc_wb_slow(jl_value_t* parent, jl_value_t* ptr) { gc_wb(parent, ptr); } @@ -1628,9 +1628,9 @@ static Value *emit_lambda_closure(jl_value_t *expr, jl_codectx_t *ctx) if (vari.closureidx != -1) { int idx = vari.closureidx; #ifdef OVERLAP_TUPLE_LEN - val = emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen); + val = emit_nthptr((Value*)ctx->envArg, idx, tbaa_tuplelen); #else - val = emit_nthptr((Value*)ctx->envArg, idx+2, tbaa_tuplelen); + val = emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen); #endif } else { @@ -1713,8 +1713,7 @@ static Value *emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t *ctx) if (strct->getType() == jl_pvalue_llvmt) { Value *addr = builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), - ConstantInt::get(T_size, - sty->fields[idx].offset + sizeof(void*))); + ConstantInt::get(T_size, sty->fields[idx].offset)); JL_GC_POP(); MDNode *tbaa = sty->mutabl ? tbaa_user : tbaa_immut; if (sty->fields[idx].isptr) { @@ -2025,7 +2024,11 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, else if (f->fptr == &jl_f_apply && nargs==3 && ctx->vaStack && symbol_eq(args[3], ctx->vaName) && expr_type(args[2],ctx) == (jl_value_t*)jl_function_type) { Value *theF = emit_expr(args[2],ctx); - Value *theFptr = emit_nthptr_recast(theF,1, tbaa_func, jl_pfptr_llvmt); + Value *theFptr = emit_nthptr_recast( + theF, + offsetof(jl_function_t,fptr)/sizeof(void*), + tbaa_func, + jl_pfptr_llvmt); Value *nva = emit_n_varargs(ctx); #ifdef _P64 nva = builder.CreateTrunc(nva, T_int32); @@ -2062,7 +2065,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, if (idx > 0 && (idx < tlen || (idx == tlen && !isseqt))) { // known to be in bounds JL_GC_POP(); - return emit_tupleref(arg1,ConstantInt::get(T_size,idx),tty,ctx); + return emit_tupleref(arg1,ConstantInt::get(T_size,idx-1),tty,ctx); } if (idx==0 || (!isseqt && idx > tlen)) { // known to be out of bounds @@ -2092,9 +2095,9 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, unbox = true; jl_add_linfo_root(ctx->linfo, tty); } - emit_bounds_check(arg1, unbox ? tty : NULL, idx, tlen, ctx); + Value *idx0 = emit_bounds_check(arg1, unbox ? tty : NULL, idx, tlen, ctx); JL_GC_POP(); - return emit_tupleref(arg1,idx,tty,ctx); + return emit_tupleref(arg1,idx0,tty,ctx); } } else if (f->fptr == &jl_f_tuple) { @@ -2132,7 +2135,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, } assert(tpl != NULL); Value *elt = emit_unbox(ety,emit_unboxed(args[i+1],ctx),jl_tupleref(rt1,i)); - tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i+1),elt,rt1,ctx); + tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i),elt,rt1,ctx); } JL_GC_POP(); if (ty->isEmptyTy()) @@ -2150,15 +2153,15 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, make_gcroot(arg1, ctx); bool rooted = false; #ifdef OVERLAP_TUPLE_LEN - size_t nwords = nargs+1; + size_t nwords = nargs; #else - size_t nwords = nargs+2; + size_t nwords = nargs+1; #endif Value *tup = emit_allocobj(sizeof(void*)*nwords); #ifdef OVERLAP_TUPLE_LEN - builder.CreateStore(arg1, emit_nthptr_addr(tup, 1)); + builder.CreateStore(arg1, emit_nthptr_addr(tup, 0)); #else - builder.CreateStore(arg1, emit_nthptr_addr(tup, 2)); + builder.CreateStore(arg1, emit_nthptr_addr(tup, 1)); emit_write_barrier(ctx, tup, arg1); #endif ctx->argDepth = last_depth; @@ -2167,22 +2170,14 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, CreateStore(builder. CreateOr(builder.CreatePtrToInt(literal_pointer_val((jl_value_t*)jl_tuple_type), T_int64), ConstantInt::get(T_int64, nargs<<52)), - builder.CreateBitCast(emit_nthptr_addr(tup, (size_t)0), - T_pint64)); + builder.CreateBitCast(emit_typeptr_addr(tup), T_pint64)); #else - builder.CreateStore(literal_pointer_val((jl_value_t*)jl_tuple_type), - emit_nthptr_addr(tup, (size_t)0)); + builder.CreateStore(literal_pointer_val((jl_value_t*)jl_tuple_type), emit_typeptr_addr(tup)); builder.CreateStore(ConstantInt::get(T_size, nargs), - builder.CreateBitCast(emit_nthptr_addr(tup, (size_t)1), T_psize)); -#endif -#ifdef OVERLAP_TUPLE_LEN - size_t offs = 1; -#else - size_t offs = 2; + builder.CreateBitCast(tup, T_psize)); #endif for(i=1; i < nargs; i++) { - builder.CreateStore(V_null, - emit_nthptr_addr(tup, i+offs)); + builder.CreateStore(V_null, emit_nthptr_addr(tup, i+TUPLE_DATA_OFFSET)); } for(i=1; i < nargs; i++) { if (might_need_root(args[i+1]) && !rooted) { @@ -2195,7 +2190,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, rooted = true; } Value *argi = boxed(argval,ctx); - builder.CreateStore(argi, emit_nthptr_addr(tup, i+offs)); + builder.CreateStore(argi, emit_nthptr_addr(tup, i+TUPLE_DATA_OFFSET)); emit_write_barrier(ctx, tup, argi); } ctx->argDepth = last_depth; @@ -2364,12 +2359,10 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, if (llvm_st == jl_pvalue_llvmt) { if (is_structtype_all_pointers(stt)) { idx = emit_bounds_check(strct, NULL, idx, ConstantInt::get(T_size, nfields), ctx); - Value *fld = - tbaa_decorate(tbaa_user, builder. - CreateLoad(builder. - CreateGEP(builder. - CreateBitCast(strct, jl_ppvalue_llvmt), - builder.CreateAdd(idx,ConstantInt::get(T_size,1))))); + Value *fld = tbaa_decorate(tbaa_user, builder.CreateLoad( + builder.CreateGEP( + builder.CreateBitCast(strct, jl_ppvalue_llvmt), + idx))); if ((unsigned)stt->ninitialized != jl_tuple_len(stt->types)) { null_pointer_check(fld, ctx); } @@ -2397,7 +2390,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, // frobbing the stack Value *fld; if (nfields == 0) { - emit_bounds_check(tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlnull_var))), + idx = emit_bounds_check(tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlnull_var))), NULL, idx, ConstantInt::get(T_size, nfields), ctx); fld = UndefValue::get(jl_pvalue_llvmt); } @@ -2413,8 +2406,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, jl_add_linfo_root(ctx->linfo, (jl_value_t*)stt); } idx = emit_bounds_check(tempSpace, (jl_value_t*)stt, idx, ConstantInt::get(T_size, nfields), ctx); - Value *ptr = builder.CreateGEP(tempSpace, ConstantInt::get(T_size, 0)); - fld = typed_load(ptr, idx, jt, ctx, stt->mutabl ? tbaa_user : tbaa_immut); + fld = typed_load(tempSpace, idx, jt, ctx, stt->mutabl ? tbaa_user : tbaa_immut); builder.CreateCall(Intrinsic::getDeclaration(jl_Module,Intrinsic::stackrestore), stacksave); } @@ -2640,7 +2632,7 @@ static Value *emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_ #endif // extract pieces of the function object // TODO: try extractvalue instead - theFptr = emit_nthptr_recast(theFunc, 1, tbaa_func, jl_pfptr_llvmt); + theFptr = emit_nthptr_recast(theFunc, offsetof(jl_function_t,fptr)/sizeof(void*), tbaa_func, jl_pfptr_llvmt); theF = theFunc; } else { @@ -2674,7 +2666,7 @@ static Value *emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_ myargs = builder.CreateGEP(ctx->argTemp, ConstantInt::get(T_size, argStart+1+ctx->argSpaceOffs)); } - theFptr = emit_nthptr_recast(theFunc, 1, tbaa_func, jl_pfptr_llvmt); + theFptr = emit_nthptr_recast(theFunc, offsetof(jl_function_t,fptr)/sizeof(void*), tbaa_func, jl_pfptr_llvmt); Value *r1 = builder.CreateCall3(prepare_call(theFptr), theFunc, myargs, ConstantInt::get(T_int32,nargs)); builder.CreateBr(mergeBB1); @@ -2764,21 +2756,21 @@ static Value *var_binding_pointer(jl_sym_t *s, jl_binding_t **pbnd, assert(((Value*)ctx->envArg)->getType() == jl_pvalue_llvmt); if (isBoxed(s, ctx)) { #ifdef OVERLAP_TUPLE_LEN - return emit_nthptr_addr(emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen), 1); + return builder.CreatePointerCast(emit_nthptr((Value*)ctx->envArg, idx, tbaa_tuplelen), jl_ppvalue_llvmt); #else - return emit_nthptr_addr(emit_nthptr((Value*)ctx->envArg, idx+2, tbaa_tuplelen), 1); + return builder.CreatePointerCast(emit_nthptr((Value*)ctx->envArg, idx+1, tbaa_tuplelen), jl_ppvalue_llvmt); #endif } #ifdef OVERLAP_TUPLE_LEN - return emit_nthptr_addr((Value*)ctx->envArg, idx+1); + return emit_nthptr_addr((Value*)ctx->envArg, idx); #else - return emit_nthptr_addr((Value*)ctx->envArg, idx+2); + return emit_nthptr_addr((Value*)ctx->envArg, idx+1); #endif } Value *l = vi.memvalue; if (l == NULL) return NULL; if (isBoxed(s, ctx)) { - return emit_nthptr_addr(builder.CreateLoad(l,false), 1); + return builder.CreatePointerCast(builder.CreateLoad(l,false), jl_ppvalue_llvmt); } return l; } @@ -2902,9 +2894,8 @@ static Value *emit_assignment(Value *bp, jl_value_t *r, jl_value_t *declType, bo } else { rval = boxed(emit_expr(r, ctx, true),ctx,rt); - if (!is_stack(bp)) { - Value* box = builder.CreateGEP(bp, ConstantInt::get(T_size, -1)); - emit_write_barrier(ctx, box, rval); + if (!is_stack(bp)) { // bp is a jl_box_t* + emit_write_barrier(ctx, bp, rval); } } if (builder.GetInsertBlock()->getTerminator() == NULL) { @@ -3189,8 +3180,8 @@ static Value *emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, } else { bp = var_binding_pointer((jl_sym_t*)mn, &bnd, false, ctx); - if (isBoxed((jl_sym_t*)mn, ctx)) { - bp_owner = builder.CreateBitCast(emit_nthptr_addr(bp, -1), jl_pvalue_llvmt); + if (isBoxed((jl_sym_t*)mn, ctx)) { // bp is a jl_box_t* + bp_owner = builder.CreateBitCast(bp, jl_pvalue_llvmt); } } } @@ -3653,7 +3644,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (isref & (2<getType() == jl_pvalue_llvmt) { - builder.CreateStore(val, builder.CreateBitCast( - emit_nthptr_addr(mem, (size_t)1), val->getType()->getPointerTo())); + builder.CreateStore(val, builder.CreateBitCast(mem, val->getType()->getPointerTo())); val = mem; } else { @@ -3727,7 +3716,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t finalize_gc_frame(&ctx); if (isref&1) { - //r = builder.CreateConstGEP1_32(r, 1); // skip type-tag -- broken if you do, broken if you don't builder.CreateRet(r); } else { @@ -4347,7 +4335,7 @@ static Function *emit_function(jl_lambda_info_t *lam) // fetch env out of function object if we need it if (hasCapt) { - ctx.envArg = emit_nthptr(fArg, 2, tbaa_func); + ctx.envArg = emit_nthptr(fArg, offsetof(jl_function_t,env)/sizeof(jl_value_t*), tbaa_func); } // step 8. set up GC frame @@ -4780,14 +4768,10 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig) extern "C" DLLEXPORT jl_value_t *jl_new_box(jl_value_t *v) { - jl_value_t *box = (jl_value_t*)alloc_2w(); -#ifdef OVERLAP_TUPLE_LEN - box->type = (size_t)jl_box_any_type; -#else - box->type = jl_box_any_type; -#endif - if (v) gc_wb(box, v); - ((jl_value_t**)box)[1] = v; + jl_value_t *box = (jl_value_t*)alloc_1w(); + jl_set_typeof(box, jl_box_any_type); + // if (v) gc_wb(box, v); // write block not needed: box was just allocated + box->fieldptr[0] = v; return box; } @@ -4911,9 +4895,9 @@ static void init_julia_llvm_env(Module *m) assert(jl_func_sig != NULL); jl_pfptr_llvmt = PointerType::get(PointerType::get(jl_func_sig, 0), 0); - Type* vaelts[] = {valueStructElts[0], T_pint8 + Type* vaelts[] = {T_pint8 #ifdef STORE_ARRAY_LEN - , T_size + , T_size #endif }; Type* jl_array_llvmt = @@ -5114,13 +5098,14 @@ static void init_julia_llvm_env(Module *m) jlcall_func_to_llvm("jl_apply_generic", (void*)&jl_apply_generic, m); jlgetfield_func = jlcall_func_to_llvm("jl_f_get_field", (void*)&jl_f_get_field, m); - std::vector wbargs(0); - wbargs.push_back(jl_pvalue_llvmt); - wbargs.push_back(jl_pvalue_llvmt); queuerootfun = Function::Create(FunctionType::get(T_void, args_1ptr, false), Function::ExternalLinkage, "gc_queue_root", m); add_named_global(queuerootfun, (void*)&gc_queue_root); + + std::vector wbargs(0); + wbargs.push_back(jl_pvalue_llvmt); + wbargs.push_back(jl_pvalue_llvmt); wbfunc = Function::Create(FunctionType::get(T_void, wbargs, false), Function::ExternalLinkage, "gc_wb_slow", m); @@ -5256,6 +5241,12 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlallocobj_func, (void*)&allocobj); std::vector empty_args(0); + jlalloc1w_func = + Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), + Function::ExternalLinkage, + "alloc_1w", m); + add_named_global(jlalloc1w_func, (void*)&alloc_1w); + jlalloc2w_func = Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), Function::ExternalLinkage, @@ -5268,12 +5259,6 @@ static void init_julia_llvm_env(Module *m) "alloc_3w", m); add_named_global(jlalloc3w_func, (void*)&alloc_3w); - jlalloc4w_func = - Function::Create(FunctionType::get(jl_pvalue_llvmt, empty_args, false), - Function::ExternalLinkage, - "alloc_4w", m); - add_named_global(jlalloc4w_func, (void*)&alloc_4w); - std::vector atargs(0); atargs.push_back(T_size); jl_alloc_tuple_func = diff --git a/src/dump.c b/src/dump.c index aca39b0b668cb..e4a05df548df1 100644 --- a/src/dump.c +++ b/src/dump.c @@ -900,7 +900,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) int has_instance = (flags>>3)&1; if (has_instance) { dt->instance = jl_deserialize_value(s, &dt->instance); - dt->instance->type = (jl_value_t*)dt; + jl_set_typeof(dt->instance, dt); } assert(tree_literal_values==NULL && mode != MODE_AST); backref_list.items[pos] = dt; @@ -910,7 +910,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) arraylist_push(&flagref_list, (void*)(uptrint_t)pos); if (has_instance) { arraylist_push(&flagref_list, dt); - arraylist_push(&flagref_list, &dt->instance->type); + arraylist_push(&flagref_list, &jl_typetagof(dt->instance)->type); arraylist_push(&flagref_list, (void*)(uptrint_t)-1); } } @@ -1074,7 +1074,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } } if (mode == MODE_MODULE) { - aty = jl_deserialize_value(s, &a->type); + aty = jl_deserialize_value(s, &jl_typetagof(a)->type); assert(aty == jl_typeof(a)); } return (jl_value_t*)a; @@ -1101,7 +1101,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)e; } else if (vtag == (jl_value_t*)jl_tvar_type) { - jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, 4); + jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, NWORDS(sizeof(jl_tvar_t))); if (usetable) arraylist_push(&backref_list, tv); tv->name = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1115,7 +1115,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_function_type) { jl_function_t *f = - (jl_function_t*)newobj((jl_value_t*)jl_function_type, 3); + (jl_function_t*)newobj((jl_value_t*)jl_function_type, NWORDS(sizeof(jl_function_t))); if (usetable) arraylist_push(&backref_list, f); f->linfo = (jl_lambda_info_t*)jl_deserialize_value(s, (jl_value_t**)&f->linfo); @@ -1128,7 +1128,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t else if (vtag == (jl_value_t*)jl_lambda_info_type) { jl_lambda_info_t *li = (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - LAMBDA_INFO_NW); + NWORDS(sizeof(jl_lambda_info_t))); if (usetable) arraylist_push(&backref_list, li); li->ast = jl_deserialize_value(s, &li->ast); @@ -1276,8 +1276,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t case 4: v = jl_box32(dt, *(int32_t*)data); break; case 8: v = jl_box64(dt, *(int64_t*)data); break; default: - v = (jl_value_t*)allocobj(sizeof(void*)+nby); - v->type = (jl_value_t*)dt; + v = (jl_value_t*)allocobj(nby); + jl_set_typeof(v, dt); memcpy(jl_data_ptr(v), data, nby); } } @@ -1311,14 +1311,14 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } // TODO: put WeakRefs on the weak_refs list if (mode == MODE_MODULE) { - dt = (jl_datatype_t*)jl_deserialize_value(s, (jl_value_t**)&v->type); + dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_typetagof(v)->type); assert((jl_value_t*)dt == jl_typeof(v)); } return v; } else if (vtag == (jl_value_t*)Singleton_tag) { assert(mode != MODE_MODULE_LAMBDAS); - jl_value_t *v = (jl_value_t*)allocobj(sizeof(void*)); + jl_value_t *v = (jl_value_t*)allocobj(0); if (usetable) { uptrint_t pos = backref_list.len; arraylist_push(&backref_list, (void*)v); @@ -1703,14 +1703,14 @@ jl_module_t *jl_restore_new_module(const char *fname) jl_value_t **loc = (jl_value_t**)flagref_list.items[i++]; int offs = (int)(intptr_t)flagref_list.items[i++]; if (t != dt) { - dt->type = (jl_value_t*)(ptrint_t)2; // invalidate the old value to help catch errors + jl_set_typeof(dt, (ptrint_t)2); // invalidate the old value to help catch errors if ((jl_value_t*)dt == o) { if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } if (t->instance != v) { - v->type = (jl_value_t*)(ptrint_t)1; // invalidate the old value to help catch errors + jl_set_typeof(v, (ptrint_t)1); // invalidate the old value to help catch errors if (v == o) { if (loc) *loc = v; if (offs > 0) backref_list.items[offs] = v; diff --git a/src/gc.c b/src/gc.c index 2cd122dd7c62e..583f820fc1a5c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -416,8 +416,10 @@ static inline int gc_setmark_pool(void *o, int mark_mode) } -static inline int gc_setmark(void *o, int sz, int mark_mode) +static inline int gc_setmark(jl_value_t *v, int sz, int mark_mode) { + jl_typetag_t *o = jl_typetagof(v); + sz += sizeof(jl_typetag_t); #ifdef MEMDEBUG return gc_setmark_big(o, mark_mode); #endif @@ -427,7 +429,7 @@ static inline int gc_setmark(void *o, int sz, int mark_mode) return gc_setmark_big(o, mark_mode); } -#define gc_typeof(v) ((jl_value_t*)(((uptrint_t)jl_typeof(v))&(~(uintptr_t)3))) +#define gc_typeof(v) jl_typeof(v) #define gc_val_buf(o) ((buff_t*)(((void**)(o))-1)) inline void gc_setmark_buf(void *o, int mark_mode) @@ -608,7 +610,7 @@ void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, int isaligned, jl_ { maybe_collect(); - if (gc_bits(owner) == GC_MARKED) { + if (gc_bits(jl_typetagof(owner)) == GC_MARKED) { perm_scanned_bytes += sz - oldsz; live_bytes += sz - oldsz; } @@ -666,8 +668,8 @@ static arraylist_t weak_refs; DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { - jl_weakref_t *wr = (jl_weakref_t*)alloc_2w(); - wr->type = (jl_value_t*)jl_weakref_type; + jl_weakref_t *wr = (jl_weakref_t*)alloc_1w(); + jl_set_typeof(wr, jl_weakref_type); wr->value = value; arraylist_push(&weak_refs, wr); return wr; @@ -684,9 +686,9 @@ static void sweep_weak_refs(void) return; do { wr = (jl_weakref_t*)lst[n]; - if (gc_marked(wr)) { + if (gc_marked(jl_typetagof(wr))) { // weakref itself is alive - if (!gc_marked(wr->value)) + if (!gc_marked(jl_typetagof(wr->value))) wr->value = (jl_value_t*)jl_nothing; n++; } @@ -940,7 +942,7 @@ static void sweep_malloced_arrays(void) mallocarray_t **pma = &mallocarrays; while (ma != NULL) { mallocarray_t *nxt = ma->next; - if (gc_marked(ma->a)) { + if (gc_marked(jl_typetagof(ma->a))) { pma = &ma->next; } else { @@ -1357,27 +1359,31 @@ void reset_remset(void) remset->len = 0; } -DLLEXPORT void gc_queue_root(void *ptr) +DLLEXPORT void gc_queue_root(jl_value_t *ptr) { - assert(gc_bits(ptr) != GC_QUEUED); - gc_bits(ptr) = GC_QUEUED; + jl_typetag_t *o = jl_typetagof(ptr); + assert(gc_bits(o) != GC_QUEUED); + gc_bits(o) = GC_QUEUED; arraylist_push(remset, ptr); } -void gc_queue_binding(void *bnd) + +void gc_queue_binding(jl_binding_t *bnd) { - assert(gc_bits(bnd) != GC_QUEUED); - gc_bits(bnd) = GC_QUEUED; - arraylist_push(&rem_bindings, (void*)((void**)bnd + 1)); + buff_t *buf = gc_val_buf(bnd); + assert(gc_bits(buf) != GC_QUEUED); + gc_bits(buf) = GC_QUEUED; + arraylist_push(&rem_bindings, bnd); } static int push_root(jl_value_t *v, int d, int); -static inline int gc_push_root(void *v, int d) +static inline int gc_push_root(void *v, int d) // v isa jl_value_t* { - assert((v) != NULL); - verify_val(v); - int bits = gc_bits(v); - if (!gc_marked(v)) { - return push_root((jl_value_t*)(v),d, bits); + assert(v != NULL); + jl_typetag_t* o = jl_typetagof(v); + verify_val(o); + int bits = gc_bits(o); + if (!gc_marked(o)) { + return push_root((jl_value_t*)v, d, bits); } return bits; } @@ -1385,12 +1391,13 @@ static inline int gc_push_root(void *v, int d) void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now { // int64_t s = perm_scanned_bytes; - if (!gc_marked(v)) { + jl_typetag_t *o = jl_typetagof(v); + if (!gc_marked(o)) { // objprofile_count(jl_typeof(v), 1, 16); #ifdef MEMDEBUG - gc_setmark_big(v, GC_MARKED_NOESC); + gc_setmark_big(o, GC_MARKED_NOESC); #else - gc_setmark_pool(v, GC_MARKED_NOESC); + gc_setmark_pool(o, GC_MARKED_NOESC); #endif } // perm_scanned_bytes = s; @@ -1429,7 +1436,7 @@ NOINLINE static int gc_mark_module(jl_module_t *m, int d) for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - gc_setmark_buf(b, gc_bits(m)); + gc_setmark_buf(b, gc_bits(jl_typetagof(m))); #ifdef GC_VERIFY void* vb = gc_val_buf(b); verify_parent1("module", m, &vb, "binding_buff"); @@ -1461,7 +1468,7 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) { if (ta->stkbuf != NULL || ta == jl_current_task) { if (ta->stkbuf != NULL) { - gc_setmark_buf(ta->stkbuf, gc_bits(ta)); + gc_setmark_buf(ta->stkbuf, gc_bits(jl_typetagof(ta))); } #ifdef COPY_STACKS ptrint_t offset; @@ -1518,7 +1525,7 @@ static int push_root(jl_value_t *v, int d, int bits) int refyoung = 0; if (vt == (jl_value_t*)jl_weakref_type) { - bits = gc_setmark(v, jl_datatype_size(jl_weakref_type), GC_MARKED_NOESC); + bits = gc_setmark(v, sizeof(jl_weakref_t), GC_MARKED_NOESC); goto ret; } if ((jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) { @@ -1551,6 +1558,7 @@ static int push_root(jl_value_t *v, int d, int bits) } else if (((jl_datatype_t*)(vt))->name == jl_array_typename) { jl_array_t *a = (jl_array_t*)v; + jl_typetag_t *o = jl_typetagof(v); int todo = !(bits & GC_MARKED); if (a->pooled) #ifdef MEMDEBUG @@ -1561,18 +1569,18 @@ static int push_root(jl_value_t *v, int d, int bits) MARK(a, bits = _gc_setmark_pool(a, GC_MARKED_NOESC); if (a->how == 2 && todo) { - objprofile_count(MATY, gc_bits(a) == GC_MARKED, array_nbytes(a)); - if (gc_bits(a) == GC_MARKED) + objprofile_count(MATY, gc_bits(o) == GC_MARKED, array_nbytes(a)); + if (gc_bits(o) == GC_MARKED) perm_scanned_bytes += array_nbytes(a); else scanned_bytes += array_nbytes(a); }); else MARK(a, - bits = gc_setmark_big(a, GC_MARKED_NOESC); + bits = gc_setmark_big(o, GC_MARKED_NOESC); if (a->how == 2 && todo) { - objprofile_count(MATY, gc_bits(a) == GC_MARKED, array_nbytes(a)); - if (gc_bits(a) == GC_MARKED) + objprofile_count(MATY, gc_bits(o) == GC_MARKED, array_nbytes(a)); + if (gc_bits(o) == GC_MARKED) perm_scanned_bytes += array_nbytes(a); else scanned_bytes += array_nbytes(a); @@ -1587,7 +1595,7 @@ static int push_root(jl_value_t *v, int d, int bits) void* val_buf = gc_val_buf((char*)a->data - a->offset*a->elsize); verify_parent1("array", v, &val_buf, "buffer ('loc' addr is meaningless)"); #endif - gc_setmark_buf((char*)a->data - a->offset*a->elsize, gc_bits(v)); + gc_setmark_buf((char*)a->data - a->offset*a->elsize, gc_bits(o)); } if (a->ptrarray && a->data!=NULL) { size_t l = jl_array_len(a); @@ -1642,7 +1650,7 @@ static int push_root(jl_value_t *v, int d, int bits) jl_fielddesc_t* fields = dt->fields; for(int i=0; i < nf; i++) { if (fields[i].isptr) { - jl_value_t **slot = (jl_value_t**)((char*)v + fields[i].offset + sizeof(void*)); + jl_value_t **slot = (jl_value_t**)((char*)v + fields[i].offset); jl_value_t *fld = *slot; if (fld) { verify_parent2("object", v, slot, "field(%d)", i); @@ -1685,8 +1693,10 @@ static void visit_mark_stack_inc(int mark_mode) { while(mark_sp > 0 && !should_timeout()) { jl_value_t* v = mark_stack[--mark_sp]; - assert(gc_bits(v) == GC_QUEUED || gc_bits(v) == GC_MARKED || gc_bits(v) == GC_MARKED_NOESC); - push_root(v, 0, gc_bits(v)); + assert(gc_bits(jl_typetagof(v)) == GC_QUEUED || + gc_bits(jl_typetagof(v)) == GC_MARKED || + gc_bits(jl_typetagof(v)) == GC_MARKED_NOESC); + push_root(v, 0, gc_bits(jl_typetagof(v))); } } @@ -1767,9 +1777,9 @@ static void post_mark(arraylist_t *list, int dryrun) for(size_t i=0; i < list->len; i+=2) { jl_value_t *v = (jl_value_t*)list->items[i]; jl_value_t *fin = (jl_value_t*)list->items[i+1]; - int isfreed = !gc_marked(v); + int isfreed = !gc_marked(jl_typetagof(v)); gc_push_root(fin, 0); - int isold = list == &finalizer_list && gc_bits(v) == GC_MARKED && gc_bits(fin) == GC_MARKED; + int isold = list == &finalizer_list && gc_bits(jl_typetagof(v)) == GC_MARKED && gc_bits(jl_typetagof(fin)) == GC_MARKED; if (!dryrun && (isfreed || isold)) { // remove from this list if (i < list->len - 2) { @@ -2074,10 +2084,9 @@ void jl_gc_collect(int full) reset_remset(); // avoid counting remembered objects & bindings twice in perm_scanned_bytes for(int i = 0; i < last_remset->len; i++) { - uintptr_t item = (uintptr_t)last_remset->items[i]; - void* ptr = (void*)(item & ~(uintptr_t)1); - objprofile_count(jl_typeof(ptr), 2, 0); - gc_bits(ptr) = GC_MARKED; + jl_value_t *item = (jl_value_t*)last_remset->items[i]; + objprofile_count(jl_typeof(item), 2, 0); + gc_bits(jl_typetagof(item)) = GC_MARKED; } for (int i = 0; i < rem_bindings.len; i++) { void *ptr = rem_bindings.items[i]; @@ -2085,9 +2094,8 @@ void jl_gc_collect(int full) } for (int i = 0; i < last_remset->len; i++) { - uintptr_t item = (uintptr_t)last_remset->items[i]; - void* ptr = (void*)(item & ~(uintptr_t)1); - push_root((jl_value_t*)ptr, 0, gc_bits(ptr)); + jl_value_t *item = (jl_value_t*)last_remset->items[i]; + push_root(item, 0, GC_MARKED); } // 2. mark every object in a remembered binding @@ -2198,7 +2206,7 @@ void jl_gc_collect(int full) // so that we don't trigger the barrier again on them. if (sweep_mask == GC_MARKED_NOESC) { for (int i = 0; i < remset->len; i++) { - gc_bits(((uintptr_t)remset->items[i] & ~(uintptr_t)1)) = GC_QUEUED; + gc_bits(jl_typetagof(remset->items[i])) = GC_QUEUED; } for (int i = 0; i < rem_bindings.len; i++) { void *ptr = rem_bindings.items[i]; @@ -2301,51 +2309,54 @@ void *reallocb(void *b, size_t sz) } } -DLLEXPORT void *allocobj(size_t sz) +#define jl_valueof(v) (((jl_typetag_t*)(v))->value) + +DLLEXPORT jl_value_t *allocobj(size_t sz) { + sz += sizeof(void*); #ifdef MEMDEBUG - return alloc_big(sz); + return jl_valueof(alloc_big(sz)); #endif if (sz <= 2048) - return pool_alloc(&pools[szclass(sz)]); + return jl_valueof(pool_alloc(&pools[szclass(sz)])); else - return alloc_big(sz); + return jl_valueof(alloc_big(sz)); } -DLLEXPORT void *alloc_2w(void) +DLLEXPORT jl_value_t *alloc_1w(void) { #ifdef MEMDEBUG - return alloc_big(2*sizeof(void*)); + return jl_valueof(alloc_big(2*sizeof(void*))); #endif #ifdef _P64 - return _pool_alloc(&pools[2], 2*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[2], 2*sizeof(void*))); #else - return _pool_alloc(&pools[0], 2*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[0], 2*sizeof(void*))); #endif } -DLLEXPORT void *alloc_3w(void) +DLLEXPORT jl_value_t *alloc_2w(void) { #ifdef MEMDEBUG - return alloc_big(3*sizeof(void*)); + return jl_valueof(alloc_big(3*sizeof(void*))); #endif #ifdef _P64 - return _pool_alloc(&pools[4], 3*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[4], 3*sizeof(void*))); #else - return _pool_alloc(&pools[1], 3*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[1], 3*sizeof(void*))); #endif } -DLLEXPORT void *alloc_4w(void) +DLLEXPORT jl_value_t *alloc_3w(void) { #ifdef MEMDEBUG - return alloc_big(4*sizeof(void*)); + return jl_valueof(alloc_big(4*sizeof(void*))); #endif #ifdef _P64 - return _pool_alloc(&pools[6], 4*sizeof(void*)); + return jl_valueof(_pool_alloc(&pools[6], 4*sizeof(void*))); #else - return pool_alloc(&pools[2]); + return jl_valueof(pool_alloc(&pools[2])); #endif } @@ -2519,7 +2530,7 @@ static void big_obj_stats(void) mallocarray_t *ma = mallocarrays; while (ma != NULL) { - if (gc_marked(ma->a)) { + if (gc_marked(jl_typetagof(ma->a))) { nused++; nbytes += array_nbytes(ma->a); } diff --git a/src/gf.c b/src/gf.c index 2c3c368397ae2..1cb8ec4cf324a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -22,7 +22,7 @@ extern "C" { static jl_methtable_t *new_method_table(jl_sym_t *name) { jl_methtable_t *mt = (jl_methtable_t*)allocobj(sizeof(jl_methtable_t)); - mt->type = (jl_value_t*)jl_methtable_type; + jl_set_typeof(mt, jl_methtable_type); mt->name = name; mt->defs = (jl_methlist_t*)JL_NULL; mt->cache = (jl_methlist_t*)JL_NULL; @@ -1291,7 +1291,7 @@ jl_methlist_t *jl_method_list_insert(jl_methlist_t **pml, jl_tuple_t *type, l = l->next; } jl_methlist_t *newrec = (jl_methlist_t*)allocobj(sizeof(jl_methlist_t)); - newrec->type = (jl_value_t*)jl_method_type; + jl_set_typeof(newrec, jl_method_type); newrec->sig = type; newrec->tvars = tvars; newrec->va = (jl_tuple_len(type) > 0 && diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 9f2e9850e12aa..2254009508514 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -152,7 +152,7 @@ static Constant *julia_const_to_llvm(jl_value_t *e) case 1: { uint8_t data8 = *(uint8_t*)jl_data_ptr(e); return ConstantInt::get(T_int8, data8); - } + } case 2: { uint16_t data16 = *(uint16_t*)jl_data_ptr(e); #ifndef DISABLE_FLOAT16 @@ -161,21 +161,21 @@ static Constant *julia_const_to_llvm(jl_value_t *e) } #endif return ConstantInt::get(T_int16, data16); - } + } case 4: { uint32_t data32 = *(uint32_t*)jl_data_ptr(e); if (jl_is_float(e)) { return ConstantFP::get(jl_LLVMContext,LLVM_FP(APFloat::IEEEsingle,APInt(32,data32))); } return ConstantInt::get(T_int32, data32); - } + } case 8: { uint64_t data64 = *(uint64_t*)jl_data_ptr(e); if (jl_is_float(e)) { return ConstantFP::get(jl_LLVMContext,LLVM_FP(APFloat::IEEEdouble,APInt(64,data64))); } return ConstantInt::get(T_int64, data64); - } + } default: size_t nw = (nb+sizeof(uint64_t)-1)/sizeof(uint64_t); uint64_t *data = (uint64_t*)jl_data_ptr(e); @@ -200,7 +200,7 @@ static Constant *julia_const_to_llvm(jl_value_t *e) return ConstantInt::get(IntegerType::get(jl_LLVMContext,8*nb),val); } } - else if (jl_isbits(jt)) { + if (jl_isbits(jt)) { size_t nf = jl_tuple_len(bt->names), i; size_t llvm_nf = 0; Constant **fields = (Constant**)alloca(nf * sizeof(Constant*)); @@ -297,9 +297,9 @@ static Value *emit_unbox(Type *to, Value *x, jl_value_t *jt) Type *ety = jl_llvmtuple_eltype(to,jt,i); if (ety == T_void) continue; - Value *ref = emit_tupleref(x,ConstantInt::get(T_size,i+1),jt,NULL); + Value *ref = emit_tupleref(x,ConstantInt::get(T_size,i),jt,NULL); Value *elt = emit_unbox(ety,ref,jl_tupleref(jt,i)); - tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i+1),elt,jt,NULL); + tpl = emit_tupleset(tpl,ConstantInt::get(T_size,i),elt,jt,NULL); } return tpl; } @@ -663,11 +663,11 @@ static Value *emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ctx) ConstantInt::get(T_size, sizeof(void*)+size)); builder.CreateStore(literal_pointer_val((jl_value_t*)ety), - emit_nthptr_addr(strct, (size_t)0)); + emit_typeptr_addr(strct)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); - builder.CreateMemCpy(builder.CreateBitCast(emit_nthptr_addr(strct, (size_t)1), T_pint8), + builder.CreateMemCpy(builder.CreateBitCast(strct, T_pint8), thePtr, size, 1); return mark_julia_type(strct, ety); } @@ -723,7 +723,7 @@ static Value *emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, jl_co im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); builder.CreateMemCpy(builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1), - builder.CreateBitCast(emit_nthptr_addr(val, (size_t)1), T_pint8), size, 1); + builder.CreateBitCast(val, T_pint8), size, 1); } else { if (val == NULL) { diff --git a/src/jlapi.c b/src/jlapi.c index df615d51f9da9..fc12852a1e561 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -277,6 +277,17 @@ DLLEXPORT const char* jl_ver_string(void) return JULIA_VERSION_STRING; } +// Create function versions of some useful macros +#undef jl_typetagof +DLLEXPORT jl_typetag_t *jl_typetagof(jl_value_t *v) { + return jl_typetagof__MACRO(v); +} + +#undef jl_typeof +DLLEXPORT jl_value_t *jl_typeof(jl_value_t *v) { + return jl_typeof__MACRO(v); +} + #ifdef __cplusplus } #endif diff --git a/src/jltypes.c b/src/jltypes.c index 43541e49166c6..372d05a8a5da7 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2974,13 +2974,13 @@ void jl_init_types(void) { // create base objects jl_datatype_type = jl_new_uninitialized_datatype(10); - jl_datatype_type->type = (jl_value_t*)jl_datatype_type; + jl_set_typeof(jl_datatype_type, jl_datatype_type); jl_typename_type = jl_new_uninitialized_datatype(4); jl_sym_type = jl_new_uninitialized_datatype(0); jl_symbol_type = jl_sym_type; jl_tuple_type = jl_alloc_tuple(1); - jl_tuple_type->type = (jl_value_t*)jl_tuple_type; + jl_set_typeof(jl_tuple_type, jl_tuple_type); #ifdef OVERLAP_TUPLE_LEN jl_tuple_set_len_unsafe(jl_tuple_type, 1); #endif diff --git a/src/julia.h b/src/julia.h index c2f42fe4cd404..cdb1f8642b4f1 100644 --- a/src/julia.h +++ b/src/julia.h @@ -39,6 +39,9 @@ extern "C" { #define NORETURN #endif +#define container_of(ptr, type, member) \ + ((type *) ((char *)(ptr) - offsetof(type, member))) + #ifdef _MSC_VER #if _WIN64 #define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) __declspec(align(8)) x @@ -53,19 +56,40 @@ extern "C" { // core data types ------------------------------------------------------------ -#ifdef OVERLAP_TUPLE_LEN -#define JL_DATA_TYPE \ - size_t type : 52; \ - size_t _resvd : 12; -#else #define JL_DATA_TYPE \ - struct _jl_value_t *type; -#endif + struct _jl_value_t *fieldptr0[0]; typedef struct _jl_value_t { JL_DATA_TYPE + struct _jl_value_t *fieldptr[]; } jl_value_t; +typedef struct { + union { + jl_value_t *type; + uintptr_t type_bits; + struct { + uintptr_t gc_bits:2; +#ifdef OVERLAP_TUPLE_LEN +#ifdef _P64 + uintptr_t unmarked:50; +#else +#error OVERLAP_TUPLE_LEN requires 64-bit pointers +#endif + uintptr_t length:12; +#endif + }; + }; + jl_value_t value[0]; +} jl_typetag_t; + +#define jl_typetagof__MACRO(v) container_of((v),jl_typetag_t,value) +#define jl_typeof__MACRO(v) ((jl_value_t*)(jl_typetagof__MACRO(v)->type_bits&~(size_t)3)) +#define jl_typetagof jl_typetagof__MACRO +#define jl_typeof jl_typeof__MACRO +#define jl_set_typeof(v,t) (jl_typetagof(v)->type = (jl_value_t*)(t)) +#define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) + typedef struct _jl_sym_t { JL_DATA_TYPE struct _jl_sym_t *left; @@ -80,11 +104,8 @@ typedef struct _jl_gensym_t { } jl_gensym_t; typedef struct { -#ifdef OVERLAP_TUPLE_LEN - size_t type : 52; - size_t length : 12; -#else JL_DATA_TYPE +#ifndef OVERLAP_TUPLE_LEN size_t length; #endif jl_value_t *data[]; @@ -172,8 +193,6 @@ typedef struct _jl_lambda_info_t { int32_t specFunctionID; // index that this specFunction will have in the codegen table } jl_lambda_info_t; -#define LAMBDA_INFO_NW (NWORDS(sizeof(jl_lambda_info_t))-1) - typedef struct _jl_function_t { JL_DATA_TYPE jl_fptr_t fptr; @@ -431,38 +450,36 @@ extern jl_sym_t *inert_sym; // GC write barrier -DLLEXPORT void gc_queue_root(void *root); -void gc_queue_binding(void *bnd); +DLLEXPORT void gc_queue_root(jl_value_t *root); // root isa jl_value_t* +void gc_queue_binding(jl_binding_t *bnd); void gc_setmark_buf(void *buf, int); -DLLEXPORT void gc_wb_slow(void* parent, void* ptr); -static inline void gc_wb_binding(void *bnd, void *val) +static inline void gc_wb_binding(jl_binding_t *bnd, void *val) // val isa jl_value_t* { - bnd = (void*)((void**)bnd - 1); - if (__unlikely((*(uintptr_t*)bnd & 1) == 1 && (*(uintptr_t*)val & 1) == 0)) + if (__unlikely((*((uintptr_t*)bnd-1) & 1) == 1 && (*(uintptr_t*)jl_typetagof(val) & 1) == 0)) gc_queue_binding(bnd); } -static inline void gc_wb(void *parent, void *ptr) +static inline void gc_wb(void *parent, void *ptr) // parent and ptr isa jl_value_t* { - if (__unlikely((*((uintptr_t*)parent) & 1) == 1 && - (*((uintptr_t*)ptr) & 1) == 0)) - gc_queue_root(parent); + if (__unlikely((*((uintptr_t*)jl_typetagof(parent)) & 1) == 1 && + (*((uintptr_t*)jl_typetagof(ptr)) & 1) == 0)) + gc_queue_root((jl_value_t*)parent); } -static inline void gc_wb_buf(void *parent, void *bufptr) +static inline void gc_wb_buf(void *parent, void *bufptr) // parent isa jl_value_t* { // if parent is marked and buf is not - if (__unlikely((*((uintptr_t*)parent) & 1) == 1)) + if (__unlikely((*((uintptr_t*)jl_typetagof(parent)) & 1) == 1)) // (*((uintptr_t*)bufptr) & 3) != 1)) - gc_setmark_buf(bufptr, *(uintptr_t*)parent & 3); + gc_setmark_buf(bufptr, *(uintptr_t*)jl_typetagof(parent) & 3); } -static inline void gc_wb_back(void *ptr) +static inline void gc_wb_back(void *ptr) // ptr isa jl_value_t* { // if ptr is marked - if(__unlikely((*((uintptr_t*)ptr) & 1) == 1)) { - gc_queue_root(ptr); + if(__unlikely((*((uintptr_t*)jl_typetagof(ptr)) & 1) == 1)) { + gc_queue_root((jl_value_t*)ptr); } } @@ -470,16 +487,13 @@ static inline void gc_wb_back(void *ptr) // object accessors ----------------------------------------------------------- #ifdef OVERLAP_TUPLE_LEN -#define jl_typeof(v) ((jl_value_t*)((uptrint_t)((jl_value_t*)(v))->type & 0x000ffffffffffffeULL)) +#define jl_tuple_len(t) (jl_typetagof(t)->length) +#define jl_tuple_set_len_unsafe(t,n) (jl_typetagof(t)->length=(n)) #else -#define jl_typeof(v) ((jl_value_t*)((uptrint_t)((jl_value_t*)(v))->type & ((uintptr_t)~3))) -#endif - -#define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) - #define jl_tuple_len(t) (((jl_tuple_t*)(t))->length) #define jl_tuple_set_len_unsafe(t,n) (((jl_tuple_t*)(t))->length=(n)) -#define jl_tuple_data(t) (((jl_tuple_t*)(t))->data) +#endif +#define jl_tuple_data(t) (((jl_tuple_t*)(t))->data) #define TUPLE_DATA_OFFSET offsetof(jl_tuple_t,data)/sizeof(void*) #ifdef STORE_ARRAY_LEN @@ -520,8 +534,8 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) return (jl_value_t*)x; } -# define jl_t0(t) jl_tupleref(t,0) -# define jl_t1(t) jl_tupleref(t,1) +#define jl_t0(t) jl_tupleref(t,0) +#define jl_t1(t) jl_tupleref(t,1) #define jl_exprarg(e,n) (((jl_value_t**)jl_array_data(((jl_expr_t*)(e))->args))[n]) #define jl_exprargset(e, n, v) jl_cellset(((jl_expr_t*)(e))->args, n, v) @@ -530,9 +544,9 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_symbolnode_sym(s) ((jl_sym_t*)jl_fieldref(s,0)) #define jl_symbolnode_type(s) (jl_fieldref(s,1)) -#define jl_linenode_line(x) (((ptrint_t*)x)[1]) -#define jl_labelnode_label(x) (((ptrint_t*)x)[1]) -#define jl_gotonode_label(x) (((ptrint_t*)x)[1]) +#define jl_linenode_line(x) (((ptrint_t*)x)[0]) +#define jl_labelnode_label(x) (((ptrint_t*)x)[0]) +#define jl_gotonode_label(x) (((ptrint_t*)x)[0]) #define jl_globalref_mod(s) ((jl_module_t*)jl_fieldref(s,0)) #define jl_globalref_name(s) ((jl_sym_t*)jl_fieldref(s,1)) @@ -541,14 +555,14 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_cell_data(a) ((jl_value_t**)((jl_array_t*)a)->data) -#define jl_string_data(s) ((char*)((jl_array_t*)((jl_value_t**)(s))[1])->data) -#define jl_iostr_data(s) ((char*)((jl_array_t*)((jl_value_t**)(s))[1])->data) +#define jl_string_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) +#define jl_iostr_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) #define jl_gf_mtable(f) ((jl_methtable_t*)((jl_function_t*)(f))->env) #define jl_gf_name(f) (jl_gf_mtable(f)->name) // get a pointer to the data in a datatype -#define jl_data_ptr(v) (&((void**)(v))[1]) +#define jl_data_ptr(v) (((jl_value_t*)v)->fieldptr) // struct type info #define jl_field_offset(st,i) (((jl_datatype_t*)st)->fields[i].offset) @@ -804,13 +818,13 @@ DLLEXPORT ssize_t jl_unbox_gensym(jl_value_t *v); #ifdef _P64 #define jl_box_long(x) jl_box_int64(x) -#define jl_box_ulong(x) jl_box_uint64(x) +#define jl_box_ulong(x) jl_box_uint64(x) #define jl_unbox_long(x) jl_unbox_int64(x) #define jl_is_long(x) jl_is_int64(x) #define jl_long_type jl_int64_type #else #define jl_box_long(x) jl_box_int32(x) -#define jl_box_ulong(x) jl_box_uint32(x) +#define jl_box_ulong(x) jl_box_uint32(x) #define jl_unbox_long(x) jl_unbox_int32(x) #define jl_is_long(x) jl_is_int32(x) #define jl_long_type jl_int32_type @@ -1177,12 +1191,12 @@ void jl_gc_free_array(jl_array_t *a); void jl_gc_track_malloced_array(jl_array_t *a); void jl_gc_count_allocd(size_t sz); void jl_gc_run_all_finalizers(void); -DLLEXPORT void *alloc_2w(void); -DLLEXPORT void *alloc_3w(void); -DLLEXPORT void *alloc_4w(void); +DLLEXPORT jl_value_t *alloc_1w(void); +DLLEXPORT jl_value_t *alloc_2w(void); +DLLEXPORT jl_value_t *alloc_3w(void); void *allocb(size_t sz); void *reallocb(void*, size_t); -DLLEXPORT void *allocobj(size_t sz); +DLLEXPORT jl_value_t *allocobj(size_t sz); DLLEXPORT void jl_clear_malloc_data(void); DLLEXPORT int64_t jl_gc_num_pause(void); @@ -1198,11 +1212,11 @@ DLLEXPORT int64_t jl_gc_num_full_sweep(void); #define jl_gc_unpreserve() #define jl_gc_n_preserved_values() (0) -STATIC_INLINE void *alloc_2w() { return allocobj(2*sizeof(void*)); } -STATIC_INLINE void *alloc_3w() { return allocobj(3*sizeof(void*)); } -STATIC_INLINE void *alloc_4w() { return allocobj(4*sizeof(void*)); } +STATIC_INLINE jl_value_t *alloc_1w() { return allocobj(1*sizeof(void*)); } +STATIC_INLINE jl_value_t *alloc_2w() { return allocobj(2*sizeof(void*)); } +STATIC_INLINE jl_value_t *alloc_3w() { return allocobj(3*sizeof(void*)); } #define allocb(nb) malloc(nb) -#define allocobj(nb) malloc(nb) +#define allocobj(nb) (((jl_typetag_t*)malloc((nb)+sizeof(jl_typetag_t)))->value) #endif // async signal handling ------------------------------------------------------ diff --git a/src/julia_internal.h b/src/julia_internal.h index 0c3e2618dc7fb..d7a5c7e25d16e 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -13,22 +13,22 @@ STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) jl_value_t *jv = NULL; switch (nfields) { case 1: - jv = (jl_value_t*)alloc_2w(); break; + jv = (jl_value_t*)alloc_1w(); break; case 2: - jv = (jl_value_t*)alloc_3w(); break; + jv = (jl_value_t*)alloc_2w(); break; case 3: - jv = (jl_value_t*)alloc_4w(); break; + jv = (jl_value_t*)alloc_3w(); break; default: - jv = (jl_value_t*)allocobj((1+nfields) * sizeof(void*)); + jv = (jl_value_t*)allocobj(nfields * sizeof(void*)); } - jv->type = type; + jl_set_typeof(jv, type); return jv; } STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) { - jl_value_t *jv = (jl_value_t*)allocobj(sizeof(void*) + type->size); - jv->type = (jl_value_t*)type; + jl_value_t *jv = (jl_value_t*)allocobj(type->size); + jl_set_typeof(jv, type); return jv; } diff --git a/src/module.c b/src/module.c index 28d9d608c2c96..dcba8ac31a071 100644 --- a/src/module.c +++ b/src/module.c @@ -17,7 +17,7 @@ jl_module_t *jl_current_module=NULL; jl_module_t *jl_new_module(jl_sym_t *name) { jl_module_t *m = (jl_module_t*)allocobj(sizeof(jl_module_t)); - m->type = (jl_value_t*)jl_module_type; + jl_set_typeof(m, jl_module_type); JL_GC_PUSH1(&m); assert(jl_is_symbol(name)); m->name = name; diff --git a/src/task.c b/src/task.c index a3d592e22616e..03ffb4eeb7003 100644 --- a/src/task.c +++ b/src/task.c @@ -817,7 +817,7 @@ DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { size_t pagesz = jl_page_size; jl_task_t *t = (jl_task_t*)allocobj(sizeof(jl_task_t)); - t->type = (jl_value_t*)jl_task_type; + jl_set_typeof(t, jl_task_type); ssize = LLT_ALIGN(ssize, pagesz); t->ssize = ssize; t->current_module = NULL; @@ -923,7 +923,7 @@ void jl_init_tasks(void *stack, size_t ssize) runnable_sym = jl_symbol("runnable"); jl_current_task = (jl_task_t*)allocobj(sizeof(jl_task_t)); - jl_current_task->type = (jl_value_t*)jl_task_type; + jl_set_typeof(jl_current_task, jl_task_type); #ifdef COPY_STACKS jl_current_task->ssize = 0; // size of saved piece jl_current_task->bufsz = 0; diff --git a/ui/repl.c b/ui/repl.c index 8d468815e6507..c294b73d3827b 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -422,7 +422,7 @@ static int true_main(int argc, char *argv[]) int i; for (i=0; i < argc; i++) { jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - s->type = (jl_value_t*)jl_utf8_string_type; + jl_set_typeof(s,jl_utf8_string_type); jl_arrayset(args, s, i); } } From cbf9da5504e86fb22464406a60b5a100a1090ba5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 19 Mar 2015 15:03:06 -0400 Subject: [PATCH 2/2] some renaming and fix overloads changed by merge rename jl_typetag_t => jl_taggedvalue_t rename jl_typetagof => jl_astaggedvalue --- src/alloc.c | 6 +++--- src/array.c | 10 +++++----- src/builtins.c | 4 ++-- src/cgutils.cpp | 4 ++-- src/codegen.cpp | 6 +++--- src/dump.c | 6 +++--- src/gc.c | 46 +++++++++++++++++++++++----------------------- src/jlapi.c | 6 +++--- src/julia.h | 28 ++++++++++++++-------------- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 3d4a67ac84661..e11112b5e0449 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -495,20 +495,20 @@ static jl_sym_t *mk_symbol(const char *str) #endif jl_sym_t *sym; size_t len = strlen(str); - size_t nb = (sizeof(jl_typetag_t)+sizeof(jl_sym_t)+len+1+7)&-8; + size_t nb = (sizeof(jl_taggedvalue_t)+sizeof(jl_sym_t)+len+1+7)&-8; if (nb >= SYM_POOL_SIZE) { jl_exceptionf(jl_argumenterror_type, "Symbol length exceeds maximum length"); } #ifdef MEMDEBUG - sym = (jl_sym_t*)((jl_typetag_t*)malloc(nb))->value; + sym = (jl_sym_t*)((jl_taggedvalue_t*)malloc(nb))->value; #else if (sym_pool == NULL || pool_ptr+nb > sym_pool+SYM_POOL_SIZE) { sym_pool = (char*)malloc(SYM_POOL_SIZE); pool_ptr = sym_pool; } - sym = (jl_sym_t*)((jl_typetag_t*)malloc(nb))->value; + sym = (jl_sym_t*)((jl_taggedvalue_t*)malloc(nb))->value; pool_ptr += nb; #endif jl_set_typeof(sym, jl_sym_type); diff --git a/src/array.c b/src/array.c index fcce1ee085333..d5d27cb48258d 100644 --- a/src/array.c +++ b/src/array.c @@ -14,7 +14,7 @@ extern "C" { #endif -#define JL_ARRAY_ALIGN(jl_value, nbytes) (LLT_ALIGN((jl_value)+sizeof(jl_typetag_t), nbytes)-sizeof(jl_typetag_t)) +#define JL_ARRAY_ALIGN(jl_value, nbytes) (LLT_ALIGN((jl_value)+sizeof(jl_taggedvalue_t), nbytes)-sizeof(jl_taggedvalue_t)) // array constructors --------------------------------------------------------- @@ -101,7 +101,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, memset(data, 0, tot); JL_GC_POP(); } - a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; + a->pooled = tsz + sizeof(jl_taggedvalue_t) <= 2048; a->data = data; if (elsz == 1) ((char*)data)[tot-1] = '\0'; @@ -153,7 +153,7 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *di int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), 16); a = (jl_array_t*)allocobj(tsz); jl_set_typeof(a, atype); - a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; + a->pooled = tsz + sizeof(jl_taggedvalue_t) <= 2048; a->ndims = ndims; a->offset = 0; a->data = NULL; @@ -221,7 +221,7 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); a = (jl_array_t*)allocobj(tsz); jl_set_typeof(a, atype); - a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; + a->pooled = tsz + sizeof(jl_taggedvalue_t) <= 2048; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; @@ -272,7 +272,7 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims, int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), 16); a = (jl_array_t*)allocobj(tsz); jl_set_typeof(a, atype); - a->pooled = tsz + sizeof(jl_typetag_t) <= 2048; + a->pooled = tsz + sizeof(jl_taggedvalue_t) <= 2048; a->data = data; #ifdef STORE_ARRAY_LEN a->length = nel; diff --git a/src/builtins.c b/src/builtins.c index a6a1e591cb440..37c569002b39b 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1291,8 +1291,8 @@ size_t jl_static_show_x(JL_STREAM *out, jl_value_t *v, int depth) else if (jl_typeof(v) == NULL) { n += jl_printf(out, ""); } - else if (jl_typetagof(v)->type_bits < 4096U) { - n += jl_printf(out, "", (int)jl_typetagof(v)->type_bits); + else if (jl_astaggedvalue(v)->type_bits < 4096U) { + n += jl_printf(out, "", (int)jl_astaggedvalue(v)->type_bits); } else if (jl_is_lambda_info(v)) { jl_lambda_info_t *li = (jl_lambda_info_t*)v; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index a3e8ca8561907..9910ff36d9039 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -821,7 +821,7 @@ static Value *emit_nthptr_recast(Value *v, Value *idx, MDNode *tbaa, Type *ptype static Value *emit_typeptr_addr(Value *p) { - ssize_t offset = offsetof(jl_typetag_t,type) - offsetof(jl_typetag_t,value); + ssize_t offset = offsetof(jl_taggedvalue_t,type) - offsetof(jl_taggedvalue_t,value); offset /= (signed)sizeof(jl_value_t*); // important: division must be signed return emit_nthptr_addr(p, offset); } @@ -1152,7 +1152,7 @@ static Value *emit_tuplelen(Value *t,jl_value_t *jt) return builder.CreateLShr(builder.CreatePtrToInt(lenbits, T_int64), ConstantInt::get(T_int32, 52)); #else - return emit_nthptr_recast(t, offsetof(jl_tuple_t,length)/sizeof(jl_value_t*), tbaa_tuplelen, T_psize); + return emit_nthptr_recast(t, (ssize_t)(offsetof(jl_tuple_t,length)/sizeof(jl_value_t*)), tbaa_tuplelen, T_psize); #endif } else { //unboxed diff --git a/src/codegen.cpp b/src/codegen.cpp index 2161aa26998b5..1960a815965b9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2026,7 +2026,7 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, Value *theF = emit_expr(args[2],ctx); Value *theFptr = emit_nthptr_recast( theF, - offsetof(jl_function_t,fptr)/sizeof(void*), + (ssize_t)(offsetof(jl_function_t,fptr)/sizeof(void*)), tbaa_func, jl_pfptr_llvmt); Value *nva = emit_n_varargs(ctx); @@ -2632,7 +2632,7 @@ static Value *emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_ #endif // extract pieces of the function object // TODO: try extractvalue instead - theFptr = emit_nthptr_recast(theFunc, offsetof(jl_function_t,fptr)/sizeof(void*), tbaa_func, jl_pfptr_llvmt); + theFptr = emit_nthptr_recast(theFunc, (ssize_t)(offsetof(jl_function_t,fptr)/sizeof(void*)), tbaa_func, jl_pfptr_llvmt); theF = theFunc; } else { @@ -2666,7 +2666,7 @@ static Value *emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_ myargs = builder.CreateGEP(ctx->argTemp, ConstantInt::get(T_size, argStart+1+ctx->argSpaceOffs)); } - theFptr = emit_nthptr_recast(theFunc, offsetof(jl_function_t,fptr)/sizeof(void*), tbaa_func, jl_pfptr_llvmt); + theFptr = emit_nthptr_recast(theFunc, (ssize_t)(offsetof(jl_function_t,fptr)/sizeof(void*)), tbaa_func, jl_pfptr_llvmt); Value *r1 = builder.CreateCall3(prepare_call(theFptr), theFunc, myargs, ConstantInt::get(T_int32,nargs)); builder.CreateBr(mergeBB1); diff --git a/src/dump.c b/src/dump.c index e4a05df548df1..cb2ef196263f9 100644 --- a/src/dump.c +++ b/src/dump.c @@ -910,7 +910,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) arraylist_push(&flagref_list, (void*)(uptrint_t)pos); if (has_instance) { arraylist_push(&flagref_list, dt); - arraylist_push(&flagref_list, &jl_typetagof(dt->instance)->type); + arraylist_push(&flagref_list, &jl_astaggedvalue(dt->instance)->type); arraylist_push(&flagref_list, (void*)(uptrint_t)-1); } } @@ -1074,7 +1074,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } } if (mode == MODE_MODULE) { - aty = jl_deserialize_value(s, &jl_typetagof(a)->type); + aty = jl_deserialize_value(s, &jl_astaggedvalue(a)->type); assert(aty == jl_typeof(a)); } return (jl_value_t*)a; @@ -1311,7 +1311,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } // TODO: put WeakRefs on the weak_refs list if (mode == MODE_MODULE) { - dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_typetagof(v)->type); + dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type); assert((jl_value_t*)dt == jl_typeof(v)); } return v; diff --git a/src/gc.c b/src/gc.c index 583f820fc1a5c..7b327ad2340d6 100644 --- a/src/gc.c +++ b/src/gc.c @@ -343,7 +343,7 @@ static inline void objprofile_count(void* ty, int old, int sz) //static inline void gc_setmark_other(jl_value_t *v, int mark_mode) // unused function //{ -// jl_typetag_t *o = jl_typetagof(v); +// jl_taggedvalue_t *o = jl_astaggedvalue(v); // _gc_setmark(o, mark_mode); // verify_val(o); //} @@ -418,8 +418,8 @@ static inline int gc_setmark_pool(void *o, int mark_mode) static inline int gc_setmark(jl_value_t *v, int sz, int mark_mode) { - jl_typetag_t *o = jl_typetagof(v); - sz += sizeof(jl_typetag_t); + jl_taggedvalue_t *o = jl_astaggedvalue(v); + sz += sizeof(jl_taggedvalue_t); #ifdef MEMDEBUG return gc_setmark_big(o, mark_mode); #endif @@ -610,7 +610,7 @@ void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, int isaligned, jl_ { maybe_collect(); - if (gc_bits(jl_typetagof(owner)) == GC_MARKED) { + if (gc_bits(jl_astaggedvalue(owner)) == GC_MARKED) { perm_scanned_bytes += sz - oldsz; live_bytes += sz - oldsz; } @@ -686,9 +686,9 @@ static void sweep_weak_refs(void) return; do { wr = (jl_weakref_t*)lst[n]; - if (gc_marked(jl_typetagof(wr))) { + if (gc_marked(jl_astaggedvalue(wr))) { // weakref itself is alive - if (!gc_marked(jl_typetagof(wr->value))) + if (!gc_marked(jl_astaggedvalue(wr->value))) wr->value = (jl_value_t*)jl_nothing; n++; } @@ -942,7 +942,7 @@ static void sweep_malloced_arrays(void) mallocarray_t **pma = &mallocarrays; while (ma != NULL) { mallocarray_t *nxt = ma->next; - if (gc_marked(jl_typetagof(ma->a))) { + if (gc_marked(jl_astaggedvalue(ma->a))) { pma = &ma->next; } else { @@ -1361,7 +1361,7 @@ void reset_remset(void) DLLEXPORT void gc_queue_root(jl_value_t *ptr) { - jl_typetag_t *o = jl_typetagof(ptr); + jl_taggedvalue_t *o = jl_astaggedvalue(ptr); assert(gc_bits(o) != GC_QUEUED); gc_bits(o) = GC_QUEUED; arraylist_push(remset, ptr); @@ -1379,7 +1379,7 @@ static int push_root(jl_value_t *v, int d, int); static inline int gc_push_root(void *v, int d) // v isa jl_value_t* { assert(v != NULL); - jl_typetag_t* o = jl_typetagof(v); + jl_taggedvalue_t* o = jl_astaggedvalue(v); verify_val(o); int bits = gc_bits(o); if (!gc_marked(o)) { @@ -1391,7 +1391,7 @@ static inline int gc_push_root(void *v, int d) // v isa jl_value_t* void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now { // int64_t s = perm_scanned_bytes; - jl_typetag_t *o = jl_typetagof(v); + jl_taggedvalue_t *o = jl_astaggedvalue(v); if (!gc_marked(o)) { // objprofile_count(jl_typeof(v), 1, 16); #ifdef MEMDEBUG @@ -1436,7 +1436,7 @@ NOINLINE static int gc_mark_module(jl_module_t *m, int d) for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - gc_setmark_buf(b, gc_bits(jl_typetagof(m))); + gc_setmark_buf(b, gc_bits(jl_astaggedvalue(m))); #ifdef GC_VERIFY void* vb = gc_val_buf(b); verify_parent1("module", m, &vb, "binding_buff"); @@ -1468,7 +1468,7 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) { if (ta->stkbuf != NULL || ta == jl_current_task) { if (ta->stkbuf != NULL) { - gc_setmark_buf(ta->stkbuf, gc_bits(jl_typetagof(ta))); + gc_setmark_buf(ta->stkbuf, gc_bits(jl_astaggedvalue(ta))); } #ifdef COPY_STACKS ptrint_t offset; @@ -1558,7 +1558,7 @@ static int push_root(jl_value_t *v, int d, int bits) } else if (((jl_datatype_t*)(vt))->name == jl_array_typename) { jl_array_t *a = (jl_array_t*)v; - jl_typetag_t *o = jl_typetagof(v); + jl_taggedvalue_t *o = jl_astaggedvalue(v); int todo = !(bits & GC_MARKED); if (a->pooled) #ifdef MEMDEBUG @@ -1693,10 +1693,10 @@ static void visit_mark_stack_inc(int mark_mode) { while(mark_sp > 0 && !should_timeout()) { jl_value_t* v = mark_stack[--mark_sp]; - assert(gc_bits(jl_typetagof(v)) == GC_QUEUED || - gc_bits(jl_typetagof(v)) == GC_MARKED || - gc_bits(jl_typetagof(v)) == GC_MARKED_NOESC); - push_root(v, 0, gc_bits(jl_typetagof(v))); + assert(gc_bits(jl_astaggedvalue(v)) == GC_QUEUED || + gc_bits(jl_astaggedvalue(v)) == GC_MARKED || + gc_bits(jl_astaggedvalue(v)) == GC_MARKED_NOESC); + push_root(v, 0, gc_bits(jl_astaggedvalue(v))); } } @@ -1777,9 +1777,9 @@ static void post_mark(arraylist_t *list, int dryrun) for(size_t i=0; i < list->len; i+=2) { jl_value_t *v = (jl_value_t*)list->items[i]; jl_value_t *fin = (jl_value_t*)list->items[i+1]; - int isfreed = !gc_marked(jl_typetagof(v)); + int isfreed = !gc_marked(jl_astaggedvalue(v)); gc_push_root(fin, 0); - int isold = list == &finalizer_list && gc_bits(jl_typetagof(v)) == GC_MARKED && gc_bits(jl_typetagof(fin)) == GC_MARKED; + int isold = list == &finalizer_list && gc_bits(jl_astaggedvalue(v)) == GC_MARKED && gc_bits(jl_astaggedvalue(fin)) == GC_MARKED; if (!dryrun && (isfreed || isold)) { // remove from this list if (i < list->len - 2) { @@ -2086,7 +2086,7 @@ void jl_gc_collect(int full) for(int i = 0; i < last_remset->len; i++) { jl_value_t *item = (jl_value_t*)last_remset->items[i]; objprofile_count(jl_typeof(item), 2, 0); - gc_bits(jl_typetagof(item)) = GC_MARKED; + gc_bits(jl_astaggedvalue(item)) = GC_MARKED; } for (int i = 0; i < rem_bindings.len; i++) { void *ptr = rem_bindings.items[i]; @@ -2206,7 +2206,7 @@ void jl_gc_collect(int full) // so that we don't trigger the barrier again on them. if (sweep_mask == GC_MARKED_NOESC) { for (int i = 0; i < remset->len; i++) { - gc_bits(jl_typetagof(remset->items[i])) = GC_QUEUED; + gc_bits(jl_astaggedvalue(remset->items[i])) = GC_QUEUED; } for (int i = 0; i < rem_bindings.len; i++) { void *ptr = rem_bindings.items[i]; @@ -2309,7 +2309,7 @@ void *reallocb(void *b, size_t sz) } } -#define jl_valueof(v) (((jl_typetag_t*)(v))->value) +#define jl_valueof(v) (((jl_taggedvalue_t*)(v))->value) DLLEXPORT jl_value_t *allocobj(size_t sz) { @@ -2530,7 +2530,7 @@ static void big_obj_stats(void) mallocarray_t *ma = mallocarrays; while (ma != NULL) { - if (gc_marked(jl_typetagof(ma->a))) { + if (gc_marked(jl_astaggedvalue(ma->a))) { nused++; nbytes += array_nbytes(ma->a); } diff --git a/src/jlapi.c b/src/jlapi.c index fc12852a1e561..8eacd176559f0 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -278,9 +278,9 @@ DLLEXPORT const char* jl_ver_string(void) } // Create function versions of some useful macros -#undef jl_typetagof -DLLEXPORT jl_typetag_t *jl_typetagof(jl_value_t *v) { - return jl_typetagof__MACRO(v); +#undef jl_astaggedvalue +DLLEXPORT jl_taggedvalue_t *jl_astaggedvalue(jl_value_t *v) { + return jl_astaggedvalue__MACRO(v); } #undef jl_typeof diff --git a/src/julia.h b/src/julia.h index cdb1f8642b4f1..897629cb7a851 100644 --- a/src/julia.h +++ b/src/julia.h @@ -81,13 +81,13 @@ typedef struct { }; }; jl_value_t value[0]; -} jl_typetag_t; +} jl_taggedvalue_t; -#define jl_typetagof__MACRO(v) container_of((v),jl_typetag_t,value) -#define jl_typeof__MACRO(v) ((jl_value_t*)(jl_typetagof__MACRO(v)->type_bits&~(size_t)3)) -#define jl_typetagof jl_typetagof__MACRO +#define jl_astaggedvalue__MACRO(v) container_of((v),jl_taggedvalue_t,value) +#define jl_typeof__MACRO(v) ((jl_value_t*)(jl_astaggedvalue__MACRO(v)->type_bits&~(size_t)3)) +#define jl_astaggedvalue jl_astaggedvalue__MACRO #define jl_typeof jl_typeof__MACRO -#define jl_set_typeof(v,t) (jl_typetagof(v)->type = (jl_value_t*)(t)) +#define jl_set_typeof(v,t) (jl_astaggedvalue(v)->type = (jl_value_t*)(t)) #define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) typedef struct _jl_sym_t { @@ -456,29 +456,29 @@ void gc_setmark_buf(void *buf, int); static inline void gc_wb_binding(jl_binding_t *bnd, void *val) // val isa jl_value_t* { - if (__unlikely((*((uintptr_t*)bnd-1) & 1) == 1 && (*(uintptr_t*)jl_typetagof(val) & 1) == 0)) + if (__unlikely((*((uintptr_t*)bnd-1) & 1) == 1 && (*(uintptr_t*)jl_astaggedvalue(val) & 1) == 0)) gc_queue_binding(bnd); } static inline void gc_wb(void *parent, void *ptr) // parent and ptr isa jl_value_t* { - if (__unlikely((*((uintptr_t*)jl_typetagof(parent)) & 1) == 1 && - (*((uintptr_t*)jl_typetagof(ptr)) & 1) == 0)) + if (__unlikely((*((uintptr_t*)jl_astaggedvalue(parent)) & 1) == 1 && + (*((uintptr_t*)jl_astaggedvalue(ptr)) & 1) == 0)) gc_queue_root((jl_value_t*)parent); } static inline void gc_wb_buf(void *parent, void *bufptr) // parent isa jl_value_t* { // if parent is marked and buf is not - if (__unlikely((*((uintptr_t*)jl_typetagof(parent)) & 1) == 1)) + if (__unlikely((*((uintptr_t*)jl_astaggedvalue(parent)) & 1) == 1)) // (*((uintptr_t*)bufptr) & 3) != 1)) - gc_setmark_buf(bufptr, *(uintptr_t*)jl_typetagof(parent) & 3); + gc_setmark_buf(bufptr, *(uintptr_t*)jl_astaggedvalue(parent) & 3); } static inline void gc_wb_back(void *ptr) // ptr isa jl_value_t* { // if ptr is marked - if(__unlikely((*((uintptr_t*)jl_typetagof(ptr)) & 1) == 1)) { + if(__unlikely((*((uintptr_t*)jl_astaggedvalue(ptr)) & 1) == 1)) { gc_queue_root((jl_value_t*)ptr); } } @@ -487,8 +487,8 @@ static inline void gc_wb_back(void *ptr) // ptr isa jl_value_t* // object accessors ----------------------------------------------------------- #ifdef OVERLAP_TUPLE_LEN -#define jl_tuple_len(t) (jl_typetagof(t)->length) -#define jl_tuple_set_len_unsafe(t,n) (jl_typetagof(t)->length=(n)) +#define jl_tuple_len(t) (jl_astaggedvalue(t)->length) +#define jl_tuple_set_len_unsafe(t,n) (jl_astaggedvalue(t)->length=(n)) #else #define jl_tuple_len(t) (((jl_tuple_t*)(t))->length) #define jl_tuple_set_len_unsafe(t,n) (((jl_tuple_t*)(t))->length=(n)) @@ -1216,7 +1216,7 @@ STATIC_INLINE jl_value_t *alloc_1w() { return allocobj(1*sizeof(void*)); } STATIC_INLINE jl_value_t *alloc_2w() { return allocobj(2*sizeof(void*)); } STATIC_INLINE jl_value_t *alloc_3w() { return allocobj(3*sizeof(void*)); } #define allocb(nb) malloc(nb) -#define allocobj(nb) (((jl_typetag_t*)malloc((nb)+sizeof(jl_typetag_t)))->value) +#define allocobj(nb) (((jl_taggedvalue_t*)malloc((nb)+sizeof(jl_taggedvalue_t)))->value) #endif // async signal handling ------------------------------------------------------