// This file is a part of Julia. License is MIT: https://julialang.org/license

/*

implementations of built-in functions

*/

#include "dtypes.h"

#include "julia_atomics.h"

#include "platform.h"

#include <stdlib.h>

#include <stdio.h>

#include <string.h>

#include <stdarg.h>

#include <setjmp.h>

#include <sys/types.h>

#include <errno.h>

#include <fcntl.h>

#include <inttypes.h>

#if defined(_OS_WINDOWS_)

#include <malloc.h>

#else

#include <unistd.h>

#endif

#include <ctype.h>

#include "julia.h"

#include "julia_internal.h"

#include "builtin_proto.h"

#include "intrinsics.h"

#include "julia_assert.h"

#ifdef __cplusplus

extern "C" {

#endif

jl_fptr_args_t const jl_builtin_f_addrs[jl_n_builtins] = {

#define BUILTIN_ADDRS(cname,jlname) &jl_f_##cname,

JL_BUILTIN_FUNCTIONS(BUILTIN_ADDRS)

#undef BUILTIN_ADDRS

};

const char *const jl_builtin_f_names[jl_n_builtins] = {

#define BUILTIN_F_NAMES(cname,jlname) XSTR(jl_f_##cname),

JL_BUILTIN_FUNCTIONS(BUILTIN_F_NAMES)

#undef BUILTIN_F_NAMES

};

jl_value_t *jl_builtin_instances[jl_n_builtins];

static const char *const jl_builtin_names[jl_n_builtins] = {

#define BUILTIN_NAMES(cname,jlname) jlname,

JL_BUILTIN_FUNCTIONS(BUILTIN_NAMES)

#undef BUILTIN_NAMES

};

// egal and object_id ---------------------------------------------------------

static int bits_equal(const void *a, const void *b, int sz) JL_NOTSAFEPOINT

{

switch (sz) {

case 1: return *(uint8_t*)a == *(uint8_t*)b;

// Let compiler constant folds the following, though we may not know alignment of them

case 2: return memcmp(a, b, 2) == 0;

case 4: return memcmp(a, b, 4) == 0;

case 8: return memcmp(a, b, 8) == 0;

default: return memcmp(a, b, sz) == 0;

}

}

// The frequently used jl_egal function deserves special attention when it

// comes to performance which is made challenging by the fact that the

// function has to handle quite a few different cases and because it is

// called recursively. To optimize performance many special cases are

// handle with separate comparisons which can dramatically reduce the run

// time of the function. The compiler can translate these simple tests

// with little effort, e.g., few registers are used.

//

// The complex cases require more effort and more registers to be translated

// efficiently. The effected cases include comparing tuples and fields. If

// the code to perform these operation would be inlined in the jl_egal

// function then the compiler would generate at the or close to the top of

// the function a prologue which saves all the callee-save registers and at

// the end the respective epilogue. The result is that even the fast cases

// are slowed down.

//

// The solution is to keep the code in jl_egal simple and split out the

// (more) complex cases into their own functions which are marked with

// NOINLINE.

static int NOINLINE compare_svec(jl_svec_t *a, jl_svec_t *b) JL_NOTSAFEPOINT

{

size_t i, l = jl_svec_len(a);

if (l != jl_svec_len(b))

return 0;

for (i = 0; i < l; i++) {

if (!jl_egal(jl_svecref(a, i), jl_svecref(b, i)))

return 0;

}

return 1;

}

// See comment above for an explanation of NOINLINE.

static int NOINLINE compare_fields(const jl_value_t *a, const jl_value_t *b, jl_datatype_t *dt) JL_NOTSAFEPOINT

{

size_t nf = jl_datatype_nfields(dt);

// npointers is used at end, but fetched here for locality with nfields.

int npointers = ((jl_datatype_t*)dt)->layout->npointers;

for (size_t f = 0; f < nf; f++) {

size_t offs = jl_field_offset(dt, f);

char *ao = (char*)a + offs;

char *bo = (char*)b + offs;

if (jl_field_isptr(dt, f)) {

// Save ptr recursion until the end -- only recurse if otherwise equal

// Note that we also skip comparing the pointers for null here, because

// null fields are rare so it can save CPU to delay this read too.

continue;

}

else {

jl_datatype_t *ft = (jl_datatype_t*)jl_field_type_concrete(dt, f);

if (jl_is_uniontype(ft)) {

size_t idx = jl_field_size(dt, f) - 1;

uint8_t asel = ((uint8_t*)ao)[idx];

uint8_t bsel = ((uint8_t*)bo)[idx];

if (asel != bsel)

return 0;

ft = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)ft, asel);

}

else if (ft->layout->first_ptr >= 0) {

// If the field is a inline immutable that can be undef

// we need to check for undef first since undef struct

// may have fields that are different but should still be treated as equal.

int32_t idx = ft->layout->first_ptr;

jl_value_t *ptra = ((jl_value_t**)ao)[idx];

jl_value_t *ptrb = ((jl_value_t**)bo)[idx];

if ((ptra == NULL) != (ptrb == NULL)) {

return 0;

}

else if (ptra == NULL) { // implies ptrb == NULL

continue; // skip this field (it is #undef)

}

}

if (!ft->layout->flags.haspadding && ft->layout->flags.isbitsegal) {

if (!bits_equal(ao, bo, ft->layout->size))

return 0;

}

else {

assert(jl_datatype_nfields(ft) > 0);

if (!compare_fields((jl_value_t*)ao, (jl_value_t*)bo, ft))

return 0;

}

}

}

// If we've gotten here, the objects are bitwise equal, besides their pointer fields.

// Now, we will recurse into jl_egal for the pointed-to elements, which might be

// arbitrarily expensive.

for (size_t p = 0; p < npointers; p++) {

size_t offs = jl_ptr_offset(dt, p);

jl_value_t *af = ((jl_value_t**)a)[offs];

jl_value_t *bf = ((jl_value_t**)b)[offs];

if (af != bf) {

if (af == NULL || bf == NULL)

return 0;

if (!jl_egal(af, bf))

return 0;

}

}

return 1;

}

static int egal_types(const jl_value_t *a, const jl_value_t *b, jl_typeenv_t *env, int tvar_names) JL_NOTSAFEPOINT

{

if (a == b)

return 1;

uintptr_t dtag = jl_typetagof(a);

if (dtag != jl_typetagof(b))

return 0;

if (dtag == jl_datatype_tag << 4) {

jl_datatype_t *dta = (jl_datatype_t*)a;

jl_datatype_t *dtb = (jl_datatype_t*)b;

if (dta->name != dtb->name)

return 0;

size_t i, l = jl_nparams(dta);

if (jl_nparams(dtb) != l)

return 0;

for (i = 0; i < l; i++) {

if (!egal_types(jl_tparam(dta, i), jl_tparam(dtb, i), env, tvar_names))

return 0;

}

return 1;

}

if (dtag == jl_tvar_tag << 4) {

jl_typeenv_t *pe = env;

while (pe != NULL) {

if (pe->var == (jl_tvar_t*)a)

return pe->val == b;

pe = pe->prev;

}

return 0;

}

if (dtag == jl_unionall_tag << 4) {

jl_unionall_t *ua = (jl_unionall_t*)a;

jl_unionall_t *ub = (jl_unionall_t*)b;

if (tvar_names && ua->var->name != ub->var->name)

return 0;

if (!(egal_types(ua->var->lb, ub->var->lb, env, tvar_names) && egal_types(ua->var->ub, ub->var->ub, env, tvar_names)))

return 0;

jl_typeenv_t e = { ua->var, (jl_value_t*)ub->var, env };

return egal_types(ua->body, ub->body, &e, tvar_names);

}

if (dtag == jl_uniontype_tag << 4) {

return egal_types(((jl_uniontype_t*)a)->a, ((jl_uniontype_t*)b)->a, env, tvar_names) &&

egal_types(((jl_uniontype_t*)a)->b, ((jl_uniontype_t*)b)->b, env, tvar_names);

}

if (dtag == jl_typeeq_tag << 4)

return egal_types(((jl_typeeq_t*)a)->T, ((jl_typeeq_t*)b)->T, env, tvar_names);

if (dtag == jl_vararg_tag << 4) {

jl_vararg_t *vma = (jl_vararg_t*)a;

jl_vararg_t *vmb = (jl_vararg_t*)b;

jl_value_t *vmaT = vma->T ? vma->T : (jl_value_t*)jl_any_type;

jl_value_t *vmbT = vmb->T ? vmb->T : (jl_value_t*)jl_any_type;

if (!egal_types(vmaT, vmbT, env, tvar_names))

return 0;

if (vma->N && vmb->N)

return egal_types(vma->N, vmb->N, env, tvar_names);

return !vma->N && !vmb->N;

}

assert(dtag == jl_symbol_tag << 4 || dtag == jl_module_tag << 4 || !((jl_datatype_t*)jl_typeof(a))->name->mutabl);

return jl_egal__bitstag(a, b, dtag);

}

JL_DLLEXPORT int jl_types_egal(jl_value_t *a, jl_value_t *b)

{

return egal_types(a, b, NULL, 0);

}

JL_DLLEXPORT int (jl_egal)(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT

{

// warning: a,b may NOT have been gc-rooted by the caller

return jl_egal(a, b);

}

JL_DLLEXPORT int jl_egal__unboxed(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT

{

// warning: a,b may NOT have been gc-rooted by the caller

return jl_egal__unboxed_(a, b, dtag);

}

JL_DLLEXPORT int jl_egal__bitstag(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, uintptr_t dtag) JL_NOTSAFEPOINT

{

if (dtag < jl_max_tags << 4) {

switch ((enum jl_small_typeof_tags)(dtag >> 4)) {

case jl_int8_tag:

case jl_uint8_tag:

case jl_addrspacecore_tag:

return *(uint8_t*)a == *(uint8_t*)b;

case jl_int16_tag:

case jl_uint16_tag:

return *(uint16_t*)a == *(uint16_t*)b;

case jl_int32_tag:

case jl_uint32_tag:

case jl_char_tag:

case jl_intrinsic_tag:

return *(uint32_t*)a == *(uint32_t*)b;

case jl_int64_tag:

case jl_uint64_tag:

return *(uint64_t*)a == *(uint64_t*)b;

case jl_ssavalue_tag:

case jl_slotnumber_tag:

case jl_argument_tag:

return *(size_t*)a == *(size_t*)b;

case jl_gotoifnot_tag:

return compare_fields(a, b, jl_gotoifnot_type);

case jl_returnnode_tag:

return compare_fields(a, b, jl_returnnode_type);

case jl_enternode_tag:

return compare_fields(a, b, jl_enternode_type);

case jl_pinode_tag:

return compare_fields(a, b, jl_pinode_type);

case jl_phinode_tag:

return compare_fields(a, b, jl_phinode_type);

case jl_phicnode_tag:

return compare_fields(a, b, jl_phicnode_type);

case jl_upsilonnode_tag:

return compare_fields(a, b, jl_upsilonnode_type);

case jl_globalref_tag:

return compare_fields(a, b, jl_globalref_type);

case jl_gotonode_tag:

return *(size_t*)a == *(size_t*)b;

case jl_quotenode_tag:

return compare_fields(a, b, jl_quotenode_type);

case jl_unionall_tag:

return egal_types(a, b, NULL, 1);

case jl_uniontype_tag:

return compare_fields(a, b, jl_uniontype_type);

case jl_typeeq_tag:

return egal_types(a, b, NULL, 1);

case jl_vararg_tag:

return compare_fields(a, b, jl_vararg_type);

case jl_task_tag:

case jl_tvar_tag:

case jl_symbol_tag:

case jl_module_tag:

case jl_bool_tag:

case jl_nothing_tag:

return 0;

case jl_simplevector_tag:

return compare_svec((jl_svec_t*)a, (jl_svec_t*)b);

case jl_string_tag: {

size_t l = jl_string_len(a);

if (jl_string_len(b) != l)

return 0;

return !memcmp(jl_string_data(a), jl_string_data(b), l);

}

case jl_datatype_tag: {

jl_datatype_t *dta = (jl_datatype_t*)a;

jl_datatype_t *dtb = (jl_datatype_t*)b;

if (dta->name != dtb->name)

return 0;

if (dta->name != jl_tuple_typename && (dta->isconcretetype || dtb->isconcretetype))

return 0;

return compare_svec(dta->parameters, dtb->parameters);

}

#ifndef NDEBUG

default:

#endif

case jl_max_tags:

case jl_null_tag:

case jl_typeofbottom_tag:

case jl_tags_count:

abort();

}

}

return jl_egal__bits(a, b, (jl_datatype_t*)dtag);

}

inline int jl_egal__bits(const jl_value_t *a JL_MAYBE_UNROOTED, const jl_value_t *b JL_MAYBE_UNROOTED, jl_datatype_t *dt) JL_NOTSAFEPOINT

{

size_t sz = jl_datatype_size(dt);

if (sz == 0)

return 1;

size_t nf = jl_datatype_nfields(dt);

if (nf == 0 || (!dt->layout->flags.haspadding && dt->layout->flags.isbitsegal))

return bits_equal(a, b, sz);

return compare_fields(a, b, dt);

}

// object_id ------------------------------------------------------------------

static uintptr_t bits_hash(const void *b, size_t sz) JL_NOTSAFEPOINT

{

switch (sz) {

case 1: return int32hash(*(const int8_t*)b);

case 2: return int32hash(jl_load_unaligned_i16(b));

case 4: return int32hash(jl_load_unaligned_i32(b));

#ifdef _P64

case 8: return int64hash(jl_load_unaligned_i64(b));

#else

case 8: return int64to32hash(jl_load_unaligned_i64(b));

#endif

default:

#ifdef _P64

return memhash((const char*)b, sz);

#else

return memhash32((const char*)b, sz);

#endif

}

}

static uintptr_t NOINLINE hash_svec(jl_svec_t *v) JL_NOTSAFEPOINT

{

uintptr_t h = 0;

size_t i, l = jl_svec_len(v);

for (i = 0; i < l; i++) {

jl_value_t *x = jl_svecref(v, i);

uintptr_t u = (x == NULL) ? 0 : jl_object_id(x);

h = bitmix(h, u);

}

return h;

}

static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOTSAFEPOINT;

typedef struct _varidx {

jl_tvar_t *var;

struct _varidx *prev;

} jl_varidx_t;

static uintptr_t type_object_id_(jl_value_t *v, jl_varidx_t *env) JL_NOTSAFEPOINT

{

if (v == NULL)

return 0;

jl_datatype_t *tv = (jl_datatype_t*)jl_typeof(v);

if (tv == jl_tvar_type) {

jl_varidx_t *pe = env;

int i = 0;

while (pe != NULL) {

if (pe->var == (jl_tvar_t*)v)

return (i<<8) + 42;

i++;

pe = pe->prev;

}

uintptr_t bits = jl_astaggedvalue(v)->header;

if (bits & GC_IN_IMAGE)

return ((uintptr_t*)v)[-2];

return inthash((uintptr_t)v);

}

if (tv == jl_uniontype_type) {

return bitmix(bitmix(jl_object_id((jl_value_t*)tv),

type_object_id_(((jl_uniontype_t*)v)->a, env)),

type_object_id_(((jl_uniontype_t*)v)->b, env));

}

if (tv == jl_typeeq_type) {

return bitmix(jl_object_id((jl_value_t*)tv),

type_object_id_(((jl_typeeq_t*)v)->T, env));

}

if (tv == jl_unionall_type) {

jl_unionall_t *u = (jl_unionall_t*)v;

uintptr_t h = u->var->name->hash;

h = bitmix(h, type_object_id_(u->var->lb, env));

h = bitmix(h, type_object_id_(u->var->ub, env));

jl_varidx_t e = { u->var, env };

return bitmix(h, type_object_id_(u->body, &e));

}

if (tv == jl_datatype_type) {

jl_datatype_t *dtv = (jl_datatype_t*)v;

if (dtv->isconcretetype)

return dtv->hash;

uintptr_t h = ~dtv->name->hash;

size_t i, l = jl_nparams(v);

for (i = 0; i < l; i++) {

h = bitmix(h, type_object_id_(jl_tparam(v, i), env));

}

return h;

}

if (tv == jl_vararg_type) {

jl_vararg_t *vm = (jl_vararg_t*)v;

jl_value_t *t = vm->T ? vm->T : (jl_value_t*)jl_any_type;

jl_value_t *n = vm->N ? vm->N : jl_nothing;

return bitmix(type_object_id_(t, env),

type_object_id_(n, env));

}

if (tv == jl_symbol_type)

return ((jl_sym_t*)v)->hash;

if (tv == jl_module_type)

return ((jl_module_t*)v)->hash;

assert(!tv->name->mutabl);

return immut_id_(tv, v, tv->hash);

}

static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOTSAFEPOINT

{

size_t sz = jl_datatype_size(dt);

if (sz == 0)

return ~h;

size_t f, nf = jl_datatype_nfields(dt);

if (nf == 0 || (!dt->layout->flags.haspadding && dt->layout->flags.isbitsegal && dt->layout->npointers == 0)) {

// operate element-wise if there are unused bits inside,

// otherwise just take the whole data block at once

// a few select pointers (notably symbol) also have special hash values

// which may affect the stability of the objectid hash, even though

// they don't affect egal comparison

return bits_hash(v, sz) ^ h;

}

if (dt == jl_unionall_type)

return type_object_id_(v, NULL);

for (f = 0; f < nf; f++) {

size_t offs = jl_field_offset(dt, f);

char *vo = (char*)v + offs;

uintptr_t u;

if (jl_field_isptr(dt, f)) {

jl_value_t *f = *(jl_value_t**)vo;

u = (f == NULL) ? 0 : jl_object_id(f);

}

else {

jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type_concrete(dt, f);

if (jl_is_uniontype(fieldtype)) {

uint8_t sel = ((uint8_t*)vo)[jl_field_size(dt, f) - 1];

fieldtype = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)fieldtype, sel);

}

assert(jl_is_datatype(fieldtype) && !fieldtype->name->abstract && !fieldtype->name->mutabl);

int32_t first_ptr = fieldtype->layout->first_ptr;

if (first_ptr >= 0 && ((jl_value_t**)vo)[first_ptr] == NULL) {

// If the field is a inline immutable that can be can be undef

// we need to check to check for undef first since undef struct

// may have fields that are different but should still be treated as equal.

u = 0;

}

else {

u = immut_id_(fieldtype, (jl_value_t*)vo, 0);

}

}

h = bitmix(h, u);

}

return h;

}

static uintptr_t NOINLINE jl_object_id__cold(uintptr_t tv, jl_value_t *v) JL_NOTSAFEPOINT

{

jl_datatype_t *dt = (jl_datatype_t*)jl_to_typeof(tv);

if (dt->name->mutabl) {

if (dt == jl_string_type) {

#ifdef _P64

return memhash_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677);

#else

return memhash32_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677);

#endif

}

if (dt == jl_simplevector_type)

return hash_svec((jl_svec_t*)v);

if (dt == jl_datatype_type) {

jl_datatype_t *dtv = (jl_datatype_t*)v;

uintptr_t h = ~dtv->name->hash;

return bitmix(h, hash_svec(dtv->parameters));

}

if (dt == jl_module_type) {

jl_module_t *m = (jl_module_t*)v;

return m->hash;

}

uintptr_t bits = jl_astaggedvalue(v)->header;

if (bits & GC_IN_IMAGE)

return ((uintptr_t*)v)[-2];

return inthash((uintptr_t)v);

}

return immut_id_(dt, v, dt->hash);

}

JL_DLLEXPORT inline uintptr_t jl_object_id_(uintptr_t tv, jl_value_t *v) JL_NOTSAFEPOINT

{

if (tv == jl_symbol_tag << 4) {

return ((jl_sym_t*)v)->hash;

}

else if (tv == jl_datatype_tag << 4) {

jl_datatype_t *dtv = (jl_datatype_t*)v;

if (dtv->isconcretetype)

return dtv->hash;

}

else if (tv == (uintptr_t)jl_typename_type) {

return ((jl_typename_t*)v)->hash;

}

return jl_object_id__cold(tv, v);

}

JL_DLLEXPORT uintptr_t jl_object_id(jl_value_t *v) JL_NOTSAFEPOINT

{

return jl_object_id_(jl_typetagof(v), v);

}

// eq hash table --------------------------------------------------------------

#include "iddict.c"

#include "idset.c"

// object model and type primitives -------------------------------------------

JL_CALLABLE(jl_f_is)

{

JL_NARGS(===, 2, 2);

return jl_egal(args[0], args[1]) ? jl_true : jl_false;

}

JL_CALLABLE(jl_f_typeof)

{

JL_NARGS(typeof, 1, 1);

return jl_typeof(args[0]);

}

JL_CALLABLE(jl_f_sizeof)

{

JL_NARGS(sizeof, 1, 1);

jl_value_t *x = args[0];

if (jl_is_unionall(x) || jl_is_uniontype(x)) {

x = jl_unwrap_unionall(x);

size_t elsize = 0;

int isinline = jl_uniontype_size(x, &elsize);

if (isinline)

return jl_box_long(elsize);

if (!jl_is_datatype(x))

jl_error("Argument is an abstract type and does not have a definite size.");

}

if (jl_is_datatype(x)) {

jl_datatype_t *dx = (jl_datatype_t*)x;

if (!jl_struct_try_layout(dx)) {

if (dx->name->abstract)

jl_errorf("Abstract type %s does not have a definite size.", jl_symbol_name(dx->name->name));

else

jl_errorf("Argument is an incomplete %s type and does not have a definite size.", jl_symbol_name(dx->name->name));

}

if (jl_is_layout_opaque(dx->layout)) // includes all GenericMemory{kind,T}

jl_errorf("Type %s does not have a definite size.", jl_symbol_name(dx->name->name));

return jl_box_long(jl_datatype_size(x));

}

if (x == jl_bottom_type)

jl_error("The empty type does not have a definite size since it does not have instances.");

if (jl_is_string(x))

return jl_box_long(jl_string_len(x));

if (jl_is_symbol(x))

return jl_box_long(strlen(jl_symbol_name((jl_sym_t*)x)));

if (jl_is_svec(x))

return jl_box_long((1+jl_svec_len(x))*sizeof(void*));

jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(x);

assert(jl_is_datatype(dt));

assert(!dt->name->abstract);

size_t sz = dt->layout->size;

if (jl_is_genericmemory(x))

sz = (sz + (dt->layout->flags.arrayelem_isunion ? 1 : 0)) * ((jl_genericmemory_t*)x)->length;

return jl_box_long(sz);

}

JL_CALLABLE(jl_f_issubtype)

{

JL_NARGS(<:, 2, 2);

jl_value_t *a = args[0], *b = args[1];

JL_TYPECHK(<:, type, a);

JL_TYPECHK(<:, type, b);

return (jl_subtype(a,b) ? jl_true : jl_false);

}

JL_CALLABLE(jl_f_isa)

{

JL_NARGS(isa, 2, 2);

JL_TYPECHK(isa, type, args[1]);

return (jl_isa(args[0],args[1]) ? jl_true : jl_false);

}

JL_CALLABLE(jl_f_typeassert)

{

JL_NARGS(typeassert, 2, 2);

JL_TYPECHK(typeassert, type, args[1]);

if (!jl_isa(args[0],args[1]))

jl_type_error("typeassert", args[1], args[0]);

return args[0];

}

JL_CALLABLE(jl_f_throw)

{

JL_NARGS(throw, 1, 1);

jl_throw(args[0]);

return jl_nothing;

}

JL_CALLABLE(jl_f_throw_methoderror)

{

JL_NARGSV(throw_methoderror, 1);

size_t world = jl_get_tls_world_age();

jl_method_error(args[0], &args[1], nargs, world);

return jl_nothing;

}

JL_CALLABLE(jl_f_ifelse)

{

JL_NARGS(ifelse, 3, 3);

JL_TYPECHK(ifelse, bool, args[0]);

return (args[0] == jl_false ? args[2] : args[1]);

}

JL_CALLABLE(jl_f_current_scope)

{

JL_NARGS(current_scope, 0, 0);

return jl_current_task->scope;

}

// apply ----------------------------------------------------------------------

static NOINLINE jl_svec_t *_copy_to(size_t newalloc, jl_value_t **oldargs, size_t oldalloc)

{

size_t j;

jl_svec_t *newheap = jl_alloc_svec_uninit(newalloc);

jl_value_t **newargs = jl_svec_data(newheap);

for (j = 0; j < oldalloc; j++)

newargs[j] = oldargs[j];

for (; j < newalloc; j++)

newargs[j] = NULL;

return newheap;

}

STATIC_INLINE void _grow_to(jl_value_t **root, jl_value_t ***oldargs, jl_svec_t **arg_heap, size_t *n_alloc, size_t newalloc, size_t extra)

{

size_t oldalloc = *n_alloc;

if (oldalloc >= newalloc)

return;

if (extra)

// grow by an extra 50% if newalloc is still only a guess

newalloc += oldalloc / 2 + 16;

JL_GC_PROMISE_ROOTED(*oldargs);

jl_svec_t *newheap = _copy_to(newalloc, *oldargs, oldalloc);

*root = (jl_value_t*)newheap;

*arg_heap = newheap;

*oldargs = jl_svec_data(newheap);

*n_alloc = newalloc;

}

static jl_value_t *jl_arrayref(jl_array_t *a, size_t i)

{

return jl_memoryrefget(jl_memoryrefindex(a->ref, i), 0);

}

JL_CALLABLE(jl_f__apply_iterate)

{

JL_NARGSV(_apply_iterate, 2);

jl_value_t *iterate = args[0];

jl_value_t *f = args[1];

assert(iterate);

args += 1;

nargs -= 1;

if (nargs == 2) {

// some common simple cases

if (f == BUILTIN(svec)) {

if (jl_is_svec(args[1]))

return args[1];

if (jl_is_genericmemory(args[1])) {

jl_genericmemory_t *mem = (jl_genericmemory_t*)args[1];

size_t n = mem->length;

jl_svec_t *t = jl_alloc_svec(n);

JL_GC_PUSH1(&t);

for (size_t i = 0; i < n; i++) {

jl_svecset(t, i, jl_genericmemoryref(mem, i));

}

JL_GC_POP();

return (jl_value_t*)t;

}

if (jl_is_array(args[1])) {

size_t n = jl_array_len(args[1]);

jl_svec_t *t = jl_alloc_svec(n);

JL_GC_PUSH1(&t);

for (size_t i = 0; i < n; i++) {

jl_svecset(t, i, jl_arrayref((jl_array_t*)args[1], i));

}

JL_GC_POP();

return (jl_value_t*)t;

}

}

else if (f == BUILTIN(tuple)) {

if (jl_is_tuple(args[1]))

return args[1];

if (jl_is_svec(args[1]))

return jl_f_tuple(NULL, jl_svec_data(args[1]), jl_svec_len(args[1]));

}

// optimization for `f(svec...)`

if (jl_is_svec(args[1]))

return jl_apply_generic(f, jl_svec_data(args[1]), jl_svec_len(args[1]));

}

// estimate how many real arguments we appear to have

size_t precount = 1;

size_t extra = 0;

size_t i;

for (i = 1; i < nargs; i++) {

if (jl_is_svec(args[i])) {

precount += jl_svec_len(args[i]);

}

else if (jl_is_tuple(args[i]) || jl_is_namedtuple(args[i])) {

precount += jl_nfields(args[i]);

}

else if (jl_is_genericmemory(args[i])) {

precount += ((jl_genericmemory_t*)args[i])->length;

}

else if (jl_is_array(args[i])) {

precount += jl_array_len(args[i]);

}

else {

extra += 1;

}

}

// allocate space for the argument array and gc roots for it

// based on our previous estimates

// use the stack if we have a good estimate that it is small

// otherwise, use the heap and grow it incrementally

// and if there are any extra elements, we'll also need a couple extra roots

int onstack = (precount + 32 * extra < jl_page_size / sizeof(jl_value_t*));

size_t stackalloc = onstack ? (precount + 4 * extra + (extra ? 16 : 0)) : 1;

size_t n_alloc;

jl_value_t **roots;

JL_GC_PUSHARGS(roots, stackalloc + (extra ? 2 : 0));

jl_value_t **newargs;

jl_svec_t *arg_heap = NULL;

if (onstack) {

newargs = roots;

n_alloc = stackalloc;

}

else {

// put arguments on the heap if there are too many

newargs = NULL;

n_alloc = precount;

if (extra)

// grow by an extra 50% if newalloc is still only a guess

n_alloc += n_alloc / 2 + 16;

arg_heap = jl_alloc_svec(n_alloc);

roots[0] = (jl_value_t*)arg_heap;

newargs = jl_svec_data(arg_heap);

}

newargs[0] = f;

precount -= 1;

size_t n = 1;

for (i = 1; i < nargs; i++) {

jl_value_t *ai = args[i];

if (jl_is_svec(ai)) {

jl_svec_t *t = (jl_svec_t*)ai;

size_t j, al = jl_svec_len(t);

precount = (precount > al) ? precount - al : 0;

_grow_to(&roots[0], &newargs, &arg_heap, &n_alloc, n + precount + al, extra);

assert(newargs != NULL); // inform GCChecker that we didn't write a NULL here

for (j = 0; j < al; j++) {

newargs[n++] = jl_svecref(t, j);

// GC Note: here we assume that the return value of `jl_svecref`

// will not be young if `arg_heap` becomes old

// since they are allocated before `arg_heap`. Otherwise,

// we need to add write barrier for !onstack

}

}

else if (jl_is_tuple(ai) || jl_is_namedtuple(ai)) {

size_t j, al = jl_nfields(ai);

precount = (precount > al) ? precount - al : 0;

_grow_to(&roots[0], &newargs, &arg_heap, &n_alloc, n + precount + al, extra);

assert(newargs != NULL); // inform GCChecker that we didn't write a NULL here

for (j = 0; j < al; j++) {

// jl_fieldref may allocate.

newargs[n++] = jl_fieldref(ai, j);

if (arg_heap)

jl_gc_wb(arg_heap, newargs[n - 1]);

}

}

else if (jl_is_genericmemory(ai)) {

jl_genericmemory_t *mem = (jl_genericmemory_t*)ai;

size_t j, al = mem->length;

precount = (precount > al) ? precount - al : 0;

_grow_to(&roots[0], &newargs, &arg_heap, &n_alloc, n + precount + al, extra);

assert(newargs != NULL); // inform GCChecker that we didn't write a NULL here

const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(mem))->layout;

if (layout->flags.arrayelem_isboxed) {

for (j = 0; j < al; j++) {

jl_value_t *arg = jl_genericmemory_ptr_ref(mem, j);

// apply with array splatting may have embedded NULL value (#11772)

if (__unlikely(arg == NULL))

jl_throw(jl_undefref_exception);

newargs[n++] = arg;

if (arg_heap)

jl_gc_wb(arg_heap, arg);

}

}

else {

for (j = 0; j < al; j++) {

newargs[n++] = jl_genericmemoryref(mem, j);

if (arg_heap)

jl_gc_wb(arg_heap, newargs[n - 1]);

}

}

}

else if (jl_is_array(ai)) {

jl_array_t *aai = (jl_array_t*)ai;

size_t j, al = jl_array_len(aai);

precount = (precount > al) ? precount - al : 0;

_grow_to(&roots[0], &newargs, &arg_heap, &n_alloc, n + precount + al, extra);

assert(newargs != NULL); // inform GCChecker that we didn't write a NULL here

const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(aai->ref.mem))->layout;

if (layout->flags.arrayelem_isboxed) {

for (j = 0; j < al; j++) {

jl_value_t *arg = jl_array_ptr_ref(aai, j);

// apply with array splatting may have embedded NULL value (#11772)

if (__unlikely(arg == NULL))

jl_throw(jl_undefref_exception);

newargs[n++] = arg;

if (arg_heap)

jl_gc_wb(arg_heap, arg);

}

}

else {

for (j = 0; j < al; j++) {

newargs[n++] = jl_arrayref(aai, j);

if (arg_heap)

jl_gc_wb(arg_heap, newargs[n - 1]);

}

}

}

else {

assert(extra > 0);

jl_value_t *args[2];

args[0] = ai;

jl_value_t *next = jl_apply_generic(iterate, args, 1);

while (next != jl_nothing) {

roots[stackalloc] = next;

jl_value_t *value = jl_get_nth_field_checked(next, 0);

roots[stackalloc + 1] = value;

jl_value_t *state = jl_get_nth_field_checked(next, 1);

roots[stackalloc] = state;

_grow_to(&roots[0], &newargs, &arg_heap, &n_alloc, n + precount + 1, extra);

JL_GC_ASSERT_LIVE(value);

newargs[n++] = value;

if (arg_heap)

jl_gc_wb(arg_heap, value);

roots[stackalloc + 1] = NULL;

JL_GC_ASSERT_LIVE(state);

args[1] = state;

next = jl_apply_generic(iterate, args, 2);

}

roots[stackalloc] = NULL;

extra -= 1;

}

}

if (arg_heap) {

// optimization: keep only the first root, free the others

#ifndef __clang_gcanalyzer__

((void**)roots)[-2] = (void*)JL_GC_ENCODE_PUSHARGS(1);

#endif

}

jl_value_t *result = jl_apply(newargs, n);

JL_GC_POP();

return result;

}

// this is like a regular call, but always runs in the newest world

JL_CALLABLE(jl_f_invokelatest)

{

JL_NARGSV(invokelatest, 1);

jl_task_t *ct = jl_current_task;

size_t last_age = ct->world_age;

if (!ct->ptls->in_pure_callback)

ct->world_age = jl_atomic_load_acquire(&jl_world_counter);

jl_value_t *ret = jl_apply(args, nargs);

ct->world_age = last_age;

return ret;

}

// Like invokelatest, but runs in the specified world.

// If world > jl_atomic_load_acquire(&jl_world_counter), run in the latest world.

JL_CALLABLE(jl_f_invoke_in_world)

{

JL_NARGSV(invoke_in_world, 2);

jl_task_t *ct = jl_current_task;

size_t last_age = ct->world_age;

JL_TYPECHK(invoke_in_world, ulong, args[0]);

size_t world = jl_unbox_ulong(args[0]);

if (!ct->ptls->in_pure_callback) {

ct->world_age = jl_atomic_load_acquire(&jl_world_counter);

if (ct->world_age > world)

ct->world_age = world;

}

jl_value_t *ret = jl_apply(&args[1], nargs - 1);

ct->world_age = last_age;

return ret;

}

JL_CALLABLE(jl_f__call_in_world_total)

{

JL_NARGSV(_call_in_world_total, 2);

JL_TYPECHK(_call_in_world_total, ulong, args[0]);

jl_task_t *ct = jl_current_task;

int last_in = ct->ptls->in_pure_callback;

jl_value_t *ret = NULL;

size_t last_age = ct->world_age;

JL_TRY {

ct->ptls->in_pure_callback = 1;

size_t world = jl_unbox_ulong(args[0]);

ct->world_age = jl_atomic_load_acquire(&jl_world_counter);

if (ct->world_age > world)

ct->world_age = world;

ret = jl_apply(&args[1], nargs - 1);

ct->world_age = last_age;

ct->ptls->in_pure_callback = last_in;

}

JL_CATCH {

ct->ptls->in_pure_callback = last_in;

jl_rethrow();

}

return ret;

}

// tuples ---------------------------------------------------------------------

static jl_value_t *arg_tuple(jl_value_t *a1, jl_value_t **args, size_t nargs)

{

size_t i;

jl_datatype_t *tt = jl_inst_arg_tuple_type(a1, args, nargs, 0);

JL_GC_PROMISE_ROOTED(tt); // it is a concrete type

if (tt->instance != NULL)

return tt->instance;

jl_task_t *ct = jl_current_task;

jl_value_t *jv = jl_gc_alloc(ct->ptls, jl_datatype_size(tt), tt);

for (i = 0; i < nargs; i++)

set_nth_field(tt, jv, i, i == 0 ? a1 : args[i - 1], 0);

return jv;

}

JL_CALLABLE(jl_f_tuple)

{

if (nargs == 0)

return (jl_value_t*)jl_emptytuple;

return arg_tuple(args[0], &args[1], nargs);

}

JL_CALLABLE(jl_f_svec)

{

size_t i;

if (nargs == 0)

return (jl_value_t*)jl_emptysvec;

jl_svec_t *t = jl_alloc_svec_uninit(nargs);

for (i = 0; i < nargs; i++) {

jl_svecset(t, i, args[i]);

}

return (jl_value_t*)t;

}