// This file is a part of Julia. License is MIT: https://julialang.org/license

/*

task.c

lightweight processes (symmetric coroutines)

*/

// this is needed for !COPY_STACKS to work on linux

#ifdef _FORTIFY_SOURCE

// disable __longjmp_chk validation so that we can jump between stacks

// (which would normally be invalid to do with setjmp / longjmp)

#pragma push_macro("_FORTIFY_SOURCE")

#undef _FORTIFY_SOURCE

#include <setjmp.h>

#pragma pop_macro("_FORTIFY_SOURCE")

#endif

#include "platform.h"

#include <stdlib.h>

#include <string.h>

#include <signal.h>

#include <unistd.h>

#include <errno.h>

#include <inttypes.h>

#include "julia.h"

#include "julia_internal.h"

#include "threading.h"

#include "julia_assert.h"

#ifdef _COMPILER_TSAN_ENABLED_

#include <sanitizer/tsan_interface.h>

#endif

#ifdef __cplusplus

extern "C" {

#endif

#if defined(_COMPILER_ASAN_ENABLED_)

#if __GLIBC__

#include <dlfcn.h>

// Bypass the ASAN longjmp wrapper - we are unpoisoning the stack ourselves,

// since ASAN normally unpoisons far too much.

// c.f. interceptor in jl_dlopen as well

void (*real_siglongjmp)(jmp_buf _Buf, int _Value) = NULL;

#endif

static inline void sanitizer_start_switch_fiber(jl_ptls_t ptls, jl_ucontext_t *from, jl_ucontext_t *to) {

if (to->copy_stack)

__sanitizer_start_switch_fiber(&from->asan_fake_stack, (char*)ptls->stackbase - ptls->stacksize, ptls->stacksize);

else

__sanitizer_start_switch_fiber(&from->asan_fake_stack, to->stkbuf, to->bufsz);

}

static inline void sanitizer_start_switch_fiber_killed(jl_ptls_t ptls, jl_ucontext_t *to) {

if (to->copy_stack)

__sanitizer_start_switch_fiber(NULL, (char*)ptls->stackbase - ptls->stacksize, ptls->stacksize);

else

__sanitizer_start_switch_fiber(NULL, to->stkbuf, to->bufsz);

}

static inline void sanitizer_finish_switch_fiber(jl_ucontext_t *last, jl_ucontext_t *current) {

__sanitizer_finish_switch_fiber(current->asan_fake_stack, NULL, NULL);

//(const void**)&last->stkbuf,

//&last->bufsz);

}

#else

static inline void sanitizer_start_switch_fiber(jl_ptls_t ptls, jl_ucontext_t *from, jl_ucontext_t *to) JL_NOTSAFEPOINT {}

static inline void sanitizer_start_switch_fiber_killed(jl_ptls_t ptls, jl_ucontext_t *to) JL_NOTSAFEPOINT {}

static inline void sanitizer_finish_switch_fiber(jl_ucontext_t *last, jl_ucontext_t *current) JL_NOTSAFEPOINT {}

#endif

#if defined(_COMPILER_TSAN_ENABLED_)

// must defined as macros, since the function containing them must not return before the longjmp

#define tsan_destroy_ctx(_ptls, _ctx) do { \

jl_ucontext_t *_tsan_macro_ctx = (_ctx); \

if (_tsan_macro_ctx != &(_ptls)->root_task->ctx) { \

__tsan_destroy_fiber(_tsan_macro_ctx->tsan_state); \

} \

_tsan_macro_ctx->tsan_state = NULL; \

} while (0)

#define tsan_switch_to_ctx(_ctx) do { \

jl_ucontext_t *_tsan_macro_ctx = (_ctx); \

__tsan_switch_to_fiber(_tsan_macro_ctx->tsan_state, 0); \

} while (0)

#else

// just do minimal type-checking on the arguments

#define tsan_destroy_ctx(_ptls, _ctx) do { \

jl_ucontext_t *_tsan_macro_ctx = (_ctx); \

(void)_tsan_macro_ctx; \

} while (0)

#define tsan_switch_to_ctx(_ctx) do { \

jl_ucontext_t *_tsan_macro_ctx = (_ctx); \

(void)_tsan_macro_ctx; \

} while (0)

#endif

// empirically, jl_finish_task needs about 64k stack space to infer/run

// and additionally, gc-stack reserves 64k for the guard pages

#if defined(MINSIGSTKSZ)

#define MINSTKSZ (MINSIGSTKSZ > 131072 ? MINSIGSTKSZ : 131072)

#else

#define MINSTKSZ 131072

#endif

#ifdef _COMPILER_ASAN_ENABLED_

#define ROOT_TASK_STACK_ADJUSTMENT 0

#else

#define ROOT_TASK_STACK_ADJUSTMENT 3000000

#endif

static void jl_set_fiber(jl_ucontext_t *t);

static void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t);

static void jl_start_fiber_swap(jl_ucontext_t *savet, jl_ucontext_t *t);

static void jl_start_fiber_set(jl_ucontext_t *t);

#ifdef ALWAYS_COPY_STACKS

# ifndef COPY_STACKS

# error "ALWAYS_COPY_STACKS requires COPY_STACKS"

# endif

static int always_copy_stacks = 1;

#else

static int always_copy_stacks = 0;

#endif

#if defined(_COMPILER_ASAN_ENABLED_)

extern void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset);

JL_NO_ASAN void *memcpy_noasan(void *dest, const void *src, size_t n) {

char *d = (char*)dest;

const char *s = (const char *)src;

for (size_t i = 0; i < n; ++i)

d[i] = s[i];

return dest;

}

JL_NO_ASAN void *memcpy_a16_noasan(uint64_t *dest, const uint64_t *src, size_t nb) {

uint64_t *end = (uint64_t*)((char*)src + nb);

while (src < end)

*(dest++) = *(src++);

return dest;

}

/* Copy stack are allocated as regular bigval objects and do no go through free_stack,

which would otherwise unpoison it before returning to the GC pool */

static void asan_free_copy_stack(void *stkbuf, size_t bufsz) {

__asan_unpoison_stack_memory((uintptr_t)stkbuf, bufsz);

}

#else

static void asan_free_copy_stack(void *stkbuf, size_t bufsz) {}

#endif

#ifdef COPY_STACKS

static void JL_NO_ASAN JL_NO_MSAN memcpy_stack_a16(uint64_t *to, uint64_t *from, size_t nb)

{

#if defined(_COMPILER_ASAN_ENABLED_)

/* Asan keeps shadow memory for everything on the stack. However, in general,

this function may touch invalid portions of the stack, since it just moves

the stack around. To keep ASAN's stack tracking capability intact, we need

to move the shadow memory along with the stack memory itself. */

size_t shadow_offset;

size_t shadow_scale;

__asan_get_shadow_mapping(&shadow_scale, &shadow_offset);

uintptr_t from_addr = (((uintptr_t)from) >> shadow_scale) + shadow_offset;

uintptr_t to_addr = (((uintptr_t)to) >> shadow_scale) + shadow_offset;

// Make sure that the shadow scale is compatible with the alignment, so

// we can copy whole bytes.

assert(shadow_scale <= 4);

size_t shadow_nb = nb >> shadow_scale;

// Copy over the shadow memory

memcpy_noasan((char*)to_addr, (char*)from_addr, shadow_nb);

memcpy_a16_noasan(jl_assume_aligned(to, 16), jl_assume_aligned(from, 16), nb);

#elif defined(_COMPILER_MSAN_ENABLED_)

# warning This function is incompletely implemented for MSAN (TODO).

memcpy((char*)jl_assume_aligned(to, 16), (char*)jl_assume_aligned(from, 16), nb);

#else

memcpy((char*)jl_assume_aligned(to, 16), (char*)jl_assume_aligned(from, 16), nb);

//uint64_t *end = (uint64_t*)((char*)from + nb);

//while (from < end)

// *(to++) = *(from++);

#endif

}

static void NOINLINE save_stack(jl_ptls_t ptls, jl_task_t *lastt, jl_task_t **pt)

{

char *frame_addr = (char*)((uintptr_t)jl_get_frame_addr() & ~15);

char *stackbase = (char*)ptls->stackbase;

assert(stackbase > frame_addr);

size_t nb = stackbase - frame_addr;

void *buf;

if (lastt->ctx.bufsz < nb) {

asan_free_copy_stack(lastt->ctx.stkbuf, lastt->ctx.bufsz);

buf = (void*)jl_gc_alloc_buf(ptls, nb);

lastt->ctx.stkbuf = buf;

lastt->ctx.bufsz = nb;

}

else {

buf = lastt->ctx.stkbuf;

}

*pt = NULL; // clear the gc-root for the target task before copying the stack for saving

lastt->ctx.copy_stack = nb;

lastt->sticky = 1;

memcpy_stack_a16((uint64_t*)buf, (uint64_t*)frame_addr, nb);

}

JL_NO_ASAN static void NOINLINE JL_NORETURN restore_stack(jl_ucontext_t *t, jl_ptls_t ptls, char *p)

{

size_t nb = t->copy_stack;

char *_x = (char*)ptls->stackbase - nb;

if (!p) {

// switch to a stackframe that's beyond the bounds of the last switch

p = _x - 4096;

if ((char*)&_x > p) {

p = (char*)alloca((char*)&_x - p);

}

restore_stack(t, ptls, p); // pass p to ensure the compiler can't tailcall this or avoid the alloca

}

void *_y = t->stkbuf;

assert(_x != NULL && _y != NULL);

#if defined(_OS_WINDOWS_) // this platform does not implement CFI_NORETURN correctly or at all in libunwind (or equivalent) which requires a workaround

#if defined(_CPU_X86_) || defined(_CPU_X86_64_)

void *volatile *return_address = (void *volatile *)__builtin_frame_address(0) + 1;

assert(*return_address == __builtin_return_address(0));

*return_address = NULL;

#else

#pragma message("warning: CFI_NORETURN not implemented for this platform, so profiling of copy_stacks may segfault in this build")

#endif

#else

CFI_NORETURN

#endif

memcpy_stack_a16((uint64_t*)_x, (uint64_t*)_y, nb); // destroys all but the current stackframe

#if defined(_OS_WINDOWS_)

jl_setcontext(t->copy_ctx);

#else

jl_longjmp(t->copy_ctx->uc_mcontext, 1);

#endif

abort(); // unreachable

}

JL_NO_ASAN static void restore_stack2(jl_ucontext_t *t, jl_ptls_t ptls, jl_ucontext_t *lastt)

{

assert(t->copy_stack && !lastt->copy_stack);

size_t nb = t->copy_stack;

if (nb > 1) {

char *_x = (char*)ptls->stackbase - nb;

void *_y = t->stkbuf;

assert(_x != NULL && _y != NULL);

memcpy_stack_a16((uint64_t*)_x, (uint64_t*)_y, nb);

}

#if defined(_OS_WINDOWS_)

// jl_swapcontext and setjmp are the same on Windows, so we can just use jl_swapcontext directly

tsan_switch_to_ctx(t);

jl_swapcontext(lastt->ctx, t->copy_ctx);

#else

#if defined(JL_TASK_SWITCH_LIBUNWIND)

volatile int returns = 0;

int r = unw_getcontext(lastt->ctx);

if (++returns == 2) // r is garbage after the first return

return;

if (r != 0 || returns != 1)

abort();

#elif defined(JL_TASK_SWITCH_ASM)

if (jl_setjmp(lastt->ctx->uc_mcontext, 0))

return;

#else

#error COPY_STACKS is incompatible with this platform

#endif

tsan_switch_to_ctx(t);

jl_longjmp(t->copy_ctx->uc_mcontext, 1);

#endif

}

JL_NO_ASAN static void NOINLINE restore_stack3(jl_ucontext_t *t, jl_ptls_t ptls, char *p)

{

#if !defined(JL_TASK_SWITCH_ASM)

char *_x = (char*)ptls->stackbase;

if (!p) {

// switch to a stackframe that's well beyond the bounds of the next switch

p = _x - 4096;

if ((char*)&_x > p) {

p = (char*)alloca((char*)&_x - p);

}

restore_stack3(t, ptls, p); // pass p to ensure the compiler can't tailcall this or avoid the alloca

}

#endif

#if defined(_OS_WINDOWS_) // this platform does not implement CFI_NORETURN correctly or at all in libunwind (or equivalent) which requires a workaround

#if defined(_CPU_X86_) || defined(_CPU_X86_64_)

void *volatile *return_address = (void *volatile *)__builtin_frame_address(0) + 1;

assert(*return_address == __builtin_return_address(0));

*return_address = NULL;

#endif

#else

CFI_NORETURN

#endif

tsan_switch_to_ctx(t);

jl_start_fiber_set(t); // (doesn't return)

abort();

}

#endif

/* Rooted by the base module */

static _Atomic(jl_value_t*) task_done_hook_func JL_GLOBALLY_ROOTED = NULL;

void JL_NORETURN jl_finish_task(jl_task_t *ct)

{

JL_PROBE_RT_FINISH_TASK(ct);

JL_SIGATOMIC_BEGIN();

if (ct->metrics_enabled) {

// [task] user_time -finished-> wait_time

assert(jl_atomic_load_relaxed(&ct->first_enqueued_at) != 0);

uint64_t now = jl_hrtime();

jl_atomic_store_relaxed(&ct->finished_at, now);

jl_atomic_fetch_add_relaxed(&ct->running_time_ns, now - jl_atomic_load_relaxed(&ct->last_started_running_at));

}

if (jl_atomic_load_relaxed(&ct->_isexception))

jl_atomic_store_release(&ct->_state, JL_TASK_STATE_FAILED);

else

jl_atomic_store_release(&ct->_state, JL_TASK_STATE_DONE);

if (ct->ctx.copy_stack) { // early free of stkbuf

asan_free_copy_stack(ct->ctx.stkbuf, ct->ctx.bufsz);

ct->ctx.stkbuf = NULL;

}

// ensure that state is cleared

ct->ptls->in_finalizer = 0;

ct->ptls->in_pure_callback = 0;

ct->world_age = jl_atomic_load_acquire(&jl_world_counter);

// let the runtime know this task is dead and find a new task to run

jl_value_t *done = jl_atomic_load_relaxed(&task_done_hook_func);

if (done == NULL) {

done = (jl_value_t*)jl_get_global_value(jl_base_module, jl_symbol("task_done_hook"), ct->world_age);

if (done != NULL)

jl_atomic_store_release(&task_done_hook_func, done);

}

if (done != NULL) {

jl_value_t *args[2] = {done, (jl_value_t*)ct};

JL_TRY {

jl_apply(args, 2);

}

JL_CATCH {

jl_no_exc_handler(jl_current_exception(ct), ct);

}

}

jl_gc_debug_fprint_critical_error(ios_safe_stderr);

abort();

}

JL_DLLEXPORT void jl_active_task_stack(jl_task_t *task,

char **active_start, char **active_end,

char **total_start, char **total_end)

{

if (!task->ctx.started) {

*total_start = *active_start = 0;

*total_end = *active_end = 0;

return;

}

jl_ptls_t ptls2 = task->ptls;

if (task->ctx.copy_stack && ptls2) {

*total_start = *active_start = (char*)ptls2->stackbase - ptls2->stacksize;

*total_end = *active_end = (char*)ptls2->stackbase;

}

else if (task->ctx.stkbuf) {

*total_start = *active_start = (char*)task->ctx.stkbuf;

#ifndef _OS_WINDOWS_

jl_ptls_t ptls0 = jl_atomic_load_relaxed(&jl_all_tls_states)[0];

if (ptls0->root_task == task) {

// See jl_init_root_task(). The root task of the main thread

// has its buffer enlarged by an artificial 3000000 bytes, but

// that means that the start of the buffer usually points to

// inaccessible memory. We need to correct for this.

*active_start += ROOT_TASK_STACK_ADJUSTMENT;

*total_start += ROOT_TASK_STACK_ADJUSTMENT;

}

#endif

*total_end = *active_end = (char*)task->ctx.stkbuf + task->ctx.bufsz;

#ifdef COPY_STACKS

// save_stack stores the stack of an inactive task in stkbuf, and the

// actual number of used bytes in copy_stack.

if (task->ctx.copy_stack > 1)

*active_end = (char*)task->ctx.stkbuf + task->ctx.copy_stack;

#endif

}

else {

// no stack allocated yet

*total_start = *active_start = 0;

*total_end = *active_end = 0;

return;

}

if (task == jl_current_task) {

// scan up to current `sp` for current thread and task

*active_start = (char*)jl_get_frame_addr();

}

}

// Marked noinline so we can consistently skip the associated frame.

// `skip` is number of additional frames to skip.

NOINLINE static void record_backtrace(jl_ptls_t ptls, int skip) JL_NOTSAFEPOINT

{

// storing bt_size in ptls ensures roots in bt_data will be found

ptls->bt_size = rec_backtrace(ptls->bt_data, JL_MAX_BT_SIZE, skip + 1);

}

JL_DLLEXPORT void jl_set_next_task(jl_task_t *task) JL_NOTSAFEPOINT

{

jl_current_task->ptls->next_task = task;

}

JL_DLLEXPORT jl_task_t *jl_get_next_task(void) JL_NOTSAFEPOINT

{

jl_task_t *ct = jl_current_task;

if (ct->ptls->next_task)

return ct->ptls->next_task;

return ct;

}

#ifdef _COMPILER_TSAN_ENABLED_

const char tsan_state_corruption[] = "TSAN state corrupted. Exiting HARD!\n";

#endif

JL_NO_ASAN static void ctx_switch(jl_task_t *lastt)

{

jl_ptls_t ptls = lastt->ptls;

jl_task_t **pt = &ptls->next_task;

jl_task_t *t = *pt;

assert(t != lastt);

// none of these locks should be held across a task switch

assert(ptls->locks.len == 0);

#ifdef _COMPILER_TSAN_ENABLED_

if (lastt->ctx.tsan_state != __tsan_get_current_fiber()) {

// Something went really wrong - don't even assume that we can

// use assert/abort which involve lots of signal handling that

// looks at the tsan state.

write(STDERR_FILENO, tsan_state_corruption, sizeof(tsan_state_corruption) - 1);

_exit(1);

}

#endif

int killed = jl_atomic_load_relaxed(&lastt->_state) != JL_TASK_STATE_RUNNABLE;

if (!t->ctx.started && !t->ctx.copy_stack) {

// may need to allocate the stack

if (t->ctx.stkbuf == NULL) {

t->ctx.stkbuf = jl_malloc_stack(&t->ctx.bufsz, t);

if (t->ctx.stkbuf == NULL) {

#ifdef COPY_STACKS

// fall back to stack copying if mmap fails

t->ctx.copy_stack = 1;

t->ctx.bufsz = 0;

t->sticky = 1;

#else

jl_throw(jl_memory_exception);

#endif

}

}

}

union {

_jl_ucontext_t ctx;

jl_stack_context_t copy_ctx;

} lasttstate;

if (killed) {

*pt = NULL; // can't fail after here: clear the gc-root for the target task now

lastt->gcstack = NULL;

lastt->eh = NULL;

if (!lastt->ctx.copy_stack && lastt->ctx.stkbuf) {

// early free of stkbuf back to the pool

jl_release_task_stack(ptls, lastt);

}

}

else {

if (lastt->ctx.copy_stack) { // save the old copy-stack

#ifdef _OS_WINDOWS_

lasttstate.copy_ctx.uc_stack.ss_sp = (char*)ptls->stackbase - ptls->stacksize;

lasttstate.copy_ctx.uc_stack.ss_size = ptls->stacksize;

#endif

#ifdef COPY_STACKS

if (jl_setjmp(lasttstate.copy_ctx.uc_mcontext, 0)) {

#ifdef MIGRATE_TASKS

ptls = lastt->ptls;

#endif

lastt->ctx.copy_ctx = NULL;

sanitizer_finish_switch_fiber(&ptls->previous_task->ctx, &lastt->ctx);

return;

}

save_stack(ptls, lastt, pt); // allocates (gc-safepoint, and can also fail)

lastt->ctx.copy_ctx = &lasttstate.copy_ctx;

#else

abort();

#endif

}

else {

*pt = NULL; // can't fail after here: clear the gc-root for the target task now

lastt->ctx.ctx = &lasttstate.ctx;

}

}

// this task's stack or scope field could have been modified after

// it was marked by an incremental collection

// move the barrier back instead of walking the shadow stack again here to check if that is required

// even if killed (dropping the stack) and just the scope field matters,

// let the gc figure that out next time it does a quick mark

jl_gc_wb_back(lastt);

// set up global state for new task and clear global state for old task

t->ptls = ptls;

jl_atomic_store_relaxed(&ptls->current_task, t);

JL_GC_PROMISE_ROOTED(t);

jl_signal_fence();

jl_set_pgcstack(&t->gcstack);

jl_signal_fence();

lastt->ptls = NULL;

#ifdef MIGRATE_TASKS

ptls->previous_task = lastt;

#endif

if (t->ctx.started) {

if (t->ctx.copy_stack) {

#ifdef COPY_STACKS

if (lastt->ctx.copy_stack) {

// Switching from copystack to copystack. Clear any shadow stack

// memory above the saved shadow stack.

uintptr_t stacktop = (uintptr_t)ptls->stackbase - t->ctx.copy_stack;

uintptr_t stackbottom = ((uintptr_t)jl_get_frame_addr() & ~15);

if (stackbottom < stacktop)

asan_unpoison_stack_memory(stackbottom, stacktop - stackbottom);

}

if (!killed && !lastt->ctx.copy_stack) {

sanitizer_start_switch_fiber(ptls, &lastt->ctx, &t->ctx);

restore_stack2(&t->ctx, ptls, &lastt->ctx); // half jl_swap_fiber and half restore_stack

}

else {

tsan_switch_to_ctx(&t->ctx);

if (killed) {

sanitizer_start_switch_fiber_killed(ptls, &t->ctx);

tsan_destroy_ctx(ptls, &lastt->ctx);

}

else {

sanitizer_start_switch_fiber(ptls, &lastt->ctx, &t->ctx);

}

if (lastt->ctx.copy_stack) {

restore_stack(&t->ctx, ptls, NULL); // (doesn't return)

abort();

}

else {

restore_stack(&t->ctx, ptls, (char*)1); // (doesn't return)

abort();

}

}

#endif

}

else {

if (lastt->ctx.copy_stack) {

// Switching away from a copystack to a non-copystack. Clear

// the whole shadow stack now, because otherwise we won't know

// how much stack memory to clear the next time we switch to

// a copystack.

uintptr_t stacktop = (uintptr_t)ptls->stackbase;

uintptr_t stackbottom = ((uintptr_t)jl_get_frame_addr() & ~15);

// We're not restoring the stack, but we still need to unpoison the

// stack, so it starts with a pristine stack.

asan_unpoison_stack_memory(stackbottom, stacktop - stackbottom);

}

if (killed) {

sanitizer_start_switch_fiber_killed(ptls, &t->ctx);

tsan_switch_to_ctx(&t->ctx);

tsan_destroy_ctx(ptls, &lastt->ctx);

jl_set_fiber(&t->ctx); // (doesn't return)

abort(); // unreachable

}

else {

sanitizer_start_switch_fiber(ptls, &lastt->ctx, &t->ctx);

if (lastt->ctx.copy_stack) {

// Resume at the jl_setjmp earlier in this function,

// don't do a full task swap

tsan_switch_to_ctx(&t->ctx);

jl_set_fiber(&t->ctx); // (doesn't return)

abort();

}

else {

jl_swap_fiber(&lastt->ctx, &t->ctx);

}

}

}

}

else {

#ifdef _COMPILER_TSAN_ENABLED_

t->ctx.tsan_state = __tsan_create_fiber(0);

#endif

if (lastt->ctx.copy_stack) {

uintptr_t stacktop = (uintptr_t)ptls->stackbase;

uintptr_t stackbottom = ((uintptr_t)jl_get_frame_addr() & ~15);

// We're not restoring the stack, but we still need to unpoison the

// stack, so it starts with a pristine stack.

asan_unpoison_stack_memory(stackbottom, stacktop - stackbottom);

}

if (t->ctx.copy_stack) {

#ifdef COPY_STACKS

tsan_switch_to_ctx(&t->ctx);

// create a temporary non-copy_stack context for starting this fiber

jl_ucontext_t ctx = t->ctx;

ctx.ctx = NULL;

ctx.stkbuf = (char*)ptls->stackbase - ptls->stacksize;

ctx.bufsz = ptls->stacksize;

ctx.copy_stack = 0;

ctx.started = 0;

if (killed) {

sanitizer_start_switch_fiber_killed(ptls, &t->ctx);

tsan_destroy_ctx(ptls, &lastt->ctx);

if (lastt->ctx.copy_stack)

restore_stack3(&ctx, ptls, NULL); // (doesn't return)

else

jl_start_fiber_set(&ctx);

abort();

}

sanitizer_start_switch_fiber(ptls, &lastt->ctx, &t->ctx);

if (lastt->ctx.copy_stack) {

restore_stack3(&ctx, ptls, NULL); // (doesn't return)

abort();

}

else {

jl_start_fiber_swap(&lastt->ctx, &ctx);

}

#else

abort();

#endif

}

else {

if (killed) {

sanitizer_start_switch_fiber_killed(ptls, &t->ctx);

tsan_switch_to_ctx(&t->ctx);

tsan_destroy_ctx(ptls, &lastt->ctx);

jl_start_fiber_set(&t->ctx); // (doesn't return)

abort();

}

sanitizer_start_switch_fiber(ptls, &lastt->ctx, &t->ctx);

if (lastt->ctx.copy_stack) {

// copy_stack resumes at the jl_setjmp earlier in this function, so don't swap here

tsan_switch_to_ctx(&t->ctx);

jl_start_fiber_set(&t->ctx); // (doesn't return)

abort();

}

else {

jl_start_fiber_swap(&lastt->ctx, &t->ctx);

}

}

}

#ifdef MIGRATE_TASKS

ptls = lastt->ptls;

#endif

assert(ptls);

assert(lastt == jl_atomic_load_relaxed(&ptls->current_task));

lastt->ctx.ctx = NULL;

sanitizer_finish_switch_fiber(&ptls->previous_task->ctx, &lastt->ctx);

}

JL_DLLEXPORT void jl_switch(void) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER

{

jl_task_t *ct = jl_current_task;

jl_ptls_t ptls = ct->ptls;

jl_task_t *t = ptls->next_task;

if (t == ct) {

return;

}

int8_t gc_state = jl_gc_unsafe_enter(ptls);

if (t->ctx.started && t->ctx.stkbuf == NULL)

jl_error("attempt to switch to exited task");

if (ptls->in_finalizer)

jl_error("task switch not allowed from inside gc finalizer");

if (ptls->in_pure_callback)

jl_error("task switch not allowed from inside staged nor pure functions");

if (!jl_set_task_tid(t, jl_atomic_load_relaxed(&ct->tid))) // manually yielding to a task

jl_error("cannot switch to task running on another thread");

JL_PROBE_RT_PAUSE_TASK(ct);

// Store old values on the stack and reset

sig_atomic_t defer_signal = ptls->defer_signal;

int finalizers_inhibited = ptls->finalizers_inhibited;

ptls->finalizers_inhibited = 0;

jl_timing_block_t *blk = jl_timing_block_task_exit(ct, ptls);

ctx_switch(ct);

#ifdef MIGRATE_TASKS

ptls = ct->ptls;

t = ptls->previous_task;

ptls->previous_task = NULL;

assert(t != ct);

assert(jl_atomic_load_relaxed(&t->tid) == ptls->tid);

if (!t->sticky && !t->ctx.copy_stack)

jl_atomic_store_release(&t->tid, -1);

#else

assert(ptls == ct->ptls);

#endif

// Pop old values back off the stack

assert(ct == jl_current_task &&

0 != ct->ptls &&

0 == ptls->finalizers_inhibited);

ptls->finalizers_inhibited = finalizers_inhibited;

jl_timing_block_task_enter(ct, ptls, blk); (void)blk;

sig_atomic_t other_defer_signal = ptls->defer_signal;

ptls->defer_signal = defer_signal;

if (other_defer_signal && !defer_signal)

jl_sigint_safepoint(ptls);

JL_PROBE_RT_RUN_TASK(ct);

jl_gc_unsafe_leave(ptls, gc_state);

}

JL_DLLEXPORT void jl_switchto(jl_task_t **pt) JL_NOTSAFEPOINT_ENTER // n.b. this does not actually enter a safepoint

{

jl_set_next_task(*pt);

jl_switch();

}

JL_DLLEXPORT JL_NORETURN void jl_no_exc_handler(jl_value_t *e, jl_task_t *ct)

{

// NULL exception objects are used when rethrowing. we don't have a handler to process

// the exception stack, so at least report the exception at the top of the stack.

if (!e)

e = jl_current_exception(ct);

// Write error to memory first

ios_t s;

ios_mem(&s, 1024);

jl_safe_fprintf(&s, "fatal: error thrown and no exception handler available.\n");

jl_static_show((JL_STREAM*)&s, e);

jl_safe_fprintf(&s, "\n");

jl_fprint_backtrace(&s);

// Then to STDERR

ios_write_direct(ios_stderr, &s);

// Finally write to system log (if supported)

#ifdef _OS_WINDOWS_

HANDLE event_source = RegisterEventSourceW(NULL, L"julia");

if (event_source != INVALID_HANDLE_VALUE) {

ios_putc('\0', &s);

const wchar_t *strings[] = { ios_utf8_to_wchar(s.buf) };

ReportEventW(

event_source, EVENTLOG_ERROR_TYPE, /* category */ 0, /* event_id */ (DWORD)0xE0000000L,

/* user_sid */ NULL, /* n_strings */ 1, /* data_size */ 0, strings, /* data */ NULL

);

free((void *)strings[0]);

if (jl_options.alert_on_critical_error) {

MessageBoxW(NULL, /* message */ L"fatal: error thrown and no exception handler available\n\n"

L"See Application log in Event Viewer for more information.",

/* title */ L"fatal error in libjulia", MB_OK | MB_ICONEXCLAMATION | MB_SYSTEMMODAL);

}

}

#endif

ios_close(&s);

if (ct == NULL)

jl_raise(6);

jl_exit(1);

}

/* throw_internal - yield to exception handler */

#ifdef ENABLE_TIMINGS

#define pop_timings_stack() \

jl_timing_block_t *cur_block = ptls->timing_stack; \

while (cur_block && eh->timing_stack != cur_block) { \

cur_block = jl_timing_block_pop(cur_block); \

} \

assert(cur_block == eh->timing_stack);

#else

#define pop_timings_stack() /* Nothing */

#endif

static void JL_NORETURN throw_internal(jl_task_t *ct, jl_value_t *exception JL_MAYBE_UNROOTED)

{

JL_GC_PUSH1(&exception);

jl_ptls_t ptls = ct->ptls;

ptls->io_wait = 0;

jl_gc_unsafe_enter(ptls);

if (exception) {

/* The temporary ptls->bt_data is rooted by special purpose code in the\

GC. This exists only for the purpose of preserving bt_data until we

set ptls->bt_size=0 below. */

jl_push_excstack(ct, &ct->excstack, exception,

ptls->bt_data, ptls->bt_size);

ptls->bt_size = 0;

}

assert(ct->excstack && ct->excstack->top);

jl_handler_t *eh = ct->eh;

if (eh != NULL) {

pop_timings_stack()

asan_unpoison_task_stack(ct, &eh->eh_ctx);

jl_longjmp(eh->eh_ctx, 1);

}

else {

jl_no_exc_handler(exception, ct);

}

assert(0);

jl_unreachable();

}

// record backtrace and raise an error

JL_DLLEXPORT void jl_throw(jl_value_t *e JL_MAYBE_UNROOTED)

{

assert(e != NULL);

jl_jmp_buf *safe_restore = jl_get_safe_restore();

jl_task_t *ct = jl_get_current_task();

if (safe_restore) {

asan_unpoison_task_stack(ct, safe_restore);

jl_longjmp(*safe_restore, 1);

}

if (ct == NULL) // During startup, or on other threads

jl_no_exc_handler(e, ct);

record_backtrace(ct->ptls, 1);

throw_internal(ct, e);

}

// rethrow with current excstack state

JL_DLLEXPORT void jl_rethrow(void)

{

jl_task_t *ct = jl_current_task;

jl_excstack_t *excstack = ct->excstack;

if (!excstack || excstack->top == 0)

jl_error("rethrow() not allowed outside a catch block");

throw_internal(ct, NULL);

}

JL_DLLEXPORT void jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED)

{

// TODO: Should uses of `rethrow(exc)` be replaced with a normal throw, now

// that exception stacks allow root cause analysis?

jl_task_t *ct = jl_current_task;

jl_excstack_t *excstack = ct->excstack;

if (!excstack || excstack->top == 0)

jl_error("rethrow(exc) not allowed outside a catch block");

// overwrite exception on top of stack. see jl_excstack_exception

jl_excstack_raw(excstack)[excstack->top-1].jlvalue = e;

JL_GC_PROMISE_ROOTED(e);

throw_internal(ct, NULL);

}

/* This is xoshiro256++ 1.0, used for tasklocal random number generation in Julia.

This implementation is intended for embedders and internal use by the runtime, and is

based on the reference implementation at https://prng.di.unimi.it

Credits go to David Blackman and Sebastiano Vigna for coming up with this PRNG.

They described xoshiro256++ in "Scrambled Linear Pseudorandom Number Generators",

ACM Trans. Math. Softw., 2021.

There is a pure Julia implementation in stdlib that tends to be faster when used from

within Julia, due to inlining and more aggressive architecture-specific optimizations.

*/

uint64_t jl_genrandom(uint64_t rngState[4]) JL_NOTSAFEPOINT

{

uint64_t s0 = rngState[0];

uint64_t s1 = rngState[1];

uint64_t s2 = rngState[2];

uint64_t s3 = rngState[3];

uint64_t t = s1 << 17;

uint64_t tmp = s0 + s3;

uint64_t res = ((tmp << 23) | (tmp >> 41)) + s0;

s2 ^= s0;

s3 ^= s1;

s1 ^= s2;

s0 ^= s3;

s2 ^= t;

s3 = (s3 << 45) | (s3 >> 19);

rngState[0] = s0;

rngState[1] = s1;

rngState[2] = s2;

rngState[3] = s3;

return res;

}

/*

The jl_rng_split function forks a task's RNG state in a way that is essentially

guaranteed to avoid collisions between the RNG streams of all tasks. The main

RNG is the xoshiro256++ RNG whose state is stored in rngState[0..3]. There is

also a small internal RNG used for task forking stored in rngState[4]. This

state is used to iterate a linear congruential generator (LCG), which is then

combined with xoshiro256's state and put through four different variations of

the strongest PCG output function, referred to as PCG-RXS-M-XS-64 [1].

The goal of jl_rng_split is to perturb the state of each child task's RNG in

such a way that for an entire tree of tasks spawned starting with a given root

task state, no two tasks have the same RNG state. Moreover, we want to do this

in a way that is deterministic and repeatable based on (1) the root task's seed,

(2) how many random numbers are generated, and (3) the task tree structure. The

RNG state of a parent task is allowed to affect the initial RNG state of a child

task, but the mere fact that a child was spawned should not alter the RNG output

of the parent. This second requirement rules out using the main RNG to seed

children: if we use the main RNG, we either advance it, which affects the

parent's RNG stream or, if we don't advance it, then every child would have an

identical RNG stream. Therefore some separate state must be maintained and

changed upon forking a child task while leaving the main RNG state unchanged.

The basic approach is a generalization and simplification of that used in the

DotMix [2] and SplitMix [3] RNG systems: each task is uniquely identified by a

sequence of "pedigree" numbers, indicating where in the task tree it was

spawned. This vector of pedigree coordinates is then reduced to a single value

by computing a "dot product" with a shared vector of random weights. I write

"dot product" in quotes because what we use is not an actual dot product. The

linear dot product construction used in both DotMix and SplitMix was found by

@foobar_iv2 [4] to allow easy construction of linear relationships between the

main RNG states of tasks, which was in turn reflected in observable linear

relationships between the outputs of their RNGs. This relationship was between a

minimum of four tasks, so doesn't constitute a collision, per se, but is clearly

undesirable and highlights a hazard of the plain dot product construction.

As in DotMix and SplitMix, each task is assigned unique task "pedigree"

coordinates. Our pedigree construction is a bit different and uses only binary

coordinates rather than arbitrary integers. Each pedigree is an infinite

sequence of ones and zeros with only finitely many ones. Each task has a "fork

index": the root task has index 0; the fork index of the jth child task of a

parent task with fork index i is i+j. The root task's coordinates are all zeros;

each child task's coordinates are the same as its parents except at its fork

index, where the parent has a zero while the child has a one; each task's

coordinates after its fork index are all zeros. The last common ancestor of two

tasks has coordinates that are the longest common prefix of their coordinates.

Also as in DotMix and SplitMix, we generate a sequence of pseudorandom "weights"

to combine with the coordinates of each task. This sequence is common across all

tasks, and different mix values for tasks stem entirely from task coordinates

being different. In DotMix and SplitMix the mix function is a literal dot

product: the pseudorandom weights are multiplied by corresponding task

coordinate and summed. While this does provably make collisions as unlikely as

random seeding, this linear construction can be used to create linearly

correlated states between more than two tasks. However, it turns out that the

compression mixing construction need not be linear, nor commutative, nor

associative. In fact, the mixing function need only be bijective in both

arguments. This allows us to use a much more non-trivial mixing function and

avoid any linear or other obvious correlations between related sets of tasks.

We maintain an LCG in rngState[4] to generate pseudorandom weights. An LCG by

itself is a very bad RNG, but we combine this one with xoshiro256 state

registers in a non-trivial way and then apply the PCG-RXS-M-XS-64 output

function to that. Even if the xoshiro256 states are all zeros, which they should

never be, the output would be the same as PCG-RXS-M-XS-64, which is a solid

statistical RNG. Each time a child is forked, we update the LCG in both parent

and child tasks, corresponding to increasing the fork index. In the parent,

that's all we have to do -- the main RNG state remains unchanged. Recall that

spawning a child should not affect subsequent RNG draws in the parent. The next

time the parent forks a child, the mixing weight used will be different. In the

child, we use the LCG state to perturb the child's main RNG state registers,

rngState[0..3].

To generalize SplitMix's optimized dot product construction, we also compute

each task's compression function value incrementally by combining the parent's

compression value with pseudorandom weight corresponding with the child's fork

index. Formally, if the parent's compression value is c then we can compute the

child's compression value as c′ = f(c, wᵢ) where w is the vector of pseudorandom

weights. What is f? It can be any function that is bijective in each argument

for all values of the other argument:

* For all c: w ↦ f(c, w) is bijective

* For all w: c ↦ f(c, w) is bijective

The proof that these requirements are sufficient to ensure collision resistance

is in the linked discussion [4]. DotMix/SplitMix are a special case where f is

just addition. Instead we use a much less simple mixing function:

1. We use (2c+1)(2w+1)÷2 % 2^64 to mix the bits of c and w

2. We then apply the PCG-RXS-M-XS-64 output function

The first step thoroughly mixes the bits of the previous compression value and

the pseudorandom weight value using multiplication, which is non-commutative

with xoshiro's operations (xor, shift, rotate). This mixing function is a

bijection on each argument witnessed by these inverses:

* c′ ↦ (2c′+1)(2w+1)⁻¹÷2 % 2^64

* w′ ↦ (2c+1)⁻¹(2w′+1)÷2 % 2^64

Here (2w+1)⁻¹ is the modular inverse of (2w+1) mod 2^64, guaranteed to exist

since 2w+1 is odd. The second PCG output step is a bijection and designed to be

significantly non-linear -- non-linear enough to mask the linearity of the LCG

that drives the PCG-RXS-M-XS-64 RNG and allows it to pass statistical RNG test

suites despite having the same size state and output. In particular, since this

mixing function is highly non-associative and non-linear, we (hopefully) don't

have any discernible relationship between these values:

* c₀₀ = c

* c₁₀ = f(c, wᵢ)

* c₀₁ = f(c, wⱼ)

* c₁₁ = f(f(c, wᵢ), wⱼ)

When f is simply `+` then these have a very obvious linear relationship:

c₀₀ + c₁₁ == c₁₀ + c₀₁

This relationship holds regardless of what wᵢ and wⱼ are and allows easy

creation of correlated tasks with the way we were previously using the

DotMix/SplitMix construction. SplitMix itself does not output the raw dot

product, probably because the authors were aware of this linearity issue;

instead: they apply the MurmurHash3 finalizer to the dot-product to get an