// This file is a part of Julia. License is MIT: https://julialang.org/license

/*

saving and restoring system images

This performs serialization and deserialization of system and package images. It creates and saves a compact binary

blob, making deserialization "simple" and fast: we "only" need to deal with uniquing, pointer relocation,

method root insertion, registering with the garbage collector, making note of special internal types, and

backedges/invalidation. Special objects include things like builtin functions, C-implemented types (those in jltypes.c),

the metadata for documentation, optimal layouts, integration with native system image generation, and preparing other

preprocessing directives.

During serialization, the flow has several steps:

- step 1 inserts relevant items into `serialization_order`, an `obj` => `id::Int` mapping. `id` is assigned by

order of insertion. This stage is implemented by `jl_queue_for_serialization` and its callees;

while it would be simplest to use recursion, this risks stack overflow, so recursion is mimicked

using a work-queue managed by `jl_serialize_reachable`.

It's worth emphasizing that the only goal of this stage is to insert objects into `serialization_order`.

In later stages, such objects get written in order of `id`.

- step 2 (the biggest of four steps) takes all items in `serialization_order` and actually serializes them ordered

by `id`. The system is serialized into several distinct streams (see `jl_serializer_state`), a "main stream"

(the `s` field) as well as parallel streams for writing specific categories of additional internal data (e.g.,

global data invisible to codegen, as well as deserialization "touch-up" tables, see below). These different streams

will be concatenated in later steps. Certain key items (e.g., builtin types & functions associated with `INSERT_TAG`

below, integers smaller than 512) get serialized via a hard-coded tag table.

Serialization builds "touch up" tables used during deserialization. Pointers and items requiring gc

registration get encoded as `(location, target)` pairs in `relocs_list` and `gctags_list`, respectively.

`location` is the site that needs updating (e.g., the address of a pointer referencing an object), and is

set to `position(s)`, the offset of the object from the beginning of the deserialized blob.

`target` is a bitfield-encoded index into lists of different categories of data (e.g., mutable data, constant data,

symbols, functions, etc.) to which the pointer at `location` refers. The different lists and their bitfield flags

are given by the `RefTags` enum: if `t` is the category tag (one of the `RefTags` enums) and `i` is the index into

one of the corresponding categorical list, then `index = t << RELOC_TAG_OFFSET + i`. The simplest source for the

details of this encoding can be found in the pair of functions `get_reloc_for_item` and `get_item_for_reloc`.

`uniquing` also holds the serialized location of external DataTypes, MethodInstances, and singletons

in the serialized blob (i.e., new-at-the-time-of-serialization specializations).

Most of step 2 is handled by `jl_write_values`, followed by special handling of the dedicated parallel streams.

- step 3 combines the different sections (fields of `jl_serializer_state`) into one

Much of the "real work" during deserialization is done by `get_item_for_reloc`. But a few items require specific

attention:

- uniquing: during deserialization, the target item (an "external" type or MethodInstance) must be checked against

the running system to see whether such an object already exists (i.e., whether some other previously-loaded package

or workload has created such types/MethodInstances previously) or whether it needs to be created de-novo.

In either case, all references at `location` must be updated to the one in the running system.

`new_dt_objs` is a hash set of newly allocated datatype-reachable objects

- method root insertion: when new specializations generate new roots, these roots must be inserted into

method root tables

- backedges & invalidation: external edges have to be checked against the running system and any invalidations executed.

Encoding of a pointer:

- in the location of the pointer, we initially write zero padding

- for both relocs_list and gctags_list, we write loc/backrefid (for gctags_list this is handled by the caller of write_gctaggedfield,

for relocs_list it's handled by write_pointerfield)

- when writing to disk, both call get_reloc_for_item, and its return value (subject to modification by gc bits)

ends up being written into the data stream (s->s), and the data stream's position written to s->relocs

External links:

- location holds the offset

- loc/0 in relocs_list

*/

#include <stdlib.h>

#include <string.h>

#include <stdio.h> // printf

#include <inttypes.h> // PRIxPTR

#include <zstd.h>

#include "julia.h"

#include "julia_internal.h"

#include "julia_gcext.h"

#include "builtin_proto.h"

#include "processor.h"

#include "serialize.h"

#ifdef _OS_WINDOWS_

#include <memoryapi.h>

#else

#include <dlfcn.h>

#include <sys/mman.h>

#endif

#include "valgrind.h"

#include "julia_assert.h"

#include "eytzinger.h"

static const size_t WORLD_AGE_REVALIDATION_SENTINEL = 0x1;

JL_DLLEXPORT size_t jl_require_world = ~(size_t)0;

JL_DLLEXPORT _Atomic(size_t) jl_first_image_replacement_world = ~(size_t)0;

// This structure is used to store hash tables for the memoization

// of queries in staticdata.c (currently only `type_in_worklist`).

typedef struct {

htable_t type_in_worklist;

} jl_query_cache;

static void init_query_cache(jl_query_cache *cache) JL_NOTSAFEPOINT

{

htable_new(&cache->type_in_worklist, 0);

}

static void destroy_query_cache(jl_query_cache *cache) JL_NOTSAFEPOINT

{

htable_free(&cache->type_in_worklist);

}

#include "staticdata_utils.c"

#include "precompile_utils.c"

#ifdef __cplusplus

extern "C" {

#endif

// TODO: put WeakRefs on the weak_refs list during deserialization

// TODO: handle finalizers

#define NUM_TAGS 6

// An array of special references that need to be restored from the sysimg

static void get_tags(jl_value_t **tags[NUM_TAGS])

{

// Make sure to keep an extra slot at the end to sentinel length

unsigned int i = 0;

#define INSERT_TAG(sym) tags[i++] = (jl_value_t**)&(sym)

INSERT_TAG(jl_method_table);

INSERT_TAG(jl_module_init_order);

INSERT_TAG(jl_typeinf_func);

INSERT_TAG(jl_compile_and_emit_func);

INSERT_TAG(jl_libdl_dlopen_func);

// n.b. must update NUM_TAGS when you add something here

#undef INSERT_TAG

assert(i == NUM_TAGS - 1);

tags[i] = NULL;

}

// hash of definitions for predefined tagged object

static htable_t symbol_table;

static uintptr_t nsym_tag;

// array of definitions for the predefined tagged object types

// (reverse of symbol_table)

static arraylist_t deser_sym;

static htable_t serialization_order; // to break cycles, mark all objects that are serialized

static htable_t nullptrs;

// FIFO queue for objects to be serialized. Anything requiring fixup upon deserialization

// must be "toplevel" in this queue. For types, parameters and field types must appear

// before the "wrapper" type so they can be properly recached against the running system.

static arraylist_t serialization_queue;

static arraylist_t layout_table; // cache of `position(s)` for each `id` in `serialization_order`

static arraylist_t object_worklist; // used to mimic recursion by jl_serialize_reachable

// Track image locations and some associated metadata for all loaded sys / pkgimages.

// (image_tree.ranges[i].data is an image_metadata_t* for build_ids[i])

static eyt_tree_t image_tree;

typedef struct {

uintptr_t base; // base address of the image

void *relocs_base; // reloc_t* for GC sweep

jl_module_t *top_mod; // owning top-level module

size_t idx; // range index in image_tree (for serialization)

} image_metadata_t;

void jl_init_staticdata(void)

{

eyt_tree_init(&image_tree);

}

// HT_NOTFOUND is a valid integer ID, so we store the integer ids mangled.

// This pair of functions mangles/demanges

static size_t from_seroder_entry(void *entry) JL_NOTSAFEPOINT

{

return (size_t)((char*)entry - (char*)HT_NOTFOUND - 1);

}

static void *to_seroder_entry(size_t idx) JL_NOTSAFEPOINT

{

return (void*)((char*)HT_NOTFOUND + 1 + idx);

}

static htable_t new_methtables;

//static size_t precompilation_world;

// Query if a Julia object is in a permalloc region (due to part of a sys- pkg-image)

static size_t n_linkage_blobs(void) JL_NOTSAFEPOINT

{

return image_tree.nranges;

}

static image_metadata_t *external_blob_metadata(jl_value_t *v) JL_NOTSAFEPOINT

{

assert((uintptr_t) v % 4 == 0 && "Object not 4-byte aligned!");

void *data = eyt_tree_find_data(&image_tree, (uintptr_t)v);

return data == EYT_NOTFOUND ? NULL : (image_metadata_t*)data;

}

static size_t external_blob_index(jl_value_t *v) JL_NOTSAFEPOINT

{

image_metadata_t *meta = external_blob_metadata(v);

return meta ? meta->idx : (size_t)-1;

}

JL_DLLEXPORT uint8_t jl_object_in_image(jl_value_t *obj) JL_NOTSAFEPOINT

{

if (obj == NULL)

return 0;

uint8_t in_image = jl_astaggedvalue(obj)->bits.in_image != 0;

assert((uintptr_t) obj % 4 == 0 && "Object not 4-byte aligned!");

assert(eyt_tree_is_in_range(&image_tree, (uintptr_t)obj) == in_image);

return in_image;

}

// Map an object to its "owning" top module

JL_DLLEXPORT jl_value_t *jl_object_top_module(jl_value_t* v) JL_NOTSAFEPOINT

{

image_metadata_t *meta = external_blob_metadata(v);

if (meta)

return (jl_value_t*)meta->top_mod;

// The object is runtime allocated

return (jl_value_t*)jl_nothing;

}

// hash of definitions for predefined function pointers

// (reverse is jl_builtin_f_addrs)

static htable_t fptr_to_id;

void *native_functions; // opaque jl_native_code_desc_t blob used for fetching data from LLVM

// table of struct field addresses to rewrite during saving

static htable_t field_replace;

static htable_t bits_replace;

typedef struct {

ios_t *s; // the main stream

ios_t *const_data; // GC-invisible internal data (e.g., datatype layouts, list-like typename fields, foreign types, internal arrays)

ios_t *symbols; // names (char*) of symbols (some may be referenced by pointer in generated code)

ios_t *relocs; // for (de)serializing relocs_list and gctags_list

ios_t *gvar_record; // serialized array mapping gvid => spos

ios_t *fptr_record; // serialized array mapping fptrid => spos

arraylist_t memowner_list; // a list of memory locations that have shared owners

arraylist_t memref_list; // a list of memoryref locations

arraylist_t relocs_list; // a list of (location, target) pairs, see description at top

arraylist_t gctags_list; // "

arraylist_t uniquing_types; // a list of locations that reference types that must be de-duplicated

arraylist_t uniquing_super; // a list of datatypes, used in super fields, that need to be marked in uniquing_types once they are reached, for handling unique-ing of them on deserialization

arraylist_t uniquing_objs; // a list of locations that reference non-types that must be de-duplicated

arraylist_t fixup_types; // a list of locations of types requiring (re)caching

arraylist_t fixup_objs; // a list of locations of objects requiring (re)caching

// mapping from a buildid_idx to a depmods_idx

jl_array_t *buildid_depmods_idxs;

// record of build_ids for all external linkages, in order of serialization for the current sysimg/pkgimg

// conceptually, the base pointer for the jth externally-linked item is determined from

// i = findfirst(==(link_ids[j]), build_ids)

// blob_base = ((image_metadata_t*)image_tree.ranges[i].data)->base

// We need separate lists since they are intermingled at creation but split when written.

jl_array_t *link_ids_relocs;

jl_array_t *link_ids_gctags;

jl_array_t *link_ids_gvars;

jl_array_t *link_ids_external_fnvars;

jl_array_t *method_roots_list;

htable_t method_roots_index;

uint64_t worklist_key;

jl_query_cache *query_cache;

jl_ptls_t ptls;

jl_image_t *image;

int8_t incremental;

} jl_serializer_state;

static jl_value_t *jl_bigint_type = NULL;

static jl_debuginfo_t *jl_nulldebuginfo;

static int gmp_limb_size = 0;

#ifdef _P64

#define RELOC_TAG_OFFSET 61

#define DEPS_IDX_OFFSET 40 // only on 64-bit can we encode the dependency-index as part of the tagged reloc

#else

// this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data.

#define RELOC_TAG_OFFSET 29

#define DEPS_IDX_OFFSET RELOC_TAG_OFFSET

#endif

// Tags of category `t` are located at offsets `t << RELOC_TAG_OFFSET`

// Consequently there is room for 2^RELOC_TAG_OFFSET pointers, etc

enum RefTags {

DataRef, // mutable data

ConstDataRef, // constant data (e.g., layouts)

TagRef, // items serialized via their tags

SymbolRef, // symbols

FunctionRef, // functions

SysimageLinkage, // reference to the sysimage (from pkgimage)

ExternalLinkage // reference to some other pkgimage

};

#define SYS_EXTERNAL_LINK_UNIT sizeof(void*)

// Sub-divisions of some RefTags

const uintptr_t BuiltinFunctionTag = ((uintptr_t)1 << (RELOC_TAG_OFFSET - 1));

// Bit set on FunctionRef when invoke should be set to jl_fptr_args, and should

// not be zeroed when we want to disable native code.

const uintptr_t BuiltinInvokeTag = ((uintptr_t)1 << (RELOC_TAG_OFFSET - 2));

#if RELOC_TAG_OFFSET <= 32

typedef uint32_t reloc_t;

#else

typedef uint64_t reloc_t;

#endif

static void write_reloc_t(ios_t *s, uintptr_t reloc_id) JL_NOTSAFEPOINT

{

if (sizeof(reloc_t) <= sizeof(uint32_t)) {

assert(reloc_id < UINT32_MAX);

write_uint32(s, reloc_id);

}

else {

write_uint64(s, reloc_id);

}

}

// Reporting to PkgCacheInspector

typedef struct {

size_t sysdata;

size_t isbitsdata;

size_t symboldata;

size_t tagslist;

size_t reloclist;

size_t gvarlist;

size_t fptrlist;

} pkgcachesizes;

// --- Static Compile ---

static jl_image_buf_t jl_sysimage_buf = { JL_IMAGE_KIND_NONE };

static inline uintptr_t *sysimg_gvars(const char *base, const int32_t *offsets, size_t idx)

{

return (uintptr_t*)(base + offsets[idx]);

}

JL_DLLEXPORT int jl_running_on_valgrind(void)

{

return RUNNING_ON_VALGRIND;

}

// --- serializer ---

#define NBOX_C 1024

static int jl_needs_serialization(jl_serializer_state *s, jl_value_t *v) JL_NOTSAFEPOINT

{

// ignore items that are given a special relocation representation

if (s->incremental && jl_object_in_image(v))

return 0;

if (v == NULL || jl_is_symbol(v) || v == jl_nothing) {

return 0;

}

else if (jl_typetagis(v, jl_int64_tag << 4)) {

int64_t i64 = *(int64_t*)v + NBOX_C / 2;

if ((uint64_t)i64 < NBOX_C)

return 0;

}

else if (jl_typetagis(v, jl_int32_tag << 4)) {

int32_t i32 = *(int32_t*)v + NBOX_C / 2;

if ((uint32_t)i32 < NBOX_C)

return 0;

}

else if (jl_typetagis(v, jl_uint8_tag << 4)) {

return 0;

}

else if (v == (jl_value_t*)s->ptls->root_task) {

return 0;

}

return 1;

}

static int caching_tag(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT

{

if (jl_is_method_instance(v)) {

jl_method_instance_t *mi = (jl_method_instance_t*)v;

jl_value_t *m = mi->def.value;

if (jl_is_method(m) && jl_object_in_image(m))

return 1 + type_in_worklist(mi->specTypes, query_cache);

}

if (jl_is_binding(v)) {

jl_globalref_t *gr = ((jl_binding_t*)v)->globalref;

if (!gr)

return 0;

if (!jl_object_in_image((jl_value_t*)gr->mod))

return 0;

return 1;

}

if (jl_is_datatype(v)) {

jl_datatype_t *dt = (jl_datatype_t*)v;

if (jl_is_tuple_type(dt) ? !dt->isconcretetype : dt->hasfreetypevars)

return 0; // aka !is_cacheable from jltypes.c

if (jl_object_in_image((jl_value_t*)dt->name))

return 1 + type_in_worklist(v, query_cache);

}

jl_value_t *dtv = jl_typeof(v);

if (jl_is_datatype_singleton((jl_datatype_t*)dtv)) {

return 1 - type_in_worklist(dtv, query_cache); // these are already recached in the datatype in the image

}

return 0;

}

static int needs_recaching(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT

{

return caching_tag(v, query_cache) == 2;

}

static int needs_uniquing(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT

{

assert(!jl_object_in_image(v));

return caching_tag(v, query_cache) == 1;

}

static void record_field_change(jl_value_t **addr, jl_value_t *newval) JL_NOTSAFEPOINT

{

if (*addr != newval)

ptrhash_put(&field_replace, (void*)addr, newval);

}

static jl_value_t *get_replaceable_field(jl_value_t **addr, int mutabl) JL_GC_DISABLED

{

jl_value_t *fld = (jl_value_t*)ptrhash_get(&field_replace, addr);

if (fld == HT_NOTFOUND) {

fld = *addr;

if (mutabl && fld && jl_is_cpointer_type(jl_typeof(fld)) && jl_unbox_voidpointer(fld) != NULL && jl_unbox_voidpointer(fld) != (void*)(uintptr_t)-1) {

void **nullval = ptrhash_bp(&nullptrs, (void*)jl_typeof(fld));

if (*nullval == HT_NOTFOUND) {

void *C_NULL = NULL;

*nullval = (void*)jl_new_bits(jl_typeof(fld), &C_NULL);

}

fld = (jl_value_t*)*nullval;

}

return fld;

}

return fld;

}

static uintptr_t jl_fptr_id(void *fptr)

{

if (fptr == NULL)

return 0;

void *val = ptrhash_get(&fptr_to_id, fptr);

if (val == HT_NOTFOUND)

return 0;

return (uintptr_t)val;

}

static int effects_foldable(uint32_t effects)

{

// N.B.: This needs to be kept in sync with Core.Compiler.is_foldable(effects, true)

return ((effects & 0x7) == 0) && // is_consistent(effects)

(((effects >> 10) & 0x03) == 0) && // is_noub(effects)

(((effects >> 3) & 0x03) == 0) && // is_effect_free(effects)

((effects >> 6) & 0x01); // is_terminates(effects)

}

// `jl_queue_for_serialization` adds items to `serialization_order`

#define jl_queue_for_serialization(s, v) jl_queue_for_serialization_((s), (jl_value_t*)(v), 1, 0)

static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED;

static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_t *m) JL_GC_DISABLED

{

jl_queue_for_serialization(s, m->name);

jl_queue_for_serialization(s, m->parent);

if (!jl_options.strip_metadata)

jl_queue_for_serialization(s, m->file);

jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindingkeyset));

if (jl_options.trim) {

jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1);

jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings);

for (size_t i = 0; i < jl_svec_len(table); i++) {

jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i);

if ((void*)b == jl_nothing)

break;

jl_value_t *val = jl_get_binding_value_in_world(b, jl_atomic_load_relaxed(&jl_world_counter));

// keep binding objects that are defined in the latest world and ...

if (val &&

// ... point to modules ...

(jl_is_module(val) ||

// ... or point to __init__ methods ...

!strcmp(jl_symbol_name(b->globalref->name), "__init__") ||

// ... or point to Base functions accessed by the runtime

(m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") ||

!strcmp(jl_symbol_name(b->globalref->name), "task_done_hook") ||

!strcmp(jl_symbol_name(b->globalref->name), "_uv_hook_close"))))) {

jl_queue_for_serialization(s, b);

}

}

}

else {

jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings));

}

for (size_t i = 0; i < module_usings_length(m); i++) {

jl_queue_for_serialization(s, module_usings_getmod(m, i));

}

if (jl_options.trim || jl_options.strip_ir) {

record_field_change((jl_value_t**)&m->usings_backedges, jl_nothing);

record_field_change((jl_value_t**)&m->scanned_methods, jl_nothing);

}

else {

jl_queue_for_serialization(s, m->usings_backedges);

jl_queue_for_serialization(s, m->scanned_methods);

}

}

static int codeinst_may_be_runnable(jl_code_instance_t *ci, int incremental) {

size_t max_world = jl_atomic_load_relaxed(&ci->max_world);

if (max_world == ~(size_t)0)

return 1;

if (incremental)

return 0;

return jl_atomic_load_relaxed(&ci->min_world) <= jl_typeinf_world && jl_typeinf_world <= max_world;

}

// Anything that requires uniquing or fixing during deserialization needs to be "toplevel"

// in serialization (i.e., have its own entry in `serialization_order`). Consequently,

// objects that act as containers for other potentially-"problematic" objects must add such "children"

// to the queue.

// Most objects use preorder traversal. But things that need uniquing require postorder:

// you want to handle uniquing of `Dict{String,Float64}` before you tackle `Vector{Dict{String,Float64}}`.

// Uniquing is done in `serialization_order`, so the very first mention of such an object must

// be the "source" rather than merely a cross-reference.

static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED

{

jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);

jl_queue_for_serialization_(s, (jl_value_t*)t, 1, immediate);

const jl_datatype_layout_t *layout = t->layout;

if (!recursive)

goto done_fields;

if (s->incremental && jl_is_datatype(v) && immediate) {

jl_datatype_t *dt = (jl_datatype_t*)v;

// ensure all type parameters are recached

jl_queue_for_serialization_(s, (jl_value_t*)dt->parameters, 1, 1);

if (jl_is_datatype_singleton(dt) && needs_uniquing(dt->instance, s->query_cache)) {

assert(jl_needs_serialization(s, dt->instance)); // should be true, since we visited dt

// do not visit dt->instance for our template object as it leads to unwanted cycles here

// (it may get serialized from elsewhere though)

record_field_change(&dt->instance, jl_nothing);

}

goto done_fields; // for now

}

if (jl_is_method_instance(v)) {

jl_method_instance_t *mi = (jl_method_instance_t*)v;

if (s->incremental) {

jl_value_t *def = mi->def.value;

if (needs_uniquing(v, s->query_cache)) {

// we only need 3 specific fields of this (the rest are not used)

jl_queue_for_serialization(s, mi->def.value);

jl_queue_for_serialization(s, mi->specTypes);

jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals);

goto done_fields;

}

else if (jl_is_method(def) && jl_object_in_image(def)) {

// we only need 3 specific fields of this (the rest are restored afterward, if valid)

// in particular, cache is repopulated by jl_mi_cache_insert for all foreign function,

// so must not be present here

record_field_change((jl_value_t**)&mi->cache, NULL);

}

else {

assert(!needs_recaching(v, s->query_cache));

}

// Any back-edges will be re-validated and added by staticdata.jl, so

// drop them from the image here

record_field_change((jl_value_t**)&mi->backedges, NULL);

// n.b. opaque closures cannot be inspected and relied upon like a

// normal method since they can get improperly introduced by generated

// functions, so if they appeared at all, we will probably serialize

// them wrong and segfault. The jl_code_for_staged function should

// prevent this from happening, so we do not need to detect that user

// error now.

}

// don't recurse into all backedges memory (yet)

jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1);

if (backedges) {

assert(!jl_options.trim && !jl_options.strip_ir);

jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1);

size_t i = 0, n = jl_array_nrows(backedges);

while (i < n) {

jl_value_t *invokeTypes;

jl_code_instance_t *caller;

i = get_next_edge((jl_array_t*)backedges, i, &invokeTypes, &caller);

if (invokeTypes)

jl_queue_for_serialization(s, invokeTypes);

}

}

}

if (jl_is_binding(v)) {

jl_binding_t *b = (jl_binding_t*)v;

if (s->incremental && needs_uniquing(v, s->query_cache)) {

jl_queue_for_serialization(s, b->globalref->mod);

jl_queue_for_serialization(s, b->globalref->name);

goto done_fields;

}

if (jl_options.trim || jl_options.strip_ir) {

record_field_change((jl_value_t**)&b->backedges, NULL);

}

else {

// don't recurse into all backedges memory (yet)

jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1);

if (backedges) {

jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1);

for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i++) {

jl_value_t *b = jl_array_ptr_ref(backedges, i);

if (!jl_is_code_instance(b) && !jl_is_method_instance(b) && !jl_is_method(b)) // otherwise usually a Binding?

jl_queue_for_serialization(s, b);

}

}

}

}

if (s->incremental && jl_is_globalref(v)) {

jl_globalref_t *gr = (jl_globalref_t*)v;

if (jl_object_in_image((jl_value_t*)gr->mod)) {

record_field_change((jl_value_t**)&gr->binding, NULL);

}

}

if (jl_is_typename(v)) {

jl_typename_t *tn = (jl_typename_t*)v;

// don't recurse into several fields (yet)

jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&tn->cache), 0, 1);

jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&tn->linearcache), 0, 1);

if (s->incremental) {

assert(!jl_object_in_image((jl_value_t*)tn->module));

assert(!jl_object_in_image((jl_value_t*)tn->wrapper));

}

}

if (jl_is_mtable(v)) {

jl_methtable_t *mt = (jl_methtable_t*)v;

// Any back-edges will be re-validated and added by staticdata.jl, so

// drop them from the image here

if (s->incremental || jl_options.trim || jl_options.strip_ir) {

record_field_change((jl_value_t**)&mt->backedges, jl_an_empty_memory_any);

}

else {

// don't recurse into all backedges memory (yet)

jl_value_t *allbackedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1);

jl_queue_for_serialization_(s, allbackedges, 0, 1);

for (size_t i = 0, n = ((jl_genericmemory_t*)allbackedges)->length; i < n; i += 2) {

jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i);

jl_queue_for_serialization(s, tn);

jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i + 1);

if (backedges && backedges != jl_nothing) {

jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1);

jl_queue_for_serialization(s, backedges);

for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) {

jl_value_t *t = jl_array_ptr_ref(backedges, i);

assert(!jl_is_code_instance(t));

jl_queue_for_serialization(s, t);

}

}

}

}

}

if (jl_is_code_instance(v)) {

jl_code_instance_t *ci = (jl_code_instance_t*)v;

jl_method_instance_t *mi = jl_get_ci_mi(ci);

if (s->incremental) {

// make sure we don't serialize other reachable cache entries of foreign methods

// Should this now be:

// if (ci !in ci->defs->cache)

// record_field_change((jl_value_t**)&ci->next, NULL);

// Why are we checking that the method/module this originates from is in_image?

// and then disconnect this CI?

if (jl_object_in_image((jl_value_t*)mi->def.value)) {

// TODO: if (ci in ci->defs->cache)

record_field_change((jl_value_t**)&ci->next, NULL);

}

}

jl_value_t *inferred = jl_atomic_load_relaxed(&ci->inferred);

if (inferred && inferred != jl_nothing && !jl_is_uint8(inferred)) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized)

jl_method_t *def = mi->def.method;

if (jl_is_method(def)) { // don't delete toplevel code

int is_relocatable = !s->incremental || jl_is_code_info(inferred) ||

(jl_is_string(inferred) && jl_string_len(inferred) > 0 && jl_string_data(inferred)[jl_string_len(inferred) - 1]);

int may_discard_trees = !jl_get_type_infer_preserve_ir();

int discard = 0;

if (!is_relocatable) {

discard = 1;

}

else if (def->source == NULL) {

// don't delete code from optimized opaque closures that can't be reconstructed (and builtins)

}

else if (may_discard_trees && // if allowed to delete

(!codeinst_may_be_runnable(ci, s->incremental) || // delete all code that cannot run

jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return)) { // delete all code that just returns a constant

discard = 1;

}

else if (may_discard_trees &&

native_functions && // don't delete any code if making a ji file

(ci->owner == jl_nothing) && // don't delete code for external interpreters

!effects_foldable(jl_atomic_load_relaxed(&ci->ipo_purity_bits)) && // don't delete code we may want for irinterp

jl_ir_inlining_cost(inferred) == UINT16_MAX) { // don't delete inlineable code

// delete the code now: if we thought it was worth keeping, it would have been converted to object code

discard = 1;

}

if (discard) {

// keep only the inlining cost, so inference can later decide if it is worth getting the source back

if (jl_is_string(inferred) || jl_is_code_info(inferred))

inferred = jl_box_uint8(jl_encode_inlining_cost(jl_ir_inlining_cost(inferred)));

else

inferred = jl_nothing;

record_field_change((jl_value_t**)&ci->inferred, inferred);

}

else if (s->incremental && jl_is_string(inferred)) {

// New roots for external methods

if (jl_object_in_image((jl_value_t*)def)) {

void **pfound = ptrhash_bp(&s->method_roots_index, def);

if (*pfound == HT_NOTFOUND) {

*pfound = def;

size_t nwithkey = nroots_with_key(def, s->worklist_key);

if (nwithkey) {

jl_array_ptr_1d_push(s->method_roots_list, (jl_value_t*)def);

jl_array_t *newroots = jl_alloc_vec_any(nwithkey);

jl_array_ptr_1d_push(s->method_roots_list, (jl_value_t*)newroots);

rle_iter_state rootiter = rle_iter_init(0);

uint64_t *rletable = NULL;

size_t nblocks2 = 0;

size_t nroots = jl_array_nrows(def->roots);

size_t k = 0;

if (def->root_blocks) {

rletable = jl_array_data(def->root_blocks, uint64_t);

nblocks2 = jl_array_nrows(def->root_blocks);

}

while (rle_iter_increment(&rootiter, nroots, rletable, nblocks2)) {

if (rootiter.key == s->worklist_key) {

jl_value_t *newroot = jl_array_ptr_ref(def->roots, rootiter.i);

jl_queue_for_serialization(s, newroot);

jl_array_ptr_set(newroots, k++, newroot);

}

}

assert(k == nwithkey);

}

}

}

}

}

}

}

if (immediate) // must be things that can be recursively handled, and valid as type parameters

assert(jl_is_immutable(t) || jl_is_typevar(v) || jl_is_symbol(v) || jl_is_svec(v));

if (layout->npointers == 0) {

// bitstypes do not require recursion

}

else if (jl_is_svec(v)) {

size_t i, l = jl_svec_len(v);

jl_value_t **data = jl_svec_data(v);

for (i = 0; i < l; i++) {

jl_queue_for_serialization_(s, data[i], 1, immediate);

}

}

else if (jl_is_array(v)) {

jl_array_t *ar = (jl_array_t*)v;

jl_value_t *mem = get_replaceable_field((jl_value_t**)&ar->ref.mem, 1);

jl_queue_for_serialization_(s, mem, 1, immediate);

}

else if (jl_is_genericmemory(v)) {

jl_genericmemory_t *m = (jl_genericmemory_t*)v;

const char *data = (const char*)m->ptr;

if (jl_genericmemory_how(m) == JL_GENERICMEMORY_STRINGOWNED) {

assert(jl_is_string(jl_genericmemory_data_owner_field(m)));

}

else if (layout->flags.arrayelem_isboxed) {

size_t i, l = m->length;

for (i = 0; i < l; i++) {

jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[i], 1);

jl_queue_for_serialization_(s, fld, 1, immediate);

}

}

else if (layout->first_ptr >= 0) {

uint16_t elsz = layout->size;

size_t i, l = m->length;

size_t j, np = layout->npointers;

for (i = 0; i < l; i++) {

for (j = 0; j < np; j++) {

uint32_t ptr = jl_ptr_offset(t, j);

jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[ptr], 1);

jl_queue_for_serialization_(s, fld, 1, immediate);

}

data += elsz;

}

}

}

else if (jl_is_module(v)) {

jl_queue_module_for_serialization(s, (jl_module_t*)v);

}

else if (layout->nfields > 0) {

if (jl_options.trim) {

if (jl_is_method(v)) {

jl_method_t *m = (jl_method_t *)v;

if (jl_is_svec(jl_atomic_load_relaxed(&m->specializations)))

jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->specializations), 0, 1);

}

else if (jl_is_mtable(v)) {

jl_methtable_t *mt = (jl_methtable_t*)v;

jl_methtable_t *newmt = (jl_methtable_t*)ptrhash_get(&new_methtables, mt);

if (newmt != HT_NOTFOUND)

record_field_change((jl_value_t **)&mt->defs, (jl_value_t*)jl_atomic_load_relaxed(&newmt->defs));

else

record_field_change((jl_value_t **)&mt->defs, jl_nothing);

}

else if (jl_is_mcache(v)) {

jl_methcache_t *mc = (jl_methcache_t*)v;

jl_value_t *cache = jl_atomic_load_relaxed(&mc->cache);

if (!jl_typetagis(cache, jl_typemap_entry_type) || ((jl_typemap_entry_t*)cache)->sig != jl_tuple_type) { // aka Builtins (maybe sometimes OpaqueClosure too)

record_field_change((jl_value_t **)&mc->cache, jl_nothing);

}

record_field_change((jl_value_t **)&mc->leafcache, jl_an_empty_memory_any);

}

// TODO: prune any partitions and partition data that has been deleted in the current world

//else if (jl_is_binding(v)) {

// jl_binding_t *b = (jl_binding_t*)v;

//}

//else if (jl_is_binding_partition(v)) {

// jl_binding_partition_t *bpart = (jl_binding_partition_t*)v;

//}

}

char *data = (char*)jl_data_ptr(v);

size_t i, np = layout->npointers;

size_t fldidx = 1;

for (i = 0; i < np; i++) {

uint32_t ptr = jl_ptr_offset(t, i);

size_t offset = jl_ptr_offset(t, i) * sizeof(jl_value_t*);

while (offset >= (fldidx == layout->nfields ? jl_datatype_size(t) : jl_field_offset(t, fldidx)))

fldidx++;

int mutabl = !jl_field_isconst(t, fldidx - 1);

jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[ptr], mutabl);

jl_queue_for_serialization_(s, fld, 1, immediate);

}

}

done_fields: ;

// We've encountered an item we need to cache

void **bp = ptrhash_bp(&serialization_order, v);

assert(*bp == (void*)(uintptr_t)-2);

arraylist_push(&serialization_queue, (void*) v);

size_t idx = serialization_queue.len - 1;

assert(serialization_queue.len < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many items to serialize");

*bp = to_seroder_entry(idx);

// DataType is very unusual, in that some of the fields need to be pre-order, and some

// (notably super) must not be (even if `jl_queue_for_serialization_` would otherwise

// try to promote itself to be immediate)

if (s->incremental && jl_is_datatype(v) && immediate && recursive) {

jl_datatype_t *dt = (jl_datatype_t*)v;

void *bp = ptrhash_get(&serialization_order, (void*)dt->super);

if (bp != (void*)-2) {

// if super is already on the stack of things to handle when this returns, do

// not try to handle it now

jl_queue_for_serialization_(s, (jl_value_t*)dt->super, 1, immediate);

}

immediate = 0;

char *data = (char*)jl_data_ptr(v);

size_t i, np = layout->npointers;

for (i = 0; i < np; i++) {

uint32_t ptr = jl_ptr_offset(t, i);

if (ptr * sizeof(jl_value_t*) == offsetof(jl_datatype_t, super))

continue; // skip the super field, since it might not be quite validly ordered

int mutabl = 1;

jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[ptr], mutabl);

jl_queue_for_serialization_(s, fld, 1, immediate);

}

}

}

static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, int recursive, int immediate) JL_GC_DISABLED

{

if (!jl_needs_serialization(s, v))

return;

jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);

// check early from errors, so we have a little bit of contextual state for debugging them

if (t == jl_task_type) {

jl_error("Task cannot be serialized");

}

if (s->incremental && needs_uniquing(v, s->query_cache) && t == jl_binding_type) {

jl_binding_t *b = (jl_binding_t*)v;

if (b->globalref == NULL)

jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity

}

if (jl_is_foreign_type(t) == 1) {

jl_error("Cannot serialize instances of foreign datatypes");

}

// Items that require postorder traversal must visit their children prior to insertion into

// the worklist/serialization_order (and also before their first use)

if (s->incremental && !immediate) {

if (jl_is_datatype(v) && needs_uniquing(v, s->query_cache))

immediate = 1;

if (jl_is_datatype_singleton((jl_datatype_t*)t) && needs_uniquing(v, s->query_cache))

immediate = 1;

}

void **bp = ptrhash_bp(&serialization_order, v);

assert(!immediate || *bp != (void*)(uintptr_t)-2);

if (*bp == HT_NOTFOUND)

*bp = (void*)(uintptr_t)-1; // now enqueued

else if (!s->incremental || !immediate || !recursive || *bp != (void*)(uintptr_t)-1)

return;

if (immediate) {

*bp = (void*)(uintptr_t)-2; // now immediate

jl_insert_into_serialization_queue(s, v, recursive, immediate);

}

else {

arraylist_push(&object_worklist, (void*)v);

}

}

// Do a pre-order traversal of the to-serialize worklist, in the identical order

// to the calls to jl_queue_for_serialization would occur in a purely recursive

// implementation, but without potentially running out of stack.

static void jl_serialize_reachable(jl_serializer_state *s) JL_GC_DISABLED

{

size_t i, prevlen = 0;

while (object_worklist.len) {

// reverse!(object_worklist.items, prevlen:end);

// prevlen is the index of the first new object

size_t mid = prevlen + (object_worklist.len - prevlen) / 2;

for (i = prevlen; i < mid; i++) {

size_t j = object_worklist.len - i + prevlen - 1;

void *tmp = object_worklist.items[i];

object_worklist.items[i] = object_worklist.items[j];

object_worklist.items[j] = tmp;

}

prevlen = --object_worklist.len;

jl_value_t *v = (jl_value_t*)object_worklist.items[prevlen];

void **bp = ptrhash_bp(&serialization_order, (void*)v);

assert(*bp != HT_NOTFOUND && *bp != (void*)(uintptr_t)-2);

if (*bp == (void*)(uintptr_t)-1) { // might have been eagerly handled for post-order while in the lazy pre-order queue

*bp = (void*)(uintptr_t)-2;

jl_insert_into_serialization_queue(s, v, 1, 0);

}

else {

assert(s->incremental);

}

}

}

static void ios_ensureroom(ios_t *s, size_t newsize) JL_NOTSAFEPOINT

{

size_t prevsize = s->size;

if (prevsize < newsize) {

ios_trunc(s, newsize);

assert(s->size == newsize);

memset(&s->buf[prevsize], 0, newsize - prevsize);

}

}

static void write_padding(ios_t *s, size_t nb) JL_NOTSAFEPOINT

{

static const char zeros[16] = {0};

while (nb > 16) {

ios_write(s, zeros, 16);

nb -= 16;

}

if (nb != 0)

ios_write(s, zeros, nb);

}

static void write_pointer(ios_t *s) JL_NOTSAFEPOINT

{

assert((ios_pos(s) & (sizeof(void*) - 1)) == 0 && "stream misaligned for writing a word-sized value");

write_uint(s, 0);

}

// Records the buildid holding `v` and returns the tagged offset within the corresponding image

static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED

{

image_metadata_t *meta = external_blob_metadata(v);

if (meta) {

size_t i = meta->idx;

// We found the sysimg/pkg that this item links against

// Compute the relocation code

size_t offset = (uintptr_t)v - meta->base;

assert((offset % SYS_EXTERNAL_LINK_UNIT) == 0);

offset /= SYS_EXTERNAL_LINK_UNIT;

assert(n_linkage_blobs() == jl_array_nrows(s->buildid_depmods_idxs));

size_t depsidx = jl_array_data(s->buildid_depmods_idxs, uint32_t)[i]; // map from build_id_idx -> deps_idx

assert(depsidx < INT32_MAX);

if (depsidx < ((uintptr_t)1 << (RELOC_TAG_OFFSET - DEPS_IDX_OFFSET)) && offset < ((uintptr_t)1 << DEPS_IDX_OFFSET))

// if it fits in a SysimageLinkage type, use that representation