// Copyright (c) Microsoft. All rights reserved.

// Licensed under the MIT license. See LICENSE file in the project root for full license information.

//

//

// Copyright (c) Microsoft. All rights reserved.

// Licensed under the MIT license. See LICENSE file in the project root for full license information.

//

// optimize for speed

#ifndef _DEBUG

#ifdef _MSC_VER

#pragma optimize( "t", on )

#endif

#endif

#ifdef __GNUC__

#define inline __attribute__((always_inline)) inline

#else

#define inline __forceinline

#endif // __GNUC__

#include "gc.h"

//#define DT_LOG

#include "gcrecord.h"

#ifdef _MSC_VER

#pragma warning(disable:4293)

#pragma warning(disable:4477)

#endif //_MSC_VER

inline void FATAL_GC_ERROR()

{

#ifndef DACCESS_COMPILE

GCToOSInterface::DebugBreak();

#endif // DACCESS_COMPILE

_ASSERTE(!"Fatal Error in GC.");

GCToEEInterface::HandleFatalError((unsigned int)COR_E_EXECUTIONENGINE);

}

#ifdef MULTIPLE_HEAPS

// This turns on instrumentation that collects info for heap balancing.

// Define it and make sure you have HEAP_BALANCE_LOG/HEAP_BALANCE_TEMP_LOG

// level logging enabled *only*.

//#define HEAP_BALANCE_INSTRUMENTATION

#endif //MULTIPLE_HEAPS

#ifdef _MSC_VER

#pragma inline_depth(20)

#endif

/* the following section defines the optional features */

// FEATURE_STRUCTALIGN was added by Midori. In CLR we are not interested

// in supporting custom alignments on LOH. Currently FEATURE_LOH_COMPACTION

// and FEATURE_STRUCTALIGN are mutually exclusive. It shouldn't be much

// work to make FEATURE_STRUCTALIGN not apply to LOH so they can be both

// turned on.

#define FEATURE_LOH_COMPACTION

#ifdef FEATURE_64BIT_ALIGNMENT

// We need the following feature as part of keeping 64-bit types aligned in the GC heap.

#define RESPECT_LARGE_ALIGNMENT //used to keep "double" objects aligned during

//relocation

#endif //FEATURE_64BIT_ALIGNMENT

#define SHORT_PLUGS //used to keep ephemeral plugs short so they fit better into the oldest generation free items

#ifdef SHORT_PLUGS

#define DESIRED_PLUG_LENGTH (1000)

#endif //SHORT_PLUGS

#define FEATURE_PREMORTEM_FINALIZATION

#define GC_HISTORY

#ifndef FEATURE_REDHAWK

#define HEAP_ANALYZE

#define COLLECTIBLE_CLASS

#endif // !FEATURE_REDHAWK

#ifdef HEAP_ANALYZE

#define initial_internal_roots (1024*16)

#endif // HEAP_ANALYZE

#define MARK_LIST //used sorted list to speed up plan phase

#define BACKGROUND_GC //concurrent background GC (requires WRITE_WATCH)

#ifdef SERVER_GC

#define MH_SC_MARK //scalable marking

//#define SNOOP_STATS //diagnostic

#define PARALLEL_MARK_LIST_SORT //do the sorting and merging of the multiple mark lists in server gc in parallel

#endif //SERVER_GC

//This is used to mark some type volatile only when the scalable marking is used.

#if defined (SERVER_GC) && defined (MH_SC_MARK)

#define SERVER_SC_MARK_VOLATILE(x) VOLATILE(x)

#else //SERVER_GC&&MH_SC_MARK

#define SERVER_SC_MARK_VOLATILE(x) x

#endif //SERVER_GC&&MH_SC_MARK

//#define MULTIPLE_HEAPS //Allow multiple heaps for servers

#define INTERIOR_POINTERS //Allow interior pointers in the code manager

#define CARD_BUNDLE //enable card bundle feature.(requires WRITE_WATCH)

// If this is defined we use a map for segments in order to find the heap for

// a segment fast. But it does use more memory as we have to cover the whole

// heap range and for each entry we allocate a struct of 5 ptr-size words

// (3 for WKS as there's only one heap).

#define SEG_MAPPING_TABLE

// If allocating the heap mapping table for the available VA consumes too

// much memory, you can enable this to allocate only the portion that

// corresponds to rw segments and grow it when needed in grow_brick_card_table.

// However in heap_of you will need to always compare the address with

// g_lowest/highest before you can look at the heap mapping table.

#define GROWABLE_SEG_MAPPING_TABLE

#ifdef BACKGROUND_GC

#define MARK_ARRAY //Mark bit in an array

#define BGC_SERVO_TUNING

#endif //BACKGROUND_GC

#if defined(BACKGROUND_GC) || defined (CARD_BUNDLE) || defined(FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)

#define WRITE_WATCH //Write Watch feature

#endif //BACKGROUND_GC || CARD_BUNDLE

#ifdef WRITE_WATCH

#define array_size 100

#endif //WRITE_WATCH

//#define SHORT_PLUGS //keep plug short

#define FFIND_DECAY 7 //Number of GC for which fast find will be active

#ifndef MAX_LONGPATH

#define MAX_LONGPATH 1024

#endif // MAX_LONGPATH

//#define DEBUG_WRITE_WATCH //Additional debug for write watch

//#define STRESS_PINNING //Stress pinning by pinning randomly

//#define TRACE_GC //debug trace gc operation

//#define SIMPLE_DPRINTF

//#define TIME_GC //time allocation and garbage collection

//#define TIME_WRITE_WATCH //time GetWriteWatch and ResetWriteWatch calls

//#define COUNT_CYCLES //Use cycle counter for timing

//#define JOIN_STATS //amount of time spent in the join

//also, see TIME_SUSPEND in switches.h.

//#define SYNCHRONIZATION_STATS

//#define SEG_REUSE_STATS

#if defined (SYNCHRONIZATION_STATS) || defined (STAGE_STATS)

#define BEGIN_TIMING(x) \

int64_t x##_start; \

x##_start = GCToOSInterface::QueryPerformanceCounter()

#define END_TIMING(x) \

int64_t x##_end; \

x##_end = GCToOSInterface::QueryPerformanceCounter(); \

x += x##_end - x##_start

#else

#define BEGIN_TIMING(x)

#define END_TIMING(x)

#define BEGIN_TIMING_CYCLES(x)

#define END_TIMING_CYCLES(x)

#endif //SYNCHRONIZATION_STATS || STAGE_STATS

/* End of optional features */

#ifdef GC_CONFIG_DRIVEN

void GCLogConfig (const char *fmt, ... );

#define cprintf(x) {GCLogConfig x;}

#endif //GC_CONFIG_DRIVEN

#ifdef _DEBUG

#define TRACE_GC

#endif

// For the bestfit algorithm when we relocate ephemeral generations into an

// existing gen2 segment.

// We recorded sizes from 2^6, 2^7, 2^8...up to 2^30 (1GB). So that's 25 sizes total.

#define MIN_INDEX_POWER2 6

#ifdef SERVER_GC

#ifdef BIT64

#define MAX_INDEX_POWER2 30

#else

#define MAX_INDEX_POWER2 26

#endif // BIT64

#else //SERVER_GC

#ifdef BIT64

#define MAX_INDEX_POWER2 28

#else

#define MAX_INDEX_POWER2 24

#endif // BIT64

#endif //SERVER_GC

#define MAX_NUM_BUCKETS (MAX_INDEX_POWER2 - MIN_INDEX_POWER2 + 1)

#define MAX_NUM_FREE_SPACES 200

#define MIN_NUM_FREE_SPACES 5

//Please leave these definitions intact.

// hosted api

#ifdef memcpy

#undef memcpy

#endif //memcpy

#ifdef FEATURE_STRUCTALIGN

#define REQD_ALIGN_DCL ,int requiredAlignment

#define REQD_ALIGN_ARG ,requiredAlignment

#define REQD_ALIGN_AND_OFFSET_DCL ,int requiredAlignment,size_t alignmentOffset

#define REQD_ALIGN_AND_OFFSET_DEFAULT_DCL ,int requiredAlignment=DATA_ALIGNMENT,size_t alignmentOffset=0

#define REQD_ALIGN_AND_OFFSET_ARG ,requiredAlignment,alignmentOffset

#else // FEATURE_STRUCTALIGN

#define REQD_ALIGN_DCL

#define REQD_ALIGN_ARG

#define REQD_ALIGN_AND_OFFSET_DCL

#define REQD_ALIGN_AND_OFFSET_DEFAULT_DCL

#define REQD_ALIGN_AND_OFFSET_ARG

#endif // FEATURE_STRUCTALIGN

#ifdef MULTIPLE_HEAPS

#define THREAD_NUMBER_DCL ,int thread

#define THREAD_NUMBER_ARG ,thread

#define THREAD_NUMBER_FROM_CONTEXT int thread = sc->thread_number;

#define THREAD_FROM_HEAP int thread = heap_number;

#define HEAP_FROM_THREAD gc_heap* hpt = gc_heap::g_heaps[thread];

#else

#define THREAD_NUMBER_DCL

#define THREAD_NUMBER_ARG

#define THREAD_NUMBER_FROM_CONTEXT

#define THREAD_FROM_HEAP

#define HEAP_FROM_THREAD gc_heap* hpt = 0;

#endif //MULTIPLE_HEAPS

//These constants are ordered

const int policy_sweep = 0;

const int policy_compact = 1;

const int policy_expand = 2;

#ifdef TRACE_GC

#define SEG_REUSE_LOG_0 7

#define SEG_REUSE_LOG_1 (SEG_REUSE_LOG_0 + 1)

#define DT_LOG_0 (SEG_REUSE_LOG_1 + 1)

#define BGC_TUNING_LOG (DT_LOG_0 + 1)

#define GTC_LOG (DT_LOG_0 + 2)

#define GC_TABLE_LOG (DT_LOG_0 + 3)

#define JOIN_LOG (DT_LOG_0 + 4)

#define SPINLOCK_LOG (DT_LOG_0 + 5)

#define SNOOP_LOG (DT_LOG_0 + 6)

// NOTE! This is for HEAP_BALANCE_INSTRUMENTATION

// This particular one is special and needs to be well formatted because we

// do post processing on it with tools\GCLogParser. If you need to add some

// detail to help with investigation that's not 't processed by tooling

// prefix it with TEMP so that line will be written to the results as is in

// the result. I have some already logged with HEAP_BALANCE_TEMP_LOG.

#define HEAP_BALANCE_LOG (DT_LOG_0 + 7)

#define HEAP_BALANCE_TEMP_LOG (DT_LOG_0 + 8)

#ifndef DACCESS_COMPILE

#ifdef SIMPLE_DPRINTF

//#define dprintf(l,x) {if (trace_gc && ((l<=print_level)||gc_heap::settings.concurrent)) {printf ("\n");printf x ; fflush(stdout);}}

void GCLog (const char *fmt, ... );

//#define dprintf(l,x) {if (trace_gc && (l<=print_level)) {GCLog x;}}

//#define dprintf(l,x) {if ((l==SEG_REUSE_LOG_0) || (l==SEG_REUSE_LOG_1) || (trace_gc && (l<=3))) {GCLog x;}}

//#define dprintf(l,x) {if (l == DT_LOG_0) {GCLog x;}}

//#define dprintf(l,x) {if (trace_gc && ((l <= 2) || (l == BGC_LOG) || (l==GTC_LOG))) {GCLog x;}}

//#define dprintf(l,x) {if ((l == 1) || (l == 2222)) {GCLog x;}}

#define dprintf(l,x) {if ((l <= 1) || (l == GTC_LOG)) {GCLog x;}}

//#define dprintf(l,x) {if (l == HEAP_BALANCE_LOG) {GCLog x;}}

//#define dprintf(l,x) {if ((l==GTC_LOG) || (l <= 1)) {GCLog x;}}

//#define dprintf(l,x) {if (trace_gc && ((l <= print_level) || (l==GTC_LOG))) {GCLog x;}}

//#define dprintf(l,x) {if (l==GTC_LOG) {printf ("\n");printf x ; fflush(stdout);}}

#else //SIMPLE_DPRINTF

// Nobody used the logging mechanism that used to be here. If we find ourselves

// wanting to inspect GC logs on unmodified builds, we can use this define here

// to do so.

#define dprintf(l, x)

//#define dprintf(l,x) STRESS_LOG_VA(x);

#endif //SIMPLE_DPRINTF

#else //DACCESS_COMPILE

#define dprintf(l,x)

#endif //DACCESS_COMPILE

#else //TRACE_GC

#define dprintf(l,x)

#endif //TRACE_GC

#if !defined(FEATURE_REDHAWK) && !defined(BUILD_AS_STANDALONE)

#undef assert

#define assert _ASSERTE

#undef ASSERT

#define ASSERT _ASSERTE

#endif // FEATURE_REDHAWK

struct GCDebugSpinLock {

VOLATILE(int32_t) lock; // -1 if free, 0 if held

#ifdef _DEBUG

VOLATILE(Thread *) holding_thread; // -1 if no thread holds the lock.

VOLATILE(BOOL) released_by_gc_p; // a GC thread released the lock.

#endif

#if defined (SYNCHRONIZATION_STATS)

// number of times we went into SwitchToThread in enter_spin_lock.

unsigned int num_switch_thread;

// number of times we went into WaitLonger.

unsigned int num_wait_longer;

// number of times we went to calling SwitchToThread in WaitLonger.

unsigned int num_switch_thread_w;

// number of times we went to calling DisablePreemptiveGC in WaitLonger.

unsigned int num_disable_preemptive_w;

#endif

GCDebugSpinLock()

: lock(-1)

#ifdef _DEBUG

, holding_thread((Thread*) -1)

#endif

#if defined (SYNCHRONIZATION_STATS)

, num_switch_thread(0), num_wait_longer(0), num_switch_thread_w(0), num_disable_preemptive_w(0)

#endif

{

}

#if defined (SYNCHRONIZATION_STATS)

void init()

{

num_switch_thread = 0;

num_wait_longer = 0;

num_switch_thread_w = 0;

num_disable_preemptive_w = 0;

}

#endif

};

typedef GCDebugSpinLock GCSpinLock;

class mark;

class heap_segment;

class CObjectHeader;

class l_heap;

class sorted_table;

class c_synchronize;

class seg_free_spaces;

class gc_heap;

#ifdef BACKGROUND_GC

class exclusive_sync;

class recursive_gc_sync;

#endif //BACKGROUND_GC

#ifdef MULTIPLE_HEAPS

// card marking stealing only makes sense in server GC

// but it works and is easier to debug for workstation GC

// so turn it on for server GC, turn on for workstation GC if necessary

#define FEATURE_CARD_MARKING_STEALING

#endif //MULTIPLE_HEAPS

#ifdef FEATURE_CARD_MARKING_STEALING

class card_marking_enumerator;

#define CARD_MARKING_STEALING_ARG(a) ,a

#define CARD_MARKING_STEALING_ARGS(a,b,c) ,a,b,c

#else // FEATURE_CARD_MARKING_STEALING

#define CARD_MARKING_STEALING_ARG(a)

#define CARD_MARKING_STEALING_ARGS(a,b,c)

#endif // FEATURE_CARD_MARKING_STEALING

// The following 2 modes are of the same format as in clr\src\bcl\system\runtime\gcsettings.cs

// make sure you change that one if you change this one!

enum gc_pause_mode

{

pause_batch = 0, //We are not concerned about pause length

pause_interactive = 1, //We are running an interactive app

pause_low_latency = 2, //short pauses are essential

//avoid long pauses from blocking full GCs unless running out of memory

pause_sustained_low_latency = 3,

pause_no_gc = 4

};

enum gc_loh_compaction_mode

{

loh_compaction_default = 1, // the default mode, don't compact LOH.

loh_compaction_once = 2, // only compact once the next time a blocking full GC happens.

loh_compaction_auto = 4 // GC decides when to compact LOH, to be implemented.

};

enum set_pause_mode_status

{

set_pause_mode_success = 0,

set_pause_mode_no_gc = 1 // NoGCRegion is in progress, can't change pause mode.

};

/*

Latency modes required user to have specific GC knowledge (eg, budget, full blocking GC).

We are trying to move away from them as it makes a lot more sense for users to tell

us what's the most important out of the perf aspects that make sense to them.

In general there are 3 such aspects:

+ memory footprint

+ throughput

+ pause predictibility

Currently the following levels are supported. We may (and will likely) add more

in the future.

+----------+--------------------+---------------------------------------+

| Level | Optimization Goals | Latency Charactaristics |

+==========+====================+=======================================+

| 0 | memory footprint | pauses can be long and more frequent |

+----------+--------------------+---------------------------------------+

| 1 | balanced | pauses are more predictable and more |

| | | frequent. the longest pauses are |

| | | shorter than 1. |

+----------+--------------------+---------------------------------------+

*/

enum gc_latency_level

{

latency_level_first = 0,

latency_level_memory_footprint = latency_level_first,

latency_level_balanced = 1,

latency_level_last = latency_level_balanced,

latency_level_default = latency_level_balanced

};

enum gc_tuning_point

{

tuning_deciding_condemned_gen = 0,

tuning_deciding_full_gc = 1,

tuning_deciding_compaction = 2,

tuning_deciding_expansion = 3,

tuning_deciding_promote_ephemeral = 4,

tuning_deciding_short_on_seg = 5

};

#if defined(TRACE_GC) && defined(BACKGROUND_GC)

static const char * const str_bgc_state[] =

{

"not_in_process",

"mark_handles",

"mark_stack",

"revisit_soh",

"revisit_loh",

"overflow_soh",

"overflow_loh",

"final_marking",

"sweep_soh",

"sweep_loh",

"plan_phase"

};

#endif // defined(TRACE_GC) && defined(BACKGROUND_GC)

enum allocation_state

{

a_state_start = 0,

a_state_can_allocate,

a_state_cant_allocate,

// This could be due to having to wait till a GC is done,

// or having to try a different heap.

a_state_retry_allocate,

a_state_try_fit,

a_state_try_fit_new_seg,

a_state_try_fit_after_cg,

a_state_try_fit_after_bgc,

a_state_try_free_full_seg_in_bgc,

a_state_try_free_after_bgc,

a_state_try_seg_end,

a_state_acquire_seg,

a_state_acquire_seg_after_cg,

a_state_acquire_seg_after_bgc,

a_state_check_and_wait_for_bgc,

a_state_trigger_full_compact_gc,

a_state_trigger_ephemeral_gc,

a_state_trigger_2nd_ephemeral_gc,

a_state_check_retry_seg,

a_state_max

};

enum gc_type

{

gc_type_compacting = 0,

gc_type_blocking = 1,

#ifdef BACKGROUND_GC

gc_type_background = 2,

#endif //BACKGROUND_GC

gc_type_max = 3

};

//encapsulates the mechanism for the current gc

class gc_mechanisms

{

public:

VOLATILE(size_t) gc_index; // starts from 1 for the first GC, like dd_collection_count

int condemned_generation;

BOOL promotion;

BOOL compaction;

BOOL loh_compaction;

BOOL heap_expansion;

uint32_t concurrent;

BOOL demotion;

BOOL card_bundles;

int gen0_reduction_count;

BOOL should_lock_elevation;

int elevation_locked_count;

BOOL elevation_reduced;

BOOL minimal_gc;

gc_reason reason;

gc_pause_mode pause_mode;

BOOL found_finalizers;

#ifdef BACKGROUND_GC

BOOL background_p;

bgc_state b_state;

BOOL allocations_allowed;

#endif //BACKGROUND_GC

#ifdef STRESS_HEAP

BOOL stress_induced;

#endif // STRESS_HEAP

// These are opportunistically set

uint32_t entry_memory_load;

uint64_t entry_available_physical_mem;

uint32_t exit_memory_load;

void init_mechanisms(); //for each GC

void first_init(); // for the life of the EE

void record (gc_history_global* history);

};

// This is a compact version of gc_mechanism that we use to save in the history.

class gc_mechanisms_store

{

public:

size_t gc_index;

bool promotion;

bool compaction;

bool loh_compaction;

bool heap_expansion;

bool concurrent;

bool demotion;

bool card_bundles;

bool should_lock_elevation;

int condemned_generation : 8;

int gen0_reduction_count : 8;

int elevation_locked_count : 8;

gc_reason reason : 8;

gc_pause_mode pause_mode : 8;

#ifdef BACKGROUND_GC

bgc_state b_state : 8;

#endif //BACKGROUND_GC

bool found_finalizers;

#ifdef BACKGROUND_GC

bool background_p;

#endif //BACKGROUND_GC

#ifdef STRESS_HEAP

bool stress_induced;

#endif // STRESS_HEAP

#ifdef BIT64

uint32_t entry_memory_load;

#endif // BIT64

void store (gc_mechanisms* gm)

{

gc_index = gm->gc_index;

condemned_generation = gm->condemned_generation;

promotion = (gm->promotion != 0);

compaction = (gm->compaction != 0);

loh_compaction = (gm->loh_compaction != 0);

heap_expansion = (gm->heap_expansion != 0);

concurrent = (gm->concurrent != 0);

demotion = (gm->demotion != 0);

card_bundles = (gm->card_bundles != 0);

gen0_reduction_count = gm->gen0_reduction_count;

should_lock_elevation = (gm->should_lock_elevation != 0);

elevation_locked_count = gm->elevation_locked_count;

reason = gm->reason;

pause_mode = gm->pause_mode;

found_finalizers = (gm->found_finalizers != 0);

#ifdef BACKGROUND_GC

background_p = (gm->background_p != 0);

b_state = gm->b_state;

#endif //BACKGROUND_GC

#ifdef STRESS_HEAP

stress_induced = (gm->stress_induced != 0);

#endif // STRESS_HEAP

#ifdef BIT64

entry_memory_load = gm->entry_memory_load;

#endif // BIT64

}

};

#ifdef GC_STATS

// GC specific statistics, tracking counts and timings for GCs occuring in the system.

// This writes the statistics to a file every 60 seconds, if a file is specified in

// COMPlus_GcMixLog

struct GCStatistics

: public StatisticsBase

{

// initialized to the contents of COMPlus_GcMixLog, or NULL, if not present

static char* logFileName;

static FILE* logFile;

// number of times we executed a background GC, a foreground GC, or a

// non-concurrent GC

int cntBGC, cntFGC, cntNGC;

// min, max, and total time spent performing BGCs, FGCs, NGCs

// (BGC time includes everything between the moment the BGC starts until

// it completes, i.e. the times of all FGCs occuring concurrently)

MinMaxTot bgc, fgc, ngc;

// number of times we executed a compacting GC (sweeping counts can be derived)

int cntCompactNGC, cntCompactFGC;

// count of reasons

int cntReasons[reason_max];

// count of condemned generation, by NGC and FGC:

int cntNGCGen[max_generation+1];

int cntFGCGen[max_generation];

///////////////////////////////////////////////////////////////////////////////////////////////

// Internal mechanism:

virtual void Initialize();

virtual void DisplayAndUpdate();

// Public API

static BOOL Enabled()

{ return logFileName != NULL; }

void AddGCStats(const gc_mechanisms& settings, size_t timeInMSec);

};

extern GCStatistics g_GCStatistics;

extern GCStatistics g_LastGCStatistics;

#endif // GC_STATS

typedef DPTR(class heap_segment) PTR_heap_segment;

typedef DPTR(class gc_heap) PTR_gc_heap;

typedef DPTR(PTR_gc_heap) PTR_PTR_gc_heap;

#ifdef FEATURE_PREMORTEM_FINALIZATION

typedef DPTR(class CFinalize) PTR_CFinalize;

#endif // FEATURE_PREMORTEM_FINALIZATION

//-------------------------------------

//generation free list. It is an array of free lists bucketed by size, starting at sizes lower than first_bucket_size

//and doubling each time. The last bucket (index == num_buckets) is for largest sizes with no limit

#define MAX_BUCKET_COUNT (13)//Max number of buckets for the small generations.

class alloc_list

{

uint8_t* head;

uint8_t* tail;

size_t damage_count;

public:

#ifdef FL_VERIFICATION

size_t item_count;

#endif //FL_VERIFICATION

uint8_t*& alloc_list_head () { return head;}

uint8_t*& alloc_list_tail () { return tail;}

size_t& alloc_list_damage_count(){ return damage_count; }

alloc_list()

{

head = 0;

tail = 0;

damage_count = 0;

}

};

class allocator

{

size_t num_buckets;

size_t frst_bucket_size;

alloc_list first_bucket;

alloc_list* buckets;

alloc_list& alloc_list_of (unsigned int bn);

size_t& alloc_list_damage_count_of (unsigned int bn);

public:

allocator (unsigned int num_b, size_t fbs, alloc_list* b);

allocator()

{

num_buckets = 1;

frst_bucket_size = SIZE_T_MAX;

}

unsigned int number_of_buckets() {return (unsigned int)num_buckets;}

size_t first_bucket_size() {return frst_bucket_size;}

uint8_t*& alloc_list_head_of (unsigned int bn)

{

return alloc_list_of (bn).alloc_list_head();

}

uint8_t*& alloc_list_tail_of (unsigned int bn)

{

return alloc_list_of (bn).alloc_list_tail();

}

void clear();

BOOL discard_if_no_fit_p()

{

return (num_buckets == 1);

}

// This is when we know there's nothing to repair because this free

// list has never gone through plan phase. Right now it's only used

// by the background ephemeral sweep when we copy the local free list

// to gen0's free list.

//

// We copy head and tail manually (vs together like copy_to_alloc_list)

// since we need to copy tail first because when we get the free items off

// of each bucket we check head first. We also need to copy the

// smaller buckets first so when gen0 allocation needs to thread

// smaller items back that bucket is guaranteed to have been full

// copied.

void copy_with_no_repair (allocator* allocator_to_copy)

{

assert (num_buckets == allocator_to_copy->number_of_buckets());

for (unsigned int i = 0; i < num_buckets; i++)

{

alloc_list* al = &(allocator_to_copy->alloc_list_of (i));

alloc_list_tail_of(i) = al->alloc_list_tail();

alloc_list_head_of(i) = al->alloc_list_head();

}

}

void unlink_item (unsigned int bucket_number, uint8_t* item, uint8_t* previous_item, BOOL use_undo_p);

void thread_item (uint8_t* item, size_t size);

void thread_item_front (uint8_t* itme, size_t size);

void thread_free_item (uint8_t* free_item, uint8_t*& head, uint8_t*& tail);

void copy_to_alloc_list (alloc_list* toalist);

void copy_from_alloc_list (alloc_list* fromalist);

void commit_alloc_list_changes();

};

#define NUM_GEN_POWER2 (20)

#define BASE_GEN_SIZE (1*512)

// group the frequently used ones together (need intrumentation on accessors)

class generation

{

public:

// Don't move these first two fields without adjusting the references

// from the __asm in jitinterface.cpp.

alloc_context allocation_context;

PTR_heap_segment start_segment;

uint8_t* allocation_start;

heap_segment* allocation_segment;

uint8_t* allocation_context_start_region;

allocator free_list_allocator;

size_t free_list_allocated;

size_t end_seg_allocated;

BOOL allocate_end_seg_p;

size_t condemned_allocated;

size_t sweep_allocated;

size_t free_list_space;

size_t free_obj_space;

size_t allocation_size;

uint8_t* plan_allocation_start;

size_t plan_allocation_start_size;

// this is the pinned plugs that got allocated into this gen.

size_t pinned_allocated;

size_t pinned_allocation_compact_size;

size_t pinned_allocation_sweep_size;

int gen_num;

#ifdef FREE_USAGE_STATS

size_t gen_free_spaces[NUM_GEN_POWER2];

// these are non pinned plugs only

size_t gen_plugs[NUM_GEN_POWER2];

size_t gen_current_pinned_free_spaces[NUM_GEN_POWER2];

size_t pinned_free_obj_space;

// this is what got allocated into the pinned free spaces.

size_t allocated_in_pinned_free;

size_t allocated_since_last_pin;

#endif //FREE_USAGE_STATS

};

static_assert(offsetof(dac_generation, allocation_context) == offsetof(generation, allocation_context), "DAC generation offset mismatch");

static_assert(offsetof(dac_generation, start_segment) == offsetof(generation, start_segment), "DAC generation offset mismatch");

static_assert(offsetof(dac_generation, allocation_start) == offsetof(generation, allocation_start), "DAC generation offset mismatch");

// static data remains the same after it's initialized.

// It's per generation.

// TODO: for gen_time_tuning, we should put the multipliers in static data.

struct static_data

{

size_t min_size;

size_t max_size;

size_t fragmentation_limit;

float fragmentation_burden_limit;

float limit;

float max_limit;

size_t time_clock; // time after which to collect generation, in performance counts (see QueryPerformanceCounter)

size_t gc_clock; // nubmer of gcs after which to collect generation

};

// The dynamic data fields are grouped into 3 categories:

//

// calculated logical data (like desired_allocation)

// physical data (like fragmentation)

// const data (sdata), initialized at the beginning

class dynamic_data

{

public:

ptrdiff_t new_allocation;

ptrdiff_t gc_new_allocation; // new allocation at beginning of gc

float surv;

size_t desired_allocation;

// # of bytes taken by objects (ie, not free space) at the beginning

// of the GC.

size_t begin_data_size;

// # of bytes taken by survived objects after mark.

size_t survived_size;

// # of bytes taken by survived pinned plugs after mark.

size_t pinned_survived_size;

size_t artificial_pinned_survived_size;

size_t added_pinned_size;

#ifdef SHORT_PLUGS

size_t padding_size;

#endif //SHORT_PLUGS

#if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)

// # of plugs that are not pinned plugs.

size_t num_npinned_plugs;

#endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN

//total object size after a GC, ie, doesn't include fragmentation

size_t current_size;

size_t collection_count;

size_t promoted_size;

size_t freach_previous_promotion;

size_t fragmentation; //fragmentation when we don't compact

size_t gc_clock; //gc# when last GC happened

size_t time_clock; //time when last gc started

size_t gc_elapsed_time; // Time it took for the gc to complete

float gc_speed; // speed in bytes/msec for the gc to complete

size_t min_size;

static_data* sdata;

};

#define ro_in_entry 0x1

#ifdef SEG_MAPPING_TABLE

// Note that I am storing both h0 and seg0, even though in Server GC you can get to

// the heap* from the segment info. This is because heap_of needs to be really fast

// and we would not want yet another indirection.

struct seg_mapping

{

// if an address is > boundary it belongs to h1; else h0.

// since we init h0 and h1 to 0, if we get 0 it means that

// address doesn't exist on managed segments. And heap_of

// would just return heap0 which is what it does now.

uint8_t* boundary;

#ifdef MULTIPLE_HEAPS

gc_heap* h0;

gc_heap* h1;

#endif //MULTIPLE_HEAPS

// You could have an address that's inbetween 2 segments and

// this would return a seg, the caller then will use

// in_range_for_segment to determine if it's on that seg.

heap_segment* seg0; // this is what the seg for h0 is.

heap_segment* seg1; // this is what the seg for h1 is.

// Note that when frozen objects are used we mask seg1

// with 0x1 to indicate that there is a ro segment for

// this entry.

};

#endif //SEG_MAPPING_TABLE

// alignment helpers

//Alignment constant for allocation

#define ALIGNCONST (DATA_ALIGNMENT-1)

inline

size_t Align (size_t nbytes, int alignment=ALIGNCONST)

{

return (nbytes + alignment) & ~alignment;

}

//return alignment constant for small object heap vs large object heap

inline

int get_alignment_constant (BOOL small_object_p)

{

#ifdef FEATURE_STRUCTALIGN

// If any objects on the large object heap require 8-byte alignment,

// the compiler will tell us so. Let's not guess an alignment here.

return ALIGNCONST;

#else // FEATURE_STRUCTALIGN

return small_object_p ? ALIGNCONST : 7;

#endif // FEATURE_STRUCTALIGN

}

struct etw_opt_info

{

size_t desired_allocation;

size_t new_allocation;

int gen_number;

};

// Note, I am not removing the ones that are no longer used

// because the older versions of the runtime still use them

// and ETW interprets them.

enum alloc_wait_reason

{

// When we don't care about firing an event for

// this.

awr_ignored = -1,

// when we detect we are in low memory

awr_low_memory = 0,

// when we detect the ephemeral segment is too full

awr_low_ephemeral = 1,

// we've given out too much budget for gen0.

awr_gen0_alloc = 2,

// we've given out too much budget for loh.

awr_loh_alloc = 3,

// this event is really obsolete - it's for pre-XP

// OSs where low mem notification is not supported.

awr_alloc_loh_low_mem = 4,

// we ran out of VM spaced to reserve on loh.

awr_loh_oos = 5,

// ran out of space when allocating a small object

awr_gen0_oos_bgc = 6,

// ran out of space when allocating a large object

awr_loh_oos_bgc = 7,

// waiting for BGC to let FGC happen

awr_fgc_wait_for_bgc = 8,

// wait for bgc to finish to get loh seg.

// no longer used with the introduction of loh msl.

awr_get_loh_seg = 9,

// we don't allow loh allocation during bgc planning.

// no longer used with the introduction of loh msl.

awr_loh_alloc_during_plan = 10,

// we don't allow too much loh allocation during bgc.

awr_loh_alloc_during_bgc = 11

};

struct alloc_thread_wait_data

{

int awr;

};

enum msl_take_state

{

mt_get_large_seg = 0,

mt_bgc_loh_sweep,

mt_wait_bgc,

mt_block_gc,

mt_clr_mem,

mt_clr_large_mem,

mt_t_eph_gc,

mt_t_full_gc,

mt_alloc_small,

mt_alloc_large,

mt_alloc_small_cant,

mt_alloc_large_cant,