Google Git
Sign in
chromium / external / dynamorio / e30d0f4a7d3b1f3dabfe5e76ca906c34f4909f42 / . / core / fragment.c
blob: a0d35fb7420400e4d766657543d1ae03899930df [file] [log] [blame]
/* **********************************************************
* Copyright (c) 2011-2014 Google, Inc. All rights reserved.
* Copyright (c) 2000-2010 VMware, Inc. All rights reserved.
* **********************************************************/
/*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of VMware, Inc. nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
/* Copyright (c) 2003-2007 Determina Corp. */
/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
/* Copyright (c) 2000-2001 Hewlett-Packard Company */
/*
* fragment.c - fragment related routines
*/
#include "globals.h"
#include "link.h"
#include "fragment.h"
#include "fcache.h"
#include "emit.h"
#include "monitor.h"
#include <string.h> /* for memset */
#include "instrument.h"
#include <stddef.h> /* for offsetof */
#include <limits.h> /* UINT_MAX */
#include "perscache.h"
#include "synch.h"
#ifdef UNIX
# include "nudge.h"
#endif
/* FIXME: make these runtime parameters */
#define INIT_HTABLE_SIZE_SHARED_BB (DYNAMO_OPTION(coarse_units) ? 5 : 10)
#define INIT_HTABLE_SIZE_SHARED_TRACE 10
/* the only private bbs will be selfmod, so start small */
#define INIT_HTABLE_SIZE_BB (DYNAMO_OPTION(shared_bbs) ? 5 : 10)
/* coarse-grain fragments do not use futures */
#define INIT_HTABLE_SIZE_SHARED_FUTURE (DYNAMO_OPTION(coarse_units) ? 5 : 10)
#ifdef RETURN_AFTER_CALL
/* we have small per-module hashtables */
# define INIT_HTABLE_SIZE_AFTER_CALL 5
#endif
/* private futures are only used when we have private fragments */
#define INIT_HTABLE_SIZE_FUTURE \
((DYNAMO_OPTION(shared_bbs) && DYNAMO_OPTION(shared_traces)) ? 5 : 9)
/* per-module htables */
#define INIT_HTABLE_SIZE_COARSE 5
#define INIT_HTABLE_SIZE_COARSE_TH 4
#ifdef RCT_IND_BRANCH
# include "rct.h"
/* we have small per-module hashtables */
# define INIT_HTABLE_SIZE_RCT_IBT 7
# ifndef RETURN_AFTER_CALL
# error RCT_IND_BRANCH requires RETURN_AFTER_CALL since it reuses data types
# endif
#endif
/* if shared traces, we currently have no private traces so make table tiny
* FIMXE: should start out w/ no table at all
*/
#define INIT_HTABLE_SIZE_TRACE (DYNAMO_OPTION(shared_traces) ? 6 : 9)
/* for small table sizes resize is not an expensive operation and we start smaller */
/* Current flusher, protected by thread_initexit_lock. */
DECLARE_FREQPROT_VAR(static dcontext_t *flusher, NULL);
/* Current allsynch-flusher, protected by thread_initexit_lock. */
DECLARE_FREQPROT_VAR(static dcontext_t *allsynch_flusher, NULL);
/* These global tables are kept on the heap for selfprot (case 7957) */
/* synchronization to these tables is accomplished via read-write locks,
* where the writers are removal and resizing -- addition is atomic to
* readers.
* for now none of these are read from ibl routines so we only have to
* synch with other DR routines
*/
static fragment_table_t *shared_bb;
static fragment_table_t *shared_trace;
/* if we have either shared bbs or shared traces we need this shared: */
static fragment_table_t *shared_future;
/* Thread-shared tables are allocated in a shared per_thread_t.
* The structure is also used if we're dumping shared traces.
* Kept on the heap for selfprot (case 7957)
*/
static per_thread_t *shared_pt;
#define USE_SHARED_PT() (SHARED_IBT_TABLES_ENABLED() || \
(TRACEDUMP_ENABLED() && DYNAMO_OPTION(shared_traces)))
/* We keep track of "old" IBT target tables in a linked list and
* deallocate them in fragment_exit(). */
/* FIXME Deallocate tables more aggressively using a distributed, refcounting
* algo as is used for shared deletion. */
typedef struct _dead_fragment_table_t {
fragment_entry_t *table_unaligned;
uint table_flags;
uint capacity;
uint ref_count;
struct _dead_fragment_table_t *next;
} dead_fragment_table_t;
/* We keep these list pointers on the heap for selfprot (case 8074). */
typedef struct _dead_table_lists_t {
dead_fragment_table_t *dead_tables;
dead_fragment_table_t *dead_tables_tail;
} dead_table_lists_t;
static dead_table_lists_t *dead_lists;
DECLARE_CXTSWPROT_VAR(static mutex_t dead_tables_lock, INIT_LOCK_FREE(dead_tables_lock));
#ifdef RETURN_AFTER_CALL
/* High level lock for an atomic lookup+add operation on the
* after call tables. */
DECLARE_CXTSWPROT_VAR(static mutex_t after_call_lock, INIT_LOCK_FREE(after_call_lock));
/* We use per-module tables and only need this table for non-module code;
* on Linux though this is the only table used, until we have a module list.
*/
static rct_module_table_t rac_non_module_table;
#endif
/* allows independent sequences of flushes and delayed deletions,
* though with -syscalls_synch_flush additions we now hold this
* throughout a flush.
*/
DECLARE_CXTSWPROT_VAR(mutex_t shared_cache_flush_lock,
INIT_LOCK_FREE(shared_cache_flush_lock));
/* Global count of flushes, used as a timestamp for shared deletion.
* Reads may be done w/o a lock, but writes can only be done
* via increment_global_flushtime() while holding shared_cache_flush_lock.
*/
DECLARE_FREQPROT_VAR(uint flushtime_global, 0);
#ifdef CLIENT_INTERFACE
DECLARE_CXTSWPROT_VAR(mutex_t client_flush_request_lock,
INIT_LOCK_FREE(client_flush_request_lock));
DECLARE_CXTSWPROT_VAR(client_flush_req_t *client_flush_requests, NULL);
#endif
#if defined(RCT_IND_BRANCH) && defined(UNIX)
/* On Win32 we use per-module tables; on Linux we use a single global table,
* until we have a module list.
*/
rct_module_table_t rct_global_table;
#endif
#define NULL_TAG ((app_pc)PTR_UINT_0)
/* FAKE_TAG is used as a deletion marker for unlinked entries */
#define FAKE_TAG ((app_pc)PTR_UINT_MINUS_1)
/* instead of an empty hashtable slot containing NULL, we fill it
* with a pointer to this constant fragment, which we give a tag
* of 0.
* PR 305731: rather than having a start_pc of 0, which causes
* an app targeting 0 to crash at 0, we point at a handler that
* sends the app to an ibl miss.
*/
byte * hashlookup_null_target;
#define HASHLOOKUP_NULL_START_PC ((cache_pc)hashlookup_null_handler)
static const fragment_t null_fragment = { NULL_TAG, 0, 0, 0, 0,
HASHLOOKUP_NULL_START_PC, };
/* to avoid range check on fast path using an end of table sentinel fragment */
static const fragment_t sentinel_fragment = { NULL_TAG, 0, 0, 0, 0,
HASHLOOKUP_SENTINEL_START_PC, };
/* Shared fragment IBTs: We need to preserve the open addressing traversal
* in the hashtable while marking a table entry as unlinked.
* A null_fragment won't work since it terminates the traversal,
* so we use an unlinked marker. The lookup table entry for
* an unlinked entry *always* has its start_pc_fragment set to
* an IBL target_delete entry.
*/
static const fragment_t unlinked_fragment = { FAKE_TAG, };
/* macro used in the code from time of deletion markers */
/* Shared fragment IBTs: unlinked_fragment isn't a real fragment either. So they
* are naturally deleted during a table resize. */
#define REAL_FRAGMENT(fragment) \
((fragment) != &null_fragment && \
(fragment) != &unlinked_fragment && \
(fragment) != &sentinel_fragment)
#define GET_PT(dc) ((dc) == GLOBAL_DCONTEXT ? (USE_SHARED_PT() ? shared_pt : NULL) :\
(per_thread_t *) (dc)->fragment_field)
#define TABLE_PROTECTED(ptable) \
(!TABLE_NEEDS_LOCK(ptable) || READWRITE_LOCK_HELD(&(ptable)->rwlock))
/* everything except the invisible table is in here */
#define GET_FTABLE_HELPER(pt, flags, otherwise) \
(TEST(FRAG_IS_TRACE, (flags)) ? \
(TEST(FRAG_SHARED, (flags)) ? shared_trace : &pt->trace) : \
(TEST(FRAG_SHARED, (flags)) ? \
(TEST(FRAG_IS_FUTURE, (flags)) ? shared_future : shared_bb) : \
(TEST(FRAG_IS_FUTURE, (flags)) ? &pt->future : \
(otherwise))))
#define GET_FTABLE(pt, flags) GET_FTABLE_HELPER(pt, (flags), &pt->bb)
/* indirect branch table per target type (bb vs trace) and indirect branch type */
#define GET_IBT_TABLE(pt, flags, branch_type) \
(TEST(FRAG_IS_TRACE, (flags)) ? \
(DYNAMO_OPTION(shared_trace_ibt_tables) ? \
&shared_pt->trace_ibt[(branch_type)] : \
&(pt)->trace_ibt[(branch_type)]) : \
(DYNAMO_OPTION(shared_bb_ibt_tables) ? \
&shared_pt->bb_ibt[(branch_type)] : \
&(pt)->bb_ibt[(branch_type)]))
/********************************** STATICS ***********************************/
static uint fragment_heap_size(uint flags, int direct_exits, int indirect_exits);
static void fragment_free_future(dcontext_t *dcontext, future_fragment_t *fut);
#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
static void
coarse_persisted_fill_ibl(dcontext_t *dcontext, coarse_info_t *info,
ibl_branch_type_t branch_type);
#endif
#ifdef CLIENT_INTERFACE
static void
process_client_flush_requests(dcontext_t *dcontext, dcontext_t *alloc_dcontext,
client_flush_req_t *req, bool flush);
#endif
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
/* trace logging and synch for shared trace file: */
DECLARE_CXTSWPROT_VAR(static mutex_t tracedump_mutex, INIT_LOCK_FREE(tracedump_mutex));
DECLARE_FREQPROT_VAR(static stats_int_t tcount, 0); /* protected by tracedump_mutex */
static void exit_trace_file(per_thread_t *pt);
static void output_trace(dcontext_t *dcontext, per_thread_t *pt,
fragment_t *f, stats_int_t deleted_at);
static void init_trace_file(per_thread_t *pt);
#endif
#define SHOULD_OUTPUT_FRAGMENT(flags) \
(TEST(FRAG_IS_TRACE, (flags)) && \
!TEST(FRAG_TRACE_OUTPUT, (flags)) && \
TRACEDUMP_ENABLED())
#define FRAGMENT_COARSE_WRAPPER_FLAGS \
FRAG_FAKE | FRAG_SHARED | FRAG_COARSE_GRAIN | \
FRAG_LINKED_OUTGOING | FRAG_LINKED_INCOMING
/* We use temporary fragment_t + linkstub_t structs to more easily
* use existing code when emitting coarse-grain fragments.
* Only 1-ind-exit or 1 or 2 dir exit bbs can be coarse-grain.
* The bb_building_lock protects use of this.
*/
DECLARE_FREQPROT_VAR(
static struct {
fragment_t f;
union {
struct {
direct_linkstub_t dir_exit_1;
direct_linkstub_t dir_exit_2;
} dir_exits;
indirect_linkstub_t ind_exit;
} exits;
} coarse_emit_fragment, {{0}});
#ifdef SHARING_STUDY
/***************************************************************************
* fragment_t sharing study
* Only used with -fragment_sharing_study
* When the option is off we go ahead and waste the 4 static vars
* below so we don't have to have a define and separate build.
*/
typedef struct _thread_list_t {
uint thread_num;
uint count;
struct _thread_list_t *next;
} thread_list_t;
typedef struct _shared_entry_t {
app_pc tag;
uint num_threads;
thread_list_t *threads;
uint heap_size;
uint cache_size;
struct _shared_entry_t *next;
} shared_entry_t;
# define SHARED_HASH_BITS 16
static shared_entry_t ** shared_blocks;
DECLARE_CXTSWPROT_VAR(static mutex_t shared_blocks_lock,
INIT_LOCK_FREE(shared_blocks_lock));
static shared_entry_t ** shared_traces;
DECLARE_CXTSWPROT_VAR(static mutex_t shared_traces_lock,
INIT_LOCK_FREE(shared_traces_lock));
/* assumes caller holds table's lock! */
static shared_entry_t *
shared_block_lookup(shared_entry_t **table, fragment_t *f)
{
shared_entry_t *e;
uint hindex;
hindex = HASH_FUNC_BITS((ptr_uint_t)f->tag, SHARED_HASH_BITS);
/* using collision chains */
for (e = table[hindex]; e != NULL; e = e->next) {
if (e->tag == f->tag) {
return e;
}
}
return NULL;
}
static void
reset_shared_block_table(shared_entry_t **table, mutex_t *lock)
{
shared_entry_t *e, *nxte;
uint i;
uint size = HASHTABLE_SIZE(SHARED_HASH_BITS);
mutex_lock(lock);
for (i = 0; i < size; i++) {
for (e = table[i]; e != NULL; e = nxte) {
thread_list_t *tl = e->threads;
thread_list_t *tlnxt;
nxte = e->next;
while (tl != NULL) {
tlnxt = tl->next;
global_heap_free(tl, sizeof(thread_list_t) HEAPACCT(ACCT_OTHER));
tl = tlnxt;
}
global_heap_free(e, sizeof(shared_entry_t) HEAPACCT(ACCT_OTHER));
}
}
global_heap_free(table, size*sizeof(shared_entry_t*) HEAPACCT(ACCT_OTHER));
mutex_unlock(lock);
}
static void
add_shared_block(shared_entry_t **table, mutex_t *lock, fragment_t *f)
{
shared_entry_t *e;
uint hindex;
int num_direct = 0, num_indirect = 0;
linkstub_t *l = FRAGMENT_EXIT_STUBS(f);
/* use num to avoid thread_id_t recycling problems */
uint tnum = get_thread_num(get_thread_id());
mutex_lock(lock);
e = shared_block_lookup(table, f);
if (e != NULL) {
thread_list_t *tl = e->threads;
for (; tl != NULL; tl = tl->next) {
if (tl->thread_num == tnum) {
tl->count++;
LOG(GLOBAL, LOG_ALL, 2,
"add_shared_block: tag "PFX", but re-add #%d for thread #%d\n",
e->tag, tl->count, tnum);
mutex_unlock(lock);
return;
}
}
tl = global_heap_alloc(sizeof(thread_list_t) HEAPACCT(ACCT_OTHER));
tl->thread_num = tnum;
tl->count = 1;
tl->next = e->threads;
e->threads = tl;
e->num_threads++;
LOG(GLOBAL, LOG_ALL, 2, "add_shared_block: tag "PFX" thread #%d => %d threads\n",
e->tag, tnum, e->num_threads);
mutex_unlock(lock);
return;
}
/* get num stubs to find heap size */
for (; l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
if (LINKSTUB_DIRECT(l->flags))
num_direct++;
else {
ASSERT(LINKSTUB_INDIRECT(l->flags));
num_indirect++;
}
}
/* add entry to thread hashtable */
e = (shared_entry_t *) global_heap_alloc(sizeof(shared_entry_t) HEAPACCT(ACCT_OTHER));
e->tag = f->tag;
e->num_threads = 1;
e->heap_size = fragment_heap_size(f->flags, num_direct, num_indirect);
e->cache_size = (f->size + f->fcache_extra);
e->threads = global_heap_alloc(sizeof(thread_list_t) HEAPACCT(ACCT_OTHER));
e->threads->thread_num = tnum;
e->threads->count = 1;
e->threads->next = NULL;
LOG(GLOBAL, LOG_ALL, 2, "add_shared_block: tag "PFX", heap %d, cache %d, thread #%d\n",
e->tag, e->heap_size, e->cache_size, e->threads->thread_num);
hindex = HASH_FUNC_BITS((ptr_uint_t)f->tag, SHARED_HASH_BITS);
e->next = table[hindex];
table[hindex] = e;
mutex_unlock(lock);
}
static void
print_shared_table_stats(shared_entry_t **table, mutex_t *lock, const char *name)
{
uint i;
shared_entry_t *e;
uint size = HASHTABLE_SIZE(SHARED_HASH_BITS);
uint tot = 0, shared_tot = 0, shared = 0, heap = 0, cache = 0,
creation_count = 0;
mutex_lock(lock);
for (i = 0; i < size; i++) {
for (e = table[i]; e != NULL; e = e->next) {
thread_list_t *tl = e->threads;
tot++;
shared_tot += e->num_threads;
for (; tl != NULL; tl = tl->next)
creation_count += tl->count;
if (e->num_threads > 1) {
shared++;
/* assume similar size for each thread -- cache padding
* only real difference
*/
heap += (e->heap_size * e->num_threads);
cache += (e->cache_size * e->num_threads);
}
}
}
mutex_unlock(lock);
LOG(GLOBAL, LOG_ALL, 1, "Shared %s statistics:\n", name);
LOG(GLOBAL, LOG_ALL, 1, "\ttotal blocks: %10d\n", tot);
LOG(GLOBAL, LOG_ALL, 1, "\tcreation count: %10d\n", creation_count);
LOG(GLOBAL, LOG_ALL, 1, "\tshared count: %10d\n", shared_tot);
LOG(GLOBAL, LOG_ALL, 1, "\tshared blocks: %10d\n", shared);
LOG(GLOBAL, LOG_ALL, 1, "\tshared heap: %10d\n", heap);
LOG(GLOBAL, LOG_ALL, 1, "\tshared cache: %10d\n", cache);
}
void
print_shared_stats()
{
print_shared_table_stats(shared_blocks, &shared_blocks_lock, "basic block");
print_shared_table_stats(shared_traces, &shared_traces_lock, "trace");
}
#endif /* SHARING_STUDY ***************************************************/
#ifdef FRAGMENT_SIZES_STUDY /*****************************************/
#include <math.h>
/* don't bother to synchronize these */
static int bb_sizes[200000];
static int trace_sizes[40000];
static int num_bb = 0;
static int num_traces = 0;
void
record_fragment_size(int size, bool is_trace)
{
if (is_trace) {
trace_sizes[num_traces] = size;
num_traces++;
ASSERT(num_traces < 40000);
} else {
bb_sizes[num_bb] = size;
num_bb++;
ASSERT(num_bb < 200000);
}
}
void
print_size_results()
{
LOG(GLOBAL, LOG_ALL, 1, "Basic block sizes (bytes):\n");
print_statistics(bb_sizes, num_bb);
LOG(GLOBAL, LOG_ALL, 1, "Trace sizes (bytes):\n");
print_statistics(trace_sizes, num_traces);
}
#endif /* FRAGMENT_SIZES_STUDY */ /*****************************************/
#define FRAGTABLE_WHICH_HEAP(flags) \
(TESTALL(FRAG_TABLE_INCLUSIVE_HIERARCHY | FRAG_TABLE_IBL_TARGETED, \
(flags)) ? ACCT_IBLTABLE : ACCT_FRAG_TABLE)
#ifdef HASHTABLE_STATISTICS
# define UNPROT_STAT(stats) unprot_stats->stats
/* FIXME: either put in nonpersistent heap as appropriate, or
* preserve across resets
*/
# define ALLOC_UNPROT_STATS(dcontext, table) do { \
(table)->unprot_stats = \
HEAP_TYPE_ALLOC((dcontext), unprot_ht_statistics_t, \
FRAGTABLE_WHICH_HEAP((table)->table_flags), \
UNPROTECTED); \
memset((table)->unprot_stats, 0, sizeof(unprot_ht_statistics_t)); \
} while (0)
# define DEALLOC_UNPROT_STATS(dcontext, table) \
HEAP_TYPE_FREE((dcontext), (table)->unprot_stats, unprot_ht_statistics_t, \
FRAGTABLE_WHICH_HEAP((table)->table_flags), UNPROTECTED)
# define CHECK_UNPROT_STATS(table) ASSERT(table.unprot_stats != NULL)
static void
check_stay_on_trace_stats_overflow(dcontext_t *dcontext, ibl_branch_type_t branch_type)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
hashtable_statistics_t *lookup_stats = &pt->trace_ibt[branch_type].unprot_stats->
trace_ibl_stats[branch_type];
if (lookup_stats->ib_stay_on_trace_stat < lookup_stats->ib_stay_on_trace_stat_last) {
lookup_stats->ib_stay_on_trace_stat_ovfl++;
}
lookup_stats->ib_stay_on_trace_stat_last = lookup_stats->ib_stay_on_trace_stat;
/* FIXME: ib_trace_last_ibl_exit should have an overflow check as well */
}
#endif /* HASHTABLE_STATISTICS */
/* init/update the tls slots storing this table's mask and lookup base
* N.B.: for thread-shared the caller must call for each thread
*/
/* currently we don't support a mixture */
static inline void
update_lookuptable_tls(dcontext_t *dcontext, ibl_table_t *table)
{
/* use dcontext->local_state, rather than get_local_state(), to support
* being called from other threads!
*/
local_state_extended_t *state =
(local_state_extended_t *) dcontext->local_state;
ASSERT(state != NULL);
ASSERT(DYNAMO_OPTION(ibl_table_in_tls));
/* We must hold at least the read lock here, else we could grab
* an inconsistent mask/lookuptable pair if another thread is in the middle
* of resizing the table (case 10405).
*/
ASSERT_TABLE_SYNCHRONIZED(table, READWRITE);
/* case 10296: for shared tables we must update the table
* before the mask, as the ibl lookup code accesses the mask first,
* and old mask + new table is ok since it will de-ref within the
* new table (we never shrink tables) and be a miss, whereas
* new mask + old table can de-ref beyond the end of the table,
* crashing or worse.
*/
state->table_space.table[table->branch_type].lookuptable =
table->table;
state->table_space.table[table->branch_type].hash_mask =
table->hash_mask;
}
#ifdef DEBUG
static const char *ibl_bb_table_type_names[IBL_BRANCH_TYPE_END] =
{"ret_bb", "indcall_bb", "indjmp_bb"};
static const char *ibl_trace_table_type_names[IBL_BRANCH_TYPE_END] =
{"ret_trace", "indcall_trace", "indjmp_trace"};
#endif
#ifdef DEBUG
static inline void
dump_lookuptable_tls(dcontext_t *dcontext)
{
/* use dcontext->local_state, rather than get_local_state(), to support
* being called from other threads!
*/
if (DYNAMO_OPTION(ibl_table_in_tls)) {
local_state_extended_t *state =
(local_state_extended_t *) dcontext->local_state;
ibl_branch_type_t branch_type;
ASSERT(state != NULL);
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
LOG(THREAD, LOG_FRAGMENT, 1,
"\t Table %s, table "PFX", mask "PFX"\n",
!SHARED_BB_ONLY_IB_TARGETS() ?
ibl_trace_table_type_names[branch_type] :
ibl_bb_table_type_names[branch_type],
state->table_space.table[branch_type].lookuptable,
state->table_space.table[branch_type].hash_mask);
}
}
}
#endif
/*******************************************************************************
* IBL HASHTABLE INSTANTIATION
*/
#define FRAGENTRY_FROM_FRAGMENT(f) { (f)->tag, (f)->start_pc }
/* macros w/ name and types are duplicated in fragment.h -- keep in sync */
#define NAME_KEY ibl
#define ENTRY_TYPE fragment_entry_t
/* not defining HASHTABLE_USE_LOOKUPTABLE */
/* compiler won't let me use null_fragment.tag here */
static const fragment_entry_t fe_empty = { NULL_TAG, HASHLOOKUP_NULL_START_PC };
static const fragment_entry_t fe_sentinel = { NULL_TAG, HASHLOOKUP_SENTINEL_START_PC };
#define ENTRY_TAG(fe) ((ptr_uint_t)(fe).tag_fragment)
#define ENTRY_EMPTY (fe_empty)
#define ENTRY_SENTINEL (fe_sentinel)
#define IBL_ENTRY_IS_EMPTY(fe) \
((fe).tag_fragment == fe_empty.tag_fragment && \
(fe).start_pc_fragment == fe_empty.start_pc_fragment)
#define IBL_ENTRY_IS_INVALID(fe) ((fe).tag_fragment == FAKE_TAG)
#define IBL_ENTRY_IS_SENTINEL(fe) \
((fe).tag_fragment == fe_sentinel.tag_fragment && \
(fe).start_pc_fragment == fe_sentinel.start_pc_fragment)
#define ENTRY_IS_EMPTY(fe) IBL_ENTRY_IS_EMPTY(fe)
#define ENTRY_IS_SENTINEL(fe) IBL_ENTRY_IS_SENTINEL(fe)
#define ENTRY_IS_INVALID(fe) IBL_ENTRY_IS_INVALID(fe)
#define IBL_ENTRIES_ARE_EQUAL(fe1,fe2) ((fe1).tag_fragment == (fe2).tag_fragment)
#define ENTRIES_ARE_EQUAL(table,fe1,fe2) IBL_ENTRIES_ARE_EQUAL(fe1,fe2)
#define HASHTABLE_WHICH_HEAP(flags) FRAGTABLE_WHICH_HEAP(flags)
#define HTLOCK_RANK table_rwlock
#define HASHTABLE_ENTRY_STATS 1
#include "hashtablex.h"
/* all defines are undef-ed at end of hashtablex.h */
/* required routines for hashtable interface that we don't need for this instance */
static void
hashtable_ibl_free_entry(dcontext_t *dcontext, ibl_table_t *table,
fragment_entry_t entry)
{
/* nothing to do, data is inlined */
}
/*******************************************************************************
* FRAGMENT HASHTABLE INSTANTIATION
*/
/* macros w/ name and types are duplicated in fragment.h -- keep in sync */
#define NAME_KEY fragment
#define ENTRY_TYPE fragment_t *
/* not defining HASHTABLE_USE_LOOKUPTABLE */
#define ENTRY_TAG(f) ((ptr_uint_t)(f)->tag)
/* instead of setting to 0, point at null_fragment */
#define ENTRY_EMPTY ((fragment_t *)&null_fragment)
#define ENTRY_SENTINEL ((fragment_t *)&sentinel_fragment)
#define ENTRY_IS_EMPTY(f) ((f) == (fragment_t *)&null_fragment)
#define ENTRY_IS_SENTINEL(f) ((f) == (fragment_t *)&sentinel_fragment)
#define ENTRY_IS_INVALID(f) ((f) == (fragment_t *)&unlinked_fragment)
#define ENTRIES_ARE_EQUAL(t,f,g) ((f) == (g))
#define HASHTABLE_WHICH_HEAP(flags) FRAGTABLE_WHICH_HEAP(flags)
#define HTLOCK_RANK table_rwlock
#include "hashtablex.h"
/* all defines are undef-ed at end of hashtablex.h */
static void
hashtable_fragment_resized_custom(dcontext_t *dcontext, fragment_table_t *table,
uint old_capacity, fragment_t **old_table,
fragment_t **old_table_unaligned,
uint old_ref_count, uint old_table_flags)
{
/* nothing */
}
static void
hashtable_fragment_init_internal_custom(dcontext_t *dcontext, fragment_table_t *table)
{
/* nothing */
}
#ifdef DEBUG
static void
hashtable_fragment_study_custom(dcontext_t *dcontext, fragment_table_t *table,
uint entries_inc/*amnt table->entries was pre-inced*/)
{
/* nothing */
}
#endif
/* callers should use either hashtable_ibl_preinit or hashtable_resize instead */
static void
hashtable_ibl_init_internal_custom(dcontext_t *dcontext, ibl_table_t *table)
{
ASSERT(null_fragment.tag == NULL_TAG);
ASSERT(null_fragment.start_pc == HASHLOOKUP_NULL_START_PC);
ASSERT(FAKE_TAG != NULL_TAG);
ASSERT(sentinel_fragment.tag == NULL_TAG);
ASSERT(sentinel_fragment.start_pc == HASHLOOKUP_SENTINEL_START_PC);
ASSERT(HASHLOOKUP_SENTINEL_START_PC != HASHLOOKUP_NULL_START_PC);
ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
ASSERT(TEST(FRAG_TABLE_INCLUSIVE_HIERARCHY, table->table_flags));
/* every time we resize a table we reset the flush threshold,
* since it is cleared in place after one flush
*/
table->groom_factor_percent =
TEST(FRAG_TABLE_TRACE, table->table_flags) ?
DYNAMO_OPTION(trace_ibt_groom) : DYNAMO_OPTION(bb_ibt_groom);
table->max_capacity_bits =
TEST(FRAG_TABLE_TRACE, table->table_flags) ?
DYNAMO_OPTION(private_trace_ibl_targets_max) :
DYNAMO_OPTION(private_bb_ibl_targets_max);
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
if (table->unprot_stats == NULL) {
/* first time, not a resize */
ALLOC_UNPROT_STATS(dcontext, table);
} /* else, keep original */
}
#endif /* HASHTABLE_STATISTICS */
if (SHARED_IB_TARGETS() &&
!TEST(FRAG_TABLE_SHARED, table->table_flags)) {
/* currently we don't support a mixture */
ASSERT(TEST(FRAG_TABLE_TARGET_SHARED, table->table_flags));
ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
ASSERT(table->branch_type != IBL_NONE);
/* Only data for one set of tables is stored in TLS -- for the trace
* tables in the default config OR the BB tables in shared BBs
* only mode.
*/
if ((TEST(FRAG_TABLE_TRACE, table->table_flags) ||
SHARED_BB_ONLY_IB_TARGETS()) &&
DYNAMO_OPTION(ibl_table_in_tls))
update_lookuptable_tls(dcontext, table);
}
}
/* We need our own routines to init/free our added fields */
static void
hashtable_ibl_myinit(dcontext_t *dcontext, ibl_table_t *table, uint bits,
uint load_factor_percent, hash_function_t func,
uint hash_offset, ibl_branch_type_t branch_type,
bool use_lookup, uint table_flags _IF_DEBUG(const char *table_name))
{
uint flags = table_flags;
ASSERT(dcontext != GLOBAL_DCONTEXT || TEST(FRAG_TABLE_SHARED, flags));
/* flags shared by all ibl tables */
flags |= FRAG_TABLE_INCLUSIVE_HIERARCHY;
flags |= FRAG_TABLE_IBL_TARGETED;
flags |= HASHTABLE_ALIGN_TABLE;
/* use entry stats with all our ibl-targeted tables */
flags |= HASHTABLE_USE_ENTRY_STATS;
#ifdef HASHTABLE_STATISTICS
/* indicate this is first time, not a resize */
table->unprot_stats = NULL;
#endif
table->branch_type = branch_type;
hashtable_ibl_init(dcontext, table, bits, load_factor_percent,
func, hash_offset, flags _IF_DEBUG(table_name));
/* PR 305731: rather than having a start_pc of 0, which causes an
* app targeting 0 to crash at 0, we point at a handler that sends
* the app to an ibl miss via target_delete, which restores
* registers saved in the found path.
*/
if (dcontext != GLOBAL_DCONTEXT && hashlookup_null_target == NULL) {
ASSERT(!dynamo_initialized);
hashlookup_null_target = get_target_delete_entry_pc(dcontext, table);
#if !defined(X64) && defined(LINUX)
/* see comments in x86.asm: we patch to avoid text relocations */
byte *pc = (byte *) hashlookup_null_handler;
byte *page_start = (byte *) PAGE_START(pc);
byte *page_end = (byte *) ALIGN_FORWARD(pc + JMP_LONG_LENGTH, PAGE_SIZE);
make_writable(page_start, page_end - page_start);
insert_relative_target(pc + 1, hashlookup_null_target, NOT_HOT_PATCHABLE);
make_unwritable(page_start, page_end - page_start);
#endif
}
}
static void
hashtable_ibl_myfree(dcontext_t *dcontext, ibl_table_t *table)
{
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
DEALLOC_UNPROT_STATS(dcontext, table);
}
#endif /* HASHTABLE_STATISTICS */
hashtable_ibl_free(dcontext, table);
}
static void
hashtable_fragment_free_entry(dcontext_t *dcontext, fragment_table_t *table,
fragment_t *f)
{
if (TEST(FRAG_TABLE_INCLUSIVE_HIERARCHY, table->table_flags)) {
ASSERT_NOT_REACHED(); /* case 7691 */
} else {
if (TEST(FRAG_IS_FUTURE, f->flags))
fragment_free_future(dcontext, (future_fragment_t *)f);
else
fragment_free(dcontext, f);
}
}
static inline bool
fragment_add_to_hashtable(dcontext_t *dcontext, fragment_t *e, fragment_table_t *table)
{
/* When using shared IBT tables w/trace building and BB2BB IBL, there is a
* race between adding a BB target to a table and having it marked by
* another thread as a trace head. The race exists because the two functions
* do not use a common lock.
* The race does NOT cause a correctness problem since a) the marking thread
* removes the trace head from the table and b) any subsequent add attempt
* is caught in add_ibl_target(). The table lock is used during add and
* remove operations and FRAG_IS_TRACE_HEAD is checked while holding
* the lock. So although a trace head may be present in a table temporarily --
* it's being marked while an add operation that has passed the frag flags
* check is in progress -- it will be subsequently removed by the marking
* thread.
* However, the existence of the race does mean that
* we cannot ASSERT(!(FRAG_IS_TRACE_HEAD,...)) at arbitrary spots along the
* add_ibl_target() path since such an assert could fire due to the race.
* What is likely a safe point to assert is when there is only a single
* thread in the process.
*/
DOCHECK(1, {
if (TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags) &&
get_num_threads() == 1)
ASSERT(!TEST(FRAG_IS_TRACE_HEAD, e->flags));
});
return hashtable_fragment_add(dcontext, e, table);
}
/* updates all fragments in a given fragment table which may
* have IBL routine heads inlined in the indirect exit stubs
*
* FIXME: [perf] should add a filter of which branch types need updating if
* updating all is a noticeable performance hit.
*
* FIXME: [perf] Also it maybe better to traverse all fragments in an fcache
* unit instead of entries in a half-empty hashtable
*/
static void
update_indirect_exit_stubs_from_table(dcontext_t *dcontext,
fragment_table_t *ftable)
{
fragment_t *f;
linkstub_t *l;
uint i;
for (i = 0; i < ftable->capacity; i++) {
f = ftable->table[i];
if (!REAL_FRAGMENT(f))
continue;
for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
if (LINKSTUB_INDIRECT(l->flags)) {
/* FIXME: should add a filter of which branch types need updating */
update_indirect_exit_stub(dcontext, f, l);
LOG(THREAD, LOG_FRAGMENT, 5,
"\tIBL target table resizing: updating F%d\n", f->id);
STATS_INC(num_ibl_stub_resize_updates);
}
}
}
}
static void
safely_nullify_tables(dcontext_t *dcontext, ibl_table_t *new_table,
fragment_entry_t *table, uint capacity)
{
uint i;
cache_pc target_delete = get_target_delete_entry_pc(dcontext, new_table);
ASSERT(target_delete != NULL);
ASSERT_TABLE_SYNCHRONIZED(new_table, WRITE);
for (i = 0; i < capacity; i++) {
if (IBL_ENTRY_IS_SENTINEL(table[i])) {
ASSERT(i == capacity - 1);
continue;
}
/* We need these writes to be atomic, so check that they're aligned. */
ASSERT(ALIGNED(&table[i].tag_fragment, 4));
ASSERT(ALIGNED(&table[i].start_pc_fragment, 4));
/* We update the tag first so that so that a thread that's skipping
* along a chain will exit ASAP. Breaking the chain is ok since we're
* nullifying the entire table.
*/
table[i].tag_fragment = fe_empty.tag_fragment;
/* We set the payload to target_delete to induce a cache exit.
*
* The target_delete path leads to a loss of information -- we can't
* tell what the src fragment was (the one that transitioned to the
* IBL code) and this in principle could weaken our RCT checks (see case
* 5085). In practical terms, RCT checks are unaffected since they
* are not employed on in-cache transitions such as an IBL hit.
* (All transitions to target_delete are a race along the hit path.)
* If we still want to preserve the src info, we can leave the payload
* as-is, possibly pointing to a cache address. The effect is that
* any thread accessing the old table on the IBL hit path will not exit
* the cache as early. (We should leave the fragment_t* value in the
* table untouched also so that the fragment_table_t is in a consistent
* state.)
*/
table[i].start_pc_fragment = target_delete;
}
STATS_INC(num_shared_ibt_table_flushes);
}
/* Add an item to the dead tables list */
static inline void
add_to_dead_table_list(dcontext_t *alloc_dc, ibl_table_t *ftable,
uint old_capacity,
fragment_entry_t *old_table_unaligned, uint old_ref_count,
uint old_table_flags)
{
dead_fragment_table_t *item =(dead_fragment_table_t*)
heap_alloc(GLOBAL_DCONTEXT, sizeof(dead_fragment_table_t)
HEAPACCT(ACCT_IBLTABLE));
LOG(GLOBAL, LOG_FRAGMENT, 2,
"add_to_dead_table_list %s "PFX" capacity %d\n",
ftable->name, old_table_unaligned, old_capacity);
ASSERT(old_ref_count >= 1); /* someone other the caller must be holding a reference */
/* write lock must be held so that ref_count is copied accurately */
ASSERT_TABLE_SYNCHRONIZED(ftable, WRITE);
item->capacity = old_capacity;
item->table_unaligned = old_table_unaligned;
item->table_flags = old_table_flags;
item->ref_count = old_ref_count;
item->next = NULL;
/* Add to the end of list. We use a FIFO because generally we'll be
* decrementing ref-counts for older tables before we do so for
* younger tables. A FIFO will yield faster searches than, say, a
* stack.
*/
mutex_lock(&dead_tables_lock);
if (dead_lists->dead_tables == NULL) {
ASSERT(dead_lists->dead_tables_tail == NULL);
dead_lists->dead_tables = item;
}
else {
ASSERT(dead_lists->dead_tables_tail != NULL);
ASSERT(dead_lists->dead_tables_tail->next == NULL);
dead_lists->dead_tables_tail->next = item;
}
dead_lists->dead_tables_tail = item;
mutex_unlock(&dead_tables_lock);
STATS_ADD_PEAK(num_dead_shared_ibt_tables, 1);
STATS_INC(num_total_dead_shared_ibt_tables);
}
/* forward decl */
static inline void
update_private_ptr_to_shared_ibt_table(dcontext_t *dcontext,
ibl_branch_type_t branch_type, bool trace,
bool adjust_old_ref_count, bool lock_table);
static void
hashtable_ibl_resized_custom(dcontext_t *dcontext, ibl_table_t *table,
uint old_capacity, fragment_entry_t *old_table,
fragment_entry_t *old_table_unaligned,
uint old_ref_count, uint old_table_flags)
{
dcontext_t *alloc_dc = FRAGMENT_TABLE_ALLOC_DC(dcontext, table->table_flags);
per_thread_t *pt = GET_PT(dcontext);
bool shared_ibt_table =
TESTALL(FRAG_TABLE_TARGET_SHARED | FRAG_TABLE_SHARED, table->table_flags);
ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
/* If we change an ibl-targeted table, must patch up every
* inlined indirect exit stub that targets it.
* For our per-type ibl tables however we don't bother updating
* fragments _targeted_ by the resized table, instead we need to
* update all fragments that may be a source of an inlined IBL.
*/
/* private inlined IBL heads targeting this table need to be updated */
if (DYNAMO_OPTION(inline_trace_ibl) && PRIVATE_TRACES_ENABLED()) {
/* We'll get here on a trace table resize, while we
* need to patch only when the trace_ibt tables are resized.
*/
/* We assume we don't inline IBL lookup targeting tables of basic blocks
* and so shouldn't need to do this for now. */
ASSERT(dcontext != GLOBAL_DCONTEXT && pt != NULL); /* private traces */
if (TESTALL(FRAG_TABLE_INCLUSIVE_HIERARCHY | FRAG_TABLE_TRACE,
table->table_flags)) {
/* need to update all traces that could be targeting the
* currently resized table */
LOG(THREAD, LOG_FRAGMENT, 2,
"\tIBL target table resizing: updating all private trace fragments\n");
update_indirect_exit_stubs_from_table(dcontext, &pt->trace);
}
}
/* if we change the trace table (or an IBL target trace
* table), must patch up every inlined indirect exit stub
* in all bb fragments in case the inlined target is the
* resized table
*/
if (DYNAMO_OPTION(inline_bb_ibl)) {
LOG(THREAD, LOG_FRAGMENT, 3,
"\tIBL target table resizing: updating bb fragments\n");
update_indirect_exit_stubs_from_table(dcontext, &pt->bb);
}
/* don't need to update any inlined lookups in shared fragments */
if (shared_ibt_table) {
if (old_ref_count > 0) {
/* The old table should be nullified ASAP. Since threads update
* their table pointers on-demand only when they exit the cache
* after a failed IBL lookup, they could have IBL targets for
* stale entries. This would likely occur only when there's an
* app race but in the future could occur due to cache
* management.
*/
safely_nullify_tables(dcontext, table, old_table, old_capacity);
add_to_dead_table_list(alloc_dc, table, old_capacity,
old_table_unaligned,
old_ref_count, table->table_flags);
}
/* Update the resizing thread's private ptr. */
update_private_ptr_to_shared_ibt_table(dcontext, table->branch_type,
TEST(FRAG_TABLE_TRACE,
table->table_flags),
false, /* no adjust
* old ref-count */
false /* already hold lock */);
ASSERT(table->ref_count == 1);
}
/* CHECK: is it safe to update the table without holding the lock? */
/* Using the table flags to drive the update of generated code may
* err on the side of caution, but it's the best way to guarantee
* that all of the necessary code is updated.
* We may perform extra unnecessary updates when a table that's
* accessed off of the dcontext/per_thread_t is grown, but that doesn't
* cause correctness problems and likely doesn't hurt peformance.
*/
STATS_INC(num_ibt_table_resizes);
update_generated_hashtable_access(dcontext);
}
#ifdef DEBUG
static void
hashtable_ibl_study_custom(dcontext_t *dcontext, ibl_table_t *table,
uint entries_inc/*amnt table->entries was pre-inced*/)
{
# ifdef HASHTABLE_STATISTICS
/* For trace table(s) only, use stats from emitted ibl routines */
if (TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags) &&
INTERNAL_OPTION(hashtable_ibl_stats)) {
per_thread_t *pt = GET_PT(dcontext);
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
/* This is convoluted since given a table we have to
* recover its branch type.
* FIXME: should simplify these assumptions one day
*/
/* Current table should be targeted only by one of the IBL routines */
if (!((!DYNAMO_OPTION(disable_traces) &&
table == &pt->trace_ibt[branch_type]) ||
(DYNAMO_OPTION(bb_ibl_targets) &&
table == &pt->bb_ibt[branch_type])))
continue;
/* stats for lookup routines from bb's and trace's targeting the current table */
print_hashtable_stats(dcontext, entries_inc == 0 ? "Total" : "Current",
table->name,
"trace ibl ", get_branch_type_name(branch_type),
&table->UNPROT_STAT(trace_ibl_stats[branch_type]));
print_hashtable_stats(dcontext, entries_inc == 0 ? "Total" : "Current",
table->name,
"bb ibl ",
get_branch_type_name(branch_type),
&table->UNPROT_STAT(bb_ibl_stats[branch_type]));
}
}
# endif /* HASHTABLE_STATISTICS */
}
#endif /* DEBUG */
#if defined(DEBUG) || defined(CLIENT_INTERFACE)
/* filter specifies flags for fragments which are OK to be freed */
/* NOTE - if this routine is ever used for non DEBUG purposes be aware that
* because of case 7697 we don't unlink when we free the hashtable elements.
* As such, if we aren't also freeing all fragments that could possibly link
* to fragments in this table at the same time (synchronously) we'll have
* problems (for ex. a trace only reset would need to unlink incoming, or
* allowing private->shared linking would need to ulink outgoing).
*/
static void
hashtable_fragment_reset(dcontext_t *dcontext, fragment_table_t *table)
{
int i;
fragment_t *f;
/* case 7691: we now use separate ibl table types */
ASSERT(!TEST(FRAG_TABLE_INCLUSIVE_HIERARCHY, table->table_flags));
LOG(THREAD, LOG_FRAGMENT, 2, "hashtable_fragment_reset\n");
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_fragment_load_statistics(dcontext, table);
});
if (TEST(FRAG_TABLE_SHARED, table->table_flags) &&
TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags)) {
DOLOG(4, LOG_FRAGMENT, {
hashtable_fragment_dump_table(dcontext, table);
});
}
DODEBUG({
hashtable_fragment_study(dcontext, table, 0/*table consistent*/);
/* ensure write lock is held if the table is shared, unless exiting
* or resetting (N.B.: if change reset model to not suspend all in-DR
* threads, will have to change this and handle rank order issues)
*/
if (!dynamo_exited && !dynamo_resetting)
ASSERT_TABLE_SYNCHRONIZED(table, WRITE);
});
/* Go in reverse order (for efficiency) since using
* hashtable_fragment_remove_helper to keep all reachable, which is required
* for dynamo_resetting where we unlink fragments here and need to be able to
* perform lookups.
*/
i = table->capacity - 1 - 1 /* sentinel */;
while (i >= 0) {
f = table->table[i];
if (f == &null_fragment) {
i--;
} else { /* i stays put */
/* The shared BB table is reset at process reset or shutdown, so
* trace_abort() has already been called by (or for) every thread.
* If shared traces is true, by this point none of the shared BBs
* should have FRAG_TRACE_BUILDING set since the flag is cleared
* by trace_abort(). Of course, the flag shouldn't be present
* if shared traces is false so we don't need to conditionalize
* the assert.
*/
ASSERT(!TEST(FRAG_TRACE_BUILDING, f->flags));
hashtable_fragment_remove_helper(table, i, &table->table[i]);
if (!REAL_FRAGMENT(f))
continue;
/* make sure no other hashtable has shared fragments in it
* this routine is called on shared table, but only after dynamo_exited
* the per-thread IBL tables contain pointers to shared fragments
* and are OK
*/
ASSERT(dynamo_exited || !TEST(FRAG_SHARED, f->flags) || dynamo_resetting);
# if defined(SIDELINE) && defined(PROFILE_LINKCOUNT)
if ((f->flags & FRAG_DO_NOT_SIDELINE) != 0) {
/* print out total count of exit counters */
LOG(THREAD, LOG_SIDELINE, 2, "\tSidelined trace F%d total times executed: "
LINKCOUNT_FORMAT_STRING "\n", f->id, get_total_linkcount(f));
}
# endif
if (TEST(FRAG_IS_FUTURE, f->flags)) {
DODEBUG({ ((future_fragment_t *)f)->incoming_stubs = NULL; });
fragment_free_future(dcontext, (future_fragment_t *)f);
} else {
DOSTATS({
if (dynamo_resetting)
STATS_INC(num_fragments_deleted_reset);
else
STATS_INC(num_fragments_deleted_exit);
});
/* Xref 7697 - unlinking the fragments here can screw up the
* future table as we are walking in hash order, so we don't
* unlink. See note at top of routine for issues with not
* unlinking here if this code is ever used in non debug
* builds. */
fragment_delete(dcontext, f,
FRAGDEL_NO_HTABLE | FRAGDEL_NO_UNLINK |
FRAGDEL_NEED_CHLINK_LOCK |
(dynamo_resetting ? 0 : FRAGDEL_NO_OUTPUT));
}
}
}
table->entries = 0;
table->unlinked_entries = 0;
}
#endif /* DEBUG || CLIENT_INTERFACE */
/*
*******************************************************************************/
#if defined(RETURN_AFTER_CALL) || defined (RCT_IND_BRANCH)
/*******************************************************************************
* APP_PC HASHTABLE INSTANTIATION
*/
/* FIXME: RCT tables no longer use future_fragment_t and can be moved out of fragment.c */
/* The ENTRY_* defines are undef-ed at end of hashtablex.h so we make our own.
* Would be nice to re-use ENTRY_IS_EMPTY, etc., though w/ multiple htables
* in same file can't realistically get away w/o custom defines like these:
*/
#define APP_PC_EMPTY (NULL)
/* assume 1 is always invalid address */
#define APP_PC_SENTINEL ((app_pc)PTR_UINT_1)
#define APP_PC_ENTRY_IS_EMPTY(pc) ((pc) == APP_PC_EMPTY)
#define APP_PC_ENTRY_IS_SENTINEL(pc) ((pc) == APP_PC_SENTINEL)
#define APP_PC_ENTRY_IS_REAL(pc) (!APP_PC_ENTRY_IS_EMPTY(pc) && \
!APP_PC_ENTRY_IS_SENTINEL(pc))
/* 2 macros w/ name and types are duplicated in fragment.h -- keep in sync */
#define NAME_KEY app_pc
#define ENTRY_TYPE app_pc
/* not defining HASHTABLE_USE_LOOKUPTABLE */
#define ENTRY_TAG(f) ((ptr_uint_t)(f))
#define ENTRY_EMPTY APP_PC_EMPTY
#define ENTRY_SENTINEL APP_PC_SENTINEL
#define ENTRY_IS_EMPTY(f) APP_PC_ENTRY_IS_EMPTY(f)
#define ENTRY_IS_SENTINEL(f) APP_PC_ENTRY_IS_SENTINEL(f)
#define ENTRY_IS_INVALID(f) (false) /* no invalid entries */
#define ENTRIES_ARE_EQUAL(t,f,g) ((f) == (g))
#define HASHTABLE_WHICH_HEAP(flags) (ACCT_AFTER_CALL)
#define HTLOCK_RANK app_pc_table_rwlock
#define HASHTABLE_SUPPORT_PERSISTENCE 1
#include "hashtablex.h"
/* all defines are undef-ed at end of hashtablex.h */
/* required routines for hashtable interface that we don't need for this instance */
static void
hashtable_app_pc_init_internal_custom(dcontext_t *dcontext, app_pc_table_t *htable)
{ /* nothing */
}
static void
hashtable_app_pc_resized_custom(dcontext_t *dcontext, app_pc_table_t *htable,
uint old_capacity, app_pc *old_table,
app_pc *old_table_unaligned,
uint old_ref_count, uint old_table_flags)
{ /* nothing */
}
# ifdef DEBUG
static void
hashtable_app_pc_study_custom(dcontext_t *dcontext, app_pc_table_t *htable,
uint entries_inc/*amnt table->entries was pre-inced*/)
{ /* nothing */
}
# endif
static void
hashtable_app_pc_free_entry(dcontext_t *dcontext, app_pc_table_t *htable,
app_pc entry)
{
/* nothing to do, data is inlined */
}
#endif /* defined(RETURN_AFTER_CALL) || defined (RCT_IND_BRANCH) */
/*******************************************************************************/
bool
fragment_initialized(dcontext_t *dcontext)
{
return (dcontext != GLOBAL_DCONTEXT && dcontext->fragment_field != NULL);
}
/* thread-shared initialization that should be repeated after a reset */
void
fragment_reset_init(void)
{
/* case 7966: don't initialize at all for hotp_only & thin_client */
if (RUNNING_WITHOUT_CODE_CACHE())
return;
mutex_lock(&shared_cache_flush_lock);
/* ASSUMPTION: a reset frees all deletions that use flushtimes, so we can
* reset the global flushtime here
*/
flushtime_global = 0;
mutex_unlock(&shared_cache_flush_lock);
if (SHARED_FRAGMENTS_ENABLED()) {
if (DYNAMO_OPTION(shared_bbs)) {
hashtable_fragment_init(GLOBAL_DCONTEXT, shared_bb,
INIT_HTABLE_SIZE_SHARED_BB,
INTERNAL_OPTION(shared_bb_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
FRAG_TABLE_SHARED | FRAG_TABLE_TARGET_SHARED
_IF_DEBUG("shared_bb"));
}
if (DYNAMO_OPTION(shared_traces)) {
hashtable_fragment_init(GLOBAL_DCONTEXT, shared_trace,
INIT_HTABLE_SIZE_SHARED_TRACE,
INTERNAL_OPTION(shared_trace_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
FRAG_TABLE_SHARED | FRAG_TABLE_TARGET_SHARED
_IF_DEBUG("shared_trace"));
}
/* init routine will work for future_fragment_t* same as for fragment_t* */
hashtable_fragment_init(GLOBAL_DCONTEXT, shared_future,
INIT_HTABLE_SIZE_SHARED_FUTURE,
INTERNAL_OPTION(shared_future_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
FRAG_TABLE_SHARED | FRAG_TABLE_TARGET_SHARED
_IF_DEBUG("shared_future"));
}
if (SHARED_IBT_TABLES_ENABLED()) {
ibl_branch_type_t branch_type;
ASSERT(USE_SHARED_PT());
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
hashtable_ibl_myinit(GLOBAL_DCONTEXT, &shared_pt->trace_ibt[branch_type],
DYNAMO_OPTION(shared_ibt_table_trace_init),
DYNAMO_OPTION(shared_ibt_table_trace_load),
HASH_FUNCTION_NONE,
HASHTABLE_IBL_OFFSET(branch_type),
branch_type,
false, /* no lookup table */
FRAG_TABLE_SHARED |
FRAG_TABLE_TARGET_SHARED |
FRAG_TABLE_TRACE
_IF_DEBUG(ibl_trace_table_type_names[branch_type]));
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
CHECK_UNPROT_STATS(&shared_pt->trace_ibt[branch_type]);
/* for compatibility using an entry in the per-branch type stats */
INIT_HASHTABLE_STATS(shared_pt->trace_ibt[branch_type].
UNPROT_STAT(trace_ibl_stats[branch_type]));
} else {
shared_pt->trace_ibt[branch_type].unprot_stats = NULL;
}
#endif /* HASHTABLE_STATISTICS */
}
if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
hashtable_ibl_myinit(GLOBAL_DCONTEXT, &shared_pt->bb_ibt[branch_type],
DYNAMO_OPTION(shared_ibt_table_bb_init),
DYNAMO_OPTION(shared_ibt_table_bb_load),
HASH_FUNCTION_NONE,
HASHTABLE_IBL_OFFSET(branch_type),
branch_type,
false, /* no lookup table */
FRAG_TABLE_SHARED |
FRAG_TABLE_TARGET_SHARED
_IF_DEBUG(ibl_bb_table_type_names[branch_type]));
/* mark as inclusive table for bb's - we in fact currently
* keep only frags that are not FRAG_IS_TRACE_HEAD */
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
/* for compatibility using an entry in the per-branch type stats */
CHECK_UNPROT_STATS(&shared_pt->bb_ibt[branch_type]);
/* FIXME: we don't expect trace_ibl_stats yet */
INIT_HASHTABLE_STATS(shared_pt->bb_ibt[branch_type].
UNPROT_STAT(bb_ibl_stats[branch_type]));
} else {
shared_pt->bb_ibt[branch_type].unprot_stats = NULL;
}
#endif /* HASHTABLE_STATISTICS */
}
}
}
#ifdef SHARING_STUDY
if (INTERNAL_OPTION(fragment_sharing_study)) {
uint size = HASHTABLE_SIZE(SHARED_HASH_BITS) * sizeof(shared_entry_t*);
shared_blocks = (shared_entry_t**) global_heap_alloc(size HEAPACCT(ACCT_OTHER));
memset(shared_blocks, 0, size);
shared_traces = (shared_entry_t**) global_heap_alloc(size HEAPACCT(ACCT_OTHER));
memset(shared_traces, 0, size);
}
#endif
}
/* thread-shared initialization */
void
fragment_init()
{
/* case 7966: don't initialize at all for hotp_only & thin_client
* FIXME: could set initial sizes to 0 for all configurations, instead
*/
if (RUNNING_WITHOUT_CODE_CACHE())
return;
/* make sure fields are at same place */
ASSERT(offsetof(fragment_t, flags) == offsetof(future_fragment_t, flags));
ASSERT(offsetof(fragment_t, tag) == offsetof(future_fragment_t, tag));
/* ensure we can read this w/o a lock: no cache line crossing, please */
ASSERT(ALIGNED(&flushtime_global, 4));
if (SHARED_FRAGMENTS_ENABLED()) {
/* tables are persistent across resets, only on heap for selfprot (case 7957) */
if (DYNAMO_OPTION(shared_bbs)) {
shared_bb = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fragment_table_t,
ACCT_FRAG_TABLE, PROTECTED);
}
if (DYNAMO_OPTION(shared_traces)) {
shared_trace = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fragment_table_t,
ACCT_FRAG_TABLE, PROTECTED);
}
shared_future = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fragment_table_t,
ACCT_FRAG_TABLE, PROTECTED);
}
if (USE_SHARED_PT())
shared_pt = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, per_thread_t, ACCT_OTHER, PROTECTED);
if (SHARED_IBT_TABLES_ENABLED()) {
dead_lists =
HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, dead_table_lists_t, ACCT_OTHER, PROTECTED);
memset(dead_lists, 0, sizeof(*dead_lists));
}
fragment_reset_init();
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
if (TRACEDUMP_ENABLED() && DYNAMO_OPTION(shared_traces)) {
ASSERT(USE_SHARED_PT());
shared_pt->tracefile = open_log_file("traces-shared", NULL, 0);
ASSERT(shared_pt->tracefile != INVALID_FILE);
init_trace_file(shared_pt);
}
#endif
}
/* Free all thread-shared state not critical to forward progress;
* fragment_reset_init() will be called before continuing.
*/
void
fragment_reset_free(void)
{
/* case 7966: don't initialize at all for hotp_only & thin_client */
if (RUNNING_WITHOUT_CODE_CACHE())
return;
/* We must study the ibl tables before the trace/bb tables so that we're
* not looking at freed entries
*/
if (SHARED_IBT_TABLES_ENABLED()) {
ibl_branch_type_t branch_type;
dead_fragment_table_t *current, *next;
DEBUG_DECLARE(int table_count = 0;)
DEBUG_DECLARE(stats_int_t dead_tables = GLOBAL_STAT(num_dead_shared_ibt_tables);)
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_ibl_load_statistics(GLOBAL_DCONTEXT,
&shared_pt->trace_ibt[branch_type]);
});
hashtable_ibl_myfree(GLOBAL_DCONTEXT,
&shared_pt->trace_ibt[branch_type]);
}
if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_ibl_load_statistics(GLOBAL_DCONTEXT,
&shared_pt->bb_ibt[branch_type]);
});
hashtable_ibl_myfree(GLOBAL_DCONTEXT,
&shared_pt->bb_ibt[branch_type]);
}
}
/* Delete dead tables. */
/* grab lock for consistency, although we expect a single thread */
mutex_lock(&dead_tables_lock);
current = dead_lists->dead_tables;
while (current != NULL) {
DODEBUG({table_count++;});
next = current->next;
LOG(GLOBAL, LOG_FRAGMENT, 2,
"fragment_reset_free: dead table "PFX" cap %d, freeing\n",
current->table_unaligned, current->capacity);
hashtable_ibl_free_table(GLOBAL_DCONTEXT, current->table_unaligned,
current->table_flags, current->capacity);
heap_free(GLOBAL_DCONTEXT, current, sizeof(dead_fragment_table_t)
HEAPACCT(ACCT_IBLTABLE));
STATS_DEC(num_dead_shared_ibt_tables);
STATS_INC(num_dead_shared_ibt_tables_freed);
current = next;
DODEBUG({
if (dynamo_exited)
STATS_INC(num_dead_shared_ibt_tables_freed_at_exit);
});
}
dead_lists->dead_tables = dead_lists->dead_tables_tail = NULL;
ASSERT(table_count == dead_tables);
mutex_unlock(&dead_tables_lock);
}
/* FIXME: Take in a flag "permanent" that controls whether exiting or
* resetting. If resetting only, do not free unprot stats and entry stats
* (they're already in persistent heap, but we explicitly free them).
* This will be easy w/ unprot but will take work for entry stats
* since they resize themselves.
* Or, move them both to a new unprot and nonpersistent heap so we can
* actually free the memory back to the os, if we don't care to keep
* the stats across the reset.
*/
/* N.B.: to avoid rank order issues w/ shared_vm_areas lock being acquired
* after table_rwlock we do NOT grab the write lock before calling
* reset on the shared tables! We assume that reset involves suspending
* all other threads in DR and there will be no races. If the reset model
* changes, the lock order will have to be addressed.
*/
if (SHARED_FRAGMENTS_ENABLED()) {
/* clean up pending delayed deletion, if any */
vm_area_check_shared_pending(GLOBAL_DCONTEXT/*== safe to free all*/, NULL);
if (DYNAMO_OPTION(coarse_units)) {
/* We need to free coarse units earlier than vm_areas_exit() so we
* call it here. Must call before we free fine fragments so coarse
* can clean up incoming pointers.
*/
vm_area_coarse_units_reset_free();
}
#if defined(DEBUG) || defined(CLIENT_INTERFACE)
/* We need for CLIENT_INTERFACE to get fragment deleted events. */
# if !defined(DEBUG) && defined(CLIENT_INTERFACE)
if (dr_fragment_deleted_hook_exists()) {
# endif
if (DYNAMO_OPTION(shared_bbs))
hashtable_fragment_reset(GLOBAL_DCONTEXT, shared_bb);
if (DYNAMO_OPTION(shared_traces))
hashtable_fragment_reset(GLOBAL_DCONTEXT, shared_trace);
DODEBUG({hashtable_fragment_reset(GLOBAL_DCONTEXT, shared_future);});
# if !defined(DEBUG) && defined(CLIENT_INTERFACE)
}
# endif
#endif
if (DYNAMO_OPTION(shared_bbs))
hashtable_fragment_free(GLOBAL_DCONTEXT, shared_bb);
if (DYNAMO_OPTION(shared_traces))
hashtable_fragment_free(GLOBAL_DCONTEXT, shared_trace);
hashtable_fragment_free(GLOBAL_DCONTEXT, shared_future);
/* Do NOT free RAC table as its state cannot be rebuilt.
* We also do not free other RCT tables to avoid the time to rebuild them.
*/
}
#ifdef SHARING_STUDY
if (INTERNAL_OPTION(fragment_sharing_study)) {
print_shared_stats();
reset_shared_block_table(shared_blocks, &shared_blocks_lock);
reset_shared_block_table(shared_traces, &shared_traces_lock);
}
#endif
}
/* free all state */
void
fragment_exit()
{
/* case 7966: don't initialize at all for hotp_only & thin_client
* FIXME: could set initial sizes to 0 for all configurations, instead
*/
if (RUNNING_WITHOUT_CODE_CACHE())
goto cleanup;
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
if (TRACEDUMP_ENABLED() && DYNAMO_OPTION(shared_traces)) {
/* write out all traces prior to deleting any, so links print nicely */
uint i;
fragment_t *f;
/* change_linking_lock is required for output_trace(), though there
* won't be any races at this point of exiting.
*/
acquire_recursive_lock(&change_linking_lock);
TABLE_RWLOCK(shared_trace, read, lock);
for (i = 0; i < shared_trace->capacity; i++) {
f = shared_trace->table[i];
if (!REAL_FRAGMENT(f))
continue;
if (SHOULD_OUTPUT_FRAGMENT(f->flags))
output_trace(GLOBAL_DCONTEXT, shared_pt, f, -1);
}
TABLE_RWLOCK(shared_trace, read, unlock);
release_recursive_lock(&change_linking_lock);
exit_trace_file(shared_pt);
}
#endif
#ifdef FRAGMENT_SIZES_STUDY
DOLOG(1, (LOG_FRAGMENT|LOG_STATS), {
print_size_results();
});
#endif
fragment_reset_free();
#ifdef RETURN_AFTER_CALL
if (dynamo_options.ret_after_call && rac_non_module_table.live_table != NULL) {
DODEBUG({
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_app_pc_load_statistics(GLOBAL_DCONTEXT,
rac_non_module_table.live_table);
});
hashtable_app_pc_study(GLOBAL_DCONTEXT, rac_non_module_table.live_table,
0/*table consistent*/);
});
hashtable_app_pc_free(GLOBAL_DCONTEXT, rac_non_module_table.live_table);
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, rac_non_module_table.live_table,
app_pc_table_t, ACCT_AFTER_CALL, PROTECTED);
rac_non_module_table.live_table = NULL;
}
ASSERT(rac_non_module_table.persisted_table == NULL);
DELETE_LOCK(after_call_lock);
#endif
#if defined(RCT_IND_BRANCH) && defined(UNIX)
/* we do not free these tables in fragment_reset_free() b/c we
* would just have to build them all back up again in order to
* continue execution
*/
if ((TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
rct_global_table.live_table != NULL) {
DODEBUG({
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_app_pc_load_statistics(GLOBAL_DCONTEXT,
rct_global_table.live_table);
});
hashtable_app_pc_study(GLOBAL_DCONTEXT, rct_global_table.live_table,
0/*table consistent*/);
});
hashtable_app_pc_free(GLOBAL_DCONTEXT, rct_global_table.live_table);
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, rct_global_table.live_table, app_pc_table_t,
ACCT_AFTER_CALL, PROTECTED);
rct_global_table.live_table = NULL;
} else
ASSERT(rct_global_table.live_table == NULL);
ASSERT(rct_global_table.persisted_table == NULL);
#endif /* RCT_IND_BRANCH */
if (SHARED_FRAGMENTS_ENABLED()) {
/* tables are persistent across resets, only on heap for selfprot (case 7957) */
if (DYNAMO_OPTION(shared_bbs)) {
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_bb, fragment_table_t,
ACCT_FRAG_TABLE, PROTECTED);
shared_bb = NULL;
} else
ASSERT(shared_bb == NULL);
if (DYNAMO_OPTION(shared_traces)) {
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_trace, fragment_table_t,
ACCT_FRAG_TABLE, PROTECTED);
shared_trace = NULL;
} else
ASSERT(shared_trace == NULL);
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_future, fragment_table_t,
ACCT_FRAG_TABLE, PROTECTED);
shared_future = NULL;
}
if (SHARED_IBT_TABLES_ENABLED()) {
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, dead_lists, dead_table_lists_t,
ACCT_OTHER, PROTECTED);
dead_lists = NULL;
} else
ASSERT(dead_lists == NULL);
if (USE_SHARED_PT()) {
ASSERT(shared_pt != NULL);
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_pt, per_thread_t, ACCT_OTHER, PROTECTED);
shared_pt = NULL;
} else
ASSERT(shared_pt == NULL);
if (SHARED_IBT_TABLES_ENABLED())
DELETE_LOCK(dead_tables_lock);
#ifdef SHARING_STUDY
if (INTERNAL_OPTION(fragment_sharing_study)) {
DELETE_LOCK(shared_blocks_lock);
DELETE_LOCK(shared_traces_lock);
}
#endif
cleanup:
/* FIXME: we shouldn't need these locks anyway for hotp_only & thin_client */
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
DELETE_LOCK(tracedump_mutex);
#endif
#ifdef CLIENT_INTERFACE
process_client_flush_requests(NULL, GLOBAL_DCONTEXT, client_flush_requests,
false /* no flush */);
DELETE_LOCK(client_flush_request_lock);
#endif
DELETE_LOCK(shared_cache_flush_lock);
}
/* Decrement the ref-count for any reference to table that the
* per_thread_t contains. If could_be_live is true, will acquire write
* locks for the currently live tables. */
/* NOTE: Can't inline in release build -- too many call sites? */
static /* inline */ void
dec_table_ref_count(dcontext_t *dcontext, ibl_table_t *table, bool could_be_live)
{
ibl_table_t *live_table = NULL;
ibl_branch_type_t branch_type;
/* Search live tables. A live table's ref-count is decremented
* during a thread exit. */
/* FIXME If the table is more likely to be dead, we can reverse the order
* and search dead tables first. */
if (!DYNAMO_OPTION(ref_count_shared_ibt_tables))
return;
ASSERT(TESTALL(FRAG_TABLE_SHARED | FRAG_TABLE_IBL_TARGETED,
table->table_flags));
if (could_be_live) {
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
/* We match based on lookup table addresses. We need to lock the table
* during the compare and hold the lock during the ref-count dec to
* prevent a race with it being moved to the dead list.
*/
ibl_table_t *sh_table_ptr = TEST(FRAG_TABLE_TRACE, table->table_flags) ?
&shared_pt->trace_ibt[branch_type] : &shared_pt->bb_ibt[branch_type];
TABLE_RWLOCK(sh_table_ptr, write, lock);
if (table->table == sh_table_ptr->table) {
live_table = sh_table_ptr;
break;
}
TABLE_RWLOCK(sh_table_ptr, write, unlock);
}
}
if (live_table != NULL) {
/* During shutdown, the ref-count can reach 0. The table is freed
* in the fragment_exit() path. */
ASSERT(live_table->ref_count >= 1);
live_table->ref_count--;
TABLE_RWLOCK(live_table, write, unlock);
}
else { /* Search the dead tables list. */
dead_fragment_table_t *current = dead_lists->dead_tables;
dead_fragment_table_t *prev = NULL;
ASSERT(dead_lists->dead_tables != NULL);
ASSERT(dead_lists->dead_tables_tail != NULL);
/* We expect to be removing from the head of the list but due to
* races could be removing from the middle, i.e., if a preceding
* entry is about to be removed by another thread but the
* dead_tables_lock hasn't been acquired yet by that thread.
*/
mutex_lock(&dead_tables_lock);
for (current = dead_lists->dead_tables; current != NULL;
prev = current, current = current->next) {
if (current->table_unaligned == table->table_unaligned) {
ASSERT_CURIOSITY(current->ref_count >= 1);
current->ref_count--;
if (current->ref_count == 0) {
LOG(GLOBAL, LOG_FRAGMENT, 2,
"dec_table_ref_count: table "PFX" cap %d at ref 0, freeing\n",
current->table_unaligned, current->capacity);
/* Unlink this table from the list. */
if (prev != NULL)
prev->next = current->next;
if (current == dead_lists->dead_tables) {
/* remove from the front */
ASSERT(prev == NULL);
dead_lists->dead_tables = current->next;
}
if (current == dead_lists->dead_tables_tail)
dead_lists->dead_tables_tail = prev;
hashtable_ibl_free_table(GLOBAL_DCONTEXT,
current->table_unaligned,
current->table_flags,
current->capacity);
heap_free(GLOBAL_DCONTEXT, current, sizeof(dead_fragment_table_t)
HEAPACCT(ACCT_IBLTABLE));
STATS_DEC(num_dead_shared_ibt_tables);
STATS_INC(num_dead_shared_ibt_tables_freed);
}
break;
}
}
mutex_unlock(&dead_tables_lock);
ASSERT(current != NULL);
}
}
/* Decrement the ref-count for every shared IBT table that the
* per_thread_t has a reference to. */
static void
dec_all_table_ref_counts(dcontext_t *dcontext, per_thread_t *pt)
{
/* We can also decrement ref-count for dead shared tables here. */
if (SHARED_IBT_TABLES_ENABLED()) {
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
ASSERT(pt->trace_ibt[branch_type].table != NULL);
dec_table_ref_count(dcontext, &pt->trace_ibt[branch_type],
true/*check live*/);
}
if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
ASSERT(pt->bb_ibt[branch_type].table != NULL);
dec_table_ref_count(dcontext, &pt->bb_ibt[branch_type],
true/*check live*/);
}
}
}
}
/* re-initializes non-persistent memory */
void
fragment_thread_reset_init(dcontext_t *dcontext)
{
per_thread_t *pt;
ibl_branch_type_t branch_type;
/* case 7966: don't initialize at all for hotp_only & thin_client */
if (RUNNING_WITHOUT_CODE_CACHE())
return;
pt = (per_thread_t *) dcontext->fragment_field;
/* important to init w/ cur timestamp to avoid this thread dec-ing ref
* count when it wasn't included in ref count init value!
* assumption: don't need lock to read flushtime_global atomically.
* when resetting, though, thread free & re-init is done before global free,
* so we have to explicitly set to 0 for that case.
*/
pt->flushtime_last_update = (dynamo_resetting) ? 0 : flushtime_global;
/* set initial hashtable sizes */
hashtable_fragment_init(dcontext, &pt->bb, INIT_HTABLE_SIZE_BB,
INTERNAL_OPTION(private_bb_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0, 0 _IF_DEBUG("bblock"));
/* init routine will work for future_fragment_t* same as for fragment_t* */
hashtable_fragment_init(dcontext, &pt->future, INIT_HTABLE_SIZE_FUTURE,
INTERNAL_OPTION(private_future_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */, 0
_IF_DEBUG("future"));
/* The trace table now is not used by IBL routines, and
* therefore doesn't need a lookup table, we can also use the
* alternative hash functions and use a higher load.
*/
if (PRIVATE_TRACES_ENABLED()) {
hashtable_fragment_init(dcontext, &pt->trace, INIT_HTABLE_SIZE_TRACE,
INTERNAL_OPTION(private_trace_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
FRAG_TABLE_TRACE
_IF_DEBUG("trace"));
}
/* We'll now have more control over hashtables based on branch
* type. The most important of all is of course the return
* target table. These tables should be populated only when
* we know that the entry is a valid target, a trace is
* created, and it is indeed targeted by an IBL.
*/
/* These tables are targeted by both bb and trace routines */
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (!DYNAMO_OPTION(disable_traces)
/* If no traces and no bb ibl targets we point ibl at
* an empty trace table */
|| !DYNAMO_OPTION(bb_ibl_targets)) {
if (!DYNAMO_OPTION(shared_trace_ibt_tables)) {
hashtable_ibl_myinit(dcontext, &pt->trace_ibt[branch_type],
DYNAMO_OPTION(private_trace_ibl_targets_init),
DYNAMO_OPTION(private_ibl_targets_load),
HASH_FUNCTION_NONE,
HASHTABLE_IBL_OFFSET(branch_type),
branch_type,
false, /* no lookup table */
(DYNAMO_OPTION(shared_traces) ?
FRAG_TABLE_TARGET_SHARED : 0) |
FRAG_TABLE_TRACE
_IF_DEBUG(ibl_trace_table_type_names[branch_type]));
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
CHECK_UNPROT_STATS(pt->trace_ibt[branch_type]);
/* for compatibility using an entry in the per-branch type stats */
INIT_HASHTABLE_STATS(pt->trace_ibt[branch_type].
UNPROT_STAT(trace_ibl_stats[branch_type]));
} else {
pt->trace_ibt[branch_type].unprot_stats = NULL;
}
#endif /* HASHTABLE_STATISTICS */
}
else {
/* ensure table from last time (if we had a reset) not still there */
memset(&pt->trace_ibt[branch_type], 0, sizeof(pt->trace_ibt[branch_type]));
update_private_ptr_to_shared_ibt_table(dcontext, branch_type,
true, /* trace = yes */
false, /* no adjust old
* ref-count */
true /* lock */);
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
ALLOC_UNPROT_STATS(dcontext, &pt->trace_ibt[branch_type]);
CHECK_UNPROT_STATS(pt->trace_ibt[branch_type]);
INIT_HASHTABLE_STATS(pt->trace_ibt[branch_type].
UNPROT_STAT(trace_ibl_stats[branch_type]));
} else {
pt->trace_ibt[branch_type].unprot_stats = NULL;
}
#endif
}
}
/* When targetting BBs, currently the source is assumed to be only a
* bb since traces going to a bb for the first time should mark it
* as a trace head. Therefore the tables are currently only
* targeted by bb IBL routines. It will be possible to later
* deal with trace heads and allow a trace to target a BB with
* the intent of modifying its THCI.
*
* (FIXME: having another table for THCI IBLs seems better than
* adding a counter (starting at -1) to all blocks and
* trapping when 0 for marking a trace head and again at 50
* for creating a trace. And that is all of course after proving
* that doing it in DR has significant impact.)
*
* Note that private bb2bb transitions are not captured when
* we run with -shared_bbs.
*/
/* These tables should be populated only when we know that the
* entry is a valid target, and it is indeed targeted by an
* IBL. They have to be per-type so that our security
* policies are properly checked.
*/
if (DYNAMO_OPTION(bb_ibl_targets)) {
if (!DYNAMO_OPTION(shared_bb_ibt_tables)) {
hashtable_ibl_myinit(dcontext, &pt->bb_ibt[branch_type],
DYNAMO_OPTION(private_bb_ibl_targets_init),
DYNAMO_OPTION(private_bb_ibl_targets_load),
HASH_FUNCTION_NONE,
HASHTABLE_IBL_OFFSET(branch_type),
branch_type,
false, /* no lookup table */
(DYNAMO_OPTION(shared_bbs) ?
FRAG_TABLE_TARGET_SHARED : 0)
_IF_DEBUG(ibl_bb_table_type_names[branch_type]));
/* mark as inclusive table for bb's - we in fact currently
* keep only frags that are not FRAG_IS_TRACE_HEAD */
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
/* for compatibility using an entry in the per-branch type stats */
CHECK_UNPROT_STATS(pt->bb_ibt[branch_type]);
/* FIXME: we don't expect trace_ibl_stats yet */
INIT_HASHTABLE_STATS(pt->bb_ibt[branch_type].
UNPROT_STAT(bb_ibl_stats[branch_type]));
} else {
pt->bb_ibt[branch_type].unprot_stats = NULL;
}
#endif /* HASHTABLE_STATISTICS */
}
else {
/* ensure table from last time (if we had a reset) not still there */
memset(&pt->bb_ibt[branch_type], 0, sizeof(pt->bb_ibt[branch_type]));
update_private_ptr_to_shared_ibt_table(dcontext, branch_type,
false, /* trace = no */
false, /* no adjust old
* ref-count */
true /* lock */);
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
ALLOC_UNPROT_STATS(dcontext, &pt->bb_ibt[branch_type]);
CHECK_UNPROT_STATS(pt->bb_ibt[branch_type]);
INIT_HASHTABLE_STATS(pt->bb_ibt[branch_type].
UNPROT_STAT(trace_ibl_stats[branch_type]));
} else {
pt->bb_ibt[branch_type].unprot_stats = NULL;
}
#endif
}
}
}
ASSERT(IBL_BRANCH_TYPE_END == 3);
update_generated_hashtable_access(dcontext);
}
void
fragment_thread_init(dcontext_t *dcontext)
{
/* we allocate per_thread_t in the global heap solely for self-protection,
* even when turned off, since even with a lot of threads this isn't a lot of
* pressure on the global heap
*/
per_thread_t *pt;
/* case 7966: don't initialize un-needed data for hotp_only & thin_client.
* FIXME: could set htable initial sizes to 0 for all configurations, instead.
* per_thread_t is pretty big, so we avoid it, though it costs us checks for
* hotp_only in the islinking-related routines.
*/
if (RUNNING_WITHOUT_CODE_CACHE())
return;
pt = (per_thread_t *) global_heap_alloc(sizeof(per_thread_t) HEAPACCT(ACCT_OTHER));
dcontext->fragment_field = (void *) pt;
fragment_thread_reset_init(dcontext);
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
if (TRACEDUMP_ENABLED() && PRIVATE_TRACES_ENABLED()) {
pt->tracefile = open_log_file("traces", NULL, 0);
ASSERT(pt->tracefile != INVALID_FILE);
init_trace_file(pt);
}
#endif
#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
ASSIGN_INIT_LOCK_FREE(pt->fragment_delete_mutex, fragment_delete_mutex);
#endif
pt->could_be_linking = false;
pt->wait_for_unlink = false;
pt->about_to_exit = false;
pt->flush_queue_nonempty = false;
pt->waiting_for_unlink = create_event();
pt->finished_with_unlink = create_event();
ASSIGN_INIT_LOCK_FREE(pt->linking_lock, linking_lock);
pt->finished_all_unlink = create_event();
pt->soon_to_be_linking = false;
pt->at_syscall_at_flush = false;
#ifdef PROFILE_LINKCOUNT
pt->tracedump_num_below_threshold = 0;
pt->tracedump_count_below_threshold = (linkcount_type_t) 0;
#endif
}
static bool
check_flush_queue(dcontext_t *dcontext, fragment_t *was_I_flushed);
/* frees all non-persistent memory */
void
fragment_thread_reset_free(dcontext_t *dcontext)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
DEBUG_DECLARE(ibl_branch_type_t branch_type;)
/* case 7966: don't initialize at all for hotp_only & thin_client */
if (RUNNING_WITHOUT_CODE_CACHE())
return;
/* Dec ref count on any shared tables that are pointed to. */
dec_all_table_ref_counts(dcontext, pt);
#ifdef DEBUG
/* for non-debug we do fast exit path and don't free local heap */
SELF_PROTECT_CACHE(dcontext, NULL, WRITABLE);
/* we remove flushed fragments from the htable, and they can be
* flushed after enter_threadexit() due to os_thread_stack_exit(),
* so we need to check the flush queue here
*/
mutex_lock(&pt->linking_lock);
check_flush_queue(dcontext, NULL);
mutex_unlock(&pt->linking_lock);
/* For consistency we remove entries from the IBL targets
* tables before we remove them from the trace table. However,
* we cannot free any fragments because for sure all of them will
* be present in the trace table.
*/
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (!DYNAMO_OPTION(disable_traces)
/* If no traces and no bb ibl targets we point ibl at
* an empty trace table */
|| !DYNAMO_OPTION(bb_ibl_targets)) {
if (!DYNAMO_OPTION(shared_trace_ibt_tables)) {
DOLOG(2, LOG_FRAGMENT, {
hashtable_ibl_dump_table(dcontext, &pt->trace_ibt[branch_type]);
});
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_ibl_load_statistics(dcontext, &pt->trace_ibt[branch_type]);
});
hashtable_ibl_myfree(dcontext, &pt->trace_ibt[branch_type]);
} else {
# ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
print_hashtable_stats(dcontext, "Total",
shared_pt->trace_ibt[branch_type].name,
"trace ibl ",
get_branch_type_name(branch_type),
&pt->trace_ibt[branch_type].
UNPROT_STAT(trace_ibl_stats[branch_type]));
DEALLOC_UNPROT_STATS(dcontext, &pt->trace_ibt[branch_type]);
}
# endif
memset(&pt->trace_ibt[branch_type], 0,
sizeof(pt->trace_ibt[branch_type]));
}
}
if (DYNAMO_OPTION(bb_ibl_targets)) {
if (!DYNAMO_OPTION(shared_bb_ibt_tables)) {
DOLOG(2, LOG_FRAGMENT, {
hashtable_ibl_dump_table(dcontext, &pt->bb_ibt[branch_type]);
});
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_ibl_load_statistics(dcontext, &pt->bb_ibt[branch_type]);
});
hashtable_ibl_myfree(dcontext, &pt->bb_ibt[branch_type]);
} else {
# ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_stats)) {
print_hashtable_stats(dcontext, "Total",
shared_pt->bb_ibt[branch_type].name,
"bb ibl ",
get_branch_type_name(branch_type),
&pt->bb_ibt[branch_type].
UNPROT_STAT(bb_ibl_stats[branch_type]));
DEALLOC_UNPROT_STATS(dcontext, &pt->bb_ibt[branch_type]);
}
# endif
memset(&pt->bb_ibt[branch_type], 0, sizeof(pt->bb_ibt[branch_type]));
}
}
}
/* case 7653: we can't free the main tables prior to freeing the contents
* of all of them, as link freeing involves looking up in the other tables.
*/
if (PRIVATE_TRACES_ENABLED()) {
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_fragment_load_statistics(dcontext, &pt->trace);
});
hashtable_fragment_reset(dcontext, &pt->trace);
}
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_fragment_load_statistics(dcontext, &pt->bb);
});
hashtable_fragment_reset(dcontext, &pt->bb);
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_fragment_load_statistics(dcontext, &pt->future);
});
hashtable_fragment_reset(dcontext, &pt->future);
if (PRIVATE_TRACES_ENABLED())
hashtable_fragment_free(dcontext, &pt->trace);
hashtable_fragment_free(dcontext, &pt->bb);
hashtable_fragment_free(dcontext, &pt->future);
SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
#else
/* Case 10807: Clients need to be informed of fragment deletions
* so we'll reset the relevant hash tables for CI release builds.
*/
# ifdef CLIENT_INTERFACE
if (PRIVATE_TRACES_ENABLED())
hashtable_fragment_reset(dcontext, &pt->trace);
hashtable_fragment_reset(dcontext, &pt->bb);
# endif
#endif /* !DEBUG */
}
/* atexit cleanup */
void
fragment_thread_exit(dcontext_t *dcontext)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
/* case 7966: don't initialize at all for hotp_only & thin_client */
if (RUNNING_WITHOUT_CODE_CACHE())
return;
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
if (TRACEDUMP_ENABLED() && PRIVATE_TRACES_ENABLED()) {
/* write out all traces prior to deleting any, so links print nicely */
uint i;
fragment_t *f;
for (i = 0; i < pt->trace.capacity; i++) {
f = pt->trace.table[i];
if (!REAL_FRAGMENT(f))
continue;
if (SHOULD_OUTPUT_FRAGMENT(f->flags))
output_trace(dcontext, pt, f, -1);
}
exit_trace_file(pt);
}
#endif
fragment_thread_reset_free(dcontext);
/* events are global */
destroy_event(pt->waiting_for_unlink);
destroy_event(pt->finished_with_unlink);
destroy_event(pt->finished_all_unlink);
DELETE_LOCK(pt->linking_lock);
#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
DELETE_LOCK(pt->fragment_delete_mutex);
#endif
global_heap_free(pt, sizeof(per_thread_t) HEAPACCT(ACCT_OTHER));
dcontext->fragment_field = NULL;
}
#ifdef UNIX
void
fragment_fork_init(dcontext_t *dcontext)
{
/* FIXME: what about global file? */
# if defined(INTERNAL) || defined(CLIENT_INTERFACE)
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
if (TRACEDUMP_ENABLED() && PRIVATE_TRACES_ENABLED()) {
/* new log dir has already been created, so just open a new log file */
pt->tracefile = open_log_file("traces", NULL, 0);
ASSERT(pt->tracefile != INVALID_FILE);
init_trace_file(pt);
}
# endif
}
#endif
/* fragment_t heap layout looks like this:
*
* fragment_t/trace_t
* translation_info_t*, if necessary
* array composed of different sizes of linkstub_t subclasses:
* direct_linkstub_t
* cbr_fallthrough_linkstub_t
* indirect_linkstub_t
* post_linkstub_t, if necessary
*/
static uint
fragment_heap_size(uint flags, int direct_exits, int indirect_exits)
{
uint total_sz;
ASSERT((direct_exits + indirect_exits > 0) || TEST(FRAG_COARSE_GRAIN, flags));
total_sz = FRAGMENT_STRUCT_SIZE(flags) +
linkstubs_heap_size(flags, direct_exits, indirect_exits);
/* we rely on a small heap size for our ushort offset at the end */
ASSERT(total_sz <= USHRT_MAX);
return total_sz;
}
/* Allocates memory for a fragment_t and linkstubs and initializes them, but
* does not do any fcache-related initialization.
*/
static fragment_t *
fragment_create_heap(dcontext_t *dcontext,
int direct_exits, int indirect_exits, uint flags)
{
dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, flags);
uint heapsz = fragment_heap_size(flags, direct_exits, indirect_exits);
/* linkstubs are in an array immediately after the fragment_t/trace_t struct */
fragment_t *f = (fragment_t *)
nonpersistent_heap_alloc(alloc_dc, heapsz
HEAPACCT(TEST(FRAG_IS_TRACE, flags) ?
ACCT_TRACE : ACCT_FRAGMENT));
LOG(THREAD, LOG_FRAGMENT, 5,
"fragment heap size for flags 0x%08x, exits %d %d, is %d => "PFX"\n",
flags, direct_exits, indirect_exits, heapsz, f);
return f;
}
static void
fragment_init_heap(fragment_t *f, app_pc tag, int direct_exits, int indirect_exits,
uint flags)
{
ASSERT(f != NULL);
f->flags = flags; /* MUST set before calling fcache_add_fragment or
* FRAGMENT_EXIT_STUBS */
f->tag = tag;
/* Let fragment_create() fill in; other users are building fake fragments */
DODEBUG({ f->id = -1; });
f->next_vmarea = NULL; /* must be set by caller */
f->prev_vmarea = NULL; /* must be set by caller */
f->also.also_vmarea = NULL; /* must be set by caller */
linkstubs_init(FRAGMENT_EXIT_STUBS(f), direct_exits, indirect_exits, f);
/* initialize non-ibt entry to top of fragment (caller responsible for
* setting up prefix)
*/
f->prefix_size = 0;
#ifdef FRAGMENT_SIZES_STUDY
record_fragment_size(f->size, (flags & FRAG_IS_TRACE) != 0);
#endif
f->in_xlate.incoming_stubs = NULL;
#ifdef CUSTOM_TRACES_RET_REMOVAL
f->num_calls = 0;
f->num_rets = 0;
#endif
/* trace-only fields */
if (TEST(FRAG_IS_TRACE, flags)) {
trace_only_t *t = TRACE_FIELDS(f);
t->bbs = NULL;
/* real num_bbs won't be set until after the trace is emitted,
* but we need a non-zero value for linkstub_fragment()
*/
t->num_bbs = 1;
#ifdef PROFILE_RDTSC
t->count = 0UL;
t->total_time = (uint64) 0;
#endif
#ifdef SIDELINE_COUNT_STUDY
t->count_old_pre = (linkcount_type_t) 0;
t->count_old_post = (linkcount_type_t) 0;
#endif
}
}
/* Create a new fragment_t with empty prefix and return it.
* The fragment_t is allocated on the global or local heap, depending on the flags,
* unless FRAG_COARSE_GRAIN is set, in which case the fragment_t is a unique
* temporary struct that is NOT heap allocated and is only safe to use
* so long as the bb_building_lock is held!
*/
fragment_t *
fragment_create(dcontext_t *dcontext, app_pc tag, int body_size,
int direct_exits, int indirect_exits, int exits_size, uint flags)
{
fragment_t *f;
DEBUG_DECLARE(stats_int_t next_id;)
DOSTATS({
/* should watch this stat and if it gets too high need to re-do
* who needs the post-linkstub offset
*/
if (linkstub_frag_offs_at_end(flags, direct_exits, indirect_exits))
STATS_INC(num_fragment_post_linkstub);
});
/* ensure no races during a reset */
ASSERT(!dynamo_resetting);
if (TEST(FRAG_COARSE_GRAIN, flags)) {
ASSERT(DYNAMO_OPTION(coarse_units));
ASSERT_OWN_MUTEX(USE_BB_BUILDING_LOCK(), &bb_building_lock);
ASSERT(!TEST(FRAG_IS_TRACE, flags));
ASSERT(TEST(FRAG_SHARED, flags));
ASSERT(fragment_prefix_size(flags) == 0);
ASSERT((direct_exits == 0 && indirect_exits == 1) ||
(indirect_exits == 0 && (direct_exits == 1 || direct_exits == 2)));
/* FIXME: eliminate this temp fragment and linkstubs and
* have custom emit and link code that does not require such data
* structures? It would certainly be faster code.
* But would still want to record each exit's target in a convenient
* data structure, for later linking, unless we try to link in
* the same pass in which we emit indirect stubs.
* We could also use fragment_create() and free the resulting struct
* somewhere and switch to a wrapper at that point.
*/
memset(&coarse_emit_fragment, 0, sizeof(coarse_emit_fragment));
f = (fragment_t *) &coarse_emit_fragment;
/* We do not mark as FRAG_FAKE since this is pretty much a real
* fragment_t, and we do want to walk its linkstub_t structs, which
* are present.
*/
} else {
f = fragment_create_heap(dcontext, direct_exits, indirect_exits, flags);
}
fragment_init_heap(f, tag, direct_exits, indirect_exits, flags);
/* To make debugging easier we assign coarse-grain ids in the same namespace
* as fine-grain fragments, though we won't remember them at all.
*/
STATS_INC_ASSIGN(num_fragments, next_id);
IF_X64(ASSERT_TRUNCATE(f->id, int, next_id));
DOSTATS({ f->id = (int) next_id; });
DO_GLOBAL_STATS({
if (!TEST(FRAG_IS_TRACE, f->flags)) {
RSTATS_INC(num_bbs);
IF_X64(if (FRAG_IS_32(f->flags)) STATS_INC(num_32bit_bbs);)
}
});
DOSTATS({
/* avoid double-counting for adaptive working set */
if (!fragment_lookup_deleted(dcontext, tag) && !TEST(FRAG_COARSE_GRAIN, flags))
STATS_INC(num_unique_fragments);
});
/* FIXME: make fragment count a release-build stat so we can do this in
* release builds
*/
DOSTATS({
if (stats != NULL &&
(uint) GLOBAL_STAT(num_fragments) == INTERNAL_OPTION(reset_at_fragment_count)) {
schedule_reset(RESET_ALL);
}
});
/* size is a ushort
* our offsets are ushorts as well: they assume body_size is small enough, not size
*/
#ifdef CLIENT_INTERFACE
if (body_size + exits_size + fragment_prefix_size(flags) > MAX_FRAGMENT_SIZE) {
FATAL_USAGE_ERROR(INSTRUMENTATION_TOO_LARGE, 2,
get_application_name(), get_application_pid());
}
#endif
ASSERT(body_size + exits_size + fragment_prefix_size(flags) <= MAX_FRAGMENT_SIZE);
/* currently MAX_FRAGMENT_SIZE is USHRT_MAX, but future proofing */
ASSERT_TRUNCATE(f->size, ushort,
(body_size + exits_size + fragment_prefix_size(flags)));
f->size = (ushort) (body_size + exits_size + fragment_prefix_size(flags));
/* fcache_add will fill in start_pc, next_fcache,
* prev_fcache, and fcache_extra
*/
fcache_add_fragment(dcontext, f);
/* after fcache_add_fragment so we can call get_fragment_coarse_info */
DOSTATS({
if (TEST(FRAG_SHARED, flags)) {
STATS_INC(num_shared_fragments);
if (TEST(FRAG_IS_TRACE, flags))
STATS_INC(num_shared_traces);
else if (TEST(FRAG_COARSE_GRAIN, flags)) {
coarse_info_t *info = get_fragment_coarse_info(f);
if (get_executable_area_coarse_info(f->tag) != info)
STATS_INC(num_coarse_secondary);
STATS_INC(num_coarse_fragments);
} else
STATS_INC(num_shared_bbs);
} else {
STATS_INC(num_private_fragments);
if (TEST(FRAG_IS_TRACE, flags))
STATS_INC(num_private_traces);
else
STATS_INC(num_private_bbs);
}
});
/* wait until initialized fragment completely before dumping any stats */
DOLOG(1, LOG_FRAGMENT|LOG_VMAREAS, {
if (INTERNAL_OPTION(global_stats_interval) &&
(f->id % INTERNAL_OPTION(global_stats_interval) == 0)) {
LOG(GLOBAL, LOG_FRAGMENT, 1, "Created %d fragments\n", f->id);
dump_global_stats(false);
}
if (INTERNAL_OPTION(thread_stats_interval) &&
INTERNAL_OPTION(thread_stats)) {
/* FIXME: why do we need a new dcontext? */
dcontext_t *dcontext = get_thread_private_dcontext();
if (THREAD_STATS_ON(dcontext) &&
THREAD_STAT(dcontext, num_fragments) % INTERNAL_OPTION(thread_stats_interval) == 0) {
dump_thread_stats(dcontext, false);
}
}
});
#ifdef WINDOWS
DOLOG(1, LOG_FRAGMENT|LOG_VMAREAS, {
if (f->id % 50000 == 0) {
LOG(GLOBAL, LOG_VMAREAS, 1,
"50K fragment check point: here are the loaded modules:\n");
print_modules(GLOBAL, DUMP_NOT_XML);
LOG(GLOBAL, LOG_VMAREAS, 1,
"50K fragment check point: here are the executable areas:\n");
print_executable_areas(GLOBAL);
}
});
#endif
return f;
}
/* Creates a new fragment_t+linkstubs from the passed-in fragment and
* fills in linkstub_t and fragment_t fields, copying the fcache-related fields
* from the passed-in fragment (so be careful how the fields are used).
* Meant to be used to create a full fragment from a coarse-grain fragment.
* Caller is responsible for freeing via fragment_free() w/ the same dcontext
* passed in here.
*/
fragment_t *
fragment_recreate_with_linkstubs(dcontext_t *dcontext, fragment_t *f_src)
{
uint num_dir, num_indir;
uint size;
fragment_t *f_tgt;
instrlist_t *ilist;
linkstub_t *l;
cache_pc body_end_pc;
/* Not FAKE since has linkstubs, but still fake in a sense since no fcache
* slot -- need to mark that?
*/
uint flags = (f_src->flags & ~FRAG_FAKE);
ASSERT_CURIOSITY(TEST(FRAG_COARSE_GRAIN, f_src->flags)); /* only use so far */
/* FIXME case 9325: build from tag here? Need to exactly re-mangle + re-instrument.
* We use _exact to get any elided final jmp not counted in size
*/
ilist = decode_fragment_exact(dcontext, f_src, NULL, NULL, f_src->flags,
&num_dir, &num_indir);
f_tgt = fragment_create_heap(dcontext, num_dir, num_indir, flags);
fragment_init_heap(f_tgt, f_src->tag, num_dir, num_indir, flags);
f_tgt->start_pc = f_src->start_pc;
/* Can't call this until we have start_pc set */
body_end_pc = set_linkstub_fields(dcontext, f_tgt, ilist, num_dir, num_indir,
false/*do not emit*/);
/* Calculate total size */
IF_X64(ASSERT_TRUNCATE(size, uint, (body_end_pc - f_tgt->start_pc)));
size = (uint) (body_end_pc - f_tgt->start_pc);
for (l = FRAGMENT_EXIT_STUBS(f_tgt); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
if (!EXIT_HAS_LOCAL_STUB(l->flags, f_tgt->flags))
continue; /* it's kept elsewhere */
size += linkstub_size(dcontext, f_tgt, l);
#ifdef CUSTOM_EXIT_STUBS
size += l->fixed_stub_offset;
#endif
}
ASSERT_TRUNCATE(f_tgt->size, ushort, size);
f_tgt->size = (ushort) size;
ASSERT(TEST(FRAG_FAKE, f_src->flags) || size == f_src->size);
ASSERT_TRUNCATE(f_tgt->prefix_size, byte, fragment_prefix_size(f_src->flags));
f_tgt->prefix_size = (byte) fragment_prefix_size(f_src->flags);
ASSERT(TEST(FRAG_FAKE, f_src->flags) || f_src->prefix_size == f_tgt->prefix_size);
f_tgt->fcache_extra = f_src->fcache_extra;
instrlist_clear_and_destroy(dcontext, ilist);
return f_tgt;
}
/* Frees the storage associated with f.
* Callers should use fragment_delete() instead of this routine, unless they
* obtained their fragment_t from fragment_recreate_with_linkstubs().
*/
void
fragment_free(dcontext_t *dcontext, fragment_t *f)
{
dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, f->flags);
uint heapsz;
int direct_exits = 0;
int indirect_exits = 0;
linkstub_t *l = FRAGMENT_EXIT_STUBS(f);
for (; l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
if (LINKSTUB_DIRECT(l->flags))
direct_exits++;
else {
ASSERT(LINKSTUB_INDIRECT(l->flags));
indirect_exits++;
}
}
heapsz = fragment_heap_size(f->flags, direct_exits, indirect_exits);
STATS_INC(num_fragments_deleted);
if (HAS_STORED_TRANSLATION_INFO(f)) {
ASSERT(FRAGMENT_TRANSLATION_INFO(f) != NULL);
translation_info_free(dcontext, FRAGMENT_TRANSLATION_INFO(f));
} else
ASSERT(FRAGMENT_TRANSLATION_INFO(f) == NULL);
/* N.B.: monitor_remove_fragment() was called in fragment_delete,
* which is assumed to have been called prior to fragment_free
*/
linkstub_free_exitstubs(dcontext, f);
if ((f->flags & FRAG_IS_TRACE) != 0) {
trace_only_t *t = TRACE_FIELDS(f);
if (t->bbs != NULL) {
nonpersistent_heap_free(alloc_dc, t->bbs, t->num_bbs*sizeof(trace_bb_info_t)
HEAPACCT(ACCT_TRACE));
}
nonpersistent_heap_free(alloc_dc, f, heapsz HEAPACCT(ACCT_TRACE));
}
else {
nonpersistent_heap_free(alloc_dc, f, heapsz HEAPACCT(ACCT_FRAGMENT));
}
}
/* Returns the end of the fragment body + any local stubs (excluding selfmod copy) */
cache_pc
fragment_stubs_end_pc(fragment_t *f)
{
if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags))
return FRAGMENT_SELFMOD_COPY_PC(f);
else
return f->start_pc + f->size;
}
/* Returns the end of the fragment body (excluding exit stubs and selfmod copy) */
cache_pc
fragment_body_end_pc(dcontext_t *dcontext, fragment_t *f)
{
linkstub_t *l;
for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
if (EXIT_HAS_LOCAL_STUB(l->flags, f->flags)) {
return EXIT_STUB_PC(dcontext, f, l);
}
}
/* must be no stubs after fragment body */
return fragment_stubs_end_pc(f);
}
#ifdef PROFILE_LINKCOUNT
linkcount_type_t
get_total_linkcount(fragment_t *f)
{
/* return total count of exit counters */
linkstub_t *l;
linkcount_type_t total = (linkcount_type_t) 0;
for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
total += l->count;
}
return total;
}
#endif
#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
/* synchronization routines needed for sideline threads so they don't get
* fragments they are referencing deleted */
void
fragment_get_fragment_delete_mutex(dcontext_t *dcontext)
{
if (dynamo_exited || dcontext == GLOBAL_DCONTEXT)
return;
mutex_lock(&(((per_thread_t *) dcontext->fragment_field)->fragment_delete_mutex));
}
void
fragment_release_fragment_delete_mutex(dcontext_t *dcontext)
{
if (dynamo_exited || dcontext == GLOBAL_DCONTEXT)
return;
mutex_unlock(&(((per_thread_t *) dcontext->fragment_field)->fragment_delete_mutex));
}
#endif
/* cleaner to have own flags since there are no negative versions
* of FRAG_SHARED and FRAG_IS_TRACE for distinguishing from "don't care"
*/
enum {
LOOKUP_TRACE = 0x001,
LOOKUP_BB = 0x002,
LOOKUP_PRIVATE = 0x004,
LOOKUP_SHARED = 0x008,
};
/* A lookup constrained by bb/trace and/or shared/private */
static inline fragment_t *
fragment_lookup_type(dcontext_t *dcontext, app_pc tag, uint lookup_flags)
{
fragment_t *f;
LOG(THREAD, LOG_MONITOR, 6, "fragment_lookup_type "PFX" 0x%x\n",
tag, lookup_flags);
if (dcontext != GLOBAL_DCONTEXT && TEST(LOOKUP_PRIVATE, lookup_flags)) {
/* FIXME: add a hashtablex.h wrapper that checks #entries and
* grabs lock for us for all lookups?
*/
/* look at private tables */
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
/* case 147: traces take precedence over bbs */
if (PRIVATE_TRACES_ENABLED() && TEST(LOOKUP_TRACE, lookup_flags)) {
/* now try trace table */
f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, &pt->trace);
if (f->tag != NULL) {
ASSERT(f->tag == tag);
DOLOG(2, LOG_FRAGMENT, {
if (DYNAMO_OPTION(shared_traces)) {
/* ensure private trace never shadows shared trace */
fragment_t *sf;
read_lock(&shared_trace->rwlock);
sf = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag,
shared_trace);
read_unlock(&shared_trace->rwlock);
ASSERT(sf->tag == NULL);
}
});
ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
return f;
}
}
if (TEST(LOOKUP_BB, lookup_flags) && pt->bb.entries > 0) {
/* basic block table last */
f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, &pt->bb);
if (f->tag != NULL) {
ASSERT(f->tag == tag);
DOLOG(2, LOG_FRAGMENT, {
if (DYNAMO_OPTION(shared_bbs)) {
/* ensure private bb never shadows shared bb, except for
* temp privates for trace building
*/
fragment_t *sf;
read_lock(&shared_bb->rwlock);
sf = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag,
shared_bb);
read_unlock(&shared_bb->rwlock);
ASSERT(sf->tag == NULL || TEST(FRAG_TEMP_PRIVATE, f->flags));
}
});
ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
return f;
}
}
}
if (TEST(LOOKUP_SHARED, lookup_flags)) {
if (DYNAMO_OPTION(shared_traces) && TEST(LOOKUP_TRACE, lookup_flags)) {
/* MUST look at shared trace table before shared bb table,
* since a shared trace can shadow a shared trace head
*/
read_lock(&shared_trace->rwlock);
f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, shared_trace);
read_unlock(&shared_trace->rwlock);
if (f->tag != NULL) {
ASSERT(f->tag == tag);
ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
return f;
}
}
if (DYNAMO_OPTION(shared_bbs) && TEST(LOOKUP_BB, lookup_flags)) {
/* MUST look at private trace table before shared bb table,
* since a private trace can shadow a shared trace head
*/
read_lock(&shared_bb->rwlock);
f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, shared_bb);
read_unlock(&shared_bb->rwlock);
if (f->tag != NULL) {
ASSERT(f->tag == tag);
ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
return f;
}
}
}
return NULL;
}
/* lookup a fragment tag */
fragment_t *
fragment_lookup(dcontext_t *dcontext, app_pc tag)
{
return fragment_lookup_type(dcontext, tag,
LOOKUP_TRACE|LOOKUP_BB|LOOKUP_PRIVATE|LOOKUP_SHARED);
}
/* lookup a fragment tag, but only look in trace tables
* N.B.: because of shadowing this may not return what fragment_lookup() returns!
*/
fragment_t *
fragment_lookup_trace(dcontext_t *dcontext, app_pc tag)
{
return fragment_lookup_type(dcontext, tag, LOOKUP_TRACE|LOOKUP_PRIVATE|LOOKUP_SHARED);
}
/* lookup a fragment tag, but only look in bb tables
* N.B.: because of shadowing this may not return what fragment_lookup() returns!
*/
fragment_t *
fragment_lookup_bb(dcontext_t *dcontext, app_pc tag)
{
return fragment_lookup_type(dcontext, tag, LOOKUP_BB|LOOKUP_PRIVATE|LOOKUP_SHARED);
}
/* lookup a fragment tag, but only look in shared bb table
* N.B.: because of shadowing this may not return what fragment_lookup() returns!
*/
fragment_t *
fragment_lookup_shared_bb(dcontext_t *dcontext, app_pc tag)
{
return fragment_lookup_type(dcontext, tag, LOOKUP_BB|LOOKUP_SHARED);
}
/* lookup a fragment tag, but only look in tables that are the same shared-ness
* as flags.
* N.B.: because of shadowing this may not return what fragment_lookup() returns!
*/
fragment_t *
fragment_lookup_same_sharing(dcontext_t *dcontext, app_pc tag, uint flags)
{
return fragment_lookup_type(dcontext, tag, LOOKUP_TRACE|LOOKUP_BB|
(TEST(FRAG_SHARED, flags) ?
LOOKUP_SHARED : LOOKUP_PRIVATE));
}
#ifdef DEBUG /*currently only used for debugging */
static fragment_t *
hashtable_pclookup(dcontext_t *dcontext, fragment_table_t *table, cache_pc pc)
{
uint i;
fragment_t *f;
ASSERT_TABLE_SYNCHRONIZED(table, READWRITE); /* lookup requires read (or write) lock */
for (i = 0; i < table->capacity; i++) {
f = table->table[i];
if (!REAL_FRAGMENT(f))
continue;
if (pc >= f->start_pc && pc < (f->start_pc + f->size)) {
return f;
}
}
return NULL;
}
/* lookup a fragment pc in the fcache by walking all hashtables.
* we have more efficient methods (fcache_fragment_pclookup) so this is only
* used for debugging.
*/
fragment_t *
fragment_pclookup_by_htable(dcontext_t *dcontext, cache_pc pc, fragment_t *wrapper)
{
/* if every fragment is guaranteed to end in 1+ stubs (which
* is not true for DYNAMO_OPTION(separate_private_stubs) we can
* simply decode forward until we hit the stub and recover
* the linkstub_t* from there -- much more efficient than walking
* all the hashtables, plus nicely handles invisible & removed frags!
* FIXME: measure perf hit of pclookup, implement this decode strategy.
* also we can miss invisible or removed fragments (case 122) so we
* may want this regardless of performance -- see also FIXME below.
*/
fragment_t *f;
per_thread_t *pt = NULL;
if (dcontext != GLOBAL_DCONTEXT) {
pt = (per_thread_t *) dcontext->fragment_field;
/* look at private traces first */
if (PRIVATE_TRACES_ENABLED()) {
f = hashtable_pclookup(dcontext, &pt->trace, pc);
if (f != NULL)
return f;
}
}
if (DYNAMO_OPTION(shared_traces)) {
/* then shared traces */
read_lock(&shared_trace->rwlock);
f = hashtable_pclookup(dcontext, shared_trace, pc);
read_unlock(&shared_trace->rwlock);
if (f != NULL)
return f;
}
if (DYNAMO_OPTION(shared_bbs)) {
/* then shared basic blocks */
read_lock(&shared_bb->rwlock);
f = hashtable_pclookup(dcontext, shared_bb, pc);
read_unlock(&shared_bb->rwlock);
if (f != NULL)
return f;
}
if (dcontext != GLOBAL_DCONTEXT) {
/* now private basic blocks */
f = hashtable_pclookup(dcontext, &pt->bb, pc);
if (f != NULL)
return f;
}
if (DYNAMO_OPTION(coarse_units)) {
coarse_info_t *info = get_executable_area_coarse_info(pc);
while (info != NULL) { /* loop over primary and secondary unit */
cache_pc body;
app_pc tag = fragment_coarse_pclookup(dcontext, info, pc, &body);
if (tag != NULL) {
ASSERT(wrapper != NULL);
fragment_coarse_wrapper(wrapper, tag, body);
return wrapper;
}
ASSERT(info->frozen || info->non_frozen == NULL);
info = info->non_frozen;
ASSERT(info == NULL || !info->frozen);
}
}
/* FIXME: shared fragment may have been removed from hashtable but
* still be in cache, and e.g. handle_modified_code still needs to know about it --
* should walk deletion vector
*/
return NULL;
}
#endif /* DEBUG */
/* lookup a fragment pc in the fcache */
fragment_t *
fragment_pclookup(dcontext_t *dcontext, cache_pc pc, fragment_t *wrapper)
{
/* Rather than walk every single hashtable, including the invisible table,
* and the pending-deletion list (case 3567), we find the fcache unit
* and walk it.
* An even more efficient alternative would be to decode backward, but
* that's not doable in general.
*
* If every fragment is guaranteed to end in 1+ stubs (which
* is not true for DYNAMO_OPTION(separate_{private,shared}_stubs) we can
* simply decode forward until we hit the stub and recover
* the linkstub_t* from there.
* Or we can decode until we hit a jmp and if it's to a linked fragment_t,
* search its incoming list.
* Stub decoding is complicated by CLIENT_INTERFACE custom stubs and by
* PROFILE_LINKCOUNT stub variations.
*/
return fcache_fragment_pclookup(dcontext, pc, wrapper);
}
/* Performs a pclookup and if the result is a coarse-grain fragment, allocates
* a new fragment_t+linkstubs.
* Returns in alloc whether the returned fragment_t was allocated and needs to be
* freed by the caller via fragment_free().
* If no result is found, alloc is set to false.
* FIXME: use FRAG_RECREATED flag to indicate allocated instead?
*/
fragment_t *
fragment_pclookup_with_linkstubs(dcontext_t *dcontext, cache_pc pc,
/*OUT*/bool *alloc)
{
fragment_t wrapper;
fragment_t *f = fragment_pclookup(dcontext, pc, &wrapper);
ASSERT(alloc != NULL);
if (f != NULL && TEST(FRAG_COARSE_GRAIN, f->flags)) {
ASSERT(f == &wrapper);
f = fragment_recreate_with_linkstubs(dcontext, f);
*alloc = true;
} else
*alloc = false;
return f;
}
/* add f to the ftable */
void
fragment_add(dcontext_t *dcontext, fragment_t *f)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
fragment_table_t *table = GET_FTABLE(pt, f->flags);
bool resized;
/* no future frags! */
ASSERT(!TEST(FRAG_IS_FUTURE, f->flags));
DOCHECK(1, {
fragment_t *existing = fragment_lookup(dcontext, f->tag);
ASSERT(existing == NULL ||
IF_CUSTOM_TRACES(/* we create and persist shadowed trace heads */
(TEST(FRAG_IS_TRACE_HEAD, f->flags) ||
TEST(FRAG_IS_TRACE_HEAD, existing->flags)) ||)
/* private trace or temp can shadow shared bb */
(TESTANY(FRAG_IS_TRACE | FRAG_TEMP_PRIVATE, f->flags) &&
TEST(FRAG_SHARED, f->flags) != TEST(FRAG_SHARED, existing->flags)) ||
/* shared trace can shadow shared trace head, even with
* -remove_shared_trace_heads */
(TESTALL(FRAG_IS_TRACE | FRAG_SHARED, f->flags) &&
!TEST(FRAG_IS_TRACE, existing->flags) &&
TESTALL(FRAG_SHARED | FRAG_IS_TRACE_HEAD, existing->flags)));
});
/* We'd like the shared fragment table synch to be independent of the
* bb building synch (which may become more fine-grained in the future),
* so an add needs to hold the write lock to prevent conflicts with
* other adds.
* We may be able to have a scheme where study() and remove() are writers
* but add() is a reader -- but that's confusing and prone to errors in
* the future.
* We assume that synchronizing addition of the same tag is done through
* other means -- we cannot grab this while performing the lookup
* w/o making our read locks check to see if we're the writer,
* which is a perf hit. Only the actual hashtable add is a "write".
*/
TABLE_RWLOCK(table, write, lock);
resized = fragment_add_to_hashtable(dcontext, f, table);
TABLE_RWLOCK(table, write, unlock);
/* After resizing a table that is targeted by inlined IBL heads
* the current fragment will need to be repatched; but, we don't have
* to update the stubs when using per-type trace tables since the
* trace table itself is not targeted therefore resizing it doesn't matter.
*/
#ifdef SHARING_STUDY
if (INTERNAL_OPTION(fragment_sharing_study)) {
if (TEST(FRAG_IS_TRACE, f->flags))
add_shared_block(shared_traces, &shared_traces_lock, f);
else
add_shared_block(shared_blocks, &shared_blocks_lock, f);
}
#endif
}
/* Many options, use macros in fragment.h for readability
* If output:
* dumps f to trace file
* If remove:
* removes f from ftable
* If unlink:
* if f is linked, unlinks f
* removes f from incoming link tables
* If fcache:
* deletes f from fcache unit
*/
void
fragment_delete(dcontext_t *dcontext, fragment_t *f, uint actions)
{
#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
bool acquired_shared_vm_lock = false;
bool acquired_fragdel_lock = false;
#endif
LOG(THREAD, LOG_FRAGMENT, 3,
"fragment_delete: *"PFX" F%d("PFX")."PFX" %s 0x%x\n",
f, f->id, f->tag, f->start_pc,
TEST(FRAG_IS_TRACE, f->flags) ? "trace" : "bb", actions);
DOLOG(1, LOG_FRAGMENT, {
if ((f->flags & FRAG_CANNOT_DELETE) != 0) {
LOG(THREAD, LOG_FRAGMENT, 2,
"ERROR: trying to delete undeletable F%d("PFX") 0x%x\n",
f->id, f->tag, actions);
}
});
ASSERT((f->flags & FRAG_CANNOT_DELETE) == 0);
ASSERT((f->flags & FRAG_IS_FUTURE) == 0);
/* ensure the actual free of a shared fragment is done only
* after a multi-stage flush or a reset
*/
ASSERT(!TEST(FRAG_SHARED, f->flags) || TEST(FRAG_WAS_DELETED, f->flags) ||
dynamo_exited || dynamo_resetting || is_self_allsynch_flushing());
#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
/* need to protect ability to reference frag fields and fcache space */
/* all other options are mostly notification */
if (monitor_delete_would_abort_trace(dcontext, f) && DYNAMO_OPTION(shared_traces)) {
/* must acquire shared_vm_areas lock before fragment_delete_mutex (PR 596371) */
acquired_shared_vm_lock = true;
acquire_recursive_lock(&change_linking_lock);
acquire_vm_areas_lock(dcontext, FRAG_SHARED);
}
/* XXX: I added the test for FRAG_WAS_DELETED for i#759: does sideline
* look at fragments after that is set? If so need to resolve rank order
* w/ shared_cache_lock.
*/
if (!TEST(FRAG_WAS_DELETED, f->flags) &&
(!TEST(FRAGDEL_NO_HEAP, actions) || !TEST(FRAGDEL_NO_FCACHE, actions))) {
acquired_fragdel_lock = true;
fragment_get_fragment_delete_mutex(dcontext);
}
#endif
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
if (!TEST(FRAGDEL_NO_OUTPUT, actions)) {
if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
acquire_recursive_lock(&change_linking_lock);
else {
ASSERT(!TEST(FRAG_SHARED, f->flags) ||
self_owns_recursive_lock(&change_linking_lock));
}
fragment_output(dcontext, f);
if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
release_recursive_lock(&change_linking_lock);
}
#endif
if (!TEST(FRAGDEL_NO_MONITOR, actions))
monitor_remove_fragment(dcontext, f);
if (!TEST(FRAGDEL_NO_UNLINK, actions)) {
if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
acquire_recursive_lock(&change_linking_lock);
else {
ASSERT(!TEST(FRAG_SHARED, f->flags) ||
self_owns_recursive_lock(&change_linking_lock));
}
if ((f->flags & FRAG_LINKED_INCOMING) != 0)
unlink_fragment_incoming(dcontext, f);
if ((f->flags & FRAG_LINKED_OUTGOING) != 0)
unlink_fragment_outgoing(dcontext, f);
incoming_remove_fragment(dcontext, f);
if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
release_recursive_lock(&change_linking_lock);
}
if (!TEST(FRAGDEL_NO_HTABLE, actions))
fragment_remove(dcontext, f);
if (!TEST(FRAGDEL_NO_VMAREA, actions))
vm_area_remove_fragment(dcontext, f);
if (!TEST(FRAGDEL_NO_FCACHE, actions)) {
fcache_remove_fragment(dcontext, f);
}
#ifdef SIDELINE
if (dynamo_options.sideline)
sideline_fragment_delete(f);
#endif
#ifdef CLIENT_INTERFACE
if (dr_fragment_deleted_hook_exists() &&
(!TEST(FRAGDEL_NO_HEAP, actions) || !TEST(FRAGDEL_NO_FCACHE, actions)))
instrument_fragment_deleted(dcontext, f->tag, f->flags);
#endif
#ifdef UNIX
if (INTERNAL_OPTION(profile_pcs))
pcprofile_fragment_deleted(dcontext, f);
#endif
if (!TEST(FRAGDEL_NO_HEAP, actions)) {
fragment_free(dcontext, f);
}
#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
if (acquired_fragdel_lock)
fragment_release_fragment_delete_mutex(dcontext);
if (acquired_shared_vm_lock) {
release_vm_areas_lock(dcontext, FRAG_SHARED);
release_recursive_lock(&change_linking_lock);
}
#endif
}
/* Record translation info. Typically used for pending-delete fragments
* whose original app code cannot be trusted as it has been modified (case
* 3559).
* Caller is required to take care of synch (typically this is called
* during a flush or during fragment emit)
*/
void
fragment_record_translation_info(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist)
{
ASSERT(!NEED_SHARED_LOCK(f->flags) ||
!USE_BB_BUILDING_LOCK() ||
OWN_MUTEX(&bb_building_lock) ||
OWN_MUTEX(&trace_building_lock) ||
is_self_flushing());
/* We require that either the FRAG_WAS_DELETED flag is set, to
* indicate there is allocated memory in the live field that needs
* to be freed, or that the FRAG_HAS_TRANSLATION_INFO field is
* set, indicating that there is a special appended field pointing
* to the translation info.
*/
if (TEST(FRAG_HAS_TRANSLATION_INFO, f->flags)) {
ASSERT(!TEST(FRAG_WAS_DELETED, f->flags));
*(FRAGMENT_TRANSLATION_INFO_ADDR(f)) =
record_translation_info(dcontext, f, ilist);
ASSERT(FRAGMENT_TRANSLATION_INFO(f) != NULL);
STATS_INC(num_fragment_translation_stored);
} else if (TEST(FRAG_WAS_DELETED, f->flags)) {
ASSERT(f->in_xlate.incoming_stubs == NULL);
if (INTERNAL_OPTION(safe_translate_flushed)) {
f->in_xlate.translation_info = record_translation_info(dcontext, f, ilist);
ASSERT(f->in_xlate.translation_info != NULL);
ASSERT(FRAGMENT_TRANSLATION_INFO(f) == f->in_xlate.translation_info);
STATS_INC(num_fragment_translation_stored);
#ifdef INTERNAL
DODEBUG({
if (INTERNAL_OPTION(stress_recreate_pc)) {
/* verify recreation */
stress_test_recreate(dcontext, f, NULL);
}
});
#endif
} else
f->in_xlate.translation_info = NULL;
} else
ASSERT_NOT_REACHED();
}
/* Removes the shared fragment f from all lookup tables in a safe
* manner that does not require a full flush synch.
* This routine can be called without synchronizing with other threads.
*/
void
fragment_remove_shared_no_flush(dcontext_t *dcontext, fragment_t *f)
{
DEBUG_DECLARE(bool shared_ibt_table_used = !TEST(FRAG_IS_TRACE, f->flags) ?
DYNAMO_OPTION(shared_bb_ibt_tables) :
DYNAMO_OPTION(shared_trace_ibt_tables);)
ASSERT_NOT_IMPLEMENTED(!TEST(FRAG_COARSE_GRAIN, f->flags));
/* Strategy: ensure no races in updating table or links by grabbing the high-level
* locks that are used to synchronize additions to the table itself.
* Then, simply remove directly from DR-only tables, and safely from ib tables.
* FIXME: There are still risks that the fragment's link state may change
*/
LOG(THREAD, LOG_FRAGMENT, 3, "fragment_remove_shared_no_flush: F%d\n", f->id);
ASSERT(TEST(FRAG_SHARED, f->flags));
if (TEST(FRAG_IS_TRACE, f->flags)) {
mutex_lock(&trace_building_lock);
}
/* grab bb building lock even for traces to further prevent link changes */
mutex_lock(&bb_building_lock);
if (TEST(FRAG_WAS_DELETED, f->flags)) {
/* since caller can't grab locks, we can have a race where someone
* else deletes first -- in that case nothing to do
*/
STATS_INC(shared_delete_noflush_race);
mutex_unlock(&bb_building_lock);
if (TEST(FRAG_IS_TRACE, f->flags))
mutex_unlock(&trace_building_lock);
return;
}
/* FIXME: try to share code w/ fragment_unlink_for_deletion() */
/* Make link changes atomic. We also want vm_area_remove_fragment and
* marking as deleted to be atomic so we grab vm_areas lock up front.
*/
acquire_recursive_lock(&change_linking_lock);
acquire_vm_areas_lock(dcontext, f->flags);
/* FIXME: share all this code w/ vm_area_unlink_fragments()
* The work there is just different enough to make that hard, though.
*/
if (TEST(FRAG_LINKED_OUTGOING, f->flags))
unlink_fragment_outgoing(GLOBAL_DCONTEXT, f);
if (TEST(FRAG_LINKED_INCOMING, f->flags))
unlink_fragment_incoming(GLOBAL_DCONTEXT, f);
incoming_remove_fragment(GLOBAL_DCONTEXT, f);
/* remove from ib lookup tables in a safe manner. this removes the
* frag only from this thread's tables OR from shared tables.
*/
fragment_prepare_for_removal(GLOBAL_DCONTEXT, f);
/* fragment_remove ignores the ibl tables for shared fragments */
fragment_remove(GLOBAL_DCONTEXT, f);
/* FIXME: we don't currently remove from thread-private ibl tables as that
* requires walking all of the threads. */
ASSERT_NOT_IMPLEMENTED(!IS_IBL_TARGET(f->flags) || shared_ibt_table_used);
vm_area_remove_fragment(dcontext, f);
/* case 8419: make marking as deleted atomic w/ fragment_t.also_vmarea field
* invalidation, so that users of vm_area_add_to_list() can rely on this
* flag to determine validity
*/
f->flags |= FRAG_WAS_DELETED;
release_vm_areas_lock(dcontext, f->flags);
release_recursive_lock(&change_linking_lock);
/* if a flush occurs, this fragment will be ignored -- so we must store
* translation info now, just in case
*/
if (!TEST(FRAG_HAS_TRANSLATION_INFO, f->flags))
fragment_record_translation_info(dcontext, f, NULL);
/* try to catch any potential races */
ASSERT(!TEST(FRAG_LINKED_OUTGOING, f->flags));
ASSERT(!TEST(FRAG_LINKED_INCOMING, f->flags));
mutex_unlock(&bb_building_lock);
if (TEST(FRAG_IS_TRACE, f->flags)) {
mutex_unlock(&trace_building_lock);
}
/* no locks can be held when calling this, but f is already unreachable,
* so can do this outside of locks
*/
add_to_lazy_deletion_list(dcontext, f);
}
/* Prepares a fragment for delayed deletion by unlinking it.
* Caller is responsible for calling vm_area_remove_fragment().
* Caller must hold the change_linking_lock if f is shared.
*/
void
fragment_unlink_for_deletion(dcontext_t *dcontext, fragment_t *f)
{
ASSERT(!TEST(FRAG_SHARED, f->flags) ||
self_owns_recursive_lock(&change_linking_lock));
/* this is not an error since fcache unit flushing puts lazily-deleted
* fragments onto its list to ensure they are in the same pending
* delete entry as the normal fragments -- so this routine becomes
* a nop for them
*/
if (TEST(FRAG_WAS_DELETED, f->flags)) {
LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
"NOT unlinking F%d("PFX") for deletion\n", f->id, f->start_pc);
STATS_INC(deleted_frags_re_deleted);
return;
}
LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
"unlinking F%d("PFX") for deletion\n", f->id, f->start_pc);
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
/* we output now to avoid problems reading component blocks
* of traces after source modules are unloaded
*/
fragment_output(dcontext, f);
#endif
if (TEST(FRAG_LINKED_OUTGOING, f->flags))
unlink_fragment_outgoing(dcontext, f);
if (TEST(FRAG_LINKED_INCOMING, f->flags))
unlink_fragment_incoming(dcontext, f);
/* need to remove outgoings from others' incoming and
* redirect others' outgoing to a future. former must be
* done before we remove from hashtable, and latter must be
* done now to avoid other fragments jumping into stale code,
* so we do it here and not when we do the real fragment_delete().
* we don't need to do this for private fragments, but we do anyway
* so that we can use the fragment_t.incoming_stubs field as a union.
*/
incoming_remove_fragment(dcontext, f);
if (TEST(FRAG_SHARED, f->flags)) {
/* we shouldn't need to worry about someone else changing the
* link status, since nobody else is allowed to be in DR now,
* and afterward they all must invalidate any ptrs they hold
* to flushed fragments, and flushed fragments are not
* reachable via hashtable or incoming lists!
*/
/* ASSUMPTION: monitor_remove_fragment does NOT need to
* be called for all threads, since private trace head
* ctrs are cleared lazily (only relevant here for
* -shared_traces) and invalidating last_{exit,fragment}
* is done by the trace overlap and abort in the main
* flush loop.
*/
}
/* need to remove from htable
* we used to only do fragment_prepare_for_removal() (xref case 1808)
* for private fragments, but for case 3559 we want to free up the
* incoming field at unlink time, and we must do all 3 of unlink,
* vmarea, and htable freeing at once.
*/
fragment_remove(dcontext, f);
/* let recreate_fragment_ilist() know that this fragment is
* pending deletion and might no longer match the app's state.
* for shared fragments, also lets people know f is not in a normal
* vmarea anymore (though actually moving f is up to the caller).
* additionally the flag indicates that translation info was allocated
* for this fragment.
*/
f->flags |= FRAG_WAS_DELETED;
/* the original app code cannot be used to recreate state, so we must
* store translation info now
*/
if (!TEST(FRAG_HAS_TRANSLATION_INFO, f->flags))
fragment_record_translation_info(dcontext, f, NULL);
STATS_INC(fragments_unlinked_for_deletion);
}
/* When shared IBT tables are used, update thread-private state
* to reflect the current parameter values -- hash mask, table address --
* for the shared ftable.
*/
static bool
update_private_ibt_table_ptrs(dcontext_t *dcontext, ibl_table_t *ftable
_IF_DEBUG(fragment_entry_t **orig_table))
{
bool table_change = false;
if (TEST(FRAG_TABLE_SHARED, ftable->table_flags)) {
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
if (TEST(FRAG_TABLE_TRACE, ftable->table_flags) &&
ftable->table != pt->trace_ibt[ftable->branch_type].table) {
DODEBUG({
if (orig_table != NULL)
*orig_table =
pt->trace_ibt[ftable->branch_type].table;
});
table_change = true;
}
else if (DYNAMO_OPTION(bb_ibl_targets) &&
!TEST(FRAG_TABLE_TRACE, ftable->table_flags) &&
ftable->table != pt->bb_ibt[ftable->branch_type].table) {
DODEBUG({
if (orig_table != NULL)
*orig_table =
pt->bb_ibt[ftable->branch_type].table;
});
table_change = true;
}
if (table_change) {
update_private_ptr_to_shared_ibt_table(dcontext, ftable->branch_type,
TEST(FRAG_TABLE_TRACE,
ftable->table_flags),
true, /* adjust old
* ref-count */
true /* lock */);
DODEBUG({
if (orig_table != NULL)
ASSERT(ftable->table != *orig_table);
});
}
#ifdef DEBUG
else if (orig_table != NULL)
*orig_table = NULL;
#endif
}
return table_change;
}
/* Update the thread-private ptrs for the dcontext to point to the
* currently "live" shared IBT table for branch_type.
* When adjust_ref_count==true, adjust the ref-count for the old table
* that the dcontext currently points to.
* When lock_table==true, lock the shared table prior to manipulating
* it. If this is false, the caller must have locked the table already.
* NOTE: If adjust_ref_count=true, lock_table should be true also and
* the caller should NOT hold the table lock, since the underlying
* routines that manipulate the ref count lock the table.
*/
static inline void
update_private_ptr_to_shared_ibt_table(dcontext_t *dcontext,
ibl_branch_type_t branch_type, bool trace,
bool adjust_old_ref_count, bool lock_table)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
ibl_table_t *sh_table_ptr = trace ? &shared_pt->trace_ibt[branch_type] :
&shared_pt->bb_ibt[branch_type];
ibl_table_t *pvt_table_ptr = trace ? &pt->trace_ibt[branch_type] :
&pt->bb_ibt[branch_type];
/* Point to the new table. The shared table must be locked prior to
* accessing any of its fields. */
if (lock_table)
TABLE_RWLOCK(sh_table_ptr, write, lock);
ASSERT_OWN_WRITE_LOCK(true, &sh_table_ptr->rwlock);
/* We can get here multiple times due to callers being racy */
if (pvt_table_ptr->table == sh_table_ptr->table) {
SYSLOG_INTERNAL_WARNING_ONCE("racy private ptr to shared table update");
if (lock_table)
TABLE_RWLOCK(sh_table_ptr, write, unlock);
return;
}
/* Decrement the ref-count for any old table that is pointed to. */
if (adjust_old_ref_count) {
dec_table_ref_count(dcontext, pvt_table_ptr, false/*can't be live*/);
}
/* We must hold at least the read lock when writing, else we could grab
* an inconsistent mask/lookuptable pair if another thread is in the middle
* of resizing the table (case 10405).
*/
/* Only data for one set of tables is stored in TLS -- for the trace
* tables in the default config OR the BB tables in shared BBs
* only mode.
*/
if ((trace || SHARED_BB_ONLY_IB_TARGETS()) &&
DYNAMO_OPTION(ibl_table_in_tls))
update_lookuptable_tls(dcontext, sh_table_ptr);
ASSERT(pvt_table_ptr->table != sh_table_ptr->table);
pvt_table_ptr->table = sh_table_ptr->table;
pvt_table_ptr->hash_mask = sh_table_ptr->hash_mask;
/* We copy the unaligned value over also because it's used for matching
* in the dead table list. */
pvt_table_ptr->table_unaligned = sh_table_ptr->table_unaligned;
pvt_table_ptr->table_flags = sh_table_ptr->table_flags;
sh_table_ptr->ref_count++;
ASSERT(sh_table_ptr->ref_count > 0);
DODEBUG({
LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 2,
"update_table_ptrs %s-%s table: addr "PFX", mask "PIFX"\n",
trace ? "trace" : "BB", sh_table_ptr->name,
sh_table_ptr->table, sh_table_ptr->hash_mask);
if ((trace || SHARED_BB_ONLY_IB_TARGETS()) &&
DYNAMO_OPTION(ibl_table_in_tls)) {
local_state_extended_t *state =
(local_state_extended_t *) dcontext->local_state;
LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 2,
"TLS state %s-%s table: addr "PFX", mask "PIFX"\n",
trace ? "trace" : "BB", sh_table_ptr->name,
state->table_space.table[branch_type].lookuptable,
state->table_space.table[branch_type].hash_mask);
}
});
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(hashtable_ibl_entry_stats)) {
pvt_table_ptr->entry_stats_to_lookup_table =
sh_table_ptr->entry_stats_to_lookup_table;
}
else
pvt_table_ptr->entry_stats_to_lookup_table = 0;
#endif
if (lock_table)
TABLE_RWLOCK(sh_table_ptr, write, unlock);
/* We don't need the lock for this, and holding it will have rank order
* issues with disassembling in debug builds */
if (PRIVATE_TRACES_ENABLED() || DYNAMO_OPTION(bb_ibl_targets))
update_generated_hashtable_access(dcontext);
STATS_INC(num_shared_ibt_table_ptr_resets);
}
/* When shared IBT tables are used, update thread-private state
* to reflect the current parameter values -- hash mask, table address --
* for all tables.
*/
static bool
update_all_private_ibt_table_ptrs(dcontext_t *dcontext, per_thread_t *pt)
{
bool rc = false;
if (SHARED_IBT_TABLES_ENABLED()) {
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
if (update_private_ibt_table_ptrs(dcontext,
&shared_pt->trace_ibt[branch_type]
_IF_DEBUG(NULL)))
rc = true;
}
if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
if (update_private_ibt_table_ptrs(dcontext,
&shared_pt->bb_ibt[branch_type]
_IF_DEBUG(NULL)))
rc = true;
}
}
}
return rc;
}
/* Prepares for removal of f from ftable (does not delete f) by pointing the
* fragment's lookup table entry to an entry point that leads to a cache exit.
* This routine is needed for safe removal of a fragment by a thread while
* another thread may be about to jump to it via an IBL. The lookuptable is
* left in a slightly inconsistent state but one that is accepted by the
* consistency check. See the note on hashtable_fragment_check_consistency
* in the routine.
*
* Returns true if the fragment was found & removed.
*/
static bool
fragment_prepare_for_removal_from_table(dcontext_t *dcontext, fragment_t *f,
ibl_table_t *ftable)
{
uint hindex;
fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
fragment_entry_t *pg;
/* We need the write lock since the start_pc is modified (though we
* technically may be ok in all scenarios there) and to avoid problems
* with parallel prepares (shouldn't count on the bb building lock).
* Grab the lock after all private ptrs are updated since that
* operation might grab the same lock, if this remove is from a
* shared IBT table.
*/
/* FIXME: why do we need to update here? */
update_private_ibt_table_ptrs(dcontext, ftable _IF_DEBUG(NULL));
TABLE_RWLOCK(ftable, write, lock);
pg = hashtable_ibl_lookup_for_removal(fe, ftable, &hindex);
if (pg != NULL) {
/* Note all IBL routines that could be looking up an entry
* in this table have to exit with equivalent register
* state. It is possible to enter a private bb IBL
* lookup, shared bb IBL lookup or trace bb IBL lookup and
* if a delete race is hit then they would all go to the
* pending_delete_pc that we'll now supply. They HAVE to
* be all equivalent independent of the source fragment for this to work.
*
* On the other hand we can provide different start_pc
* values if we have different tables. We currently don't
* take advantage of this but we'll leave the power in place.
*/
/* FIXME: [perf] we could memoize this value in the table itself */
cache_pc pending_delete_pc =
get_target_delete_entry_pc(dcontext, ftable);
ASSERT(IBL_ENTRIES_ARE_EQUAL(*pg, fe));
ASSERT(pending_delete_pc != NULL);
LOG(THREAD, LOG_FRAGMENT, 3,
"fragment_prepare: remove F%d("PFX") from %s[%u] (table addr "PFX"), "
"set to "PFX"\n",
f->id, f->tag, ftable->name, hindex, ftable->table,
pending_delete_pc);
/* start_pc_fragment will not match start_pc for the table
* consistency checks. However, the hashtable_fragment_check_consistency
* routine verifies that either start_pc/start_pc_fragment match OR that
* the start_pc_fragment is set to the correct target_delete
* entry point.
*
* We change the tag to FAKE_TAG, which preserves linear probing.
* In a thread-shared table, this ensures that the same tag will never
* be present in more than one entry in a table (1 real entry &
* 1+ target_delete entries).
* This isn't needed in a thread-private table but doesn't hurt.
*/
ftable->table[hindex].start_pc_fragment = pending_delete_pc;
ftable->table[hindex].tag_fragment = FAKE_TAG;
/* FIXME In a shared table, this means that the entry cannot
* be overwritten for a fragment with the same tag. */
ftable->unlinked_entries++;
ftable->entries--;
TABLE_RWLOCK(ftable, write, unlock);
ASSERT(!TEST(FRAG_CANNOT_DELETE, f->flags));
return true;
}
TABLE_RWLOCK(ftable, write, unlock);
return false;
}
/* Prepares fragment f for removal from all IBL routine targeted tables.
* Does not actually remove the entry from the table
* as that can only be done through proper cross-thread synchronization.
*
* Returns true if the fragment was found & removed.
*/
bool
fragment_prepare_for_removal(dcontext_t *dcontext, fragment_t *f)
{
per_thread_t *pt;
bool prepared = false;
ibl_branch_type_t branch_type;
if (!IS_IBL_TARGET(f->flags)) {
/* nothing to do */
return false;
}
ASSERT(TEST(FRAG_SHARED, f->flags) || dcontext != GLOBAL_DCONTEXT);
/* We need a real per_thread_t & context below so make sure we have one. */
if (dcontext == GLOBAL_DCONTEXT) {
dcontext = get_thread_private_dcontext();
ASSERT(dcontext != NULL);
}
pt = GET_PT(dcontext);
/* FIXME: as an optimization we could test if IS_IBL_TARGET() is
* set before looking it up
*/
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
per_thread_t *local_pt = pt;
/* We put traces into the trace tables and BBs into the BB tables
* and sometimes put traces into BB tables also. We never put
* BBs into a trace table.
*/
if (TEST(FRAG_IS_TRACE, f->flags)) {
if (DYNAMO_OPTION(shared_trace_ibt_tables))
local_pt = shared_pt;
if (fragment_prepare_for_removal_from_table(dcontext, f,
&local_pt->
trace_ibt[branch_type]))
prepared = true;
}
if (DYNAMO_OPTION(bb_ibl_targets) &&
(!TEST(FRAG_IS_TRACE, f->flags) ||
DYNAMO_OPTION(bb_ibt_table_includes_traces))) {
if (DYNAMO_OPTION(shared_bb_ibt_tables))
local_pt = shared_pt;
if (fragment_prepare_for_removal_from_table(dcontext, f,
&local_pt->
bb_ibt[branch_type])) {
#ifdef DEBUG
ibl_table_t *ibl_table = GET_IBT_TABLE(pt, f->flags, branch_type);
fragment_entry_t current;
TABLE_RWLOCK(ibl_table, read, lock);
current = hashtable_ibl_lookup(dcontext, (ptr_uint_t)f->tag, ibl_table);
ASSERT(IBL_ENTRY_IS_EMPTY(current));
TABLE_RWLOCK(ibl_table, read, unlock);
#endif
prepared = true;
}
}
}
return prepared;
}
#ifdef DEBUG
/* FIXME: hashtable_fragment_reset() needs to walk the tables to get these
* stats, but then we'd need to subtract 1 from all smaller counts -
* e.g. if an entry is found in 3 tables we can add (1,-1,0) then
* we'll find it again and we should add (0,1,-1) and one more time
* when we should add (0,0,1). In total all will be accounted for to
* (1,0,0) without messing much else.
*/
static inline void
fragment_ibl_stat_account(uint flags, uint ibls_targeted)
{
if (TEST(FRAG_IS_TRACE, flags)) {
switch (ibls_targeted) {
case 0: break; /* doesn't have to be a target of any IBL routine */
case 1: STATS_INC(num_traces_in_1_ibl_tables); break;
case 2: STATS_INC(num_traces_in_2_ibl_tables); break;
case 3: STATS_INC(num_traces_in_3_ibl_tables); break;
default: ASSERT_NOT_REACHED();
}
} else {
switch (ibls_targeted) {
case 0: break; /* doesn't have to be a target of any IBL routine */
case 1: STATS_INC(num_bbs_in_1_ibl_tables); break;
case 2: STATS_INC(num_bbs_in_2_ibl_tables); break;
case 3: STATS_INC(num_bbs_in_3_ibl_tables); break;
default: ASSERT_NOT_REACHED();
}
}
}
#endif
/* Removes f from any IBT tables it is in.
* If f is in a shared table, only removes if from_shared is true, in
* which case dcontext must be GLOBAL_DCONTEXT and we must have
* dynamo_all_threads_synched (case 10137).
*/
void
fragment_remove_from_ibt_tables(dcontext_t *dcontext, fragment_t *f,
bool from_shared)
{
bool shared_ibt_table =
(!TEST(FRAG_IS_TRACE, f->flags) && DYNAMO_OPTION(shared_bb_ibt_tables)) ||
(TEST(FRAG_IS_TRACE, f->flags) && DYNAMO_OPTION(shared_trace_ibt_tables));
fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
ASSERT(!from_shared || !shared_ibt_table || !IS_IBL_TARGET(f->flags) ||
(dcontext == GLOBAL_DCONTEXT && dynamo_all_threads_synched));
if (((!shared_ibt_table && dcontext != GLOBAL_DCONTEXT) ||
(from_shared && dcontext == GLOBAL_DCONTEXT && dynamo_all_threads_synched)) &&
IS_IBL_TARGET(f->flags)) {
/* trace_t tables should be all private and any deletions should follow strict
* two step deletion process, we don't need to be holding nested locks when
* removing any cached entries from the per-type IBL target tables.
*/
/* FIXME: the stats on ibls_targeted are not quite correct - we need to
* gather these independently */
DEBUG_DECLARE(uint ibls_targeted = 0;)
ibl_branch_type_t branch_type;
per_thread_t *pt = GET_PT(dcontext);
ASSERT(TEST(FRAG_IS_TRACE, f->flags) || DYNAMO_OPTION(bb_ibl_targets));
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
/* assuming a single tag can't be both a trace and bb */
ibl_table_t *ibtable = GET_IBT_TABLE(pt, f->flags, branch_type);
ASSERT(!TEST(FRAG_TABLE_SHARED, ibtable->table_flags) ||
dynamo_all_threads_synched);
TABLE_RWLOCK(ibtable, write, lock); /* satisfy asserts, even if allsynch */
if (hashtable_ibl_remove(fe, ibtable)) {
LOG(THREAD, LOG_FRAGMENT, 2,
" removed F%d("PFX") from IBT table %s\n",
f->id, f->tag,
TEST(FRAG_TABLE_TRACE, ibtable->table_flags) ?
ibl_trace_table_type_names[branch_type] :
ibl_bb_table_type_names[branch_type]);
DOSTATS({ibls_targeted++;});
}
TABLE_RWLOCK(ibtable, write, unlock);
}
DOSTATS({fragment_ibl_stat_account(f->flags, ibls_targeted);});
}
}
/* Removes ibl entries whose tags are in [start,end) */
static uint
fragment_remove_ibl_entries_in_region(dcontext_t *dcontext, app_pc start, app_pc end,
uint frag_flags)
{
uint total_removed = 0;
per_thread_t *pt = GET_PT(dcontext);
ibl_branch_type_t branch_type;
ASSERT(pt != NULL);
ASSERT(TEST(FRAG_IS_TRACE, frag_flags) || DYNAMO_OPTION(bb_ibl_targets));
ASSERT(dcontext == get_thread_private_dcontext() || dynamo_all_threads_synched);
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
ibl_table_t *ibtable = GET_IBT_TABLE(pt, frag_flags, branch_type);
uint removed = 0;
TABLE_RWLOCK(ibtable, write, lock);
if (ibtable->entries > 0) {
removed = hashtable_ibl_range_remove(dcontext, ibtable,
(ptr_uint_t)start, (ptr_uint_t)end, NULL);
/* Ensure a full remove gets everything */
ASSERT(start != UNIVERSAL_REGION_BASE || end != UNIVERSAL_REGION_END ||
(ibtable->entries == 0 &&
is_region_memset_to_char((app_pc)ibtable->table,
(ibtable->capacity-1)*sizeof(fragment_entry_t),
0)));
}
LOG(THREAD, LOG_FRAGMENT, 2,
" removed %d entries (%d left) in "PFX"-"PFX" from IBT table %s\n",
removed, ibtable->entries, start, end,
TEST(FRAG_TABLE_TRACE, ibtable->table_flags) ?
ibl_trace_table_type_names[branch_type] :
ibl_bb_table_type_names[branch_type]);
TABLE_RWLOCK(ibtable, write, unlock);
total_removed += removed;
}
return total_removed;
}
/* Removes shared (and incidentally private, but if no shared targets
* can exist, may remove nothing) ibl entries whose tags are in
* [start,end) from all tables associated w/ dcontext. If
* dcontext==GLOBAL_DCONTEXT, uses the shared tables, if they exist;
* else, uses the private tables, if any.
*/
uint
fragment_remove_all_ibl_in_region(dcontext_t *dcontext, app_pc start, app_pc end)
{
uint removed = 0;
if (DYNAMO_OPTION(bb_ibl_targets) &&
((dcontext == GLOBAL_DCONTEXT && DYNAMO_OPTION(shared_bb_ibt_tables)) ||
(dcontext != GLOBAL_DCONTEXT && !DYNAMO_OPTION(shared_bb_ibt_tables)))) {
removed +=
fragment_remove_ibl_entries_in_region(dcontext, start, end, 0/*bb table*/);
}
if (DYNAMO_OPTION(shared_traces) &&
((dcontext == GLOBAL_DCONTEXT && DYNAMO_OPTION(shared_trace_ibt_tables)) ||
(dcontext != GLOBAL_DCONTEXT && !DYNAMO_OPTION(shared_trace_ibt_tables)))) {
removed +=
fragment_remove_ibl_entries_in_region(dcontext, start, end, FRAG_IS_TRACE);
}
return removed;
}
/* Removes f from any hashtables -- BB, trace, or future -- and IBT tables
* it is in, except for shared IBT tables. */
void
fragment_remove(dcontext_t *dcontext, fragment_t *f)
{
per_thread_t *pt = GET_PT(dcontext);
fragment_table_t *table = GET_FTABLE(pt, f->flags);
ASSERT(TEST(FRAG_SHARED, f->flags) || dcontext != GLOBAL_DCONTEXT);
/* For consistency we remove entries from the IBT
* tables before we remove them from the trace table.
*/
fragment_remove_from_ibt_tables(dcontext, f, false/*leave in shared*/);
/* We need the write lock since deleting shifts elements around (though we
* technically may be ok in all scenarios there) and to avoid problems with
* multiple removes at once (shouldn't count on the bb building lock)
*/
TABLE_RWLOCK(table, write, lock);
if (hashtable_fragment_remove(f, table)) {
LOG(THREAD, LOG_FRAGMENT, 4,
"fragment_remove: removed F%d("PFX") from fcache lookup table\n",
f->id, f->tag);
TABLE_RWLOCK(table, write, unlock);
return;
}
TABLE_RWLOCK(table, write, unlock);
/* ok to not find a trace head used to start a trace -- fine to have deleted
* the trace head
*/
ASSERT(cur_trace_tag(dcontext) == f->tag
/* PR 299808: we have invisible temp trace bbs */
IF_CLIENT_INTERFACE(|| TEST(FRAG_TEMP_PRIVATE, f->flags)));
}
/* Remove f from ftable, replacing it in the hashtable with new_f,
* which has an identical tag.
* f's next field is left intact so this can be done while owner is in fcache
* f is NOT deleted in any other way!
* To delete later, caller must call fragment_delete w/ remove=false
*/
void
fragment_replace(dcontext_t *dcontext, fragment_t *f, fragment_t *new_f)
{
per_thread_t *pt = GET_PT(dcontext);
fragment_table_t *table = GET_FTABLE(pt, f->flags);
TABLE_RWLOCK(table, write, lock);
if (hashtable_fragment_replace(f, new_f, table)) {
fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
fragment_entry_t new_fe = FRAGENTRY_FROM_FRAGMENT(new_f);
LOG(THREAD, LOG_FRAGMENT, 4,
"removed F%d from fcache lookup table (replaced with F%d) "PFX"->~"PFX","PFX"\n",
f->id, new_f->id, f->tag, f->start_pc, new_f->start_pc);
/* Need to replace all entries from the IBL tables that may have this entry */
if (IS_IBL_TARGET(f->flags)) {
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
ibl_table_t *ibtable = GET_IBT_TABLE(pt, f->flags, branch_type);
/* currently we don't have shared ib target tables,
* otherwise a write lock would come in the picture here
*/
ASSERT(!TEST(FRAG_TABLE_SHARED, ibtable->table_flags));
hashtable_ibl_replace(fe, new_fe, ibtable);
}
}
} else
ASSERT_NOT_REACHED();
TABLE_RWLOCK(table, write, unlock);
/* tell monitor f has disappeared, but do not delete from incoming table
* or from fcache, also do not dump to trace file
*/
monitor_remove_fragment(dcontext, f);
}
void
fragment_shift_fcache_pointers(dcontext_t *dcontext, fragment_t *f, ssize_t shift,
cache_pc start, cache_pc end, size_t old_size)
{
per_thread_t *pt = GET_PT(dcontext);
IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* must re-relativize when copying! */
/* need to shift all stored cache_pcs.
* do not need to shift relative pcs pointing to other fragments -- they're
* all getting shifted too!
* just need to re-pc-relativize jmps to fixed locations, namely
* cti's in exit stubs, and call instructions inside fragments.
*/
LOG(THREAD, LOG_FRAGMENT, 2, "fragment_shift_fcache_pointers: F%d + "SSZFMT"\n",
f->id, shift);
ASSERT(!TEST(FRAG_IS_FUTURE, f->flags)); /* only in-cache frags */
f->start_pc += shift;
/* Should shift cached lookup entries in all IBL target tables,
* order doesn't matter here: either way we'll be inconsistent, can't do this within the cache.
*/
if (IS_IBL_TARGET(f->flags)) {
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
/* Of course, we need to shift only pointers into the cache that is getting shifted! */
ibl_table_t *ibtable = GET_IBT_TABLE(pt, f->flags, branch_type);
fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
fragment_entry_t *pg;
uint hindex;
TABLE_RWLOCK(ibtable, read, lock);
pg = hashtable_ibl_lookup_for_removal(fe, ibtable, &hindex);
if (pg != NULL)
pg->start_pc_fragment += shift;
TABLE_RWLOCK(ibtable, read, unlock);
LOG(THREAD, LOG_FRAGMENT, 2,
"fragment_table_shift_fcache_pointers: %s ibt %s shifted by %d\n",
TEST(FRAG_IS_TRACE, f->flags) ? "trace" : "BB", ibtable->name, shift);
}
}
linkstubs_shift(dcontext, f, shift);
DOLOG(6, LOG_FRAGMENT, { /* print after start_pc updated so get actual code */
LOG(THREAD, LOG_FRAGMENT, 6, "before shifting F%d ("PFX")\n", f->id, f->tag);
disassemble_fragment(dcontext, f, stats->loglevel < 3);
});
#ifdef X86
if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
/* just re-finalize to update */
finalize_selfmod_sandbox(dcontext, f);
}
#endif
/* inter-cache links must be redone, but all fragment entry pcs must be
* fixed up first, so that's done separately
*/
/* re-do pc-relative targets outside of cache */
shift_ctis_in_fragment(dcontext, f, shift, start, end, old_size);
#ifdef CHECK_RETURNS_SSE2
finalize_return_check(dcontext, f);
#endif
DOLOG(6, LOG_FRAGMENT, {
LOG(THREAD, LOG_FRAGMENT, 6, "after shifting F%d ("PFX")\n", f->id, f->tag);
disassemble_fragment(dcontext, f, stats->loglevel < 3);
});
}
/* this routine only copies data structures like bbs and statistics
*/
void
fragment_copy_data_fields(dcontext_t *dcontext, fragment_t *f_src, fragment_t *f_dst)
{
if ((f_src->flags & FRAG_IS_TRACE) != 0) {
trace_only_t *t_src = TRACE_FIELDS(f_src);
trace_only_t *t_dst = TRACE_FIELDS(f_dst);
ASSERT((f_dst->flags & FRAG_IS_TRACE) != 0);
if (t_src->bbs != NULL) {
t_dst->bbs =
nonpersistent_heap_alloc(dcontext, t_src->num_bbs*sizeof(trace_bb_info_t)
HEAPACCT(ACCT_TRACE));
memcpy(t_dst->bbs, t_src->bbs, t_src->num_bbs*sizeof(trace_bb_info_t));
t_dst->num_bbs = t_src->num_bbs;
}
#ifdef PROFILE_RDTSC
t_dst->count = t_src->count;
t_dst->total_time = t_src->total_time;
#endif
#if defined (PROFILE_LINKCOUNT) && defined(SIDELINE_COUNT_STUDY)
t_dst->count_old_pre = t_src->count_old_pre;
t_dst->count_old_post = t_src->count_old_post;
#endif
}
}
#if defined(DEBUG) && defined(INTERNAL)
static void
dump_lookup_table(dcontext_t *dcontext, ibl_table_t *ftable)
{
uint i;
cache_pc target_delete = get_target_delete_entry_pc(dcontext, ftable);
ASSERT(target_delete != NULL);
ASSERT(ftable->table != NULL);
LOG(THREAD, LOG_FRAGMENT, 1,
"%6s %10s %10s -- %s\n", "i", "tag", "target", ftable->name);
/* need read lock to traverse the table */
TABLE_RWLOCK(ftable, read, lock);
for (i = 0; i < ftable->capacity; i++) {
if (ftable->table[i].tag_fragment != 0) {
if (ftable->table[i].start_pc_fragment == target_delete) {
LOG(THREAD, LOG_FRAGMENT, 1,
"%6x "PFX" target_delete\n",
i, ftable->table[i].tag_fragment);
ASSERT(ftable->table[i].tag_fragment == FAKE_TAG);
}
else {
LOG(THREAD, LOG_FRAGMENT, 1,
"%6x "PFX" "PFX"\n",
i, ftable->table[i].tag_fragment,
ftable->table[i].start_pc_fragment);
}
}
DOCHECK(1, { hashtable_ibl_check_consistency(dcontext, ftable, i); });
}
TABLE_RWLOCK(ftable, read, unlock);
}
#endif
#ifdef DEBUG
/* used only for debugging purposes, check if IBL routine leaks due to wraparound */
/* interesting only when not using INTERNAL_OPTION(ibl_sentinel_check) */
static bool
is_fragment_index_wraparound(dcontext_t *dcontext, ibl_table_t *ftable, fragment_t *f)
{
uint hindex = HASH_FUNC((ptr_uint_t)f->tag, ftable);
uint found_at_hindex;
fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
fragment_entry_t *pg = hashtable_ibl_lookup_for_removal(fe, ftable, &found_at_hindex);
ASSERT(pg != NULL);
ASSERT(IBL_ENTRIES_ARE_EQUAL(*pg, fe));
LOG(THREAD, LOG_FRAGMENT, 3,
"is_fragment_index_wraparound F%d, tag "PFX", found_at_hindex 0x%x, preferred 0x%x\n",
f->id, f->tag, found_at_hindex, hindex);
return (found_at_hindex < hindex); /* wraparound */
}
#endif /* DEBUG */
static void
fragment_add_ibl_target_helper(dcontext_t *dcontext, fragment_t *f,
ibl_table_t *ibl_table)
{
fragment_entry_t current;
fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
/* Never add a BB to a trace table. */
ASSERT(!(!TEST(FRAG_IS_TRACE, f->flags) &&
TEST(FRAG_TABLE_TRACE, ibl_table->table_flags)));
/* adding is a write operation */
TABLE_RWLOCK(ibl_table, write, lock);
/* This is the last time the table lock is grabbed before adding the frag so
* check here to account for the race in the time between the
* FRAG_IS_TRACE_HEAD check in add_ibl_target() and now. We never add trace
* heads to an IBT target table.
*/
if (TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
TABLE_RWLOCK(ibl_table, write, unlock);
STATS_INC(num_th_bb_ibt_add_race);
return;
}
/* For shared tables, check again in case another thread snuck in
* before the preceding lock and added the target. */
if (TEST(FRAG_TABLE_SHARED, ibl_table->table_flags)) {
current = hashtable_ibl_lookup(dcontext, (ptr_uint_t)f->tag, ibl_table);
if (IBL_ENTRY_IS_EMPTY(current))
hashtable_ibl_add(dcontext, fe, ibl_table);
/* We don't ever expect to find a like-tagged fragment. A BB
* can be unlinked due to eviction or when it's marked as a trace
* head. Eviction (for example, due to cache consistency)
* sets start_pc_fragment to FAKE_TAG, so there can't be
* a tag match; &unlinked_fragment is returned, and this
* applies to traces also. For trace head marking, FAKE_TAG
* is also set so &unlinked_fragment is returned.
*
* If we didn't set FAKE_TAG for an unlinked entry, then it
* could be clobbered with a new fragment w/the same tag.
* In a shared table, unlinked entries cannot be clobbered
* except by fragments w/the same tags, so this could help
* limit the length of collision chains.
*/
}
else {
hashtable_ibl_add(dcontext, fe, ibl_table);
}
TABLE_RWLOCK(ibl_table, write, unlock);
DOSTATS({
if (!TEST(FRAG_IS_TRACE, f->flags))
STATS_INC(num_bbs_ibl_targets);
/* We assume that traces can be added to trace and BB IBT tables but
* not just to BB tables. We count only traces added to trace tables
* so that we don't double increment.
*/
else if (TEST(FRAG_IS_TRACE, f->flags) &&
TEST(FRAG_TABLE_TRACE, ibl_table->table_flags))
STATS_INC(num_traces_ibl_targets);
});
/* Adding current exit to help calculate an estimated
* indirect branch fan-out, that is function fan-in for
* returns. Note that other IBL hits to the same place
* will not have exits associated.
*/
LOG(THREAD, LOG_FRAGMENT, 2,
"fragment_add_ibl_target added F%d("PFX"), branch %d, to %s, on exit from "PFX"\n",
f->id, f->tag,
ibl_table->branch_type, ibl_table->name,
LINKSTUB_FAKE(dcontext->last_exit) ? 0 :
EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit)
);
DOLOG(4, LOG_FRAGMENT, {
dump_lookuptable_tls(dcontext);
hashtable_ibl_dump_table(dcontext, ibl_table);
dump_lookup_table(dcontext, ibl_table);
});
DODEBUG({
if (TEST(FRAG_SHARED, f->flags) && !TEST(FRAG_IS_TRACE, f->flags))
LOG(THREAD, LOG_FRAGMENT, 2,
"add_ibl_target: shared BB F%d("PFX") added\n", f->id,
f->tag);
});
}
/* IBL targeted fragments per branch type */
fragment_t *
fragment_add_ibl_target(dcontext_t *dcontext, app_pc tag,
ibl_branch_type_t branch_type)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
fragment_t *f = NULL;
fragment_t wrapper;
if (SHARED_BB_ONLY_IB_TARGETS()) {
f = fragment_lookup_bb(dcontext, tag);
if (f == NULL) {
f = fragment_coarse_lookup_wrapper(dcontext, tag, &wrapper);
if (f != NULL) {
#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
if (TEST(COARSE_FILL_IBL_MASK(branch_type),
DYNAMO_OPTION(coarse_fill_ibl))) {
/* On-demand per-type ibl filling from the persisted RAC/RCT
* table. We limit to the first thread to ask for it by
* clearing the coarse_info_t pending_table fields.
*/
/* FIXME: combine w/ the coarse lookup to do this once only */
coarse_info_t *coarse = get_fragment_coarse_info(f);
ASSERT(coarse != NULL);
if (coarse->persisted &&
exists_coarse_ibl_pending_table(dcontext, coarse, branch_type)) {
bool in_persisted_ibl = false;
mutex_lock(&coarse->lock);
if (exists_coarse_ibl_pending_table(dcontext,
coarse, branch_type)) {
ibl_table_t *ibl_table =
GET_IBT_TABLE(pt, f->flags, branch_type);
coarse_persisted_fill_ibl(dcontext, coarse, branch_type);
TABLE_RWLOCK(ibl_table, read, lock);
if (!IBL_ENTRY_IS_EMPTY(hashtable_ibl_lookup(dcontext,
(ptr_uint_t)tag, ibl_table)))
in_persisted_ibl = true;
TABLE_RWLOCK(ibl_table, read, unlock);
if (in_persisted_ibl) {
mutex_unlock(&coarse->lock);
return f;
}
}
mutex_unlock(&coarse->lock);
}
}
#endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
}
}
} else {
f = fragment_lookup_trace(dcontext, tag);
if (f == NULL && DYNAMO_OPTION(bb_ibl_targets)) {
/* Populate with bb's that are not trace heads */
f = fragment_lookup_bb(dcontext, tag);
/* We don't add trace heads OR when a trace is targetting a BB. In the
* latter case, the BB will shortly be marked as a trace head and
* removed from the IBT table so we don't needlessly add it.
*/
if (f != NULL &&
(TEST(FRAG_IS_TRACE_HEAD, f->flags) ||
TEST(FRAG_IS_TRACE, dcontext->last_fragment->flags))) {
/* FIXME: should change the logic if trace headness becomes a private property */
f = NULL; /* ignore fragment */
STATS_INC(num_ib_th_target); /* counted in num_ibt_cold_misses */
}
}
}
LOG(THREAD, LOG_FRAGMENT, 3,
"fragment_add_ibl_target tag "PFX", branch %d, F%d %s\n",
tag, branch_type, f != NULL ? f->id : 0,
(f != NULL && TEST(FRAG_IS_TRACE, f->flags)) ? "existing trace" : "");
/* a valid IBT fragment exists */
if (f != NULL) {
ibl_table_t *ibl_table = GET_IBT_TABLE(pt, f->flags, branch_type);
DEBUG_DECLARE(fragment_entry_t *orig_lookuptable = NULL;)
fragment_entry_t current;
/* Make sure this thread's local ptrs & state is current in case a
* shared table resize occurred while it was in the cache. We update
* only during an IBL miss, since that's the first time that
* accessing the old table inflicted a cost (a context switch).
*/
/* NOTE We could be more aggressive and update the private ptrs for
* all tables, not just the one being added to, by calling
* update_all_private_ibt_table_ptrs(). We could be even more
* aggressive by updating all ptrs on every cache exit in
* enter_couldbelinking() but that could also prove to be more
* expensive by invoking the update logic when an IBL miss didn't
* occur. However, more frequent updates could lead to old tables
* being freed earlier. We can revisit this if we see old tables
* piling up and not being freed in a timely manner.
*/
update_private_ibt_table_ptrs(dcontext, ibl_table
_IF_DEBUG(&orig_lookuptable));
/* We can't place a private fragment into a thread-shared table.
* Nothing prevents a sandboxed or ignore syscalls frag from being
* the target of an IB. This is covered by case 5836.
*
* We don't need to re-check in the add_ibl_target_helper because
* the shared/private property applies to all IBT tables -- either
* all are shared or none are.
*/
if (TEST(FRAG_TABLE_SHARED, ibl_table->table_flags) &&
!TEST(FRAG_SHARED, f->flags)) {
STATS_INC(num_ibt_shared_private_conflict);
return f;
}
ASSERT(TEST(FRAG_IS_TRACE, f->flags) ==
TEST(FRAG_TABLE_TRACE, ibl_table->table_flags));
/* We can't assert that an IBL target isn't a trace head due to a race
* between trace head marking and adding to a table. See the comments
* in fragment_add_to_hashtable().
*/
TABLE_RWLOCK(ibl_table, read, lock);
current = hashtable_ibl_lookup(dcontext, (ptr_uint_t)tag, ibl_table);
TABLE_RWLOCK(ibl_table, read, unlock);
/* Now that we set the fragment_t* for any unlinked entry to
* &unlinked_fragment -- regardless of why it was unlinked -- and also
* set the lookup table tag to FAKE_TAG, we should never find a fragment
* with the same tag and should never have an unlinked marker returned
* here.
*/
ASSERT(!IBL_ENTRY_IS_INVALID(current));
if (IBL_ENTRY_IS_EMPTY(current)) {
DOLOG(4, LOG_FRAGMENT, {
dump_lookuptable_tls(dcontext);
hashtable_ibl_dump_table(dcontext, ibl_table);
dump_lookup_table(dcontext, ibl_table);
});
fragment_add_ibl_target_helper(dcontext, f, ibl_table);
/* When using BB2BB IBL w/trace building, we add trace targets
* to the BB table. (We always add a trace target to the trace
* table.) We fool the helper routine into using the BB
* table by passing in a non-trace value for the flags argument.
*/
if (TEST(FRAG_IS_TRACE, f->flags) && DYNAMO_OPTION(bb_ibl_targets) &&
DYNAMO_OPTION(bb_ibt_table_includes_traces)) {
ibl_table_t *ibl_table_too =
GET_IBT_TABLE(pt, f->flags & ~FRAG_IS_TRACE, branch_type);
ASSERT(ibl_table_too != NULL);
ASSERT(!TEST(FRAG_TABLE_TRACE, ibl_table_too->table_flags));
/* Make sure this thread's local ptrs & state is up to
* date in case a resize occurred while it was in the cache. */
update_private_ibt_table_ptrs(dcontext, ibl_table_too
_IF_DEBUG(NULL));
fragment_add_ibl_target_helper(dcontext, f, ibl_table_too);
}
}
else {
DEBUG_DECLARE(const char *reason;)
#ifdef DEBUG
if (is_building_trace(dcontext)) {
reason = "trace building";
STATS_INC(num_ibt_exit_trace_building);
} else if (TEST(FRAG_WAS_DELETED, dcontext->last_fragment->flags)) {
reason = "src unlinked (frag deleted)";
STATS_INC(num_ibt_exit_src_unlinked_frag_deleted);
} else if (!TEST(LINK_LINKED, dcontext->last_exit->flags) &&
TESTALL(FRAG_SHARED | FRAG_IS_TRACE_HEAD,
dcontext->last_fragment->flags) &&
fragment_lookup_type(dcontext,
dcontext->last_fragment->tag,
LOOKUP_TRACE|LOOKUP_SHARED) != NULL) {
/* Another thread unlinked src as part of replacing it with
* a new trace while this thread was in there (see case 5634
* for details) */
reason = "src unlinked (shadowed)";
STATS_INC(num_ibt_exit_src_unlinked_shadowed);
} else if (!INTERNAL_OPTION(ibl_sentinel_check) &&
is_fragment_index_wraparound(dcontext, ibl_table, f)) {
reason = "sentinel";
STATS_INC(num_ibt_leaks_likely_sentinel);
} else if (TEST(FRAG_SELFMOD_SANDBOXED, dcontext->last_fragment->flags)) {
reason = "src sandboxed";
STATS_INC(num_ibt_exit_src_sandboxed);
} else if (TEST(FRAG_TABLE_SHARED, ibl_table->table_flags) &&
orig_lookuptable != ibl_table->table) {
/* A table resize could cause a miss when the target is
* in the new table. */
reason = "shared IBT table resize";
STATS_INC(num_ibt_exit_shared_table_resize);
} else if (DYNAMO_OPTION(bb_ibl_targets) &&
IS_SHARED_SYSCALLS_LINKSTUB(dcontext->last_exit) &&
!DYNAMO_OPTION(disable_traces) &&
!TEST(FRAG_IS_TRACE, f->flags)) {
reason = "shared syscall exit cannot target BBs";
STATS_INC(num_ibt_exit_src_trace_shared_syscall);
} else if (DYNAMO_OPTION(bb_ibl_targets) &&
TEST(FRAG_IS_TRACE, f->flags) &&
!DYNAMO_OPTION(bb_ibt_table_includes_traces)) {
reason = "BBs do not target traces";
STATS_INC(num_ibt_exit_src_trace_shared_syscall);
} else if (!INTERNAL_OPTION(link_ibl)) {
reason = "-no_link_ibl prevents ibl";
STATS_INC(num_ibt_exit_nolink);
} else {
reason = "BAD leak?";
DOLOG(3, LOG_FRAGMENT, {
hashtable_ibl_dump_table(dcontext, ibl_table);
hashtable_ibl_study(dcontext, ibl_table, 0/*table consistent*/);
});
STATS_INC(num_ibt_exit_unknown);
ASSERT_CURIOSITY_ONCE(false && "fragment_add_ibl_target unknown reason");
}
/* nothing to do, just sanity checking */
LOG(THREAD, LOG_FRAGMENT, 2,
"fragment_add_ibl_target tag "PFX", F%d already added - %s\n",
tag, f->id, reason);
#endif
}
} else {
STATS_INC(num_ibt_cold_misses);
}
#ifdef HASHTABLE_STATISTICS
if (INTERNAL_OPTION(stay_on_trace_stats)) {
/* best effort: adjust for 32bit counter overflow occasionally
* we'll get a hashtable leak only when not INTERNAL_OPTION(ibl_sentinel_check)
*/
check_stay_on_trace_stats_overflow(dcontext, branch_type);
}
#endif /* HASHTABLE_STATISTICS */
DOLOG(4, LOG_FRAGMENT, {
dump_lookuptable_tls(dcontext);
});
return f;
}
/**********************************************************************/
/* FUTURE FRAGMENTS */
/* create a new fragment with empty prefix and return it
*/
static future_fragment_t *
fragment_create_future(dcontext_t *dcontext, app_pc tag, uint flags)
{
dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, flags);
future_fragment_t *fut = (future_fragment_t*)
nonpersistent_heap_alloc(alloc_dc, sizeof(future_fragment_t)
HEAPACCT(ACCT_FRAG_FUTURE));
ASSERT(!NEED_SHARED_LOCK(flags) || self_owns_recursive_lock(&change_linking_lock));
LOG(THREAD, LOG_FRAGMENT, 4, "Created future fragment "PFX" w/ flags 0x%08x\n",
tag, flags|FRAG_FAKE|FRAG_IS_FUTURE);
STATS_INC(num_future_fragments);
DOSTATS({
if (TEST(FRAG_SHARED, flags))
STATS_INC(num_shared_future_fragments);
});
fut->tag = tag;
fut->flags = flags | FRAG_FAKE | FRAG_IS_FUTURE;
fut->incoming_stubs = NULL;
return fut;
}
static void
fragment_free_future(dcontext_t *dcontext, future_fragment_t *fut)
{
dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, fut->flags);
LOG(THREAD, LOG_FRAGMENT, 4, "Freeing future fragment "PFX"\n", fut->tag);
ASSERT(fut->incoming_stubs == NULL);
nonpersistent_heap_free(alloc_dc, fut, sizeof(future_fragment_t)
HEAPACCT(ACCT_FRAG_FUTURE));
}
future_fragment_t *
fragment_create_and_add_future(dcontext_t *dcontext, app_pc tag, uint flags)
{
per_thread_t *pt = GET_PT(dcontext);
future_fragment_t *fut = fragment_create_future(dcontext, tag, flags);
fragment_table_t *futtable = GET_FTABLE(pt, fut->flags);
ASSERT(!NEED_SHARED_LOCK(flags) || self_owns_recursive_lock(&change_linking_lock));
/* adding to the table is a write operation */
TABLE_RWLOCK(futtable, write, lock);
fragment_add_to_hashtable(dcontext, (fragment_t *)fut, futtable);
TABLE_RWLOCK(futtable, write, unlock);
return fut;
}
void
fragment_delete_future(dcontext_t *dcontext, future_fragment_t *fut)
{
per_thread_t *pt = GET_PT(dcontext);
fragment_table_t *futtable = GET_FTABLE(pt, fut->flags);
ASSERT(!NEED_SHARED_LOCK(fut->flags) ||
self_owns_recursive_lock(&change_linking_lock));
/* removing from the table is a write operation */
TABLE_RWLOCK(futtable, write, lock);
hashtable_fragment_remove((fragment_t *)fut, futtable);
TABLE_RWLOCK(futtable, write, unlock);
fragment_free_future(dcontext, fut);
}
/* We do not want to remove futures from a flushed region if they have
* incoming links (i#609).
*/
static bool
fragment_delete_future_filter(fragment_t *f)
{
future_fragment_t *fut = (future_fragment_t *) f;
ASSERT(TEST(FRAG_IS_FUTURE, f->flags));
return (fut->incoming_stubs == NULL);
}
static uint
fragment_delete_futures_in_region(dcontext_t *dcontext, app_pc start, app_pc end)
{
per_thread_t *pt = GET_PT(dcontext);
uint flags = FRAG_IS_FUTURE | (dcontext == GLOBAL_DCONTEXT ? FRAG_SHARED : 0);
fragment_table_t *futtable = GET_FTABLE(pt, flags);
uint removed;
/* Higher-level lock needed since we do lookup+add w/o holding table lock between */
ASSERT(!NEED_SHARED_LOCK(flags) || self_owns_recursive_lock(&change_linking_lock));
TABLE_RWLOCK(futtable, write, lock);
removed = hashtable_fragment_range_remove(dcontext, futtable,
(ptr_uint_t)start, (ptr_uint_t)end,
fragment_delete_future_filter);
TABLE_RWLOCK(futtable, write, unlock);
return removed;
}
future_fragment_t *
fragment_lookup_future(dcontext_t *dcontext, app_pc tag)
{
/* default is to lookup shared, since private only sometimes exists,
* and often only care about trace head, for which always use shared
*/
uint flags = SHARED_FRAGMENTS_ENABLED() ? FRAG_SHARED : 0;
per_thread_t *pt = GET_PT(dcontext);
fragment_table_t *futtable = GET_FTABLE(pt, FRAG_IS_FUTURE | flags);
fragment_t *f;
TABLE_RWLOCK(futtable, read, lock);
f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, futtable);
TABLE_RWLOCK(futtable, read, unlock);
if (f != &null_fragment)
return (future_fragment_t *) f;
return NULL;
}
future_fragment_t *
fragment_lookup_private_future(dcontext_t *dcontext, app_pc tag)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
fragment_table_t *futtable = GET_FTABLE(pt, FRAG_IS_FUTURE);
fragment_t *f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, futtable);
if (f != &null_fragment)
return (future_fragment_t *) f;
return NULL;
}
/* END FUTURE FRAGMENTS
**********************************************************************/
#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
/* FIXME: move to rct.c when we move the whole app_pc table there */
#define STATS_RCT_ADD(which, stat, val) DOSTATS({ \
if ((which) == RCT_RAC) \
STATS_ADD(rac_##stat, val); \
else \
STATS_ADD(rct_##stat, val); \
})
static inline bool
rct_is_global_table(rct_module_table_t *permod)
{
return (permod == &rac_non_module_table ||
IF_UNIX_ELSE(permod == &rct_global_table, false));
}
static inline rct_module_table_t *
rct_get_table(app_pc tag, rct_type_t which)
{
rct_module_table_t *permod = os_module_get_rct_htable(tag, which);
if (permod == NULL) { /* not a module */
if (which == RCT_RAC)
permod = &rac_non_module_table;
}
return permod;
}
/* returns NULL if not found */
static app_pc
rct_table_lookup_internal(dcontext_t *dcontext, app_pc tag,
rct_module_table_t *permod)
{
app_pc actag = NULL;
ASSERT(os_get_module_info_locked());
if (permod != NULL) {
/* Check persisted table first as it's likely to be larger and
* it needs no read lock
*/
if (permod->persisted_table != NULL) {
actag = hashtable_app_pc_rlookup(dcontext, (ptr_uint_t)tag,
permod->persisted_table);
}
if (actag == NULL && permod->live_table != NULL) {
actag = hashtable_app_pc_rlookup(dcontext, (ptr_uint_t)tag,
permod->live_table);
}
}
return actag;
}
/* returns NULL if not found */
static app_pc
rct_table_lookup(dcontext_t *dcontext, app_pc tag, rct_type_t which)
{
app_pc actag = NULL;
rct_module_table_t *permod;
ASSERT(which >= 0 && which < RCT_NUM_TYPES);
os_get_module_info_lock();
permod = rct_get_table(tag, which);
actag = rct_table_lookup_internal(dcontext, tag, permod);
os_get_module_info_unlock();
return actag;
}
/* Caller must hold the higher-level lock.
* Returns whether added a new entry or not.
*/
static bool
rct_table_add(dcontext_t *dcontext, app_pc tag, rct_type_t which)
{
rct_module_table_t *permod;
/* we use a higher-level lock to synchronize the lookup + add
* combination with other simultaneous adds as well as with removals
*/
/* FIXME We could use just the table lock for the lookup+add. This is cleaner
* than using another lock during the entire routine and acquiring & releasing
* the table lock in read mode for the lookup and then acquiring & releasing
* it again in write mode for the add. Also, any writes to the table outside
* of this routine would be blocked (as is desired). The down side is that
* reads would be blocked during the entire operation.
* The #ifdef DEBUG lookup would need to be moved to after the table lock
* is released to avoid a rank order violation (all table locks have the
* same rank). That's not problematic since it's only stat code.
*/
/* If we no longer hold this high-level lock for adds+removes we need
* to hold the new add/remove lock across persist_size->persist
*/
ASSERT_OWN_MUTEX(true, (which == RCT_RAC ? &after_call_lock : &rct_module_lock));
os_get_module_info_lock();
permod = rct_get_table(tag, which);
/* Xref case 9717, on a partial image mapping we may try to add locations
* (specifically the entry point) that our outside of any module. Technically this
* is also possible on a full mapping since we've seen entry points redirected
* (and there's nothing requiring that they be re-directed to another dll or, if
* at dr init, that we've already processed that target module, xref case 10693. */
ASSERT_CURIOSITY(permod != NULL || EXEMPT_TEST("win32.partial_map.exe"));
if (permod == NULL || rct_table_lookup_internal(dcontext, tag, permod) != NULL) {
os_get_module_info_unlock();
return false;
}
if (permod->live_table == NULL) {
/* lazily initialized */
if (rct_is_global_table(permod))
SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
permod->live_table = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, app_pc_table_t,
ACCT_AFTER_CALL, PROTECTED);
if (rct_is_global_table(permod)) {
/* For global tables we would have to move to heap, or
* else unprot every time, to maintain min and max: but
* the min-max optimization isn't going to help global
* tables so we just don't bother.
*/
permod->live_min = NULL;
permod->live_max = (app_pc) POINTER_MAX;
SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
}
hashtable_app_pc_init(GLOBAL_DCONTEXT, permod->live_table,
which == RCT_RAC ? INIT_HTABLE_SIZE_AFTER_CALL :
INIT_HTABLE_SIZE_RCT_IBT,
which == RCT_RAC ? DYNAMO_OPTION(shared_after_call_load) :
DYNAMO_OPTION(global_rct_ind_br_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
(SHARED_FRAGMENTS_ENABLED() ?
HASHTABLE_ENTRY_SHARED : 0)
| HASHTABLE_SHARED | HASHTABLE_PERSISTENT
/* I'm seeing a number of high-ave-collision
* cases on both rac and rct; there's no easy
* way to estimate final size, so going to
* relax a little as not perf-critical */
| HASHTABLE_RELAX_CLUSTER_CHECKS
_IF_DEBUG(which == RCT_RAC ? "after_call_targets" :
"rct_ind_targets"));
STATS_RCT_ADD(which, live_tables, 1);
}
ASSERT(permod->live_table != NULL);
/* adding is a write operation */
TABLE_RWLOCK(permod->live_table, write, lock);
hashtable_app_pc_add(dcontext, tag, permod->live_table);
TABLE_RWLOCK(permod->live_table, write, unlock);
/* case 7628: used for persistence optimization: but watch overhead */
if (!rct_is_global_table(permod)) {
/* See comments above */
if (permod->live_min == NULL || tag < permod->live_min)
permod->live_min = tag;
if (tag > permod->live_max)
permod->live_max = tag;
}
os_get_module_info_unlock();
STATS_RCT_ADD(which, live_entries, 1);
DOSTATS({
if (permod == &rac_non_module_table)
STATS_INC(rac_non_module_entries);
});
return true;
}
static void
rct_table_flush_entry(dcontext_t *dcontext, app_pc tag, rct_type_t which)
{
rct_module_table_t *permod;
/* need higher level lock to properly synchronize with lookup+add */
ASSERT_OWN_MUTEX(true, (which == RCT_RAC ? &after_call_lock : &rct_module_lock));
os_get_module_info_lock();
permod = rct_get_table(tag, which);
ASSERT(permod != NULL);
/* We should have removed any persist info before calling this routine */
ASSERT(permod->persisted_table == NULL);
ASSERT(permod->live_table != NULL);
if (permod->live_table != NULL) {
/* removing is a write operation */
TABLE_RWLOCK(permod->live_table, write, lock);
hashtable_app_pc_remove(tag, permod->live_table);
TABLE_RWLOCK(permod->live_table, write, unlock);
}
os_get_module_info_unlock();
}
/* Invalidates all after call or indirect branch targets from given
* range [text_start,text_end) which must be either completely
* contained in a single module or not touch any modules.
* Assuming any existing fragments that were added to IBL tables will
* be flushed independently: this routine only flushes the policy
* information.
* Note this needs to be called on app_memory_deallocation() for RAC
* (potentially from DGC), and rct_process_module_mmap() for other RCT
* entries which should be only in modules.
*
* Returns entries flushed.
*/
static uint
rct_table_invalidate_range(dcontext_t *dcontext, rct_type_t which,
app_pc text_start, app_pc text_end)
{
uint entries_removed = 0;
rct_module_table_t *permod;
/* need higher level lock to properly synchronize with lookup+add */
ASSERT_OWN_MUTEX(true, (which == RCT_RAC ? &after_call_lock : &rct_module_lock));
ASSERT(text_start < text_end);
if (DYNAMO_OPTION(rct_sticky)) {
/* case 5329 - leaving for bug-compatibility with previous releases */
/* trade-off is spurious RCT violations vs memory leak */
return 0;
}
/* We only support removing from within a single module or not touching
* any modules */
ASSERT(get_module_base(text_start) == get_module_base(text_end));
os_get_module_info_lock();
permod = rct_get_table(text_start, which);
ASSERT(permod != NULL);
/* We should have removed any persist info before calling this routine */
ASSERT(permod->persisted_table == NULL);
ASSERT(permod->live_table != NULL);
if (permod != NULL && permod->live_table != NULL) {
TABLE_RWLOCK(permod->live_table, write, lock);
entries_removed =
hashtable_app_pc_range_remove(dcontext, permod->live_table,
(ptr_uint_t)text_start, (ptr_uint_t)text_end,
NULL);
DOCHECK(1, {
uint second_pass =
hashtable_app_pc_range_remove(dcontext, permod->live_table,
(ptr_uint_t)text_start,
(ptr_uint_t)text_end, NULL);
ASSERT(second_pass == 0 && "nothing should be missed");
/* simplest sanity check that hashtable_app_pc_range_remove() works */
});
TABLE_RWLOCK(permod->live_table, write, unlock);
}
os_get_module_info_unlock();
return entries_removed;
}
static void
rct_table_free_internal(dcontext_t *dcontext, app_pc_table_t *table)
{
hashtable_app_pc_free(dcontext, table);
ASSERT(TEST(HASHTABLE_PERSISTENT, table->table_flags));
HEAP_TYPE_FREE(dcontext, table, app_pc_table_t, ACCT_AFTER_CALL, PROTECTED);
}
void
rct_table_free(dcontext_t *dcontext, app_pc_table_t *table, bool free_data)
{
DODEBUG({
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_app_pc_load_statistics(dcontext, table);
});
hashtable_app_pc_study(dcontext, table, 0/*table consistent*/);
});
if (!free_data) {
/* We don't need the free_data param anymore */
ASSERT(table->table_unaligned == NULL); /* don't try to free, part of mmap */
}
rct_table_free_internal(GLOBAL_DCONTEXT, table);
}
app_pc_table_t *
rct_table_copy(dcontext_t *dcontext, app_pc_table_t *src)
{
if (src == NULL)
return NULL;
else
return hashtable_app_pc_copy(GLOBAL_DCONTEXT, src);
}
app_pc_table_t *
rct_table_merge(dcontext_t *dcontext, app_pc_table_t *src1, app_pc_table_t *src2)
{
if (src1 == NULL) {
if (src2 == NULL)
return NULL;
return hashtable_app_pc_copy(GLOBAL_DCONTEXT, src2);
} else if (src2 == NULL)
return hashtable_app_pc_copy(GLOBAL_DCONTEXT, src1);
else
return hashtable_app_pc_merge(GLOBAL_DCONTEXT, src1, src2);
}
/* Up to caller to synchronize access to table. */
uint
rct_table_persist_size(dcontext_t *dcontext, app_pc_table_t *table)
{
/* Don't persist zero-entry tables */
if (table == NULL || table->entries == 0)
return 0;
else
return hashtable_app_pc_persist_size(dcontext, table);
}
/* Up to caller to synchronize access to table.
* Returns true iff all writes succeeded.
*/
bool
rct_table_persist(dcontext_t *dcontext, app_pc_table_t *table, file_t fd)
{
bool success = true;
ASSERT(fd != INVALID_FILE);
ASSERT(table != NULL); /* caller shouldn't call us o/w */
if (table != NULL)
success = hashtable_app_pc_persist(dcontext, table, fd);
return success;
}
app_pc_table_t *
rct_table_resurrect(dcontext_t *dcontext, byte *mapped_table, rct_type_t which)
{
return hashtable_app_pc_resurrect(GLOBAL_DCONTEXT, mapped_table
_IF_DEBUG(which == RCT_RAC ? "after_call_targets" :
"rct_ind_targets"));
}
void
rct_module_table_free(dcontext_t *dcontext, rct_module_table_t *permod, app_pc modpc)
{
ASSERT(os_get_module_info_locked());
if (permod->live_table != NULL) {
rct_table_free(GLOBAL_DCONTEXT, permod->live_table, true);
permod->live_table = NULL;
}
if (permod->persisted_table != NULL) {
/* persisted table: table data is from disk, but header is on heap */
rct_table_free(GLOBAL_DCONTEXT, permod->persisted_table, false);
permod->persisted_table = NULL;
/* coarse_info_t has a duplicated pointer to the persisted table,
* but it should always be flushed before we get here
*/
ASSERT(get_executable_area_coarse_info(modpc) == NULL);
}
}
void
rct_module_table_persisted_invalidate(dcontext_t *dcontext, app_pc modpc)
{
rct_module_table_t *permod;
uint i;
os_get_module_info_lock();
for (i = 0; i < RCT_NUM_TYPES; i++) {
permod = rct_get_table(modpc, i);
ASSERT(permod != NULL);
if (permod != NULL && permod->persisted_table != NULL) {
/* If the persisted table contains entries beyond what we will discover
* when we re-build its cache we must transfer those to the live table
* now. At first I was only keeping entire-module RCT entries, but we
* must keep all RAC and RCT entries since we may not see the triggering
* code before we hit the check point (may reset at a ret so we won't see
* the call before the ret check; same for Borland SEH). We can only not
* keep them on a module unload.
*/
/* Optimization: don't transfer if about to unload. This assumes
* there will be no persistence of a later coarse unit in a different
* region of the same module, which case 9651 primary_for_module
* currently ensures! (We already have read lock; ok to grab again.)
*/
if (!os_module_get_flag(modpc, MODULE_BEING_UNLOADED) && !dynamo_exited) {
/* FIXME case 10362: we could leave the file mapped in and
* use the persisted RCT table independently of the cache
*/
/* Merging will remove any dups, though today we never re-load
* pcaches so there shouldn't be any
*/
app_pc_table_t *merged =
/* all modinfo RCT tables are on global heap */
rct_table_merge(GLOBAL_DCONTEXT, permod->live_table,
permod->persisted_table);
if (permod->live_table != NULL)
rct_table_free(GLOBAL_DCONTEXT, permod->live_table, true);
permod->live_table = merged;
LOG(THREAD, LOG_FRAGMENT, 2,
"rct_module_table_persisted_invalidate "PFX": not unload, so "
"moving persisted %d entries to live table\n",
modpc, permod->persisted_table->entries);
# ifdef WINDOWS
/* We leave the MODULE_RCT_LOADED flag */
# endif
STATS_INC(rct_persisted_outlast_cache);
}
/* we rely on coarse_unit_reset_free() freeing the persisted table struct */
permod->persisted_table = NULL;
}
}
os_get_module_info_unlock();
}
/* Produces a new hashtable that contains all entries in the live and persisted
* tables for the module containing modpc that are within [limit_start, limit_end)
*/
app_pc_table_t *
rct_module_table_copy(dcontext_t *dcontext, app_pc modpc, rct_type_t which,
app_pc limit_start, app_pc limit_end)
{
app_pc_table_t *merged = NULL;
rct_module_table_t *permod;
mutex_t *lock = (which == RCT_RAC) ? &after_call_lock : &rct_module_lock;
mutex_lock(lock);
if (which == RCT_RAC) {
ASSERT(DYNAMO_OPTION(ret_after_call));
if (!DYNAMO_OPTION(ret_after_call))
return NULL;
} else {
ASSERT(TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump)));
if (!TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) &&
!TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump)))
return NULL;
}
os_get_module_info_lock();
permod = rct_get_table(modpc, which);
ASSERT(permod != NULL);
if (permod != NULL) {
/* FIXME: we could pass the limit range down to hashtable_app_pc_{copy,merge}
* for more efficiency and to avoid over-sizing the table, but this
* should be rare w/ -persist_rct_entire and single-+x-section modules
*/
merged = rct_table_merge(dcontext, permod->live_table, permod->persisted_table);
if (merged != NULL) {
DEBUG_DECLARE(uint removed = 0;)
TABLE_RWLOCK(merged, write, lock);
if (limit_start > permod->live_min) {
DEBUG_DECLARE(removed +=)
hashtable_app_pc_range_remove(dcontext, merged,
(ptr_uint_t)permod->live_min,
(ptr_uint_t)limit_start, NULL);
}
if (limit_end <= permod->live_max) {
DEBUG_DECLARE(removed +=)
hashtable_app_pc_range_remove(dcontext, merged,
(ptr_uint_t)limit_end,
(ptr_uint_t)permod->live_max+1, NULL);
}
TABLE_RWLOCK(merged, write, unlock);
STATS_RCT_ADD(which, module_persist_out_of_range, removed);
}
}
os_get_module_info_unlock();
mutex_unlock(lock);
return merged;
}
/* We return the persisted table so we can keep a pointer to it in the
* loaded coarse_info_t, but we must be careful to do a coordinated
* free of the duplicated pointer.
*/
bool
rct_module_table_set(dcontext_t *dcontext, app_pc modpc, app_pc_table_t *table,
rct_type_t which)
{
rct_module_table_t *permod;
bool used = false;
mutex_t *lock = (which == RCT_RAC) ? &after_call_lock : &rct_module_lock;
mutex_lock(lock);
os_get_module_info_lock();
permod = rct_get_table(modpc, which);
ASSERT(permod != NULL);
ASSERT(permod->persisted_table == NULL); /* can't resurrect twice */
ASSERT(table != NULL);
/* Case 9834: avoid double-add from earlier entire-module resurrect */
ASSERT(which == RCT_RAC || !os_module_get_flag(modpc, MODULE_RCT_LOADED));
/* FIXME case 8648: we're loosening security by allowing ret to target any
* after-call executed in any prior run of this app or whatever
* app is producing, instead of just this run.
*/
if (permod != NULL && permod->persisted_table == NULL) {
used = true;
/* There could be dups in persisted table that are already in
* live table (particularly from unloading and re-loading a pcache,
* though that never happens today). Everything should work fine,
* and if we do unload the pcache prior to module unload the merge
* into the live entries will remove the dups.
*/
permod->persisted_table = table;
ASSERT(permod->persisted_table->entries > 0);
/* FIXME: for case 9639 if we had the ibl table set up (say, for shared
* ibl tables) and stored the cache pc (though I guess coarse htable is
* set up) we could fill the ibl table here as well (but then we'd
* have to abort for hotp conflicts earlier in coarse_unit_load()).
* Instead we delay and do it in rac_persisted_fill_ibl().
*/
LOG(THREAD, LOG_FRAGMENT, 2,
"rct_module_table_resurrect: added %d %s entries\n",
permod->persisted_table->entries, which == RCT_RAC ? "RAC" : "RCT");
STATS_RCT_ADD(which, persisted_tables, 1);
STATS_RCT_ADD(which, persisted_entries, permod->persisted_table->entries);
}
os_get_module_info_unlock();
mutex_unlock(lock);
return used;
}
bool
rct_module_persisted_table_exists(dcontext_t *dcontext, app_pc modpc,
rct_type_t which)
{
bool exists = false;
rct_module_table_t *permod;
os_get_module_info_lock();
permod = rct_get_table(modpc, which);
exists = (permod != NULL && permod->persisted_table != NULL);
os_get_module_info_unlock();
return exists;
}
uint
rct_module_live_entries(dcontext_t *dcontext, app_pc modpc, rct_type_t which)
{
uint entries = 0;
rct_module_table_t *permod;
os_get_module_info_lock();
permod = rct_get_table(modpc, which);
if (permod != NULL && permod->live_table != NULL)
entries = permod->live_table->entries;
os_get_module_info_unlock();
return entries;
}
static void
coarse_persisted_fill_ibl_helper(dcontext_t *dcontext, ibl_table_t *ibl_table,
coarse_info_t *info, app_pc_table_t *ptable,
ibl_branch_type_t branch_type)
{
uint i;
fragment_t wrapper;
app_pc tag;
cache_pc body_pc;
DEBUG_DECLARE(uint added = 0;)
ASSERT(ptable != NULL);
if (ptable == NULL)
return;
ASSERT(os_get_module_info_locked());
/* Make sure this thread's local ptrs & state are up to
* date in case a resize occurred while it was in the cache. */
update_private_ibt_table_ptrs(dcontext, ibl_table _IF_DEBUG(NULL));
/* Avoid hash collision asserts while adding by sizing up front;
* FIXME: we may over-size for INDJMP table
*/
TABLE_RWLOCK(ibl_table, write, lock);
hashtable_ibl_check_size(dcontext, ibl_table, 0, ptable->entries);
TABLE_RWLOCK(ibl_table, write, unlock);
/* FIXME: we should hold ptable's read lock but it's lower ranked
* than the fragment table's lock, so we rely on os module lock
*/
for (i = 0; i < ptable->capacity; i++) {
tag = ptable->table[i];
if (APP_PC_ENTRY_IS_REAL(tag)) {
/* FIXME: should we persist the cache pcs to save time here? That won't
* be ideal if we ever use the mmapped table directly (for per-module
* tables: case 9672). We could support both tag-only and tag-cache
* pairs by having the 1st entry be a flags word.
*/
fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &body_pc);
/* may not be present, given no checks in rct_entries_in_region() */
if (body_pc != NULL &&
/* We can have same entry in both RAC and RCT table, and we use
* both tables to fill the INDJMP table
*/
(branch_type != IBL_INDJMP ||
!IBL_ENTRY_IS_EMPTY(hashtable_ibl_lookup(dcontext, (ptr_uint_t)tag,
ibl_table)))) {
fragment_coarse_wrapper(&wrapper, tag, body_pc);
fragment_add_ibl_target_helper(dcontext, &wrapper, ibl_table);
DOSTATS({ added++; });
}
}
}
LOG(THREAD, LOG_FRAGMENT, 2,
"coarse_persisted_fill_ibl %s: added %d of %d entries\n",
get_branch_type_name(branch_type), added, ptable->entries);
STATS_ADD(perscache_ibl_prefill, added);
}
/* Case 9639: fill ibl table from persisted RAC/RCT table entries */
static void
coarse_persisted_fill_ibl(dcontext_t *dcontext, coarse_info_t *info,
ibl_branch_type_t branch_type)
{
per_thread_t *pt = GET_PT(dcontext);
ibl_table_t *ibl_table;
rct_module_table_t *permod;
/* Caller must hold info lock */
ASSERT_OWN_MUTEX(true, &info->lock);
ASSERT(exists_coarse_ibl_pending_table(dcontext, info, branch_type));
ASSERT(TEST(COARSE_FILL_IBL_MASK(branch_type),
DYNAMO_OPTION(coarse_fill_ibl)));
if (!exists_coarse_ibl_pending_table(dcontext, info, branch_type))
return;
os_get_module_info_lock();
ibl_table = GET_IBT_TABLE(pt, FRAG_SHARED|FRAG_COARSE_GRAIN, branch_type);
if (branch_type == IBL_RETURN || branch_type == IBL_INDJMP) {
permod = rct_get_table(info->base_pc, RCT_RAC);
ASSERT(permod != NULL && permod->persisted_table != NULL);
if (permod != NULL && permod->persisted_table != NULL) {
LOG(THREAD, LOG_FRAGMENT, 2,
"coarse_persisted_fill_ibl %s: adding RAC %d entries\n",
get_branch_type_name(branch_type), permod->persisted_table->entries);
coarse_persisted_fill_ibl_helper(dcontext, ibl_table, info,
permod->persisted_table, branch_type);
}
}
if (branch_type == IBL_INDCALL || branch_type == IBL_INDJMP) {
permod = rct_get_table(info->base_pc, RCT_RCT);
ASSERT(permod != NULL && permod->persisted_table != NULL);
if (permod != NULL && permod->persisted_table != NULL) {
LOG(THREAD, LOG_FRAGMENT, 2,
"coarse_persisted_fill_ibl %s: adding RCT %d entries\n",
get_branch_type_name(branch_type), permod->persisted_table->entries);
coarse_persisted_fill_ibl_helper(dcontext, ibl_table, info,
permod->persisted_table, branch_type);
}
}
os_get_module_info_unlock();
/* We only fill for the 1st thread (if using per-thread ibl
* tables). We'd need per-thread flags to do otherwise, and the
* goal is only to help startup performance.
* FIXME case 9639: later threads may do startup work in various apps,
* and we may want a better solution here.
*/
info->ibl_pending_used |= COARSE_FILL_IBL_MASK(branch_type);
}
#endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
#ifdef RETURN_AFTER_CALL
/* returns NULL if not found */
app_pc
fragment_after_call_lookup(dcontext_t *dcontext, app_pc tag)
{
return rct_table_lookup(dcontext, tag, RCT_RAC);
}
void
fragment_add_after_call(dcontext_t *dcontext, app_pc tag)
{
mutex_lock(&after_call_lock);
if (!rct_table_add(dcontext, tag, RCT_RAC))
STATS_INC(num_existing_after_call);
else
STATS_INC(num_future_after_call);
mutex_unlock(&after_call_lock);
}
/* flushing a fragment invalidates the after call entry */
void
fragment_flush_after_call(dcontext_t *dcontext, app_pc tag)
{
mutex_lock(&after_call_lock);
rct_table_flush_entry(dcontext, tag, RCT_RAC);
mutex_unlock(&after_call_lock);
STATS_INC(num_future_after_call_removed);
STATS_DEC(num_future_after_call);
}
/* see comments in rct_table_invalidate_range() */
uint
invalidate_after_call_target_range(dcontext_t *dcontext,
app_pc text_start, app_pc text_end)
{
uint entries_removed;
mutex_lock(&after_call_lock);
entries_removed =
rct_table_invalidate_range(dcontext, RCT_RAC, text_start, text_end);
mutex_unlock(&after_call_lock);
STATS_ADD(num_future_after_call_removed, entries_removed);
STATS_SUB(num_future_after_call, entries_removed);
LOG(THREAD, LOG_FRAGMENT, 2,
"invalidate_rct_target_range "PFX"-"PFX": removed %d entries\n",
text_start, text_end, entries_removed);
return entries_removed;
}
#endif /* RETURN_AFTER_CALL */
/***********************************************************************/
#ifdef RCT_IND_BRANCH
/*
* RCT indirect branch policy bookkeeping. Mostly a set of wrappers
* around the basic hashtable functionality.
*
* FIXME: all of these routines should be moved to rct.c after we move
* the hashtable primitives to fragment.h as static inline's
*/
/* returns NULL if not found */
app_pc
rct_ind_branch_target_lookup(dcontext_t *dcontext, app_pc tag)
{
return rct_table_lookup(dcontext, tag, RCT_RCT);
}
/* returns true if a new entry for target was added,
* or false if target was already known
*/
/* Note - entries are expected to be within MEM_IMAGE */
bool
rct_add_valid_ind_branch_target(dcontext_t *dcontext, app_pc tag)
{
ASSERT_OWN_MUTEX(true, &rct_module_lock);
DOLOG(2, LOG_FRAGMENT, {
/* FIXME: would be nice to add a heavy weight check that we're
* really only a PE IMAGE via is_in_code_section()
*/
});
if (!rct_table_add(dcontext, tag, RCT_RCT))
return false;
else {
STATS_INC(rct_ind_branch_entries);
return true; /* new entry */
}
}
/* invalidate an indirect branch target and free any associated memory */
/* FIXME: note that this is currently not used and
* invalidate_ind_branch_target_range() will be the likely method to
* use for most cases when a whole module range is invalidated.
*/
void
rct_flush_ind_branch_target_entry(dcontext_t *dcontext, app_pc tag)
{
ASSERT_OWN_MUTEX(true, &rct_module_lock); /* synch with adding */
rct_table_flush_entry(dcontext, tag, RCT_RCT);
STATS_DEC(rct_ind_branch_entries);
STATS_INC(rct_ind_branch_entries_removed);
}
/* see comments in rct_table_invalidate_range() */
uint
invalidate_ind_branch_target_range(dcontext_t *dcontext,
app_pc text_start, app_pc text_end)
{
uint entries_removed;
ASSERT_OWN_MUTEX(true, &rct_module_lock); /* synch with adding */
entries_removed =
rct_table_invalidate_range(dcontext, RCT_RCT, text_start, text_end);
STATS_ADD(rct_ind_branch_entries_removed, entries_removed);
STATS_SUB(rct_ind_branch_entries, entries_removed);
return entries_removed;
}
#endif /* RCT_IND_BRANCH */
/****************************************************************************/
/* CACHE CONSISTENCY */
/* Handle exits from the cache from our self-modifying code sandboxing
* instrumentation.
*/
void
fragment_self_write(dcontext_t *dcontext)
{
ASSERT(!is_self_couldbelinking());
/* need to delete just this fragment, then start interpreting
* at the instr after the self-write instruction
*/
dcontext->next_tag = EXIT_TARGET_TAG(dcontext, dcontext->last_fragment,
dcontext->last_exit);
LOG(THREAD, LOG_ALL, 2, "Sandboxing exit from fragment "PFX" @"PFX"\n",
dcontext->last_fragment->tag,
EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit));
LOG(THREAD, LOG_ALL, 2, "\tset next_tag to "PFX"\n", dcontext->next_tag);
/* We come in here both for actual selfmod and for exec count thresholds,
* to avoid needing separate LINK_ flags.
*/
if (DYNAMO_OPTION(sandbox2ro_threshold) > 0) {
if (vm_area_selfmod_check_clear_exec_count(dcontext, dcontext->last_fragment)) {
/* vm_area_* deleted this fragment by flushing so nothing more to do */
return;
}
}
LOG(THREAD, LOG_ALL, 1, "WARNING: fragment "PFX" @"PFX" overwrote its own code\n",
dcontext->last_fragment->tag,
EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit));
STATS_INC(num_self_writes);
if (TEST(FRAG_WAS_DELETED, dcontext->last_fragment->flags)) {
/* Case 8177: case 3559 unionized fragment_t.in_xlate, so we cannot delete a
* fragment that has already been unlinked in the first stage of a flush.
* The flush queue check, which comes after this (b/c we want to be
* nolinking), will delete.
*/
ASSERT(((per_thread_t *) dcontext->fragment_field)->flush_queue_nonempty);
STATS_INC(num_self_writes_after_flushes);
} else {
#ifdef PROGRAM_SHEPHERDING
/* we can't call fragment_delete if vm_area_t deletes it by flushing */
if (!vm_area_fragment_self_write(dcontext, dcontext->last_fragment->tag))
#endif
{
fragment_delete(dcontext, dcontext->last_fragment, FRAGDEL_ALL);
STATS_INC(num_fragments_deleted_selfmod);
}
}
}
/* coarse_grain says to use just the min_pc and max_pc of f.
* otherwise a full walk of the original code is done to see if
* any piece of the fragment really does overlap.
* returns the tag of the bb that actually overlaps (i.e., finds the
* component bb of a trace that does the overlapping).
*/
bool
fragment_overlaps(dcontext_t *dcontext, fragment_t *f,
byte *region_start, byte *region_end, bool coarse_grain,
overlap_info_t *info_res, app_pc *bb_tag)
{
overlap_info_t info;
info.overlap = false;
if ((f->flags & FRAG_IS_TRACE) != 0) {
uint i;
trace_only_t *t = TRACE_FIELDS(f);
/* look through all blocks making up the trace */
ASSERT(t->bbs != NULL);
/* trace should have at least one bb */
ASSERT(t->num_bbs > 0);
for (i=0; i<t->num_bbs; i++) {
if (app_bb_overlaps(dcontext, t->bbs[i].tag, f->flags,
region_start, region_end, &info)) {
if (bb_tag != NULL)
*bb_tag = t->bbs[i].tag;
break;
}
}
} else {
app_bb_overlaps(dcontext, f->tag, f->flags,
region_start, region_end, &info);
if (info.overlap && bb_tag != NULL)
*bb_tag = f->tag;
}
if (info_res != NULL)
*info_res = info;
return info.overlap;
}
#ifdef DEBUG
void
study_all_hashtables(dcontext_t *dcontext)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
if (!DYNAMO_OPTION(disable_traces)) {
per_thread_t *ibl_pt = pt;
if (DYNAMO_OPTION(shared_trace_ibt_tables))
ibl_pt = shared_pt;
hashtable_ibl_study(dcontext, &ibl_pt->trace_ibt[branch_type],
0/*table consistent*/);
}
if (DYNAMO_OPTION(bb_ibl_targets)) {
per_thread_t *ibl_pt = pt;
if (DYNAMO_OPTION(shared_bb_ibt_tables))
ibl_pt = shared_pt;
hashtable_ibl_study(dcontext, &ibl_pt->bb_ibt[branch_type],
0/*table consistent*/);
}
}
if (PRIVATE_TRACES_ENABLED())
hashtable_fragment_study(dcontext, &pt->trace, 0/*table consistent*/);
hashtable_fragment_study(dcontext, &pt->bb, 0/*table consistent*/);
hashtable_fragment_study(dcontext, &pt->future, 0/*table consistent*/);
if (DYNAMO_OPTION(shared_bbs))
hashtable_fragment_study(dcontext, shared_bb, 0/*table consistent*/);
if (DYNAMO_OPTION(shared_traces))
hashtable_fragment_study(dcontext, shared_trace, 0/*table consistent*/);
if (SHARED_FRAGMENTS_ENABLED())
hashtable_fragment_study(dcontext, shared_future, 0/*table consistent*/);
# ifdef RETURN_AFTER_CALL
if (dynamo_options.ret_after_call && rac_non_module_table.live_table != NULL) {
hashtable_app_pc_study(dcontext, rac_non_module_table.live_table,
0/*table consistent*/);
}
# endif
# if defined(RCT_IND_BRANCH) && defined(UNIX)
if ((TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
rct_global_table.live_table != NULL) {
hashtable_app_pc_study(dcontext, rct_global_table.live_table,
0/*table consistent*/);
}
# endif /* RCT_IND_BRANCH */
# if defined(WIN32) && (defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH))
{
module_iterator_t *mi = module_iterator_start();
uint i;
rct_module_table_t *permod;
while (module_iterator_hasnext(mi)) {
module_area_t *data = module_iterator_next(mi);
for (i = 0; i < RCT_NUM_TYPES; i++) {
permod = os_module_get_rct_htable(data->start, i);
ASSERT(permod != NULL);
if (permod->persisted_table != NULL) {
LOG(THREAD, LOG_FRAGMENT, 2,
"%s persisted hashtable for %s "PFX"-"PFX"\n",
i == RCT_RAC ? "RAC" : "RCT",
GET_MODULE_NAME(&data->names), data->start, data->end);
hashtable_app_pc_study(dcontext, permod->persisted_table,
0/*table consistent*/);
}
if (permod->live_table != NULL) {
LOG(THREAD, LOG_FRAGMENT, 2,
"%s live hashtable for %s "PFX"-"PFX"\n",
i == RCT_RAC ? "RAC" : "RCT",
GET_MODULE_NAME(&data->names), data->start, data->end);
hashtable_app_pc_study(dcontext, permod->live_table,
0/*table consistent*/);
}
}
}
module_iterator_stop(mi);
}
# endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
}
#endif /* DEBUG */
/****************************************************************************
* FLUSHING
*
* Two-stage freeing of fragments via immediate unlinking followed by lazy
* deletion.
*/
uint
get_flushtime_last_update(dcontext_t *dcontext)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
return pt->flushtime_last_update;
}
void
set_flushtime_last_update(dcontext_t *dcontext, uint val)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
pt->flushtime_last_update = val;
}
void
set_at_syscall(dcontext_t *dcontext, bool val)
{
ASSERT(dcontext != GLOBAL_DCONTEXT);
dcontext->upcontext_ptr->at_syscall = val;
}
bool
get_at_syscall(dcontext_t *dcontext)
{
ASSERT(dcontext != GLOBAL_DCONTEXT);
return dcontext->upcontext_ptr->at_syscall;
}
/* Assumes caller takes care of synchronization.
* Returns false iff was_I_flushed ends up being deleted right now from
* a private cache OR was_I_flushed has been flushed from a shared cache
* and is pending final deletion.
*/
static bool
check_flush_queue(dcontext_t *dcontext, fragment_t *was_I_flushed)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
bool not_flushed = true;
ASSERT_OWN_MUTEX(true, &pt->linking_lock);
/* first check private queue and act on pending deletions */
if (pt->flush_queue_nonempty) {
bool local_prot = local_heap_protected(dcontext);
if (local_prot)
SELF_PROTECT_LOCAL(dcontext, WRITABLE);
/* remove local vm areas on t->Q queue and all frags in their lists */
not_flushed = not_flushed &&
vm_area_flush_fragments(dcontext, was_I_flushed);
pt->flush_queue_nonempty = false;
LOG(THREAD, LOG_FRAGMENT, 2, "Hashtable state after flushing the queue:\n");
DOLOG(2, LOG_FRAGMENT, {
study_all_hashtables(dcontext);
});
if (local_prot)
SELF_PROTECT_LOCAL(dcontext, READONLY);
}
/* now check shared queue to dec ref counts */
if (DYNAMO_OPTION(shared_deletion) &&
/* No lock needed: any racy incs to global are in safe direction, and our inc
* is atomic so we shouldn't see any partial-word-updated values here. This
* check is our shared deletion algorithm's only perf hit when there's no
* actual shared flushing.
*/
pt->flushtime_last_update < flushtime_global) {
/* dec ref count on any pending shared areas */
not_flushed = not_flushed &&
vm_area_check_shared_pending(dcontext, was_I_flushed);
/* Remove unlinked markers if called for.
* FIXME If a thread's flushtime is updated due to shared syscall sync,
* its tables won't be rehashed here -- the thread's flushtime will be
* equal to the global flushtime so the 'if' isn't entered. We have
* multiple options as to other points for rehashing -- a table add, a
* table delete, any entry into DR. For now, we've chosen a table add
* (see check_table_size()).
*/
if (SHARED_IB_TARGETS() &&
(INTERNAL_OPTION(rehash_unlinked_threshold) < 100 ||
INTERNAL_OPTION(rehash_unlinked_always))) {
ibl_branch_type_t branch_type;
for (branch_type = IBL_BRANCH_TYPE_START;
branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
ibl_table_t *table = &pt->bb_ibt[branch_type];
if (table->unlinked_entries > 0 &&
(INTERNAL_OPTION(rehash_unlinked_threshold) <
(100 * table->unlinked_entries /
(table->unlinked_entries + table->entries)) ||
INTERNAL_OPTION(rehash_unlinked_always))) {
STATS_INC(num_ibt_table_rehashes);
LOG(THREAD, LOG_FRAGMENT, 1,
"Rehash table %s: linked %u, unlinked %u\n",
table->name, table->entries, table->unlinked_entries);
hashtable_ibl_unlinked_remove(dcontext, table);
}
}
}
}
/* FIXME This is the ideal location for inserting refcounting logic
* for freeing a resized shared IBT table, as is done for shared
* deletion above.
*/
return not_flushed;
}
/* Note that an all-threads-synch flush does NOT set the self-flushing flag,
* so use is_self_allsynch_flushing() instead.
*/
bool
is_self_flushing()
{
/* race condition w/ flusher being updated -- but since only testing vs self,
* if flusher update is atomic, should be safe
*/
return (get_thread_private_dcontext() == flusher);
}
bool
is_self_allsynch_flushing()
{
/* race condition w/ allsynch_flusher being updated -- but since only testing
* vs self, if flusher update is atomic, should be safe
*/
return (allsynch_flusher != NULL &&
get_thread_private_dcontext() == allsynch_flusher);
}
/* N.B.: only accurate if called on self (else a race condition) */
bool
is_self_couldbelinking()
{
dcontext_t *dcontext = get_thread_private_dcontext();
/* if no dcontext yet then can't be couldbelinking */
return (dcontext != NULL &&
!RUNNING_WITHOUT_CODE_CACHE() /*case 7966: has no pt*/ &&
is_couldbelinking(dcontext));
}
/* N.B.: can only call if target thread is self, suspended, or waiting for flush */
bool
is_couldbelinking(dcontext_t *dcontext)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
/* the lock is only needed for self writing or another thread reading
* but if different thread we require thread is suspended or waiting for
* flush so is ok */
/* FIXME : add an assert that the thread that owns the dcontext is either
* the caller, at the flush sync wait, or is suspended by thread_synch
* routines */
return (!RUNNING_WITHOUT_CODE_CACHE() /*case 7966: has no pt*/ &&
pt != NULL/*PR 536058: no pt*/ &&
pt->could_be_linking);
}
static void
wait_for_flusher_nolinking(dcontext_t *dcontext)
{
/* FIXME: can have livelock w/ these types of synch loops,
* any way to work into deadlock-avoidance?
*/
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
ASSERT(!pt->could_be_linking);
while (pt->wait_for_unlink) {
LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
"Thread "TIDFMT" waiting for flush (flusher is %d @flushtime %d)\n",
/* safe to deref flusher since flusher is waiting for our signal */
dcontext->owning_thread, flusher->owning_thread, flushtime_global);
mutex_unlock(&pt->linking_lock);
STATS_INC(num_wait_flush);
wait_for_event(pt->finished_all_unlink);
LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
"Thread "TIDFMT" resuming after flush\n", dcontext->owning_thread);
mutex_lock(&pt->linking_lock);
}
}
static void
wait_for_flusher_linking(dcontext_t *dcontext)
{
/* FIXME: can have livelock w/ these types of synch loops,
* any way to work into deadlock-avoidance?
*/
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
ASSERT(pt->could_be_linking);
while (pt->wait_for_unlink) {
LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
"Thread "TIDFMT" waiting for flush (flusher is %d @flushtime %d)\n",
/* safe to deref flusher since flusher is waiting for our signal */
dcontext->owning_thread, flusher->owning_thread, flushtime_global);
mutex_unlock(&pt->linking_lock);
signal_event(pt->waiting_for_unlink);
STATS_INC(num_wait_flush);
wait_for_event(pt->finished_with_unlink);
LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
"Thread "TIDFMT" resuming after flush\n", dcontext->owning_thread);
mutex_lock(&pt->linking_lock);
}
}
#ifdef DEBUG
static void
check_safe_for_flush_synch(dcontext_t *dcontext)
{
/* We cannot hold any locks at synch points that wait for flushers, as we
* could prevent forward progress of a couldbelinking thread that the
* flusher will wait for.
*/
/* FIXME: will fail w/ -single_thread_in_DR, along w/ the other similar
* asserts for flushing and cache entering
*/
# ifdef DEADLOCK_AVOIDANCE
ASSERT(thread_owns_no_locks(dcontext) ||
/* if thread exits while a trace is in progress (case 8055) we're
* holding thread_initexit_lock, which prevents any flusher from
* proceeding and hitting a deadlock point, so we should be safe
*/
thread_owns_one_lock(dcontext, &thread_initexit_lock) ||
/* it is safe to be the all-thread-syncher and hit a flush synch
* point, as no flusher can be active (all threads should be suspended
* except this thread)
*/
thread_owns_two_locks(dcontext, &thread_initexit_lock,
&all_threads_synch_lock));
# endif /* DEADLOCK_AVOIDANCE */
}
#endif /* DEBUG */
#ifdef CLIENT_INTERFACE
static void
process_client_flush_requests(dcontext_t *dcontext, dcontext_t *alloc_dcontext,
client_flush_req_t *req, bool flush)
{
client_flush_req_t *iter = req;
while (iter != NULL) {
client_flush_req_t *next = iter->next;
if (flush) {
/* Note that we don't free futures from potentially linked-to region b/c we
* don't have lazy linking (xref case 2236) */
/* FIXME - if there's more then one of these would be nice to batch them
* especially for the synch all ones. */
if (iter->flush_callback != NULL) {
/* FIXME - for implementation simplicity we do a synch-all flush so
* that we can inform the client right away, it might be nice to use
* the more performant regular flush when possible. */
flush_fragments_from_region(dcontext, iter->start, iter->size,
true/*force synchall*/);
(*iter->flush_callback)(iter->flush_id);
} else {
/* do a regular flush */
flush_fragments_from_region(dcontext, iter->start, iter->size,
false/*don't force synchall*/);
}
}
HEAP_TYPE_FREE(alloc_dcontext, iter, client_flush_req_t, ACCT_CLIENT,
UNPROTECTED);
iter = next;
}
}
#endif
/* Returns false iff was_I_flushed ends up being deleted
* if cache_transition is true, assumes entering the cache now.
*/
bool
enter_nolinking(dcontext_t *dcontext, fragment_t *was_I_flushed, bool cache_transition)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
bool not_flushed = true;
/*case 7966: has no pt, no flushing either */
if (RUNNING_WITHOUT_CODE_CACHE())
return true;
DOCHECK(1, { check_safe_for_flush_synch(dcontext); });
/* FIXME: once we have this working correctly, come up with scheme
* that avoids synch in common case
*/
mutex_lock(&pt->linking_lock);
ASSERT(pt->could_be_linking);
wait_for_flusher_linking(dcontext);
not_flushed = not_flushed && check_flush_queue(dcontext, was_I_flushed);
pt->could_be_linking = false;
mutex_unlock(&pt->linking_lock);
if (!cache_transition)
return not_flushed;
/* now we act on pending actions that can only be done while nolinking
* FIXME: optimization if add more triggers here: use a single
* master trigger as a first test to avoid testing all
* conditionals every time
*/
if (reset_pending != 0) {
mutex_lock(&reset_pending_lock);
if (reset_pending != 0) {
uint target = reset_pending;
reset_pending = 0;
/* fcache_reset_all_caches_proactively() will unlock */
fcache_reset_all_caches_proactively(target);
LOG(THREAD, LOG_DISPATCH, 2,
"Just reset all caches, next_tag is "PFX"\n", dcontext->next_tag);
/* fragment is gone for sure, so return false */
return false;
}
mutex_unlock(&reset_pending_lock);
}
/* FIXME: perf opt: make global flag can check w/ making a call,
* or at least inline the call
*/
if (fcache_is_flush_pending(dcontext)) {
not_flushed = not_flushed &&
fcache_flush_pending_units(dcontext, was_I_flushed);
}
#ifdef UNIX
/* i#61/PR 211530: nudges on Linux do not use separate threads */
while (dcontext->nudge_pending != NULL) {
/* handle_nudge may not return, so we can't call it w/ inconsistent state */
pending_nudge_t local = *dcontext->nudge_pending;
heap_free(dcontext, dcontext->nudge_pending, sizeof(local) HEAPACCT(ACCT_OTHER));
dcontext->nudge_pending = local.next;
if (dcontext->interrupted_for_nudge != NULL) {
fragment_t *f = dcontext->interrupted_for_nudge;
LOG(THREAD, LOG_ASYNCH, 3, "\tre-linking outgoing for interrupted F%d\n",
f->id);
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
link_fragment_outgoing(dcontext, f, false);
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
if (TEST(FRAG_HAS_SYSCALL, f->flags)) {
mangle_syscall_code(dcontext, f, EXIT_CTI_PC(f, dcontext->last_exit),
true/*skip exit cti*/);
}
dcontext->interrupted_for_nudge = NULL;
}
handle_nudge(dcontext, &local.arg);
/* we may have done a reset, so do not enter cache now */
return false;
}
#endif
#ifdef CLIENT_INTERFACE
/* Handle flush requests queued via dr_flush_fragments()/dr_delay_flush_region() */
/* thread private list */
process_client_flush_requests(dcontext, dcontext, dcontext->client_data->flush_list,
true/*flush*/);
dcontext->client_data->flush_list = NULL;
/* global list */
if (client_flush_requests != NULL) { /* avoid acquiring lock every cxt switch */
client_flush_req_t *req;
mutex_lock(&client_flush_request_lock);
req = client_flush_requests;
client_flush_requests = NULL;
mutex_unlock(&client_flush_request_lock);
/* NOTE - we must release the lock before doing the flush. */
process_client_flush_requests(dcontext, GLOBAL_DCONTEXT, req, true/*flush*/);
/* FIXME - this is an ugly, yet effective, hack. The problem is there is no
* good way to tell currently if we flushed was_I_flushed. Since it could be
* gone by now we pretend that it was flushed if we did any flushing at all.
* Dispatch should refind the fragment if it wasn't flushed. */
if (req != NULL)
not_flushed = false;
}
#endif
return not_flushed;
}
/* Returns false iff was_I_flushed ends up being deleted */
bool
enter_couldbelinking(dcontext_t *dcontext, fragment_t *was_I_flushed, bool cache_transition)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
bool not_flushed;
/*case 7966: has no pt, no flushing either */
if (RUNNING_WITHOUT_CODE_CACHE())
return true;
DOCHECK(1, { check_safe_for_flush_synch(dcontext); });
mutex_lock(&pt->linking_lock);
ASSERT(!pt->could_be_linking);
/* ensure not still marked at_syscall */
ASSERT(!DYNAMO_OPTION(syscalls_synch_flush) || !get_at_syscall(dcontext));
/* for thread-shared flush and thread-private flush+execareas atomicity,
* to avoid non-properly-nested locks (could have flusher hold
* pt->linking_lock for the entire flush) we need an additional
* synch point here for shared flushing to synch w/ all threads.
* I suppose switching from non-nested locks to this loop isn't
* nec. helping deadlock avoidance -- can still hang -- but this
* should be better performance-wise (only a few writes and a
* conditional in the common case here, no extra locks)
*/
pt->soon_to_be_linking = true;
wait_for_flusher_nolinking(dcontext);
pt->soon_to_be_linking = false;
pt->could_be_linking = true;
not_flushed = check_flush_queue(dcontext, was_I_flushed);
mutex_unlock(&pt->linking_lock);
return not_flushed;
}
/* NOTE - this routine may be called more then one time for the same exiting thread,
* xref case 8047. Any once only reference counting, cleanup, etc. should be done in
* fragment_thread_exit(). This routine is just a stripped down version of
* enter_nolinking() to keep an exiting thread from deadlocking with flushing. */
void
enter_threadexit(dcontext_t *dcontext)
{
per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
/*case 7966: has no pt, no flushing either */
if (RUNNING_WITHOUT_CODE_CACHE() || pt == NULL/*PR 536058: no pt*/)
return;
mutex_lock(&pt->linking_lock);
/* must dec ref count on shared regions before we die */
check_flush_queue(dcontext, NULL);
pt->could_be_linking = false;
if (pt->wait_for_unlink) {
/* make sure don't get into deadlock w/ flusher */
pt->about_to_exit = true; /* let flusher know can ignore us */
signal_event(pt->waiting_for_unlink); /* wake flusher up */
}
mutex_unlock(&pt->linking_lock);
}
/* caller must hold shared_cache_flush_lock */
void
increment_global_flushtime()
{
ASSERT_OWN_MUTEX(true, &shared_cache_flush_lock);
/* reset will turn flushtime_global back to 0, so we schedule one
* when we're approaching overflow
*/
if (flushtime_global == UINT_MAX/2) {
ASSERT_NOT_TESTED(); /* FIXME: add -stress_flushtime_global_max */
SYSLOG_INTERNAL_WARNING("flushtime_global approaching UINT_MAX, resetting");
schedule_reset(RESET_ALL);
}
ASSERT(flushtime_global < UINT_MAX);
/* compiler should 4-byte-align so no cache line crossing
* (asserted in fragment_init()
*/
flushtime_global++;
LOG(GLOBAL, LOG_VMAREAS, 2, "new flush timestamp: %u\n",
flushtime_global);
}
/* The master flusher routines are split into 3:
* stage1) flush_fragments_synch_unlink_priv
* stage2) flush_fragments_unlink_shared
* stage3) flush_fragments_end_synch
* which MUST be used together. They are split to allow the caller to
* perform custom operations at certain synchronization points in the
* middle of flushing without using callback routines. The stages and
* points are:
*
* stage1: head synch, priv flushee unlink
* here caller can produce a custom list of fragments to flush while
* all threads are fully synched
* stage2: shared flushee unlink
*
* between stage2 and stage3, a region flush performs additional actions:
* region flush: exec area lock
* here caller can do custom exec area manipulations
* region flush: exec area unlock
*
* stage3: tail synch
*
* When doing a region flush with no custom region removals (the default is
* removing [base,base+size)), use the routine flush_fragments_and_remove_region().
* When doing a region flush with custom removals, use
* flush_fragments_in_region_start() and flush_fragments_in_region_finish().
* When doing a flush with no region removals at all use flush_fragments_from_region().
*
* The thread requesting a flush must be !couldbelinking and must not
* be holding any locks that are ever grabbed while a thread is
* couldbelinking (pretty much all locks), as it must wait for other
* threads who are couldbelinking. The thread_initexit_lock is held
* from stage 1 to stage 3 (if there are fragments to flush), and
* region flushing also holds the executable_areas lock between stage
* 2 and stage 3. After stage 1 no thread is couldbelinking() until
* the synchronization release in stage 3.
*
* The general delayed-deletion flushing strategy involves first
* freezing the threads via the thread_initexit_lock.
* It then walks the thread list and marks all vm areas that overlap
* base...base+size as to-be-deleted, along with unlinking all fragments in those
* vm areas and unlinking shared_syscall for that thread.
* Fragments in the target area are not actually deleted until their owning thread
* checks its pending deletion queue, which it does prior to entering the fcache.
* Also flushes shared fragments, in a similar manner, with deletion delayed
* until all threads are out of the shared cache.
*/
/* Variables shared between the 3 flush stage routines
* flush_fragments_synch_unlink_priv(),
* flush_fragments_unlink_shared(), and flush_fragments_end_synch.
* As there can only be one flush at a time, we only need one copy,
* protected by the thread_initexit_lock.
*/
/* Not persistent across code cache execution, so unprotected */
DECLARE_NEVERPROT_VAR(static thread_record_t **flush_threads, NULL);
DECLARE_NEVERPROT_VAR(static int flush_num_threads, 0);
DECLARE_NEVERPROT_VAR(static int pending_delete_threads, 0);
DECLARE_NEVERPROT_VAR(static int shared_flushed, 0);
DECLARE_NEVERPROT_VAR(static bool flush_synchall, false);
#ifdef DEBUG
DECLARE_NEVERPROT_VAR(static int num_flushed, 0);
DECLARE_NEVERPROT_VAR(static int flush_last_stage, 0);
#endif
static void
flush_fragments_free_futures(app_pc base, size_t size)
{
int i;
dcontext_t *tgt_dcontext;
ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
ASSERT((allsynch_flusher == NULL && flusher == get_thread_private_dcontext()) ||
(flusher == NULL && allsynch_flusher == get_thread_private_dcontext()));
ASSERT(flush_num_threads > 0);
ASSERT(flush_threads != NULL);
if (DYNAMO_OPTION(free_unmapped_futures) && !RUNNING_WITHOUT_CODE_CACHE()) {
/* We need to free the futures after all fragments have been unlinked,
* as unlinking will create new futures
*/
acquire_recursive_lock(&change_linking_lock);
for (i=0; i<flush_num_threads; i++) {
tgt_dcontext = flush_threads[i]->dcontext;
if (tgt_dcontext != NULL) {
fragment_delete_futures_in_region(tgt_dcontext, base, base + size);
thcounter_range_remove(tgt_dcontext, base, base + size);
}
}
if (SHARED_FRAGMENTS_ENABLED())
fragment_delete_futures_in_region(GLOBAL_DCONTEXT, base, base + size);
release_recursive_lock(&change_linking_lock);
} /* else we leak them */
}
/* This routine begins a flush that requires full thread synch: currently,
* it is used for flushing coarse-grain units and for dr_flush_region()
*/
static void
flush_fragments_synchall_start(dcontext_t *ignored, app_pc base, size_t size,
bool exec_invalid)
{
dcontext_t *my_dcontext = get_thread_private_dcontext();
app_pc exec_start = NULL, exec_end = NULL;
bool all_synched = true;
int i;
const thread_synch_state_t desired_state =
THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER;
DEBUG_DECLARE(bool ok;)
KSTART(synchall_flush);
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\nflush_fragments_synchall_start: thread "TIDFMT" suspending all threads\n",
get_thread_id());
STATS_INC(flush_synchall);
/* suspend all DR-controlled threads at safe locations */
DEBUG_DECLARE(ok =)
synch_with_all_threads(desired_state, &flush_threads, &flush_num_threads,
THREAD_SYNCH_NO_LOCKS_NO_XFER,
/* if we fail to suspend a thread (e.g., for
* privilege reasons), ignore it since presumably
* the failed thread is some injected thread not
* running to-be-flushed code, so we continue
* with the flush!
*/
THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
ASSERT(ok);
/* now we own the thread_initexit_lock */
ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
/* We do NOT set flusher as is_self_flushing() is all about
* couldbelinking, while our synch is of a different nature. The
* trace_abort() is_self_flushing() check is the only worrisome
* one: FIXME.
*/
ASSERT(flusher == NULL);
ASSERT(allsynch_flusher == NULL);
allsynch_flusher = my_dcontext;
flush_synchall = true;
ASSERT(flush_last_stage == 0);
DODEBUG({ flush_last_stage = 1; });
LOG(GLOBAL, LOG_FRAGMENT, 2,
"flush_fragments_synchall_start: walking the threads\n");
/* We rely on coarse fragments not touching more than one vmarea region
* for our ibl invalidation. It's
* ok to invalidate more than we need to so we don't care if there are
* multiple coarse units within this range. We just need the exec areas
* bounds that overlap the flush region.
*/
if (!executable_area_overlap_bounds(base, base+size, &exec_start, &exec_end,
0, true/*doesn't matter w/ 0*/)) {
/* caller checks for overlap but lock let go so can get here; go ahead
* and do synch per flushing contract.
*/
exec_start = base;
exec_end = base+size;
}
LOG(GLOBAL, LOG_FRAGMENT, 2,
"flush_fragments_synchall_start: from "PFX"-"PFX" => coarse "PFX"-"PFX"\n",
base, base+size, exec_start, exec_end);
/* FIXME: share some of this code that I duplicated from reset */
for (i = 0; i < flush_num_threads; i++) {
dcontext_t *dcontext = flush_threads[i]->dcontext;
if (dcontext != NULL) { /* include my_dcontext here */
DEBUG_DECLARE(uint removed;)
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tconsidering thread #%d "TIDFMT"\n", i, flush_threads[i]->id);
if (dcontext != my_dcontext) {
/* must translate BEFORE freeing any memory! */
if (!thread_synch_successful(flush_threads[i])) {
/* FIXME case 9480: if we do get here for low-privilege handles
* or exceeding our synch try count, best to not move the thread
* as we won't get a clean translation. Chances are it is
* not in the region being flushed.
*/
SYSLOG_INTERNAL_ERROR_ONCE("failed to synch with thread during "
"synchall flush");
LOG(THREAD, LOG_FRAGMENT|LOG_SYNCH, 2,
"failed to synch with thread #%d\n", i);
STATS_INC(flush_synchall_fail);
all_synched = false;
} else if (is_thread_currently_native(flush_threads[i])) {
/* native_exec's regain-control point is in our DLL,
* and lost-control threads are truly native, so no
* state to worry about except for hooks -- and we're
* not freeing the interception buffer.
*/
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tcurrently native so no translation needed\n");
} else if (thread_synch_state_no_xfer(dcontext)) {
/* Case 6821: do not translate other synch-all-thread users.
* They have no fragment state, so leave alone.
* All flush call points are in syscalls or from nudges.
* Xref case 8901 on ensuring nudges don't try to return
* to the code cache.
*/
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tat THREAD_SYNCH_NO_LOCKS_NO_XFER so no translation needed\n");
STATS_INC(flush_synchall_races);
} else {
translate_from_synchall_to_dispatch(flush_threads[i], desired_state);
}
}
if (dcontext == my_dcontext || thread_synch_successful(flush_threads[i])) {
last_exit_deleted(dcontext);
/* case 7394: need to abort other threads' trace building
* since the reset xfer to dispatch will disrupt it.
* also, with PR 299808, we now have thread-shared
* "undeletable" trace-temp fragments, so we need to abort
* all traces.
*/
if (is_building_trace(dcontext)) {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tsquashing trace of thread #%d\n", i);
trace_abort(dcontext);
}
}
/* Since coarse fragments never cross coarse/non-coarse executable region
* bounds, we can bound their bodies by taking
* executable_area_distinct_bounds(). This lets us remove by walking the
* ibl tables and looking only at tags, rather than walking the htable of
* each coarse unit. FIXME: not clear this is a perf win: I arbitrarily
* picked it under assumption that ibl tables are relatively small. It
* would be a clearer win if we could do the fine fragments this way
* also, but fine fragments are not constrained and could be missed using
* only a tag-based range remove.
*/
DEBUG_DECLARE(removed =)
fragment_remove_all_ibl_in_region(dcontext, exec_start, exec_end);
LOG(THREAD, LOG_FRAGMENT, 2,
"\tremoved %d ibl entries in "PFX"-"PFX"\n",
removed, exec_start, exec_end);
/* Free any fine private fragments in the region */
vm_area_allsynch_flush_fragments(dcontext, dcontext, base, base+size,
exec_invalid, all_synched/*ignored*/);
if (!SHARED_IBT_TABLES_ENABLED() && SHARED_FRAGMENTS_ENABLED()) {
/* Remove shared fine fragments from private ibl tables */
vm_area_allsynch_flush_fragments(dcontext, GLOBAL_DCONTEXT,
base, base+size,
exec_invalid, all_synched/*ignored*/);
}
}
}
/* Removed shared coarse fragments from ibl tables, before freeing any */
if (SHARED_IBT_TABLES_ENABLED() && SHARED_FRAGMENTS_ENABLED())
fragment_remove_all_ibl_in_region(GLOBAL_DCONTEXT, exec_start, exec_end);
/* Free coarse units and shared fine fragments, as well as removing shared fine
* entries in any shared ibl tables
*/
if (SHARED_FRAGMENTS_ENABLED()) {
vm_area_allsynch_flush_fragments(GLOBAL_DCONTEXT, GLOBAL_DCONTEXT,
base, base+size, exec_invalid, all_synched);
}
}
static void
flush_fragments_synchall_end(dcontext_t *ignored)
{
thread_record_t **temp_threads = flush_threads;
DEBUG_DECLARE(dcontext_t *my_dcontext = get_thread_private_dcontext();)
LOG(GLOBAL, LOG_FRAGMENT, 2,
"flush_fragments_synchall_end: resuming all threads\n");
/* We need to clear this before we release the locks. We use a temp var
* so we can use end_synch_with_all_threads.
*/
flush_threads = NULL;
ASSERT(flusher == NULL);
flush_synchall = false;
ASSERT(dynamo_all_threads_synched);
ASSERT(allsynch_flusher == my_dcontext);
allsynch_flusher = NULL;
end_synch_with_all_threads(temp_threads, flush_num_threads, true/*resume*/);
KSTOP(synchall_flush);
}
/* This routine begins a flush of the group of fragments in the memory
* region [base, base+size) by synchronizing with each thread and unlinking
* all private fragments in the region.
*
* The exec_invalid parameter must be set to indicate whether the
* executable area is being invalidated as well or this is just a capacity
* flush (or a flush to change instrumentation).
*
* If size==0 then no unlinking occurs; however, the full synch is
* performed (so the caller should check size if no action is desired on
* zero size).
*
* If size>0 and there is no executable area overlap, then no synch is
* performed and false is returned. The caller must acquire the executable
* areas lock and re-check the overlap if exec area manipulation is to be
* performed. Returns true otherwise.
*/
bool
flush_fragments_synch_unlink_priv(dcontext_t *dcontext, app_pc base, size_t size,
/* WARNING: case 8572: the caller owning this lock
* is incompatible w/ suspend-the-world flushing!
*/
bool own_initexit_lock, bool exec_invalid,
bool force_synchall _IF_DGCDIAG(app_pc written_pc))
{
dcontext_t *tgt_dcontext;
per_thread_t *tgt_pt;
int i;
LOG(THREAD, LOG_FRAGMENT, 2,
"FLUSH STAGE 1: synch_unlink_priv(thread "TIDFMT" flushtime %d): "PFX"-"PFX"\n",
dcontext->owning_thread, flushtime_global, base, base + size);
/* Case 9750: to specify a region of size 0, do not pass in NULL as the base!
* Use EMPTY_REGION_{BASE,SIZE} instead.
*/
ASSERT(base != NULL || size != 0);
/* our flushing design requires that flushers are NOT couldbelinking
* and are not holding any locks
*/
ASSERT(!is_self_couldbelinking());
#if defined(DEADLOCK_AVOIDANCE) && defined(DEBUG)
if (own_initexit_lock) {
/* We can get here while holding the all_threads_synch_lock
* for detach (& prob. TerminateProcess(0) with other threads
* still active) on XP and 2003 via os_thread_stack_exit()
* (for other thread exit) if we have executed off that stack.
* Can be seen with Araktest detach on violation using the stack
* attack button. */
ASSERT(thread_owns_first_or_both_locks_only(dcontext,
&thread_initexit_lock,
&all_threads_synch_lock));
} else {
ASSERT_OWN_NO_LOCKS();
}
#endif
ASSERT(dcontext == get_thread_private_dcontext());
/* quick check for overlap first by using read lock and avoiding
* thread_initexit_lock:
* if no overlap, must hold lock through removal of region
* if overlap, ok to release for a bit
*/
if (size > 0 && !executable_vm_area_executed_from(base, base+size)) {
/* Only a curiosity since we can have a race (not holding exec areas lock) */
ASSERT_CURIOSITY((!SHARED_FRAGMENTS_ENABLED() ||
!thread_vm_area_overlap(GLOBAL_DCONTEXT, base, base+size)) &&
!thread_vm_area_overlap(dcontext, base, base+size));
return false;
}
/* Only a curiosity since we can have a race (not holding exec areas lock) */
ASSERT_CURIOSITY(size == 0 ||
executable_vm_area_overlap(base, base+size, false/*no lock*/));
STATS_INC(num_flushes);
if (force_synchall ||
(size > 0 && executable_vm_area_coarse_overlap(base, base+size))) {
/* Coarse units do not support individual unlinking (though prior to
* freezing they do, we ignore that) and instead require all-thread-synch
* in order to flush. For that we cannot be already holding
* thread_initexit_lock! FIXME case 8572: the only caller who does hold it
* now is os_thread_stack_exit(). For now relying on that stack not
* overlapping w/ any coarse regions.
*/
ASSERT(!own_initexit_lock);
/* The synchall will flush fine as well as coarse so we'll be done */
flush_fragments_synchall_start(dcontext, base, size, exec_invalid);
return true;
}
/* Take a snapshot of the threads in the system.
* Grab the thread lock to prevent threads from being created or
* exited for the duration of this routine
* FIXME -- is that wise? could be a relatively long time!
* It's unlikely a new thread will run code in a region being
* unmapped...but we would like to prevent threads from exiting
* while we're messing with their data.
* FIXME: need to special-case every instance where thread_initexit_lock
* is grabbed, to avoid deadlocks! we already do for a thread exiting.
*/
if (!own_initexit_lock)
mutex_lock(&thread_initexit_lock);
ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
flusher = dcontext;
get_list_of_threads(&flush_threads, &flush_num_threads);
ASSERT(flush_last_stage == 0);
DODEBUG({ flush_last_stage = 1; });
/* FIXME: we can optimize this even more to not grab thread_initexit_lock */
if (RUNNING_WITHOUT_CODE_CACHE()) /* case 7966: nothing to flush, ever */
return true;
/* Set the ref count of threads who may be using a deleted fragment. We
* include ourselves in the ref count as we could be invoked with a cache
* return point before any synch (as done w/ hotpatches) and should NOT
* consider ourselves done w/ all cache fragments right now. We assume we
* will soon hit a real synch point to dec the count for us, and don't try
* to specially handle the single-threaded case here.
*/
pending_delete_threads = flush_num_threads;
DODEBUG({ num_flushed = 0; });
#ifdef WINDOWS
/* Make sure exit fcache if currently inside syscall. For thread-shared
* shared_syscall, we re-link after the shared fragments have all been
* unlinked and removed from the ibl tables. All we lose is the bounded time
* on a thread checking its private flush queue, which in a thread-shared
* configuration doesn't seem so bad. For thread-private this will work as
* well, with the same time bound lost, but we're not as worried about memory
* usage of thread-private configurations.
*/
if (DYNAMO_OPTION(shared_syscalls) && IS_SHARED_SYSCALL_THREAD_SHARED)
unlink_shared_syscall(GLOBAL_DCONTEXT);
#endif
/* i#849: unlink while we clear out ibt */
if (!special_ibl_xfer_is_thread_private())
unlink_special_ibl_xfer(GLOBAL_DCONTEXT);
for (i=0; i<flush_num_threads; i++) {
tgt_dcontext = flush_threads[i]->dcontext;
tgt_pt = (per_thread_t *) tgt_dcontext->fragment_field;
LOG(THREAD, LOG_FRAGMENT, 2,
" considering thread #%d/%d = "TIDFMT"\n", i+1, flush_num_threads,
flush_threads[i]->id);
ASSERT(is_thread_known(tgt_dcontext->owning_thread));
/* can't do anything, even check if thread has any vm areas overlapping flush
* region, until sure thread is in fcache or somewhere that won't change
* vm areas or linking state
*/
mutex_lock(&tgt_pt->linking_lock);
/* Must explicitly check for self and avoid synch then, o/w will lock up
* if ever called from a could_be_linking location (currently only
* happens w/ app syscalls)
*/
if (tgt_dcontext != dcontext && tgt_pt->could_be_linking) {
/* remember we have a global lock, thread_initexit_lock, so two threads
* cannot be here at the same time!
*/
LOG(THREAD, LOG_FRAGMENT, 2,
"\twaiting for thread "TIDFMT"\n", tgt_dcontext->owning_thread);
tgt_pt->wait_for_unlink = true;
mutex_unlock(&tgt_pt->linking_lock);
wait_for_event(tgt_pt->waiting_for_unlink);
mutex_lock(&tgt_pt->linking_lock);
tgt_pt->wait_for_unlink = false;
LOG(THREAD, LOG_FRAGMENT, 2,
"\tdone waiting for thread "TIDFMT"\n", tgt_dcontext->owning_thread);
} else {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tthread "TIDFMT" synch not required\n", tgt_dcontext->owning_thread);
}
/* it is now safe to access link, vm, and trace info in tgt_dcontext
* => FIXME: rename, since not just accessing linking info?
* FIXME: if includes vm access, are syscalls now bad?
* what about handle_modified_code, considered nolinking since straight
* from cache via fault handler? reading should be ok, but exec area splits
* and whatnot?
*/
if (tgt_pt->about_to_exit) {
/* thread is about to exit, it's waiting for us to give up
* thread_initexit_lock -- we don't need to flush it
*/
goto next_thread;
}
/* if a trace-in-progress crosses this region, must squash the trace
* (all traces are essentially frozen now since threads stop in dispatch)
*/
if (size > 0 /* else, no region to cross */ &&
is_building_trace(tgt_dcontext)) {
void *trace_vmlist = cur_trace_vmlist(tgt_dcontext);
if (trace_vmlist != NULL &&
vm_list_overlaps(tgt_dcontext, trace_vmlist, base, base+size)) {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tsquashing trace of thread "TIDFMT"\n", tgt_dcontext->owning_thread);
trace_abort(tgt_dcontext);
}
}
#ifdef WINDOWS
/* Make sure exit fcache if currently inside syscall. If thread has no
* vm area overlap, will be re-linked down below before going to the
* next thread, unless we have shared fragments, in which case we force
* the thread to check_flush_queue() on its next synch point. It will
* re-link in vm_area_flush_fragments().
*/
if (DYNAMO_OPTION(shared_syscalls) && !IS_SHARED_SYSCALL_THREAD_SHARED)
unlink_shared_syscall(tgt_dcontext);
#endif
/* i#849: unlink while we clear out ibt */
if (special_ibl_xfer_is_thread_private())
unlink_special_ibl_xfer(tgt_dcontext);
/* Optimization for shared deletion strategy: perform flush work
* for a thread waiting at a system call on behalf of that thread
* (which we do before the flush tail synch, as if we did it now we
* would need to inc flushtime_global up front, and then we'd have synch
* issues trying to prevent thread we hadn't synched with from checking
* the pending list before we put flushed fragments on it).
* We do count these threads in the ref count as we check the pending
* queue on their behalf after adding the newly flushed fragments to
* the queue, so the ref count gets decremented right away.
*
* We must do this AFTER unlinking shared_syscall's post-syscall ibl, to
* avoid races -- the thread will hit a real synch point before accessing
* any fragments or link info.
* Do this BEFORE checking whether fragments in region to catch all threads.
*/
ASSERT(!tgt_pt->at_syscall_at_flush);
if (DYNAMO_OPTION(syscalls_synch_flush) && get_at_syscall(tgt_dcontext)) {
/* we have to know exactly which threads were at_syscall here when
* we get to post-flush, so we cache in this special bool
*/
DEBUG_DECLARE(bool tables_updated;)
tgt_pt->at_syscall_at_flush = true;
#ifdef DEBUG
tables_updated =
#endif
update_all_private_ibt_table_ptrs(tgt_dcontext, tgt_pt);
STATS_INC(num_shared_flush_atsyscall);
DODEBUG({
if (tables_updated)
STATS_INC(num_shared_tables_updated_atsyscall);
});
}
/* don't need to go any further if thread has no frags in region */
if (size == 0 || !thread_vm_area_overlap(tgt_dcontext, base, base+size)) {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tthread "TIDFMT" has no fragments in region to flush\n",
tgt_dcontext->owning_thread);
#ifdef WINDOWS
/* restore, since won't be restored in vm_area_flush_fragments */
if (DYNAMO_OPTION(shared_syscalls)) {
if (SHARED_FRAGMENTS_ENABLED()) {
/* we cannot re-link shared_syscall here as that would allow
* the target thread to enter to-be-flushed fragments prior
* to their being unlinked and removed from ibl tables -- so
* we force this thread to re-link in check_flush_queue.
* we could re-link after unlink/removal of fragments,
* if it's worth optimizing == case 6194/PR 210655.
*/
tgt_pt->flush_queue_nonempty = true;
STATS_INC(num_flushq_relink_syscall);
} else if (!IS_SHARED_SYSCALL_THREAD_SHARED) {
/* no shared fragments, and no private ones being flushed,
* so this thread is all set
*/
link_shared_syscall(tgt_dcontext);
}
}
#endif
if (special_ibl_xfer_is_thread_private()) {
if (SHARED_FRAGMENTS_ENABLED()) {
/* see shared_syscall relink comment: we have to delay the relink */
tgt_pt->flush_queue_nonempty = true;
STATS_INC(num_flushq_relink_special_ibl_xfer);
} else
link_special_ibl_xfer(dcontext);
}
goto next_thread;
}
LOG(THREAD, LOG_FRAGMENT, 2, "\tflushing fragments for thread "TIDFMT"\n",
flush_threads[i]->id);
DOLOG(2, LOG_FRAGMENT, {
if (tgt_dcontext != dcontext) {
LOG(tgt_dcontext->logfile, LOG_FRAGMENT, 2,
"thread "TIDFMT" is flushing our fragments\n",
dcontext->owning_thread);
}
});
if (size > 0) {
/* unlink all frags in overlapping regions, and mark regions for deletion */
tgt_pt->flush_queue_nonempty = true;
#ifdef DEBUG
num_flushed +=
#endif
vm_area_unlink_fragments(tgt_dcontext, base, base + size, 0
_IF_DGCDIAG(written_pc));
}
next_thread:
/* for thread-shared, we CANNOT let any thread become could_be_linking, for normal
* flushing synch -- for thread-private, we can, but we CANNOT let any thread
* that we've already synched with for flushing go and change the exec areas
* vector! the simplest solution is to have thread-private, like thread-shared,
* stop all threads at a cache exit point.
* FIXME: optimize thread-private by allowing already-synched threads to
* continue but not grab executable_areas lock, while letting
* to-be-synched threads grab the lock (otherwise could wait forever)
*/
/* Since we must let go of lock for proper nesting, we use a new
* synch to stop threads at cache exit, since we need them all
* out of DR for duration of shared flush.
*/
if (tgt_dcontext != dcontext && !tgt_pt->could_be_linking)
tgt_pt->wait_for_unlink = true; /* stop at cache exit */
mutex_unlock(&tgt_pt->linking_lock);
}
return true;
}
/* This routine continues a flush of one of two groups of fragments:
* 1) if list!=NULL, the list of shared fragments beginning at list and
* chained by next_vmarea (we assume that private fragments are
* easily deleted by the owning thread and do not require a flush).
* Caller should call vm_area_remove_fragment() for each target fragment,
* building a custom list chained by next_vmarea, and pass that list to this
* routine.
* 2) otherwise, all fragments in the memory region [base, base+size).
*
* This routine MUST be called after flush_fragments_synch_unlink_priv(), and
* must be followed with flush_fragments_end_synch().
*/
void
flush_fragments_unlink_shared(dcontext_t *dcontext, app_pc base, size_t size,
fragment_t *list _IF_DGCDIAG(app_pc written_pc))
{
/* we would assert that size > 0 || list != NULL but we have to pass
* in 0 and NULL for unit flushing when no live fragments are in the unit
*/
LOG(THREAD, LOG_FRAGMENT, 2,
"FLUSH STAGE 2: unlink_shared(thread "TIDFMT"): flusher is "TIDFMT"\n",
dcontext->owning_thread, (flusher == NULL) ? -1 : flusher->owning_thread);
ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
ASSERT(flush_threads != NULL);
ASSERT(flush_num_threads > 0);
ASSERT(flush_last_stage == 1);
DODEBUG({ flush_last_stage = 2; });
if (RUNNING_WITHOUT_CODE_CACHE()) /* case 7966: nothing to flush, ever */
return;
if (flush_synchall) /* no more flush work to do */
return;
if (SHARED_FRAGMENTS_ENABLED()) {
/* Flushing shared fragments: the strategy is again immediate
* unlinking (plus hashtable removal and vm area list removal)
* with delayed deletion. Unlinking is atomic, and hashtable
* removal is for now since there are no shared bbs in ibls from
* the cache. But deletion needs to ensure that every thread who may
* hold a pointer to a shared fragment or may be inside a shared
* fragment is ok with the fragment being deleted. We use a reference count
* for each flushed region to ensure that every thread has reached a synch
* point before we actually free the fragments in the region.
*
* We do have pathological cases where a single thread at a wait syscall or
* an infinite loop in the cache will prevent all freeing. One solution
* to the syscall problem is to store a flag saying that a thread is at
* the shared_syscall routine. That routine is private and unlinked so
* we know the thread will hit a synch routine if it exits, so no race.
* We then wouldn't bother to inc the ref count for that thread.
*
* Our fallback proactive deletion is to do a full reset when the size
* of the pending deletion list gets too long, though we could be more
* efficient by only resetting the pending list, and by only suspending
* those threads that haven't dec-ed a ref count in a particular region.
*/
LOG(THREAD, LOG_FRAGMENT, 2, " flushing shared fragments\n");
if (DYNAMO_OPTION(shared_deletion)) {
/* We use shared_cache_flush_lock to make atomic the increment of
* flushtime_global and the adding of pending deletion fragments with
* that flushtime, wrt other threads checking the pending list.
*/
mutex_lock(&shared_cache_flush_lock);
}
/* Increment flush count for shared deletion algorithm and for list-based
* flushing (such as for shared cache units). We could wait
* and only do this if we actually flush shared fragments, but it's
* simpler when done in this routine, and nearly every flush does flush
* shared fragments.
*/
increment_global_flushtime();
/* Both vm_area_unlink_fragments and unlink_fragments_for_deletion call
* back to flush_invalidate_ibl_shared_target to remove shared
* fragments from private/shared ibl tables
*/
if (list == NULL) {
shared_flushed =
vm_area_unlink_fragments(GLOBAL_DCONTEXT, base, base + size,
pending_delete_threads _IF_DGCDIAG(written_pc));
} else {
shared_flushed = unlink_fragments_for_deletion(GLOBAL_DCONTEXT, list,
pending_delete_threads);
}
if (DYNAMO_OPTION(shared_deletion))
mutex_unlock(&shared_cache_flush_lock);
DODEBUG({
num_flushed += shared_flushed;
if (shared_flushed > 0)
STATS_INC(num_shared_flushes);
});
}
#ifdef WINDOWS
/* Re-link thread-shared shared_syscall */
if (DYNAMO_OPTION(shared_syscalls) && IS_SHARED_SYSCALL_THREAD_SHARED)
link_shared_syscall(GLOBAL_DCONTEXT);
#endif
if (!special_ibl_xfer_is_thread_private())
link_special_ibl_xfer(GLOBAL_DCONTEXT);
STATS_ADD(num_flushed_fragments, num_flushed);
DODEBUG({
if (num_flushed > 0) {
LOG(THREAD, LOG_FRAGMENT, 1, "Flushed %5d fragments from "PFX"-"PFX"\n",
num_flushed, base, ((char *)base)+size);
} else {
STATS_INC(num_empty_flushes);
LOG(THREAD, LOG_FRAGMENT, 2, "Flushed 0 fragments from "PFX"-"PFX"\n",
base, ((char *)base)+size);
}
});
}
/* Invalidates (does not remove) shared fragment f from the private/shared
* ibl tables. Can only be called in flush stage 2.
*/
void
flush_invalidate_ibl_shared_target(dcontext_t *dcontext, fragment_t *f)
{
ASSERT(is_self_flushing());
ASSERT(!flush_synchall);
ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
ASSERT(flush_threads != NULL);
ASSERT(flush_num_threads > 0);
ASSERT(flush_last_stage == 2);
ASSERT(TEST(FRAG_SHARED, f->flags));
/*case 7966: has no pt, no flushing either */
if (RUNNING_WITHOUT_CODE_CACHE()) {
ASSERT_NOT_REACHED(); /* shouldn't get here */
return;
}
if (!SHARED_IB_TARGETS())
return;
if (SHARED_IBT_TABLES_ENABLED()) {
/* Remove from shared ibl tables.
* dcontext need not be GLOBAL_DCONTEXT.
*/
fragment_prepare_for_removal(dcontext, f);
} else {
/* We can't tell afterward which entries to remove from the private
* ibl tables, as we no longer keep fragment_t* pointers (we used to
* look for FRAG_WAS_DELETED) and we can't do a range remove (tags
* don't map regularly to regions). So we must invalidate each
* fragment as we process it. It's ok to walk the thread list
* here since we're post-synch for all threads.
*/
int i;
for (i=0; i<flush_num_threads; i++) {
fragment_prepare_for_removal(flush_threads[i]->dcontext, f);
}
}
}
/* Must ONLY be called as the third part of flushing (after
* flush_fragments_synch_unlink_priv() and flush_fragments_unlink_shared()).
* Uses the static shared variables flush_threads and flush_num_threads.
*/
void
flush_fragments_end_synch(dcontext_t *dcontext, bool keep_initexit_lock)
{
dcontext_t *tgt_dcontext;
per_thread_t *tgt_pt;
int i;
LOG(THREAD, LOG_FRAGMENT, 2,
"FLUSH STAGE 3: end_synch(thread "TIDFMT"): flusher is "TIDFMT"\n",
dcontext->owning_thread, (flusher == NULL) ? -1 : flusher->owning_thread);
if (!is_self_flushing() && !flush_synchall/* doesn't set flusher */) {
LOG(THREAD, LOG_FRAGMENT, 2, "\tnothing was flushed\n");
ASSERT_DO_NOT_OWN_MUTEX(!keep_initexit_lock, &thread_initexit_lock);
ASSERT_OWN_MUTEX(keep_initexit_lock, &thread_initexit_lock);
return;
}
ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
ASSERT(flush_threads != NULL);
ASSERT(flush_num_threads > 0);
ASSERT(flush_last_stage == 2);
DODEBUG({ flush_last_stage = 0; });
if (flush_synchall) {
flush_fragments_synchall_end(dcontext);
return;
}
/* now can let all threads at DR synch point go
* FIXME: if implement thread-private optimization above, this would turn into
* re-setting exec areas lock to treat all threads uniformly
*/
for (i=flush_num_threads-1; i>=0; i--) {
/*case 7966: has no pt, no flushing either */
if (RUNNING_WITHOUT_CODE_CACHE())
continue;
tgt_dcontext = flush_threads[i]->dcontext;
tgt_pt = (per_thread_t *) tgt_dcontext->fragment_field;
/* re-acquire lock */
mutex_lock(&tgt_pt->linking_lock);
/* Optimization for shared deletion strategy: perform flush work
* for a thread waiting at a system call, as we didn't add it to the
* ref count in the pre-flush synch.
* We assume that shared_syscall is still unlinked at this point and
* will not be relinked until we let the thread go.
*/
if (DYNAMO_OPTION(syscalls_synch_flush) && tgt_pt->at_syscall_at_flush) {
/* Act on behalf of the thread as though it's at a synch point, but
* only wrt shared fragments (we don't free private fragments here,
* though we could -- should we? may make flush time while holding
* lock take too long? FIXME)
* Currently this works w/ syscalls from dispatch, and w/
* -shared_syscalls by using unprotected storage (thus a slight hole
* but very hard to exploit for security purposes: can get stale code
* executed, but we already have that window, or crash us).
* FIXME: Does not work w/ -ignore_syscalls, but those are private
* for now.
*/
DEBUG_DECLARE(uint pre_flushtime = flushtime_global;)
vm_area_check_shared_pending(tgt_dcontext, NULL);
/* lazy deletion may inc flushtime_global, so may have a higher
* value than our cached one, but should never be lower
*/
ASSERT(tgt_pt->flushtime_last_update >= pre_flushtime);
tgt_pt->at_syscall_at_flush = false;
}
if (tgt_dcontext != dcontext) {
if (tgt_pt->could_be_linking) {
signal_event(tgt_pt->finished_with_unlink);
} else {
/* we don't need to wait on a !could_be_linking thread
* so we use this bool to tell whether we should signal
* the event.
* FIXME: really we want a pulse that wakes ALL waiting
* threads and then resets the event!
*/
tgt_pt->wait_for_unlink = false;
if (tgt_pt->soon_to_be_linking)
signal_event(tgt_pt->finished_all_unlink);
}
}
mutex_unlock(&tgt_pt->linking_lock);
}
/* thread init/exit can proceed now */
flusher = NULL;
global_heap_free(flush_threads, flush_num_threads*sizeof(thread_record_t*)
HEAPACCT(ACCT_THREAD_MGT));
flush_threads = NULL;
if (!keep_initexit_lock)
mutex_unlock(&thread_initexit_lock);
}
/* This routine performs flush stages 1 and 2 (synch_unlink_priv()
* and unlink_shared()) and then returns after grabbing the
* executable_areas lock so that removal of this area from the global list
* is atomic w/ the flush and local removals, before letting the threads go
* -- we return while holding both the thread_initexit_lock and the exec
* areas lock
*
* FIXME: this only helps thread-shared: for thread-private, should let
* already-synched threads continue but not grab executable_areas lock, while
* letting to-be-synched threads grab the lock.
*
* returning while holding locks isn't ideal -- but other choices are worse:
* 1) grab exec areas lock and then let go of initexit lock -- bad nesting
* 2) do all work here, which has to include:
* A) remove region (nearly all uses) == flush_fragments_and_remove_region
* B) change region from rw to r (splitting if nec) (vmareas.c app rw->r)
* C) restore writability and change to selfmod (splitting if nec)
* (vmareas.c handle_modified_code)
* D) remove region and then look up a certain pc and return whether
* flushing overlapped w/ it (vmareas.c handle_modified_code)
*/
void
flush_fragments_in_region_start(dcontext_t *dcontext, app_pc base, size_t size,
bool own_initexit_lock, bool free_futures,
bool exec_invalid, bool force_synchall
_IF_DGCDIAG(app_pc written_pc))
{
KSTART(flush_region);
while (true) {
if (flush_fragments_synch_unlink_priv(dcontext, base, size, own_initexit_lock,
exec_invalid, force_synchall
_IF_DGCDIAG(written_pc))) {
break;
} else {
/* grab lock and then re-check overlap */
executable_areas_lock();
if (!executable_vm_area_executed_from(base, base+size)) {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tregion not executable, so no fragments to flush\n");
/* caller will release lock! */
STATS_INC(num_noncode_flushes);
return;
}
/* unlock and try again */
executable_areas_unlock();
}
}
flush_fragments_unlink_shared(dcontext, base, size, NULL
_IF_DGCDIAG(written_pc));
/* We need to free the futures after all fragments have been unlinked */
if (free_futures) {
flush_fragments_free_futures(base, size);
}
executable_areas_lock();
}
/* must ONLY be called as the second half of flush_fragments_in_region_start().
* uses the shared variables flush_threads and flush_num_threads
*/
void
flush_fragments_in_region_finish(dcontext_t *dcontext, bool keep_initexit_lock)
{
/* done w/ exec areas lock; also free any non-executed coarse units */
free_nonexec_coarse_and_unlock();
flush_fragments_end_synch(dcontext, keep_initexit_lock);
KSTOP(flush_region);
}
/* flush and remove region from exec list, atomically */
void
flush_fragments_and_remove_region(dcontext_t *dcontext, app_pc base, size_t size,
bool own_initexit_lock, bool free_futures)
{
flush_fragments_in_region_start(dcontext, base, size, own_initexit_lock,
free_futures, true/*exec invalid*/,
false/*don't force synchall*/ _IF_DGCDIAG(NULL));
/* ok to call on non-exec region, so don't need to test return value
* both flush routines will return quickly if nothing to flush/was flushed
*/
remove_executable_region(base, size, true/*have lock*/);
flush_fragments_in_region_finish(dcontext, own_initexit_lock);
/* verify initexit lock is in the right state */
ASSERT_OWN_MUTEX(own_initexit_lock, &thread_initexit_lock);
ASSERT_DO_NOT_OWN_MUTEX(!own_initexit_lock, &thread_initexit_lock);
}
/* Flushes fragments from the region without any changes to the exec list.
* Does not free futures and caller can't be holding the initexit lock.
* FIXME - add argument parameters (free futures etc.) as needed. */
void
flush_fragments_from_region(dcontext_t *dcontext, app_pc base, size_t size,
bool force_synchall)
{
/* we pass false to flush_fragments_in_region_start() below for owning the initexit
* lock */
ASSERT_DO_NOT_OWN_MUTEX(true, &thread_initexit_lock);
/* ok to call on non-exec region, so don't need to test return value
* both flush routines will return quickly if nothing to flush/was flushed */
flush_fragments_in_region_start(dcontext, base, size, false /*don't own initexit*/,
false/*don't free futures*/, false/*exec valid*/,
force_synchall _IF_DGCDIAG(NULL));
flush_fragments_in_region_finish(dcontext, false);
}
/* Invalidate all fragments in all caches. Currently executed
* fragments may be alive until they reach an exit.
*
* Note not necessarily immediately freeing memory like
* fcache_reset_all_caches_proactively().
*/
void
invalidate_code_cache()
{
dcontext_t *dcontext = get_thread_private_dcontext();
LOG(GLOBAL, LOG_FRAGMENT, 2, "invalidate_code_cache()\n");
flush_fragments_in_region_start(dcontext, UNIVERSAL_REGION_BASE,
UNIVERSAL_REGION_SIZE,
false, true /* remove futures */,
false /*exec valid*/, false /*don't force synchall*/
_IF_DGCDIAG(NULL));
flush_fragments_in_region_finish(dcontext, false);
}
/* Flushes all areas stored in the vector toflush.
* Synchronization of toflush is up to caller, but as locks cannot be
* held when flushing, toflush must be thread-private.
* Currently only used for pcache hotp interop (case 9970).
*/
void
flush_vmvector_regions(dcontext_t *dcontext, vm_area_vector_t *toflush,
bool free_futures, bool exec_invalid)
{
vmvector_iterator_t vmvi;
app_pc start, end;
ASSERT(toflush != NULL && !TEST(VECTOR_SHARED, toflush->flags));
ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
ASSERT(DYNAMO_OPTION(coarse_units) && DYNAMO_OPTION(use_persisted)
IF_HOTP(&& DYNAMO_OPTION(hot_patching)));
if (vmvector_empty(toflush))
return;
vmvector_iterator_start(toflush, &vmvi);
while (vmvector_iterator_hasnext(&vmvi)) {
vmvector_iterator_next(&vmvi, &start, &end);
/* FIXME case 10086: optimization: batch the flushes together so we only
* synch once. Currently this is rarely used, and with few areas in the
* vector, so we don't bother. To batch them we'd need to assume the
* worst and do a synchall up front, which could be more costly than 2
* or 3 separate flushes that do not involve coarse code.
*/
ASSERT_OWN_NO_LOCKS();
flush_fragments_in_region_start(dcontext, start, end - start,
false/*no lock*/, free_futures, exec_invalid,
false/*don't force synchall*/ _IF_DGCDIAG(NULL));
flush_fragments_in_region_finish(dcontext, false/*no lock*/);
STATS_INC(num_flush_vmvector);
}
vmvector_iterator_stop(&vmvi);
}
/****************************************************************************/
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
void
fragment_output(dcontext_t *dcontext, fragment_t *f)
{
ASSERT(!TEST(FRAG_SHARED, f->flags) ||
self_owns_recursive_lock(&change_linking_lock));
if (SHOULD_OUTPUT_FRAGMENT(f->flags)) {
per_thread_t *pt = (dcontext == GLOBAL_DCONTEXT) ? shared_pt :
(per_thread_t *) dcontext->fragment_field;
output_trace(dcontext, pt, f,
IF_DEBUG_ELSE(GLOBAL_STAT(num_fragments),
GLOBAL_STAT(num_traces)+GLOBAL_STAT(num_bbs))+1);
}
}
void
init_trace_file(per_thread_t *pt)
{
if (INTERNAL_OPTION(tracedump_binary)) {
/* first 4 bytes in binary file gives size of linkcounts
* 0 if no linkcounts
*/
tracedump_file_header_t hdr =
{CURRENT_API_VERSION, IF_X64_ELSE(true, false), 0 };
#ifdef PROFILE_LINKCOUNT
if (dynamo_options.profile_counts) {
hdr.linkcount_size = sizeof(linkcount_type_t);
ASSERT_NOT_IMPLEMENTED(false == DYNAMO_OPTION(inline_trace_ibl));
/* Cannot use PROFILE_LINKCOUNT with inline_trace_ibl */
}
#endif
os_write(pt->tracefile, &hdr, sizeof(hdr));
}
}
void
exit_trace_file(per_thread_t *pt)
{
#ifdef PROFILE_LINKCOUNT
if (dynamo_options.tracedump_threshold > 0) {
if (INTERNAL_OPTION(tracedump_binary)) {
os_write(pt->tracefile, &pt->tracedump_num_below_threshold,
sizeof(pt->tracedump_num_below_threshold));
os_write(pt->tracefile, &pt->tracedump_count_below_threshold,
sizeof(pt->tracedump_count_below_threshold));
} else {
print_file(pt->tracefile, "\nTraces below dump threshold of %d: %d\n",
dynamo_options.tracedump_threshold, pt->tracedump_num_below_threshold);
print_file(pt->tracefile, "Total count below dump threshold: "
LINKCOUNT_FORMAT_STRING"\n", pt->tracedump_count_below_threshold);
}
}
#endif
close_log_file(pt->tracefile);
}
/* Binary trace dump is used to save time and space.
* The format is in fragment.h.
* FIXME: add general symbol table support to disassembly?
* We'd dump num_targets, then fcache_enter, ibl, trace_cache_incr addrs?
* Reader would then add exit stubs & other traces to table?
* But links will target cache pcs...
*/
#define TRACEBUF_SIZE 2048
#define TRACEBUF_MAKE_ROOM(p, buf, sz) do { \
if (p+sz >= &buf[TRACEBUF_SIZE]) { \
os_write(pt->tracefile, buf, (p - buf)); \
p = buf; \
} \
} while (0)
static void
output_trace_binary(dcontext_t *dcontext, per_thread_t *pt, fragment_t *f,
stats_int_t trace_num)
{
/* FIXME:
* We do not support PROFILE_RDTSC or various small fields
*/
/* FIXME: should allocate buffer elsewhere */
byte buf[TRACEBUF_SIZE];
byte *p = buf;
trace_only_t *t = TRACE_FIELDS(f);
linkstub_t *l;
tracedump_trace_header_t hdr = {
(int) trace_num, f->tag, f->start_pc, f->prefix_size, 0, f->size,
(INTERNAL_OPTION(tracedump_origins) ? t->num_bbs : 0),
IF_X64_ELSE(!TEST(FRAG_32_BIT, f->flags), false),
};
tracedump_stub_data_t stub;
/* Should we widen the identifier? */
IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(trace_num)));
for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l))
hdr.num_exits++;
TRACEBUF_MAKE_ROOM(p, buf, sizeof(hdr));
*((tracedump_trace_header_t *)p) = hdr;
p += sizeof(hdr);
if (INTERNAL_OPTION(tracedump_origins)) {
uint i;
for (i=0; i<t->num_bbs; i++) {
instr_t *inst;
instrlist_t *ilist;
int size = 0;
TRACEBUF_MAKE_ROOM(p, buf, sizeof(app_pc));
*((app_pc *)p) = t->bbs[i].tag;
p += sizeof(app_pc);
/* we assume that the target is readable, since we dump prior
* to unloading of modules on flush events
*/
ilist = build_app_bb_ilist(dcontext, t->bbs[i].tag, INVALID_FILE);
for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
size += instr_length(dcontext, inst);
}
TRACEBUF_MAKE_ROOM(p, buf, sizeof(int));
*((int *)p) = size;
p += sizeof(int);
for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
TRACEBUF_MAKE_ROOM(p, buf, instr_length(dcontext, inst));
/* PR 302353: we can't use instr_encode() as it will
* try to re-relativize rip-rel instrs, which may fail
*/
ASSERT(instr_get_raw_bits(inst) != NULL);
memcpy(p, instr_get_raw_bits(inst), instr_length(dcontext, inst));
p += instr_length(dcontext, inst);
}
/* free the instrlist_t elements */
instrlist_clear_and_destroy(dcontext, ilist);
}
}
ASSERT(SEPARATE_STUB_MAX_SIZE == DIRECT_EXIT_STUB_SIZE(0));
for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
cache_pc stub_pc = EXIT_STUB_PC(dcontext, f, l);
stub.cti_offs = l->cti_offset;
stub.stub_pc = stub_pc;
stub.target = EXIT_TARGET_TAG(dcontext, f, l);
stub.linked = TEST(LINK_LINKED, l->flags);
stub.stub_size = EXIT_HAS_STUB(l->flags, f->flags) ?
DIRECT_EXIT_STUB_SIZE(f->flags) :
0 /* no stub neededd: -no_indirect_stubs */;
ASSERT(DIRECT_EXIT_STUB_SIZE(f->flags) <= SEPARATE_STUB_MAX_SIZE);
TRACEBUF_MAKE_ROOM(p, buf, STUB_DATA_FIXED_SIZE);
memcpy(p, &stub, STUB_DATA_FIXED_SIZE);
p += STUB_DATA_FIXED_SIZE;
#ifdef PROFILE_LINKCOUNT
if (dynamo_options.profile_counts) {
*((linkcount_type_t *)p) = l->count;
p += sizeof(linkcount_type_t);
}
#endif
if (TEST(LINK_SEPARATE_STUB, l->flags) && stub_pc != NULL) {
TRACEBUF_MAKE_ROOM(p, buf, DIRECT_EXIT_STUB_SIZE(f->flags));
ASSERT(stub_pc < f->start_pc || stub_pc >= f->start_pc+f->size);
memcpy(p, stub_pc, DIRECT_EXIT_STUB_SIZE(f->flags));
p += DIRECT_EXIT_STUB_SIZE(f->flags);
} else { /* ensure client's method of identifying separate stubs works */
ASSERT(stub_pc == NULL /* no stub at all */ ||
(stub_pc >= f->start_pc && stub_pc < f->start_pc+f->size));
}
}
if (f->size >= TRACEBUF_SIZE) {
os_write(pt->tracefile, buf, (p - buf));
p = buf;
os_write(pt->tracefile, f->start_pc, f->size);
} else {
/* single syscall for all but largest traces */
TRACEBUF_MAKE_ROOM(p, buf, f->size);
memcpy(p, f->start_pc, f->size);
p += f->size;
os_write(pt->tracefile, buf, (p - buf));
p = buf;
}
}
/* Output the contents of the specified trace.
* Does full disassembly of every instruction.
* If deleted_at != -1, it is taken to be the fragment id that
* caused the flushing of this fragment from the cache
* If f is shared, pt argument must be shared_pt, and caller must hold
* the change_linking_lock.
*/
static void
output_trace(dcontext_t *dcontext, per_thread_t *pt, fragment_t *f,
stats_int_t deleted_at)
{
trace_only_t *t = TRACE_FIELDS(f);
#ifdef WINDOWS
char buf[MAXIMUM_PATH];
#endif
stats_int_t trace_num;
bool locked_vmareas = false, ok;
dr_isa_mode_t old_mode;
ASSERT(SHOULD_OUTPUT_FRAGMENT(f->flags));
ASSERT(TEST(FRAG_IS_TRACE, f->flags));
ASSERT(!TEST(FRAG_SELFMOD_SANDBOXED, f->flags)); /* no support for selfmod */
/* already been output? caller should check this flag, just like trace flag */
ASSERT(!TEST(FRAG_TRACE_OUTPUT, f->flags));
ASSERT(!TEST(FRAG_SHARED, f->flags) ||
self_owns_recursive_lock(&change_linking_lock));
f->flags |= FRAG_TRACE_OUTPUT;
#ifdef PROFILE_LINKCOUNT
if (dynamo_options.tracedump_threshold > 0) {
linkcount_type_t count = get_total_linkcount(f);
if (count < (linkcount_type_t) dynamo_options.tracedump_threshold) {
pt->tracedump_num_below_threshold++;
pt->tracedump_count_below_threshold += count;
return;
}
}
#endif
LOG(THREAD, LOG_FRAGMENT, 4, "output_trace: F%d("PFX")\n", f->id, f->tag);
/* Recreate in same mode as original fragment */
ok = dr_set_isa_mode(dcontext, FRAG_ISA_MODE(f->flags), &old_mode);
ASSERT(ok);
/* xref 8131/8202 if dynamo_resetting we don't need to grab the tracedump
* mutex to ensure we're the only writer and grabbing here on reset path
* can lead to a rank-order violation. */
if (!dynamo_resetting) {
/* We must grab shared_vm_areas lock first to avoid rank order (i#1157) */
locked_vmareas = acquire_vm_areas_lock_if_not_already(dcontext, FRAG_SHARED);
mutex_lock(&tracedump_mutex);
}
trace_num = tcount;
tcount++;
if (!TEST(FRAG_SHARED, f->flags)) {
/* No lock is needed because we use thread-private files.
* If dumping traces for a different thread (dynamo_other_thread_exit
* for ex.) caller is responsible for the necessary synchronization. */
ASSERT(pt != shared_pt);
if (!dynamo_resetting) {
mutex_unlock(&tracedump_mutex);
if (locked_vmareas) {
locked_vmareas = false;
release_vm_areas_lock(dcontext, FRAG_SHARED);
}
}
} else {
ASSERT(pt == shared_pt);
}
/* binary dump requested? */
if (INTERNAL_OPTION(tracedump_binary)) {
output_trace_binary(dcontext, pt, f, trace_num);
goto output_trace_done;
}
/* just origins => just bb tags in text */
if (INTERNAL_OPTION(tracedump_origins) &&
!INTERNAL_OPTION(tracedump_text)) {
uint i;
print_file(pt->tracefile, "Trace %d\n", tcount);
#ifdef DEBUG
print_file(pt->tracefile, "Fragment %d\n", f->id);
#endif
for (i = 0; i < t->num_bbs; i++) {
print_file(pt->tracefile, "\tbb %d = "PFX"\n", i, t->bbs[i].tag);
}
print_file(pt->tracefile, "\n");
goto output_trace_done;
}
/* full text dump */
print_file(pt->tracefile, "=============================================="
"=============================\n\n");
print_file(pt->tracefile, "TRACE # %d\n", tcount);
#ifdef DEBUG
print_file(pt->tracefile, "Fragment # %d\n", f->id);
#endif
print_file(pt->tracefile, "Tag = "PFX"\n", f->tag);
print_file(pt->tracefile, "Thread = %d\n", get_thread_id());
if (deleted_at > -1) {
print_file(pt->tracefile,
"*** Flushed from cache when top fragment id was %d\n",
deleted_at);
}
#ifdef WINDOWS
/* FIXME: for fragments flushed by unloaded modules, naturally we
* won't be able to get the module name b/c by now it is unloaded,
* the only fix is to keep a listing of mapped modules and only
* unmap in our listing at this point
*/
get_module_name(f->tag, buf, sizeof(buf));
if (buf[0] != '\0')
print_file(pt->tracefile, "Module of basic block 0 = %s\n", buf);
else
print_file(pt->tracefile, "Module of basic block 0 = <unknown>\n");
#endif
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
if (INTERNAL_OPTION(tracedump_origins)) {
uint i;
print_file(pt->tracefile, "\nORIGINAL CODE:\n");
for (i = 0; i < t->num_bbs; i++) {
/* we only care about the printing that the build routine does */
print_file(pt->tracefile, "basic block # %d: ", i);
/* we assume that the target is readable, since we dump prior
* to unloading of modules on flush events
*/
ASSERT(is_readable_without_exception(t->bbs[i].tag, sizeof(t->bbs[i])));
disassemble_app_bb(dcontext, t->bbs[i].tag, pt->tracefile);
}
print_file(pt->tracefile, "END ORIGINAL CODE\n\n");
}
#endif
#ifdef PROFILE_RDTSC
if (dynamo_options.profile_times) {
/* Cycle counts are too high due to extra instructions needed
* to save regs and store time, so we subtract a constant for
* each fragment entry.
* Experiments using large-iter loops show that:
* empty loop overhead = 2 cycles
* 2 pushes, 2 moves, 2 pops = 7 cycles - 2 cycles = 5 cycles
* add in 1 rdtsc = 38 cycles - 2 cycles = 36 cycles
* add in 2 rdtsc = 69 cycles - 2 cycles = 67 cycles
* If we assume the time transfer happens in the middle of the rdtsc, we
* should use 36 cycles.
*/
trace_only_t *tr = TRACE_FIELDS(f);
const int adjustment = 37;
uint64 real_time = tr->total_time;
uint64 temp;
uint time_top, time_bottom;
print_file(pt->tracefile, "Size = %d (+ %d for profiling)\n",
f->size-profile_call_size(), profile_call_size());
print_file(pt->tracefile, "Profiling:\n");
print_file(pt->tracefile, "\tcount = "UINT64_FORMAT_STRING"\n", tr->count);
print_file(pt->tracefile, "\tmeasured cycles = "PFX"\n", real_time);
temp = tr->count * (uint64)adjustment;
if (real_time < temp) {
print_file(pt->tracefile,
"\t ERROR: adjustment too large, cutting off at 0, should use < %d\n",
(int)(real_time / tr->count));
real_time = 0;
} else {
real_time -= temp;
}
print_file(pt->tracefile, "\tadjusted cycles = "PFX"\n", real_time);
divide_uint64_print(real_time, kilo_hertz, false, 6,
&time_top, &time_bottom);
print_file(pt->tracefile, "\ttime = %u.%.6u ms\n",
time_top, time_bottom);
} else {
print_file(pt->tracefile, "Size = %d\n", f->size);
}
#else
print_file(pt->tracefile, "Size = %d\n", f->size);
#endif /* PROFILE_RDTSC */
#ifdef PROFILE_LINKCOUNT
if (dynamo_options.profile_counts) {
int num_exits = 0;
linkstub_t *ls;
print_file(pt->tracefile, "Exit stubs:\n");
for (ls = FRAGMENT_EXIT_STUBS(f); ls; ls = LINKSTUB_NEXT_EXIT(ls)) {
int id = -1;
app_pc target;
if ((ls->flags & LINK_INDIRECT) != 0) {
target = 0;
id = -1;
} else {
# ifdef DEBUG
fragment_t *targetf = fragment_lookup(dcontext,
EXIT_TARGET_TAG(dcontext, f, ls));
id = (targetf != NULL) ? targetf->id : -1;
# else
id = -1;
# endif
target = EXIT_TARGET_TAG(dcontext, f, ls);
}
print_file(pt->tracefile,
"\t#%d: target = "PFX" (F#%d), count = "
LINKCOUNT_FORMAT_STRING "%s\n",
num_exits, target, id, ls->count,
((ls->flags & LINK_LINKED) != 0) ? ", linked" : "");
num_exits++;
}
# ifdef SIDELINE_COUNT_STUDY
if (dynamo_options.sideline && t->count_old_pre > (linkcount_type_t)0) {
print_file(pt->tracefile, "Sideline: pre-opt count = "
LINKCOUNT_FORMAT_STRING "\n",
t->count_old_pre);
print_file(pt->tracefile, "\tpost-opt count = "
LINKCOUNT_FORMAT_STRING "\n",
t->count_old_post);
print_file(pt->tracefile, "\tnew trace count = "
LINKCOUNT_FORMAT_STRING "\n",
get_total_linkcount(f));
}
# endif
}
#endif /* PROFILE_LINKCOUNT */
#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
print_file(pt->tracefile, "Body:\n");
disassemble_fragment_body(dcontext, f, pt->tracefile);
print_file(pt->tracefile, "END TRACE %d\n\n", tcount);
#endif
output_trace_done:
dr_set_isa_mode(dcontext, old_mode, NULL);
if (TEST(FRAG_SHARED, f->flags) && !dynamo_resetting) {
ASSERT_OWN_MUTEX(true, &tracedump_mutex);
mutex_unlock(&tracedump_mutex);
if (locked_vmareas)
release_vm_areas_lock(dcontext, FRAG_SHARED);
} else {
ASSERT_DO_NOT_OWN_MUTEX(true, &tracedump_mutex);
}
}
#endif /* defined(INTERNAL) || defined(CLIENT_INTERFACE) */
/****************************************************************************/
#ifdef PROFILE_RDTSC
/* This routine is called at the start of every trace
* it assumes the caller has saved the caller-saved registers
* (eax, ecx, edx)
*
* See insert_profile_call() for the assembly code prefix that calls
* this routine. The end time is computed in the assembly code and passed
* in to have as accurate times as possible (don't want the profiling overhead
* added into the times). Also, the assembly code computes the new start
* time after this routine returns.
*
* We could generate a custom copy of this routine
* for every thread, but we don't do that now -- don't need to.
*/
void
profile_fragment_enter(fragment_t *f, uint64 end_time)
{
dcontext_t *dcontext;
#ifdef WINDOWS
/* must get last error prior to getting dcontext */
int error_code = get_last_error();
#endif
trace_only_t *t = TRACE_FIELDS(f);
/* this is done in assembly: uint64 end_time = get_time() */
dcontext = get_thread_private_dcontext();
/* increment this fragment's execution count */
t->count++;
/***********************************************************************/
/* all-in-cache sequence profiling */
/* top ten cache times */
dcontext->cache_frag_count++;
/***********************************************************************/
/* we rely on dispatch being the only way to enter the fcache.
* dispatch sets prev_fragment to null prior to entry */
if (dcontext->prev_fragment != NULL) {
trace_only_t *last_t = TRACE_FIELDS(dcontext->prev_fragment);
ASSERT((dcontext->prev_fragment->flags & FRAG_IS_TRACE) != 0);
/* charge time to last fragment */
last_t->total_time += (end_time - dcontext->start_time);
}
/* set up for next fragment */
dcontext->prev_fragment = f;
/* this is done in assembly: dcontext->start_time = get_time() */
#ifdef WINDOWS
/* retore app's error code */
set_last_error(error_code);
#endif
}
/* this routine is called from dispatch after exiting the fcache
* it finishes up the final fragment's time slot
*/
void
profile_fragment_dispatch(dcontext_t *dcontext)
{
/* end time slot ASAP, should we try to move this to assembly routines exiting
* the cache? probably not worth it, in a long-running program shouldn't
* be exiting the cache that often */
uint64 end_time = get_time();
bool tagtable = LINKSTUB_INDIRECT(dcontext->last_exit->flags);
if (dcontext->prev_fragment != NULL && (dcontext->prev_fragment->flags & FRAG_IS_TRACE) != 0) {
/* end time slot, charge time to last fragment
* there's more overhead here than other time endings, so subtract
* some time off. these numbers are pretty arbitrary:
*/
trace_only_t *last_t = TRACE_FIELDS(dcontext->prev_fragment);
const uint64 adjust = (tagtable) ? 72 : 36;
uint64 add = end_time - dcontext->start_time;
if (add < adjust) {
SYSLOG_INTERNAL_ERROR("ERROR: profile_fragment_dispatch: add was %d, tagtable %d",
(int)add, tagtable);
add = 0;
} else
add -= adjust;
ASSERT((dcontext->prev_fragment->flags & FRAG_IS_TRACE) != 0);
last_t->total_time += add;
}
}
#endif /* PROFILE_RDTSC */
/*******************************************************************************
* COARSE-GRAIN FRAGMENT HASHTABLE INSTANTIATION
*/
/* Synch model: the htable lock should be held during all
* app_to_cache_t manipulations, to be consistent (not strictly
* necessary as there are no removals from the htable except in
* all-thread-synch flushes). Copies of a looked-up cache_pc are safe
* to use w/o the htable lock, or any lock -- they're pointers into
* cache units, and there are no deletions of cache units, except in
* all-thread-synch flushes.
*/
/* 2 macros w/ name and types are duplicated in fragment.h -- keep in sync */
#define NAME_KEY coarse
#define ENTRY_TYPE app_to_cache_t
static app_to_cache_t a2c_empty = { NULL, NULL };
static app_to_cache_t a2c_sentinel = { /*assume invalid*/(app_pc)PTR_UINT_MINUS_1,
NULL };
/* FIXME: want to inline the app_to_cache_t struct just like lookuptable
* does and use it for main table -- no support for that right now
*/
/* not defining HASHTABLE_USE_LOOKUPTABLE */
#define ENTRY_TAG(f) ((ptr_uint_t)(f).app)
#define ENTRY_EMPTY (a2c_empty)
#define ENTRY_SENTINEL (a2c_sentinel)
#define ENTRY_IS_EMPTY(f) ((f).app == a2c_empty.app)
#define ENTRY_IS_SENTINEL(f) ((f).app == a2c_sentinel.app)
#define ENTRY_IS_INVALID(f) (false) /* no invalid entries */
#define ENTRIES_ARE_EQUAL(t,f,g) ((f).app == (g).app)
#define HASHTABLE_WHICH_HEAP(flags) (ACCT_FRAG_TABLE)
/* note that we give the th table a lower-ranked coarse_th_htable_rwlock */
#define COARSE_HTLOCK_RANK coarse_table_rwlock /* for use after hashtablex.h */
#define HTLOCK_RANK COARSE_HTLOCK_RANK
#define HASHTABLE_SUPPORT_PERSISTENCE 1
#include "hashtablex.h"
/* All defines are undef-ed at end of hashtablex.h
* Would be nice to re-use ENTRY_IS_EMPTY, etc., though w/ multiple htables
* in same file can't realistically get away w/o custom defines like these:
*/
#define A2C_ENTRY_IS_EMPTY(a2c) ((a2c).app == NULL)
#define A2C_ENTRY_IS_SENTINEL(a2c) ((a2c).app == a2c_sentinel.app)
#define A2C_ENTRY_IS_REAL(a2c) (!A2C_ENTRY_IS_EMPTY(a2c) && !A2C_ENTRY_IS_SENTINEL(a2c))
/* required routines for hashtable interface that we don't need for this instance */
static void
hashtable_coarse_init_internal_custom(dcontext_t *dcontext, coarse_table_t *htable)
{ /* nothing */
}
static void
hashtable_coarse_resized_custom(dcontext_t *dcontext, coarse_table_t *htable,
uint old_capacity, app_to_cache_t *old_table,
app_to_cache_t *old_table_unaligned,
uint old_ref_count, uint old_table_flags)
{ /* nothing */
}
# ifdef DEBUG
static void
hashtable_coarse_study_custom(dcontext_t *dcontext, coarse_table_t *htable,
uint entries_inc/*amnt table->entries was pre-inced*/)
{ /* nothing */
}
# endif
static void
hashtable_coarse_free_entry(dcontext_t *dcontext, coarse_table_t *htable,
app_to_cache_t entry)
{
/* nothing to do, data is inlined */
}
/* i#670: To handle differing app addresses from different module
* bases across different executions, we store the persist-time abs
* addrs in our tables and always shift on lookup. For frozen exit
* stubs, we have the frozen fcache return convert to a cur-base abs
* addr so this shift will then restore to persist-time.
*
* XXX: this is needed for -persist_trust_textrel, but for Windows
* bases will only be different when we need to apply relocations, and
* there we could just add an extra relocation per exit stub, which
* would end up being lower overhead for long-running apps, but may be
* higher than the overhead of shifting for short-running. For now,
* I'm enabling this as cross-platform, and we can do perf
* measurements later if desired.
*
* Storing persist-time abs addr versus storing module offsets: note
* that whatever we do we want a frozen-unit-only solution so we don't
* want to, say, always store module offsets for non-frozen units.
* This is because we support mixing coarse and fine and we have to
* use abs addrs in fine tables b/c they're not per-module. We could
* store offsets in frozen only, but then we'd have to do extra work
* when freezing. Plus, by storing absolute, when the module base
* lines up we can avoid the shift on the exit stubs (although today
* we only avoid exit stub overhead for trace heads when the base
* matches).
*/
static inline app_to_cache_t
coarse_lookup_internal(dcontext_t *dcontext, app_pc tag, coarse_table_t *table)
{
/* note that for mod_shift we don't need to compare to bounds b/c
* this is a table for this module only
*/
app_to_cache_t a2c =
hashtable_coarse_lookup(dcontext, (ptr_uint_t)(tag + table->mod_shift), table);
if (table->mod_shift != 0 && A2C_ENTRY_IS_REAL(a2c))
a2c.app -= table->mod_shift;
return a2c;
}
/* I would #define hashtable_coarse_lookup as DO_NOT_USE but we have to use for
* pclookup
*/
/* Pass 0 for the initial capacity to use the default.
* Initial capacities are number of entires and NOT bits in mask or anything.
*/
void
fragment_coarse_htable_create(coarse_info_t *info, uint init_capacity,
uint init_th_capacity)
{
coarse_table_t *th_htable;
coarse_table_t *htable;
uint init_size;
ASSERT(SHARED_FRAGMENTS_ENABLED());
/* Case 9537: If we start the new table small and grow it we have large
* collision chains as we map the lower address space of a large table into
* the same lower fraction of a smaller table, so we create our table fully
* sized up front.
*/
if (init_capacity != 0) {
init_size = hashtable_bits_given_entries(init_capacity,
DYNAMO_OPTION(coarse_htable_load));
} else
init_size = INIT_HTABLE_SIZE_COARSE;
LOG(GLOBAL, LOG_FRAGMENT, 2, "Coarse %s htable %d capacity => %d bits\n",
info->module, init_capacity, init_size);
htable = NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_table_t,
ACCT_FRAG_TABLE);
hashtable_coarse_init(GLOBAL_DCONTEXT, htable, init_size,
DYNAMO_OPTION(coarse_htable_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
HASHTABLE_RELAX_CLUSTER_CHECKS
_IF_DEBUG("coarse htable"));
htable->mod_shift = 0;
info->htable = (void *) htable;
/* We could create th_htable lazily independently of htable but not worth it */
if (init_th_capacity != 0) {
init_size = hashtable_bits_given_entries(init_th_capacity,
DYNAMO_OPTION(coarse_th_htable_load));
} else
init_size = INIT_HTABLE_SIZE_COARSE_TH;
LOG(GLOBAL, LOG_FRAGMENT, 2, "Coarse %s th htable %d capacity => %d bits\n",
info->module, init_th_capacity, init_size);
th_htable = NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_table_t,
ACCT_FRAG_TABLE);
hashtable_coarse_init(GLOBAL_DCONTEXT, th_htable, init_size,
DYNAMO_OPTION(coarse_th_htable_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
HASHTABLE_RELAX_CLUSTER_CHECKS
_IF_DEBUG("coarse th htable"));
th_htable->mod_shift = 0;
/* We give th table a lower lock rank for coarse_body_from_htable_entry().
* FIXME: add param to init() that takes in lock rank?
*/
ASSIGN_INIT_READWRITE_LOCK_FREE(th_htable->rwlock, coarse_th_table_rwlock);
info->th_htable = (void *) th_htable;
}
/* Adds all entries from stable into dtable, offsetting by dst_cache_offset,
* which is the offset from dst->cache_start_pc at which
* the src cache has been placed.
*/
static void
fragment_coarse_htable_merge_helper(dcontext_t *dcontext,
coarse_info_t *dst, coarse_table_t *dtable,
coarse_info_t *src, coarse_table_t *stable,
ssize_t dst_cache_offset)
{
app_to_cache_t a2c, look_a2c;
uint i;
/* assumption: dtable is private to this thread and so does not need synch */
DODEBUG({ dtable->is_local = true; });
TABLE_RWLOCK(stable, read, lock);
for (i = 0; i < stable->capacity; i++) {
a2c = stable->table[i];
if (A2C_ENTRY_IS_REAL(a2c)) {
look_a2c = coarse_lookup_internal(dcontext, a2c.app, dtable);
if (A2C_ENTRY_IS_EMPTY(look_a2c)) {
a2c.cache += dst_cache_offset;
if (!dst->frozen) { /* adjust absolute value */
ASSERT_NOT_TESTED();
a2c.cache += (dst->cache_start_pc - src->cache_start_pc);
}
hashtable_coarse_add(dcontext, a2c, dtable);
} else {
/* Our merging-with-dups strategy requires that we not
* merge them in this early
*/
ASSERT_NOT_REACHED();
}
}
}
TABLE_RWLOCK(stable, read, unlock);
DODEBUG({ dtable->is_local = false; });
}
/* Merges the main and th htables from info1 and info2 into new htables for dst.
* If !add_info2, makes room for but does not add entries from info2.
* If !add_th_htable, creates but does not add entries to dst->th_htable.
* FIXME: we can't use hashtable_coarse_merge() b/c we don't want to add
* entries from the 2nd table, and we do custom mangling of entries when adding.
* Is it worth parametrizing hashtable_coarse_merge() to share anything?
*/
void
fragment_coarse_htable_merge(dcontext_t *dcontext, coarse_info_t *dst,
coarse_info_t *info1, coarse_info_t *info2,
bool add_info2, bool add_th_htable)
{
coarse_table_t *thht_dst, *thht1, *thht2;
coarse_table_t *ht_dst, *ht1, *ht2;
uint merged_entries = 0;
ASSERT(SHARED_FRAGMENTS_ENABLED());
ASSERT(info1 != NULL && info2 != NULL);
ht1 = (coarse_table_t *) info1->htable;
ht2 = (coarse_table_t *) info2->htable;
thht1 = (coarse_table_t *) info1->th_htable;
thht2 = (coarse_table_t *) info2->th_htable;
ASSERT(dst != NULL && dst->htable == NULL && dst->th_htable == NULL);
/* We go to the trouble of determining non-dup total entries to
* avoid repeatedly increasing htable size on merges and hitting
* collision asserts. FIXME: should we shrink afterward instead?
* Or just ignore collision asserts until at full size?
*/
merged_entries = hashtable_coarse_num_unique_entries(dcontext, ht1, ht2);
STATS_ADD(coarse_merge_dups, ht1->entries + ht2->entries - merged_entries);
LOG(THREAD, LOG_FRAGMENT, 2, "Merging %s: %d + %d => %d (%d unique) entries\n",
info1->module, ht1->entries, ht2->entries, ht1->entries + ht2->entries,
merged_entries);
/* We could instead copy ht1 and then add ht2; for simplicity we re-use
* our create-empty routine and add both
*/
fragment_coarse_htable_create(dst, merged_entries,
/* Heuristic to never over-size, yet try to avoid
* collision asserts while resizing the table;
* FIXME: if we shrink the main htable afterward
* could do the same here and start at sum of
* entries */
MAX(thht1->entries, thht2->entries));
ht_dst = (coarse_table_t *) dst->htable;
thht_dst = (coarse_table_t *) dst->th_htable;
ASSERT(ht_dst != NULL && thht_dst != NULL);
/* For now we only support frozen tables; else will have to change
* the offsets to be stubs for main table and cache for th table
*/
ASSERT(info1->frozen && info2->frozen);
fragment_coarse_htable_merge_helper(dcontext, dst, ht_dst, info1, ht1, 0);
if (add_info2) {
fragment_coarse_htable_merge_helper(dcontext, dst, ht_dst, info2, ht2,
info1->cache_end_pc -
info1->cache_start_pc);
}
if (add_th_htable) {
ASSERT_NOT_TESTED();
fragment_coarse_htable_merge_helper(dcontext, dst, thht_dst, info1, thht1, 0);
fragment_coarse_htable_merge_helper(dcontext, dst, thht_dst, info2, thht2,
/* stubs_end_pc is not allocated end */
info1->mmap_pc + info1->mmap_size -
info1->stubs_start_pc);
}
}
#ifndef DEBUG
/* not in header file unless debug, in which case it's static */
static void
coarse_body_from_htable_entry(dcontext_t *dcontext, coarse_info_t *info,
app_pc tag, cache_pc res,
cache_pc *stub_pc_out/*OUT*/,
cache_pc *body_pc_out/*OUT*/);
#endif
static void
study_and_free_coarse_htable(coarse_info_t *info, coarse_table_t *htable,
bool never_persisted _IF_DEBUG(const char *name))
{
LOG(GLOBAL, LOG_FRAGMENT, 1, "Coarse %s %s hashtable stats:\n",
info->module, name);
DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
hashtable_coarse_load_statistics(GLOBAL_DCONTEXT, htable);
});
DODEBUG({
hashtable_coarse_study(GLOBAL_DCONTEXT, htable, 0/*table consistent*/);
});
DOLOG(3, LOG_FRAGMENT, {
hashtable_coarse_dump_table(GLOBAL_DCONTEXT, htable);
});
#ifdef CLIENT_INTERFACE
/* Only raise deletion events if client saw creation events: so no persisted
* units
*/
if (!info->persisted && htable == info->htable &&
dr_fragment_deleted_hook_exists()) {
app_to_cache_t a2c;
uint i;
dcontext_t *dcontext = get_thread_private_dcontext();
cache_pc body = NULL;
TABLE_RWLOCK(htable, read, lock);
for (i = 0; i < htable->capacity; i++) {
a2c = htable->table[i];
if (A2C_ENTRY_IS_REAL(a2c)) {
if (info->frozen)
body = a2c.cache;
else { /* make sure not an entrance stub w/ no body */
coarse_body_from_htable_entry(dcontext, info, a2c.app, a2c.cache,
NULL, &body);
}
if (body != NULL) {
instrument_fragment_deleted(get_thread_private_dcontext(), a2c.app,
FRAGMENT_COARSE_WRAPPER_FLAGS);
}
}
}
TABLE_RWLOCK(htable, read, unlock);
}
#endif
/* entries are inlined so nothing external to free */
if (info->persisted && !never_persisted) {
/* ensure won't try to free (part of mmap) */
ASSERT(htable->table_unaligned == NULL);
}
hashtable_coarse_free(GLOBAL_DCONTEXT, htable);
NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, htable, coarse_table_t, ACCT_FRAG_TABLE);
}
void
fragment_coarse_free_entry_pclookup_table(dcontext_t *dcontext, coarse_info_t *info)
{
if (info->pclookup_htable != NULL) {
ASSERT(DYNAMO_OPTION(coarse_pclookup_table));
study_and_free_coarse_htable(info, (coarse_table_t *) info->pclookup_htable,
true/*never persisted*/ _IF_DEBUG("pclookup"));
info->pclookup_htable = NULL;
}
}
void
fragment_coarse_htable_free(coarse_info_t *info)
{
ASSERT_OWN_MUTEX(!info->is_local, &info->lock);
if (info->htable == NULL) {
/* lazily initialized, so common to have empty units */
ASSERT(info->th_htable == NULL);
ASSERT(info->pclookup_htable == NULL);
return;
}
study_and_free_coarse_htable(info, (coarse_table_t *) info->htable, false
_IF_DEBUG("main"));
info->htable = NULL;
study_and_free_coarse_htable(info, (coarse_table_t *) info->th_htable, false
_IF_DEBUG("tracehead"));
info->th_htable = NULL;
if (info->pclookup_last_htable != NULL) {
generic_hash_destroy(GLOBAL_DCONTEXT, info->pclookup_last_htable);
info->pclookup_last_htable = NULL;
}
fragment_coarse_free_entry_pclookup_table(GLOBAL_DCONTEXT, info);
}
uint
fragment_coarse_num_entries(coarse_info_t *info)
{
coarse_table_t *htable;
ASSERT(info != NULL);
htable = (coarse_table_t *) info->htable;
if (htable == NULL)
return 0;
return htable->entries;
}
/* Add coarse fragment represented by wrapper f to the hashtable
* for unit info.
*/
void
fragment_coarse_add(dcontext_t *dcontext, coarse_info_t *info,
app_pc tag, cache_pc cache)
{
coarse_table_t *htable;
app_to_cache_t a2c = {tag, cache};
ASSERT(info != NULL && info->htable != NULL);
htable = (coarse_table_t *) info->htable;
DOCHECK(1, {
/* We have lock rank order problems b/c this lookup may acquire
* the th table read lock, and we can't split its rank from
* the main table's. So we live w/ a racy assert that could
* have false positives or negatives.
*/
cache_pc stub;
cache_pc body;
/* OK to have dup entries for entrance stubs in other units,
* or a stub already present for a trace head */
fragment_coarse_lookup_in_unit(dcontext, info, tag, &stub, &body);
ASSERT(body == NULL);
ASSERT(stub == NULL ||
coarse_is_trace_head_in_own_unit(dcontext, tag, stub,
/* case 10876: pass what we're adding */
(ptr_uint_t)cache + info->cache_start_pc,
true, info));
/* There can only be one body. But similarly to above,
* fragment_coarse_lookup may look in the secondary unit, so
* we can't do this lookup holding the write lock.
*/
if (!coarse_is_entrance_stub(cache)) {
coarse_info_t *xinfo = get_executable_area_coarse_info(tag);
fragment_t *f;
ASSERT(xinfo != NULL);
/* If an official unit, tag should not exist in any other official unit */
ASSERT((info != xinfo && info != xinfo->non_frozen) ||
fragment_coarse_lookup(dcontext, tag) == NULL);
f = fragment_lookup(dcontext, tag);
/* ok for trace to shadow coarse trace head */
ASSERT(f == NULL || TEST(FRAG_IS_TRACE, f->flags));
}
});
TABLE_RWLOCK(htable, write, lock);
/* Table is not an ibl table so we ignore resize return value */
hashtable_coarse_add(dcontext, a2c, htable);
TABLE_RWLOCK(htable, write, unlock);
#ifdef SHARING_STUDY
if (INTERNAL_OPTION(fragment_sharing_study)) {
ASSERT_NOT_IMPLEMENTED(false && "need to pass f in to add_shared_block");
}
#endif
}
/* Returns the body pc of the coarse trace head fragment corresponding to tag, or
* NULL if not found. Caller must hold the th table's read or write lock!
*/
static cache_pc
fragment_coarse_th_lookup(dcontext_t *dcontext, coarse_info_t *info, app_pc tag)
{
cache_pc res = NULL;
app_to_cache_t a2c;
coarse_table_t *htable;
ASSERT(info != NULL);
ASSERT(info->htable != NULL);
htable = (coarse_table_t *) info->th_htable;
ASSERT(TABLE_PROTECTED(htable));
a2c = coarse_lookup_internal(dcontext, tag, htable);
if (!A2C_ENTRY_IS_EMPTY(a2c)) {
ASSERT(BOOLS_MATCH(info->frozen, info->stubs_start_pc != NULL));
/* for frozen, th_htable only holds stubs */
res = ((ptr_uint_t)a2c.cache) + info->stubs_start_pc;
}
return res;
}
/* Performs two actions while holding the trace head table's write lock,
* making them atomic (solving the race in case 8795):
* 1) unlinks the coarse fragment's entrance pc and points it at the
* trace head exit routine;
* 2) adds the coarse fragment's body pc to the trace head hashtable.
*
* Case 8628: if we split the main htable into stubs and bodies then
* we can eliminate the th htable as it will be part of the body table.
*
* We could merge this info in to the thcounter table, and assume that
* most trace heads are coarse so we're not wasting much memory with
* the extra field. But then we have to walk entire stub table on
* cache flush and individually clear body pcs from monitor table, so
* better to have own th table?
*/
void
fragment_coarse_th_unlink_and_add(dcontext_t *dcontext, app_pc tag,
cache_pc stub_pc, cache_pc body_pc)
{
ASSERT(stub_pc != NULL);
if (body_pc != NULL) {
/* trace head is in this unit, so we have to add it to our th htable */
coarse_info_t *info = get_fcache_coarse_info(body_pc);
coarse_table_t *th_htable;
app_to_cache_t a2c = {tag, body_pc};
ASSERT(info != NULL && info->th_htable != NULL);
ASSERT(!info->frozen);
th_htable = (coarse_table_t *) info->th_htable;
TABLE_RWLOCK(th_htable, write, lock);
ASSERT(fragment_coarse_th_lookup(dcontext, info, tag) == NULL);
unlink_entrance_stub(dcontext, stub_pc, FRAG_IS_TRACE_HEAD, info);
/* Table is not an ibl table so we ignore resize return value */
hashtable_coarse_add(dcontext, a2c, th_htable);
TABLE_RWLOCK(th_htable, write, unlock);
LOG(THREAD, LOG_FRAGMENT, 4,
"adding to coarse th table for %s: "PFX"->"PFX"\n",
info->module, tag, body_pc);
} else {
/* ensure lives in another unit */
ASSERT(fragment_coarse_lookup(dcontext, tag) != stub_pc);
unlink_entrance_stub(dcontext, stub_pc, FRAG_IS_TRACE_HEAD, NULL);
}
}
/* Only use when building up a brand-new table. Otherwise use
* fragment_coarse_th_unlink_and_add().
*/
void
fragment_coarse_th_add(dcontext_t *dcontext, coarse_info_t *info,
app_pc tag, cache_pc cache)
{
coarse_table_t *th_htable;
app_to_cache_t a2c = {tag, cache};
ASSERT(info != NULL && info->th_htable != NULL);
ASSERT(info->frozen); /* only used when merging units */
th_htable = (coarse_table_t *) info->th_htable;
TABLE_RWLOCK(th_htable, write, lock);
ASSERT(fragment_coarse_th_lookup(dcontext, info, tag) == NULL);
/* Table is not an ibl table so we ignore resize return value */
hashtable_coarse_add(dcontext, a2c, th_htable);
TABLE_RWLOCK(th_htable, write, unlock);
}
/* The input here is the result of a lookup in the main htable.
* For a frozen unit, this actually looks up the stub pc since res is
* always the body pc.
* For a non-frozen unit this determines where to obtain the body pc.
* FIXME: case 8628 will simplify this whole thing
*/
/* exported only for assert in push_pending_freeze() */
IF_DEBUG_ELSE(,static) void
coarse_body_from_htable_entry(dcontext_t *dcontext, coarse_info_t *info,
app_pc tag, cache_pc res,
cache_pc *stub_pc_out/*OUT*/,
cache_pc *body_pc_out/*OUT*/)
{
cache_pc stub_pc = NULL, body_pc = NULL;
/* Should be passing absolute pc, not offset */
ASSERT(!info->frozen || res == NULL || res >= info->cache_start_pc);
if (info->frozen) {
/* We still need the stub table for lazy linking and for
shifting links from a trace head to a trace.
*/
body_pc = res;
if (stub_pc_out != NULL) {
TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, lock);
stub_pc = fragment_coarse_th_lookup(dcontext, info, tag);
TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, unlock);
}
} else {
/* In a non-frozen unit, htable entries are always stubs */
DOCHECK(CHKLVL_DEFAULT+1, { ASSERT(coarse_is_entrance_stub(res)); });
stub_pc = res;
if (body_pc_out != NULL) {
/* keep the th table entry and stub link status linked atomically */
TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, lock);
body_pc = fragment_coarse_th_lookup(dcontext, info, tag);
if (body_pc != NULL) {
/* We can't just check coarse_is_trace_head(res) because a
* shadowed trace head is directly linked to its trace. The
* trace has lookup order precedence, but the head must show up
* if asked about here!
*/
/* FIXME: we could have a flags OUT param and set FRAG_IS_TRACE_HEAD
* to help out fragment_lookup_fine_and_coarse*().
*/
} else {
if (entrance_stub_linked(res, info)) {
/* We only want to set body_pc if it is present in this unit */
cache_pc tgt = entrance_stub_jmp_target(res);
if (get_fcache_coarse_info(tgt) == info)
body_pc = tgt;
else
body_pc = NULL;
} else
body_pc = NULL;
DOCHECK(CHKLVL_DEFAULT+1, {
ASSERT(!coarse_is_trace_head(res) ||
/* allow targeting trace head in another unit */
body_pc == NULL);
});
}
TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, unlock);
}
}
if (stub_pc_out != NULL)
*stub_pc_out = stub_pc;
if (body_pc_out != NULL)
*body_pc_out = body_pc;
}
/* Coarse fragments have two entrance points: the actual fragment
* body, and the entrance stub that is used for indirection and
* convenient incremental building for intra-unit non-frozen linking
* as well as always used at the source end for inter-unit linking.
* This routine returns both. If the stub_pc returned is non-NULL, it
* only indicates that there is an outgoing link from a fragment
* present in this unit that targets the queried tag (and for
* intra-unit links in a frozen unit, such a link may exist but
* stub_pc will be NULL as the entrance stub indirection will have
* been replaced with a direct link). The fragment corresponding to
* the tag is only present in this unit if the body_pc returned is
* non-NULL.
*/
void
fragment_coarse_lookup_in_unit(dcontext_t *dcontext, coarse_info_t *info, app_pc tag,
/* FIXME: have separate type for stub pc vs body pc? */
cache_pc *stub_pc_out/*OUT*/,
cache_pc *body_pc_out/*OUT*/)
{
cache_pc res = NULL;
cache_pc stub_pc = NULL, body_pc = NULL;
app_to_cache_t a2c;
coarse_table_t *htable;
ASSERT(info != NULL);
if (info->htable == NULL) /* not initialized yet, so no code there */
goto coarse_lookup_return;
htable = (coarse_table_t *) info->htable;
TABLE_RWLOCK(htable, read, lock);
a2c = coarse_lookup_internal(dcontext, tag, htable);
if (!A2C_ENTRY_IS_EMPTY(a2c)) {
LOG(THREAD, LOG_FRAGMENT, 5, "%s: %s %s tag="PFX" => app="PFX" cache="PFX"\n",
__FUNCTION__, info->module, info->frozen ? "frozen" : "",
tag, a2c.app, a2c.cache);
ASSERT(BOOLS_MATCH(info->frozen, info->cache_start_pc != NULL));
/* for frozen, htable only holds body pc */
res = ((ptr_uint_t)a2c.cache) + info->cache_start_pc;
}
if (res != NULL)
coarse_body_from_htable_entry(dcontext, info, tag, res, &stub_pc, &body_pc);
else if (info->frozen) /* need a separate lookup for stub */
coarse_body_from_htable_entry(dcontext, info, tag, res, &stub_pc, &body_pc);
TABLE_RWLOCK(htable, read, unlock);
/* cannot have both a shared coarse and shared fine bb for same tag
* (can have shared trace shadowing shared coarse trace head bb, or
* private bb shadowing shared coarse bb)
*/
ASSERT(body_pc == NULL || fragment_lookup_shared_bb(dcontext, tag) == NULL);
coarse_lookup_return:
if (stub_pc_out != NULL)
*stub_pc_out = stub_pc;
if (body_pc_out != NULL)
*body_pc_out = body_pc;
}
/* Returns the body pc of the coarse fragment corresponding to tag, or
* NULL if not found
*/
cache_pc
fragment_coarse_lookup(dcontext_t *dcontext, app_pc tag)
{
cache_pc res = NULL;
coarse_info_t *info = get_executable_area_coarse_info(tag);
/* We must check each unit in turn */
while (info != NULL) { /* loop over primary and secondary unit */
fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &res);
if (res != NULL)
return res;
ASSERT(info->frozen || info->non_frozen == NULL);
info = info->non_frozen;
ASSERT(info == NULL || !info->frozen);
}
return NULL;
}
/* It's up to the caller to hold locks preventing simultaneous writes to wrapper */
void
fragment_coarse_wrapper(fragment_t *wrapper, app_pc tag, cache_pc body_pc)
{
ASSERT(wrapper != NULL);
if (wrapper == NULL)
return;
ASSERT(tag != NULL);
ASSERT(body_pc != NULL);
/* FIXME: fragile to rely on other routines not inspecting other
* fields of fragment_t -- but setting to 0 perhaps no better than garbage
* in that case. We do need to set prefix_size, incoming_stubs, and
* {next,prev}_vmarea to NULL, for sure.
*/
memset(wrapper, 0, sizeof(*wrapper));
wrapper->tag = tag;
wrapper->start_pc = body_pc;
wrapper->flags = FRAGMENT_COARSE_WRAPPER_FLAGS;
/* We don't have stub pc so can't fill in FRAG_IS_TRACE_HEAD -- so we rely
* on callers passing src info to fragment_lookup_fine_and_coarse()
*/
}
/* If finds a coarse fragment for tag, returns wrapper; else returns NULL */
fragment_t *
fragment_coarse_lookup_wrapper(dcontext_t *dcontext, app_pc tag, fragment_t *wrapper)
{
cache_pc coarse;
ASSERT(wrapper != NULL);
coarse = fragment_coarse_lookup(dcontext, tag);
if (coarse != NULL) {
fragment_coarse_wrapper(wrapper, tag, coarse);
return wrapper;
}
return NULL;
}
/* Takes in last_exit in order to mark trace headness.
* FIXME case 8600: should we replace all other lookups with this one?
* Fragile to only have select callers use this and everyone else
* ignore coarse fragments...
*/
fragment_t *
fragment_lookup_fine_and_coarse(dcontext_t *dcontext, app_pc tag,
fragment_t *wrapper, linkstub_t *last_exit)
{
fragment_t *res = fragment_lookup(dcontext, tag);
if (DYNAMO_OPTION(coarse_units)) {
ASSERT(wrapper != NULL);
if (res == NULL) {
res = fragment_coarse_lookup_wrapper(dcontext, tag, wrapper);
/* FIXME: no way to know if source is a fine fragment! */
if (res != NULL && last_exit == get_coarse_trace_head_exit_linkstub())
res->flags |= FRAG_IS_TRACE_HEAD;
} else {
/* cannot have both coarse and fine shared bb for same tag
* (can have shared trace shadowing shared coarse trace head bb,
* or private bb with same tag)
*/
ASSERT(TEST(FRAG_IS_TRACE, res->flags) ||
!TEST(FRAG_SHARED, res->flags) ||
fragment_coarse_lookup(dcontext, tag) == NULL);
}
}
return res;
}
/* Takes in last_exit in order to mark trace headness.
* FIXME: should we replace all other same_sharing lookups with this one?
* Fragile to only have select callers use this and everyone else
* ignore coarse fragments...
*/
fragment_t *
fragment_lookup_fine_and_coarse_sharing(dcontext_t *dcontext, app_pc tag,
fragment_t *wrapper, linkstub_t *last_exit,
uint share_flags)
{
fragment_t *res =
fragment_lookup_same_sharing(dcontext, tag, share_flags);
if (DYNAMO_OPTION(coarse_units) && TEST(FRAG_SHARED, share_flags)) {
ASSERT(wrapper != NULL);
if (res == NULL) {
res = fragment_coarse_lookup_wrapper(dcontext, tag, wrapper);
if (res != NULL && last_exit == get_coarse_trace_head_exit_linkstub())
res->flags |= FRAG_IS_TRACE_HEAD;
}
}
return res;
}
/* Returns the owning unit of f (versus get_executable_area_coarse_info(f->tag)
* which may return a frozen unit that must be checked for f along with its
* secondary unfrozen unit).
*/
coarse_info_t *
get_fragment_coarse_info(fragment_t *f)
{
/* We have multiple potential units per vmarea region, so we use the fcache
* pc to get the unambiguous owning unit
*/
if (!TEST(FRAG_COARSE_GRAIN, f->flags))
return NULL;
ASSERT(FCACHE_ENTRY_PC(f) != NULL);
return get_fcache_coarse_info(FCACHE_ENTRY_PC(f));
}
/* Pass in info if you know it; else this routine will look it up.
* Checks for stub targeting a trace head, or targeting a trace thus
* indicating that this is a shadowed trace head.
* If body_in or info_in is NULL, this routine will look them up.
*/
bool
coarse_is_trace_head_in_own_unit(dcontext_t *dcontext, app_pc tag, cache_pc stub,
cache_pc body_in, bool body_valid,
coarse_info_t *info_in)
{
coarse_info_t *info = info_in;
ASSERT(stub != NULL);
if (coarse_is_trace_head(stub))
return true;
if (info == NULL)
info = get_stub_coarse_info(stub);
if (info == NULL)
return false;
/* If a coarse stub is linked to a fine fragment and there exists
* a body for that target tag in the same unit as the stub, we assume that
* we have a shadowed coarse trace head.
*/
if (entrance_stub_linked(stub, info) &&
/* Target is fine if no info for it */
get_fcache_coarse_info(entrance_stub_jmp_target(stub)) == NULL) {
cache_pc body = body_in;
if (!body_valid) {
ASSERT(body == NULL);
fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &body);
}
if (body != NULL)
return true;
}
return false;
}
/* Returns whether an entry exists. */
bool
fragment_coarse_replace(dcontext_t *dcontext, coarse_info_t *info, app_pc tag,
cache_pc new_value)
{
app_to_cache_t old_entry = {tag, NULL/*doesn't matter*/};
app_to_cache_t new_entry = {tag, new_value};
coarse_table_t *htable;
bool res = false;
ASSERT(info != NULL && info->htable != NULL);
htable = (coarse_table_t *) info->htable;
TABLE_RWLOCK(htable, read, lock);
res = hashtable_coarse_replace(old_entry, new_entry, info->htable);
TABLE_RWLOCK(htable, read, unlock);
return res;
}
/**************************************************
* PC LOOKUP
*/
/* Table for storing results of prior coarse pclookups (i#658) */
#define PCLOOKUP_LAST_HTABLE_INIT_SIZE 6 /*should remain small*/
/* Alarm signals can result in many pclookups from a variety of places
* (unlike usual DGC patterns) so we bound the size of the table.
* We want to err on the side of using more memory, since failing to
* cache all the instrs that are writing DGC can result in 2x slowdowns.
* It's not worth fancy replacement: we just clear the table if it gets
* really large and start over. Remember that this is per-coarse-unit.
*/
#define PCLOOKUP_LAST_HTABLE_MAX_ENTRIES 8192
typedef struct _pclookup_last_t {
app_pc tag;
cache_pc entry;
} pclookup_last_t;
static void
pclookup_last_free(pclookup_last_t *last)
{
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, last, pclookup_last_t, ACCT_FRAG_TABLE, PROTECTED);
}
/* Returns the tag for the coarse fragment whose body contains pc.
* If that returned tag != NULL, also returns the body pc in the optional OUT param.
* FIXME: verify not called too often: watch the kstat.
*/
app_pc
fragment_coarse_pclookup(dcontext_t *dcontext, coarse_info_t *info, cache_pc pc,
/*OUT*/cache_pc *body_out)
{
app_to_cache_t a2c;
coarse_table_t *htable;
generic_table_t *pc_htable;
pclookup_last_t *last;
cache_pc body_pc;
ssize_t closest_distance = SSIZE_T_MAX;
app_pc closest = NULL;
uint i;
ASSERT(info != NULL);
if (info->htable == NULL)
return NULL;
KSTART(coarse_pclookup);
htable = (coarse_table_t *) info->htable;
if (info->pclookup_last_htable == NULL) {
/* lazily allocate table of all pclookups to avoid htable walk
* on frequent codemod instrs (i#658).
* I tried using a small array instead but chrome v8 needs at least
* 12 entries, and rather than LFU or LRU to avoid worst-case,
* it seems reasonable to simply store all lookups.
* Then the worst case is some extra memory, not 4x slowdowns.
*/
mutex_lock(&info->lock);
if (info->pclookup_last_htable == NULL) {
/* coarse_table_t isn't quite enough b/c we need the body pc,
* which would require an extra lookup w/ coarse_table_t
*/
pc_htable = generic_hash_create(GLOBAL_DCONTEXT,
PCLOOKUP_LAST_HTABLE_INIT_SIZE,
80 /* load factor: not perf-critical */,
HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
HASHTABLE_RELAX_CLUSTER_CHECKS,
(void(*)(void*)) pclookup_last_free
_IF_DEBUG("pclookup last table"));
/* Only when fully initialized can we set it, as we hold no lock for it */
info->pclookup_last_htable = (void *) pc_htable;
}
mutex_unlock(&info->lock);
}
pc_htable = (generic_table_t *) info->pclookup_last_htable;
ASSERT(pc_htable != NULL);
TABLE_RWLOCK(pc_htable, read, lock);
last = (pclookup_last_t *) generic_hash_lookup(GLOBAL_DCONTEXT, pc_htable,
(ptr_uint_t)pc);
if (last != NULL) {
closest = last->tag;
ASSERT(pc >= last->entry);
closest_distance = pc - last->entry;
}
TABLE_RWLOCK(pc_htable, read, unlock);
if (closest == NULL) {
/* do the htable walk */
TABLE_RWLOCK(htable, read, lock);
for (i = 0; i < htable->capacity; i++) {
a2c = htable->table[i];
/* must check for sentinel */
if (A2C_ENTRY_IS_REAL(a2c)) {
a2c.app -= htable->mod_shift;
ASSERT(BOOLS_MATCH(info->frozen, info->cache_start_pc != NULL));
/* for frozen, htable only holds body pc */
a2c.cache += (ptr_uint_t) info->cache_start_pc;
/* We have no body length so we must walk entire table */
coarse_body_from_htable_entry(dcontext, info, a2c.app,
a2c.cache, NULL, &body_pc);
if (body_pc != NULL &&
body_pc <= pc && (pc - body_pc) < closest_distance) {
closest_distance = pc - body_pc;
closest = a2c.app;
}
}
}
if (closest != NULL) {
/* Update the cache of results. Note that since this is coarse we
* don't have to do anything special to invalidate on codemod as the
* whole coarse unit will be thrown out.
*/
TABLE_RWLOCK(pc_htable, write, lock);
/* Check for race (i#1191) */
last = (pclookup_last_t *) generic_hash_lookup(GLOBAL_DCONTEXT, pc_htable,
(ptr_uint_t)pc);
if (last != NULL) {
closest = last->tag;
ASSERT(pc >= last->entry);
closest_distance = pc - last->entry;
} else {
last = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, pclookup_last_t,
ACCT_FRAG_TABLE, PROTECTED);
last->tag = closest;
last->entry = pc - closest_distance;
if (pc_htable->entries >= PCLOOKUP_LAST_HTABLE_MAX_ENTRIES) {
/* See notes above: we don't want an enormous table.
* We just clear rather than a fancy replacement policy.
*/
generic_hash_clear(GLOBAL_DCONTEXT, pc_htable);
}
generic_hash_add(GLOBAL_DCONTEXT, pc_htable, (ptr_uint_t)pc,
(void *)last);
STATS_INC(coarse_pclookup_cached);
}
TABLE_RWLOCK(pc_htable, write, unlock);
}
TABLE_RWLOCK(htable, read, unlock);
}
if (body_out != NULL)
*body_out = pc - closest_distance;
KSTOP(coarse_pclookup);
LOG(THREAD, LOG_FRAGMENT, 4, "%s: "PFX" => "PFX"\n", __FUNCTION__, pc, closest);
return closest;
}
/* Creates a reverse lookup table. For a non-frozen unit, the caller should only
* do this while all threads are suspended, and should free the table before
* resuming other threads.
*/
void
fragment_coarse_create_entry_pclookup_table(dcontext_t *dcontext, coarse_info_t *info)
{
app_to_cache_t main_a2c;
app_to_cache_t pc_a2c;
coarse_table_t *main_htable;
coarse_table_t *pc_htable;
cache_pc body_pc;
uint i;
ASSERT(info != NULL);
if (info->htable == NULL)
return;
if (info->pclookup_htable == NULL) {
mutex_lock(&info->lock);
if (info->pclookup_htable == NULL) {
/* set up reverse lookup table */
main_htable = (coarse_table_t *) info->htable;
pc_htable = NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_table_t,
ACCT_FRAG_TABLE);
hashtable_coarse_init(GLOBAL_DCONTEXT, pc_htable, main_htable->hash_bits,
DYNAMO_OPTION(coarse_pclookup_htable_load),
(hash_function_t)INTERNAL_OPTION(alt_hash_func),
0 /* hash_mask_offset */,
HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
HASHTABLE_RELAX_CLUSTER_CHECKS
_IF_DEBUG("coarse pclookup htable"));
pc_htable->mod_shift = 0;
/* We give pc table a lower lock rank so we can add below
* while holding the lock, though the table is still local
* while we hold info->lock and do not write it out to its
* info-> field, so we could update the add() and
* deadlock-avoidance routines instead (xref case 9522).
* FIXME: add param to init() that takes in lock rank?
*/
ASSIGN_INIT_READWRITE_LOCK_FREE(pc_htable->rwlock,
coarse_pclookup_table_rwlock);
TABLE_RWLOCK(main_htable, read, lock);
TABLE_RWLOCK(pc_htable, write, lock);
for (i = 0; i < main_htable->capacity; i++) {
main_a2c = main_htable->table[i];
/* must check for sentinel */
if (A2C_ENTRY_IS_REAL(main_a2c)) {
main_a2c.app -= main_htable->mod_shift;
ASSERT(BOOLS_MATCH(info->frozen, info->cache_start_pc != NULL));
coarse_body_from_htable_entry(dcontext, info, main_a2c.app,
main_a2c.cache
/* frozen htable only holds body pc */
+ (ptr_uint_t) info->cache_start_pc,
NULL, &body_pc);
if (body_pc != NULL) {
/* We can have two tags with the same cache pc, if
* one is a single jmp that was elided (but we only
* do that if -unsafe_freeze_elide_sole_ubr).
* It's unsafe b/c of case 9677: when translating back
* to app pc we will just take 1st one in linear walk of
* htable, which may be the wrong one!
*/
pc_a2c = hashtable_coarse_lookup(dcontext,
(ptr_uint_t)body_pc, pc_htable);
if (A2C_ENTRY_IS_EMPTY(pc_a2c)) {
pc_a2c.app = body_pc;
pc_a2c.cache = main_a2c.app;
hashtable_coarse_add(dcontext, pc_a2c, pc_htable);
} else {
ASSERT(DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr));
}
}
}
}
TABLE_RWLOCK(pc_htable, write, unlock);
TABLE_RWLOCK(main_htable, read, unlock);
/* Only when fully initialized can we set it, as we hold no lock for it */
info->pclookup_htable = (void *) pc_htable;
}
mutex_unlock(&info->lock);
}
}
/* Returns the tag for the coarse fragment whose body _begins at_ pc */
app_pc
fragment_coarse_entry_pclookup(dcontext_t *dcontext, coarse_info_t *info, cache_pc pc)
{
app_to_cache_t pc_a2c;
coarse_table_t *pc_htable;
cache_pc body_pc;
app_pc res = NULL;
ASSERT(info != NULL);
if (info->htable == NULL)
return NULL;
/* FIXME: we could use this table for non-frozen if we updated it
* when we add to main htable; for now we only support frozen use
*/
if (!DYNAMO_OPTION(coarse_pclookup_table) ||
/* we do create a table for non-frozen while we're freezing (i#735) */
(!info->frozen && info->pclookup_htable == NULL)) {
res = fragment_coarse_pclookup(dcontext, info, pc, &body_pc);
if (body_pc == pc) {
LOG(THREAD, LOG_FRAGMENT, 4, "%s: "PFX" => "PFX"\n", __FUNCTION__, pc, res);
return res;
} else
return NULL;
}
KSTART(coarse_pclookup);
if (info->pclookup_htable == NULL) {
fragment_coarse_create_entry_pclookup_table(dcontext, info);
}
pc_htable = (coarse_table_t *) info->pclookup_htable;
ASSERT(pc_htable != NULL);
TABLE_RWLOCK(pc_htable, read, lock);
pc_a2c = hashtable_coarse_lookup(dcontext, (ptr_uint_t)pc, pc_htable);
if (!A2C_ENTRY_IS_EMPTY(pc_a2c))
res = pc_a2c.cache;
TABLE_RWLOCK(pc_htable, read, unlock);
KSTOP(coarse_pclookup);
LOG(THREAD, LOG_FRAGMENT, 4, "%s: "PFX" => "PFX"\n", __FUNCTION__, pc, res);
return res;
}
/* case 9900: must have dynamo_all_threads_synched since we haven't resolved
* lock rank ordering issues with the hashtable locks
*/
static void
fragment_coarse_entry_freeze(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
pending_freeze_t *pending)
{
app_to_cache_t frozen_a2c;
app_to_cache_t looka2c = {0,0};
coarse_table_t *frozen_htable;
cache_pc tgt;
if (pending->entrance_stub) {
frozen_htable = (coarse_table_t *) freeze_info->dst_info->th_htable;
/* case 9900: rank order conflict with coarse_info_incoming_lock:
* do not grab frozen_htable write lock (only need read but we were
* grabbing write to match assert): mark is_local instead and rely
* on dynamo_all_threads_synched
*/
DODEBUG({ frozen_htable->is_local = true; });
ASSERT_NOT_IMPLEMENTED(dynamo_all_threads_synched && "case 9900");
} else {
frozen_htable = (coarse_table_t *) freeze_info->dst_info->htable;
}
ASSERT_OWN_WRITE_LOCK(!frozen_htable->is_local, &frozen_htable->rwlock);
looka2c = coarse_lookup_internal(dcontext, pending->tag, frozen_htable);
if (A2C_ENTRY_IS_EMPTY(looka2c)) {
frozen_a2c.app = pending->tag;
if (pending->entrance_stub) {
/* add inter-unit/trace-head stub to htable */
LOG(THREAD, LOG_FRAGMENT, 4,
" adding pending stub "PFX"."PFX" => "PFX"\n",
pending->tag, pending->cur_pc, freeze_info->stubs_cur_pc);
frozen_a2c.cache = (cache_pc)
(freeze_info->stubs_cur_pc - freeze_info->stubs_start_pc);
hashtable_coarse_add(dcontext, frozen_a2c, frozen_htable);
/* copy to new stubs area */
transfer_coarse_stub(dcontext, freeze_info, pending->cur_pc,
pending->trace_head, true/*replace*/);
} else {
/* fall-through optimization */
if (DYNAMO_OPTION(coarse_freeze_elide_ubr) &&
pending->link_cti_opnd != NULL &&
pending->link_cti_opnd + 4 == freeze_info->cache_cur_pc &&
pending->elide_ubr) {
ASSERT(!pending->trace_head);
LOG(THREAD, LOG_FRAGMENT, 4, " fall-through opt from prev fragment\n");
freeze_info->cache_cur_pc -= JMP_LONG_LENGTH;
pending->link_cti_opnd = NULL;
STATS_INC(coarse_freeze_fallthrough);
DODEBUG({ freeze_info->num_elisions++; });
}
LOG(THREAD, LOG_FRAGMENT, 4,
" adding pending %sfragment "PFX"."PFX" => "PFX"\n",
pending->trace_head ? "trace head " : "",
pending->tag, pending->cur_pc, freeze_info->cache_cur_pc);
/* add to htable the offset from start of cache */
frozen_a2c.cache = (cache_pc)
(freeze_info->cache_cur_pc - freeze_info->cache_start_pc);
hashtable_coarse_add(dcontext, frozen_a2c, frozen_htable);
/* copy to new cache */
transfer_coarse_fragment(dcontext, freeze_info, pending->cur_pc);
}
tgt = frozen_a2c.cache;
} else {
tgt = looka2c.cache;
/* Should not hit any links to TH, so should hit once, from htable walk */
ASSERT(!pending->trace_head || pending->entrance_stub);
/* May have added entrance stub for intra-unit TH as non-TH if it was
* linked to a trace, since didn't have body pc at the time, so we
* fix up here on the proactive add when adding its body
*/
if (pending->entrance_stub && pending->trace_head && freeze_info->unlink) {
cache_pc abs_tgt = tgt + (ptr_uint_t)freeze_info->stubs_start_pc;
transfer_coarse_stub_fix_trace_head(dcontext, freeze_info, abs_tgt);
}
}
if (pending->link_cti_opnd != NULL) {
/* fix up incoming link */
cache_pc patch_tgt = (cache_pc)
(((ptr_uint_t)(pending->entrance_stub ? freeze_info->stubs_start_pc :
freeze_info->cache_start_pc)) + tgt);
ASSERT(!pending->trace_head || pending->entrance_stub);
LOG(THREAD, LOG_FRAGMENT, 4, " patch link "PFX" => "PFX"."PFX"%s\n",
pending->link_cti_opnd, pending->tag, patch_tgt,
pending->entrance_stub ? " stub" : "");
insert_relative_target(pending->link_cti_opnd, patch_tgt, NOT_HOT_PATCHABLE);
}
if (pending->entrance_stub) {
DODEBUG({ frozen_htable->is_local = false; });
}
}
/* There are several strategies for copying each fragment and non-inter-unit stub
* to new, compact storage. Here we use a cache-driven approach, necessarily
* augmented with the htable as we cannot find tags for arbitrary fragments in
* the cache. We use a pending-add stack to avoid a second pass.
* By adding ubr targets last, we can elide fall-through jmps.
* FIXME case 9428:
* - Sorting entrance stubs for faster lazy linking lookup
* - Use 8-bit-relative jmps when possible for compaction, though this
* requires pc translation support and probably an extra pass when freezing.
*
* Tasks:
* Copy each new fragment and non-inter-unit stub to the new region
* Unlink all inter-unit entrance stubs, unless freezing and not
* writing to disk.
* Re-target intra-unit jmps from old entrance stub to new fragment location
* Re-target jmp to stub
* Re-target indirect stubs to new prefix
* Re-target inter-unit and trace head stubs to new prefix
*
* case 9900: must have dynamo_all_threads_synched since we haven't resolved
* lock rank ordering issues with the hashtable locks
*/
void
fragment_coarse_unit_freeze(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info)
{
pending_freeze_t pending_local;
pending_freeze_t *pending;
app_to_cache_t a2c;
coarse_table_t *frozen_htable;
coarse_table_t *htable;
cache_pc body_pc;
uint i;
ASSERT(freeze_info != NULL && freeze_info->src_info != NULL);
if (freeze_info->src_info->htable == NULL)
return;
LOG(THREAD, LOG_FRAGMENT, 2,
"freezing fragments in %s\n", freeze_info->src_info->module);
/* we walk just the main htable and find trace heads by asking for the body pc */
htable = (coarse_table_t *) freeze_info->src_info->htable;
DOSTATS({
LOG(THREAD, LOG_ALL, 1,
"htable pre-freezing %s\n", freeze_info->src_info->module);
hashtable_coarse_study(dcontext, htable, 0/*clean state*/);
});
frozen_htable = (coarse_table_t *) freeze_info->dst_info->htable;
/* case 9900: rank order conflict with coarse_info_incoming_lock:
* do not grab frozen_htable write lock: mark is_local instead and rely
* on dynamo_all_threads_synched
*/
DODEBUG({ frozen_htable->is_local = true; });
ASSERT_NOT_IMPLEMENTED(dynamo_all_threads_synched && "case 9900");
/* FIXME case 9522: not grabbing TABLE_RWLOCK(htable, read, lock) due to
* rank order violation with frozen_htable write lock! We go w/ the write
* lock since lookup routines check for it. To solve we'll need a way to
* tell deadlock checking that frozen_htable is private to this thread and
* that no other thread can hold its lock. For now we only support
* all-synch, but if later we want !in_place that needs no synch we'll need
* a solution.
*/
ASSERT_NOT_IMPLEMENTED(dynamo_all_threads_synched && "case 9522");
/* FIXME: we're doing htable order, better to use either tag (original app)
* or cache (execution) order?
*/
for (i = 0; i < htable->capacity || freeze_info->pending != NULL; i++) {
/* Process pending entries first; then continue through htable */
while (freeze_info->pending != NULL) {
pending = freeze_info->pending;
freeze_info->pending = pending->next;
fragment_coarse_entry_freeze(dcontext, freeze_info, pending);
HEAP_TYPE_FREE(dcontext, pending, pending_freeze_t,
ACCT_MEM_MGT/*appropriate?*/, UNPROTECTED);
}
if (i < htable->capacity) {
a2c = htable->table[i];
/* must check for sentinel */
if (!A2C_ENTRY_IS_REAL(a2c))
continue;
} else
continue;
LOG(THREAD, LOG_FRAGMENT, 4,
" %d app="PFX", cache="PFX"\n", i, a2c.app, a2c.cache);
coarse_body_from_htable_entry(dcontext, freeze_info->src_info, a2c.app,
a2c.cache, NULL, &body_pc);
if (body_pc == NULL) {
/* We add only when targeted by fragments, so we don't have to
* figure out multiple times whether intra-unit or not
*/
LOG(THREAD, LOG_FRAGMENT, 4,
" ignoring entrance stub "PFX"\n", a2c.cache);
} else {
pending_local.tag = a2c.app;
pending_local.cur_pc = body_pc;
pending_local.entrance_stub = false;
pending_local.link_cti_opnd = NULL;
pending_local.elide_ubr = true; /* doesn't matter since no link */
pending_local.trace_head =
coarse_is_trace_head_in_own_unit(dcontext, a2c.app, a2c.cache,
body_pc, true, freeze_info->src_info);
pending_local.next = NULL;
fragment_coarse_entry_freeze(dcontext, freeze_info, &pending_local);
if (pending_local.trace_head) {
/* we do need to proactively add the entrance stub, in
* case it is only targeted by an indirect branch
*/
LOG(THREAD, LOG_FRAGMENT, 4,
" adding trace head entrance stub "PFX"\n", a2c.cache);
pending_local.tag = a2c.app;
pending_local.cur_pc = a2c.cache;
pending_local.entrance_stub = true;
pending_local.link_cti_opnd = NULL;
pending_local.elide_ubr = true; /* doesn't matter since no link */
pending_local.trace_head = true;
fragment_coarse_entry_freeze(dcontext, freeze_info, &pending_local);
}
}
}
DODEBUG({ frozen_htable->is_local = false; });
DOSTATS({
/* The act of freezing tends to improve hashtable layout */
LOG(THREAD, LOG_ALL, 1,
"htable post-freezing %s\n", freeze_info->src_info->module);
hashtable_coarse_study(dcontext, frozen_htable, 0/*clean state*/);
});
}
uint
fragment_coarse_htable_persist_size(dcontext_t *dcontext, coarse_info_t *info,
bool cache_table)
{
coarse_table_t *htable = (coarse_table_t *)
(cache_table ? info->htable : info->th_htable);
return hashtable_coarse_persist_size(dcontext, htable);
}
/* Returns true iff all writes succeeded. */
bool
fragment_coarse_htable_persist(dcontext_t *dcontext, coarse_info_t *info,
bool cache_table, file_t fd)
{
coarse_table_t *htable = (coarse_table_t *)
(cache_table ? info->htable : info->th_htable);
ASSERT(fd != INVALID_FILE);
return hashtable_coarse_persist(dcontext, htable, fd);
}
void
fragment_coarse_htable_resurrect(dcontext_t *dcontext, coarse_info_t *info,
bool cache_table, byte *mapped_table)
{
coarse_table_t **htable = (coarse_table_t **)
(cache_table ? &info->htable : &info->th_htable);
ASSERT(info->frozen);
ASSERT(mapped_table != NULL);
ASSERT(*htable == NULL);
*htable = hashtable_coarse_resurrect(dcontext, mapped_table
_IF_DEBUG(cache_table ?
"persisted cache htable" :
"persisted stub htable"));
(*htable)->mod_shift = info->mod_shift;
/* generally want to keep basic alignment */
ASSERT_CURIOSITY(ALIGNED((*htable)->table, sizeof(app_pc)));
}
/*******************************************************************************/