Google Git
Sign in
chromium / external / dynamorio / f750c8d76d865ffbf0493e931ecb31bf6fff74f6 / . / core / perscache.c
blob: 9ffe341dd441529f862a1a47ee94f5c518e47047 [file] [log] [blame]
/* **********************************************************
* Copyright (c) 2011-2014 Google, Inc. All rights reserved.
* Copyright (c) 2006-2010 VMware, Inc. All rights reserved.
* **********************************************************/
/*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of VMware, Inc. nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
/* Copyright (c) 2006-2007 Determina Corp. */
/*
* perscache.c - coarse-grain units and persistent cache management
*/
#include "globals.h"
#include "link.h"
#include "fragment.h"
#include "fcache.h"
#include "monitor.h"
#include "perscache.h"
#include "instr.h"
#include "decode_fast.h"
#include "hotpatch.h"
#include "synch.h"
#include "module_shared.h"
#include <string.h> /* for memset */
#include <stddef.h> /* for offsetof */
#ifdef CLIENT_INTERFACE
# include "instrument.h"
#endif
#ifdef DEBUG
# include "disassemble.h"
#endif
#define MAX_PCACHE_OPTIONS_STRING (MAX_OPTIONS_STRING/2)
/* case 10823: align option string to keep hashtable data aligned.
* we're not using a cache-line-aligned lookuptable. */
#define OPTION_STRING_ALIGNMENT (sizeof(app_pc))
/* in general we want new data sections aligned to keep hashtable aligned */
#define CLIENT_ALIGNMENT (sizeof(app_pc))
/* used while merging */
typedef struct _jmp_tgt_list_t {
app_pc tag;
cache_pc jmp_end_pc;
struct _jmp_tgt_list_t *next;
} jmp_tgt_list_t;
/* Forward decls */
static void
persist_calculate_module_digest(module_digest_t *digest, app_pc modbase, size_t modsize,
app_pc code_start, app_pc code_end,
uint validation_option);
static bool
get_persist_dir(char *directory /* OUT */,
uint directory_len,
bool create);
#if defined(DEBUG) && defined(INTERNAL)
static void
print_module_digest(file_t f, module_digest_t *digest, const char *prefix);
#endif
static void
coarse_unit_shift_jmps(dcontext_t *dcontext, coarse_info_t *info,
ssize_t cache_shift, ssize_t stubs_shift, size_t old_mapsz);
static void
coarse_unit_merge_persist_info(dcontext_t *dcontext, coarse_info_t *dst,
coarse_info_t *info1, coarse_info_t *info2);
#ifdef DEBUG
/* used below for pcache_dir_check_permissions() */
DECLARE_CXTSWPROT_VAR(static mutex_t pcache_dir_check_lock,
INIT_LOCK_FREE(pcache_dir_check_lock));
#endif
/***************************************************************************
* COARSE-GRAIN UNITS
*/
/* case 9653/10380: only one coarse unit in a module's +x region(s) is persisted */
static void
coarse_unit_mark_primary(coarse_info_t *info)
{
if (!info->in_use)
return;
#ifdef WINDOWS
/* FIXME PR 295529: put in for Linux once we have per-module flags */
/* Go ahead and get write lock up front; else have to check again; not
* frequently called so don't need perf opt here.
*/
os_get_module_info_write_lock();
if (!os_module_get_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE)) {
os_module_set_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE);
ASSERT(os_module_get_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE));
info->primary_for_module = true;
LOG(GLOBAL, LOG_CACHE, 1, "marking "PFX"-"PFX" as primary coarse for %s\n",
info->base_pc, info->end_pc, info->module);
}
os_get_module_info_write_unlock();
#else
info->primary_for_module = true;
#endif
}
static void
coarse_unit_unmark_primary(coarse_info_t *info)
{
#ifdef WINDOWS
/* FIXME PR 295529: put in for Linux once we have per-module flags */
if (info->primary_for_module && info->in_use) {
ASSERT(os_module_get_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE));
os_module_clear_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE);
info->primary_for_module = false;
}
#else
info->primary_for_module = false;
#endif
}
void
coarse_unit_mark_in_use(coarse_info_t *info)
{
info->in_use = true;
coarse_unit_mark_primary(info);
}
coarse_info_t *
coarse_unit_create(app_pc base_pc, app_pc end_pc, module_digest_t *digest,
bool for_execution)
{
coarse_info_t *info = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_info_t,
/* FIXME: have separate heap acct? */
ACCT_VMAREAS, PROTECTED);
memset(info, 0, sizeof(*info));
ASSIGN_INIT_LOCK_FREE(info->lock, coarse_info_lock);
ASSIGN_INIT_LOCK_FREE(info->incoming_lock, coarse_info_incoming_lock);
info->base_pc = base_pc;
/* XXX i#704: handle overflow: better to store size */
info->end_pc = end_pc;
/* FIXME: set PERSCACHE_X86_{32,64} here since for x64 the live
* unit's flags are used for 32-bit code in 64-bit processes.
* app_memory_allocation() may need a "bool x86_mode" param that is at
* least passed in to here: not clear if it should be stored in the
* vmarea.
*/
DODEBUG({
info->is_local = false;
info->module = os_get_module_name_strdup(info->base_pc HEAPACCT(ACCT_VMAREAS));
if (info->module == NULL) {
/* else our LOG statements will crash */
info->module = dr_strdup("" HEAPACCT(ACCT_VMAREAS));
}
});
if (for_execution)
coarse_unit_mark_in_use(info);
if (digest != NULL) {
memcpy(&info->module_md5, digest, sizeof(info->module_md5));
} else if (TEST(PERSCACHE_MODULE_MD5_AT_LOAD,
DYNAMO_OPTION(persist_gen_validation))) {
/* case 9735: calculate the module md5 at load time so we have a consistent
* point at which to compare it when loading in a persisted cache file.
* If we inject at different points we may see different views of
* post-loader vs pre-loader module changes but we'll live with that.
* Should have consistent injection points in steady state usage.
* FIXME PR 215036: for 4.4 we'll want to not record the at-mmap md5, but
* rather the 1st-execution-time post-rebase md5.
*/
app_pc modbase = get_module_base(info->base_pc);
size_t modsize;
os_get_module_info_lock();
/* For linux we can't do module segment walking at initial mmap time
* b/c the segments are not set up: we hit SIGBUS!
*/
IF_UNIX(ASSERT_BUG_NUM(215036, true));
if (os_get_module_info(modbase, NULL, NULL, &modsize,
NULL, NULL, NULL)) {
os_get_module_info_unlock();
persist_calculate_module_digest(&info->module_md5, modbase, modsize,
info->base_pc, info->end_pc,
DYNAMO_OPTION(persist_gen_validation));
DOLOG(1, LOG_CACHE, {
print_module_digest(GLOBAL, &info->module_md5, "md5 at load time: ");
});
} else
os_get_module_info_unlock();
}
/* the rest is initialized lazily in coarse_unit_init() */
RSTATS_ADD_PEAK(num_coarse_units, 1);
return info;
}
void
coarse_unit_free(dcontext_t *dcontext, coarse_info_t *info)
{
ASSERT(info != NULL);
/* Elements should have been freed in coarse_unit_reset_free() */
ASSERT(info->htable == NULL);
ASSERT(info->th_htable == NULL);
ASSERT(info->pclookup_htable == NULL);
ASSERT(info->cache == NULL);
ASSERT(info->incoming == NULL);
ASSERT(info->stubs == NULL);
ASSERT(info->cache_start_pc == NULL);
ASSERT(info->stubs_start_pc == NULL);
DODEBUG({
if (info->module != NULL)
dr_strfree(info->module HEAPACCT(ACCT_VMAREAS));
});
DELETE_LOCK(info->lock);
DELETE_LOCK(info->incoming_lock);
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, info, coarse_info_t,
ACCT_VMAREAS, PROTECTED);
RSTATS_DEC(num_coarse_units);
}
void
coarse_unit_init(coarse_info_t *info, void *cache)
{
ASSERT(info != NULL);
ASSERT(cache != NULL);
ASSERT_OWN_MUTEX(true, &info->lock);
fragment_coarse_htable_create(info, 0, 0);
coarse_stubs_create(info, NULL, 0);
/* cache is passed in since it can't be created while holding info->lock */
info->cache = cache;
}
/* If caller holds change_linking_lock and info->lock, have_locks should be true.
* If !need_info_lock, info must be a thread-local, unlinked, private pointer!
*/
static void
coarse_unit_reset_free_internal(dcontext_t *dcontext, coarse_info_t *info,
bool have_locks, bool unlink, bool abdicate_primary,
bool need_info_lock)
{
DEBUG_DECLARE(bool ok;)
ASSERT(info != NULL);
LOG(THREAD, LOG_CACHE, 2, "coarse_unit_reset_free %s\n", info->module);
if (!have_locks) {
/* Though only called during all-threads-synch, we still grab our lock here */
/* higher rank than info, needed for unlink */
if (unlink)
acquire_recursive_lock(&change_linking_lock);
if (need_info_lock)
mutex_lock(&info->lock);
}
ASSERT(!unlink || self_owns_recursive_lock(&change_linking_lock));
ASSERT_OWN_MUTEX(need_info_lock, &info->lock);
ASSERT(need_info_lock || !unlink); /* else will get deadlock */
/* case 11064: avoid rank order */
DODEBUG({
if (!need_info_lock)
info->is_local = true;
});
if (unlink)
coarse_unit_unlink(dcontext, info);
fragment_coarse_htable_free(info);
coarse_stubs_delete(info);
fcache_coarse_cache_delete(dcontext, info);
if (info->in_use && abdicate_primary)
coarse_unit_unmark_primary(info);
if (info->frozen) {
ASSERT(info->mmap_size > 0);
if (info->persisted) {
#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
if (info->in_use) {
rct_module_table_persisted_invalidate(dcontext, info->base_pc);
}
#endif
/* We use GLOBAL_DCONTEXT always for these in case in use */
#ifdef RCT_IND_BRANCH
if (info->rct_table != NULL)
rct_table_free(GLOBAL_DCONTEXT, info->rct_table, false/*data mmapped*/);
#endif
#ifdef RETURN_AFTER_CALL
if (info->rac_table != NULL)
rct_table_free(GLOBAL_DCONTEXT, info->rac_table, false/*data mmapped*/);
#endif
ASSERT(info->mmap_pc != NULL);
if (info->mmap_ro_size > 0) {
/* two views */
DEBUG_DECLARE(ok =) unmap_file(info->mmap_pc, info->mmap_ro_size);
ASSERT(ok);
DEBUG_DECLARE(ok =) unmap_file(info->mmap_pc+info->mmap_ro_size,
info->mmap_size-info->mmap_ro_size);
ASSERT(ok);
info->mmap_ro_size = 0;
} else {
DEBUG_DECLARE(ok =) unmap_file(info->mmap_pc, info->mmap_size);
ASSERT(ok);
}
if (DYNAMO_OPTION(persist_lock_file)) {
ASSERT(info->fd != INVALID_FILE);
os_close(info->fd);
info->fd = INVALID_FILE;
}
} else {
ASSERT(info->cache_start_pc != NULL);
ASSERT(info->stubs_start_pc != NULL);
ASSERT(info->mmap_ro_size == 0);
heap_munmap(info->cache_start_pc, info->mmap_size);
if (info->has_persist_info) {
/* Persisted units point at their mmaps for these structures;
* non-persisted dynamically allocate them from DR heap.
*/
#ifdef RCT_IND_BRANCH
if (info->rct_table != NULL)
rct_table_free(dcontext, info->rct_table, true);
#endif
#ifdef RETURN_AFTER_CALL
if (info->rac_table != NULL)
rct_table_free(dcontext, info->rac_table, true);
#endif
#ifdef HOT_PATCHING_INTERFACE
if (info->hotp_ppoint_vec != NULL) {
HEAP_ARRAY_FREE(dcontext, info->hotp_ppoint_vec, app_rva_t,
info->hotp_ppoint_vec_num, ACCT_HOT_PATCHING, PROTECTED);
}
#endif
}
}
} else {
ASSERT(info->mmap_size == 0);
ASSERT(info->cache_start_pc == NULL);
ASSERT(info->stubs_start_pc == NULL);
ASSERT(!info->has_persist_info);
}
/* This struct may be re-used for a non-frozen/persisted unit if it was reset due
* to a non-cache-consistency reason. Thus we want to preserve the locks, vm
* region, and md5, but clear everything else (case 10119).
*/
memset(info, 0, offsetof(coarse_info_t, lock));
if (!have_locks) {
if (need_info_lock)
mutex_unlock(&info->lock);
if (unlink)
release_recursive_lock(&change_linking_lock);
}
}
/* If caller holds change_linking_lock and info->lock, have_locks should be true */
void
coarse_unit_reset_free(dcontext_t *dcontext, coarse_info_t *info,
bool have_locks, bool unlink, bool abdicate_primary)
{
coarse_unit_reset_free_internal(dcontext, info,
have_locks, unlink, abdicate_primary,
true/*need_info_lock*/);
}
/* currently only one such directory expected matching
* primary user token, see case 8812
*/
static file_t perscache_user_directory = INVALID_FILE;
void
perscache_init(void)
{
if (DYNAMO_OPTION(use_persisted) &&
DYNAMO_OPTION(persist_per_user) &&
DYNAMO_OPTION(validate_owner_dir)) {
char dir[MAXIMUM_PATH];
/* case 8812 we need to hold a handle to the user directory
* from startup (we could delay until we open our first pcache file)
*/
if (get_persist_dir(dir, BUFFER_SIZE_ELEMENTS(dir),
true /* note we MUST always create directory
* even if never persisting */
)) {
/* we just need READ_CONTROL (on Windows) to check
* ownership, and we are NOT OK with the directory being
* renamed (or deleted and recreated by a malactor) while
* we still have a handle to it.
*/
perscache_user_directory = os_open_directory(dir, 0);
ASSERT(perscache_user_directory != INVALID_FILE);
/* note that now that we have the actual handle open, we can validate */
/* see os_current_user_directory() for details */
if (perscache_user_directory != INVALID_FILE &&
!os_validate_user_owned(perscache_user_directory)) {
SYSLOG_INTERNAL_ERROR("%s is OWNED by an impostor!"
" Persistent cache use is disabled.",
dir);
os_close(perscache_user_directory);
perscache_user_directory = INVALID_FILE;
/* we could also turn off use_persisted */
} else {
/* either FAT32 or we are the proper owner */
/* FIXME: we have to verify that the final permissions and
* sharing attributes for cache/ and for the current
* directory, do NOT allow anyone to rename our directory
* while in use, and replace it. Otherwise we'd still
* have to verify owner for each file as well with
* -validate_owner_file. See duplicate comment in
* open_relocated_dlls_filecache_directory()
*/
}
}
}
}
void
perscache_fast_exit(void)
{
if (DYNAMO_OPTION(coarse_freeze_at_exit)) {
coarse_units_freeze_all(false/*!in place*/);
}
if (perscache_user_directory != INVALID_FILE) {
ASSERT_CURIOSITY(DYNAMO_OPTION(validate_owner_dir));
os_close(perscache_user_directory);
perscache_user_directory = INVALID_FILE;
}
ASSERT(perscache_user_directory == INVALID_FILE);
}
void
perscache_slow_exit(void)
{
DODEBUG(DELETE_LOCK(pcache_dir_check_lock););
}
/***************************************************************************
* FROZEN UNITS
*/
/* Separated out to keep priv_mcontext_t out of critical stack path */
static void
coarse_units_freeze_translate(thread_record_t *tr,
const thread_synch_state_t desired_state)
{
priv_mcontext_t mc;
bool res;
res = thread_get_mcontext(tr, &mc);
ASSERT(res);
/* We're freeing coarse fragments so we must translate all
* threads who are currently in a coarse unit, or about
* to enter one (case 10030). We don't translate threads
* in fine-grained caches as an optimization.
* If we did one unit at a time, could compare to just that unit.
*/
if (!res || !in_fcache((cache_pc)mc.pc) ||
get_fcache_coarse_info((cache_pc)mc.pc) != NULL) {
/* FIXME optimization: pass cxt for translation */
translate_from_synchall_to_dispatch(tr, desired_state);
} else {
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tin fine-grained cache so no translation needed\n");
}
}
/* If !in_place this routine freezes (if not already) and persists.
* FIXME case 9975: provide support for freezing in place and
* persisting in one call? Should we support loading in a newly
* persisted version to replace the in-memory unit?
*/
void
coarse_units_freeze_all(bool in_place)
{
thread_record_t **threads = NULL;
int i, num_threads = 0;
bool own_synch;
dcontext_t *my_dcontext = get_thread_private_dcontext();
const thread_synch_state_t desired_state =
THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER;
if (!DYNAMO_OPTION(coarse_units) || !DYNAMO_OPTION(coarse_enable_freeze) ||
RUNNING_WITHOUT_CODE_CACHE())
return;
KSTART(coarse_freeze_all);
/* on a detach we don't need to synch or xlate the threads */
own_synch = !dynamo_all_threads_synched;
ASSERT(own_synch IF_WINDOWS(|| doing_detach));
STATS_INC(coarse_freezes);
if (own_synch) {
/* Called from nudge threads from the code cache, so
* if the calling fragment could be coarse, we have to
* terminate this thread. Case 8711 does not allow calls from
* coarse fragments so we're fine for now.
*/
if (!synch_with_all_threads(desired_state, &threads, &num_threads,
/* FIXME: can we set mcontext->pc to next_tag and
* use THREAD_SYNCH_VALID_MCONTEXT? not if nudge
* comes here */
THREAD_SYNCH_NO_LOCKS_NO_XFER, /* Case 6821 */
/* if we fail to suspend a thread (e.g., for
* privilege reasons) just abort */
THREAD_SYNCH_SUSPEND_FAILURE_ABORT
/* if we get in a race with detach, or are having
* synch issues for whatever reason, bail out sooner
* rather than later */
| THREAD_SYNCH_SMALL_LOOP_MAX)) {
/* just give up */
ASSERT(!OWN_MUTEX(&all_threads_synch_lock) &&
!OWN_MUTEX(&thread_initexit_lock));
ASSERT(threads == NULL);
ASSERT(!dynamo_all_threads_synched);
STATS_INC(coarse_freeze_abort);
LOG(GLOBAL, LOG_CACHE, 2,
"coarse_unit_freeze: aborting due to thread synch failure\n");
SYSLOG_INTERNAL_WARNING("coarse freeze aborted due to thread synch failure");
KSTOP(coarse_freeze_all);
return;
}
}
ASSERT(dynamo_all_threads_synched);
ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
DOSTATS({
SYSLOG_INTERNAL_INFO("freezing all coarse units @ "SSZFMT" fragments",
GLOBAL_STAT(num_fragments));
});
/* This routine does the actual freeze and persist calls
* FIXME case 9641: should we end the synch after freezing
* so other threads can make progress while we persist?
*/
vm_area_coarse_units_freeze(in_place);
if (in_place && own_synch) {
DEBUG_DECLARE(uint removed;)
for (i = 0; i < num_threads; i++) {
dcontext_t *dcontext = threads[i]->dcontext;
if (dcontext != NULL && dcontext != my_dcontext) {
/* FIXME: share these checks w/ other synchall-and-abort users
* (reset) and synchall-and-don't-abort (flush).
*/
/* Should have aborted if we had any synch failures */
ASSERT(thread_synch_successful(threads[i]));
if (is_thread_currently_native(threads[i])) {
/* Whether in native_exec or we lost control, since we're not
* freeing the interception buffer, no state to worry about.
*/
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tcurrently native so no translation needed\n");
} else if (thread_synch_state_no_xfer(dcontext)) {
/* Case 6821: do not translate other synch-all-thread users. */
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tat THREAD_SYNCH_NO_LOCKS_NO_XFER so no translation needed\n");
} else {
/* subroutine to avoid priv_mcontext_t on our stack when we
* freeze + merge&load */
coarse_units_freeze_translate(threads[i], desired_state);
}
last_exit_deleted(dcontext);
if (is_building_trace(dcontext)) {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tsquashing trace of thread "TIDFMT"\n", i);
trace_abort(dcontext);
}
if (DYNAMO_OPTION(bb_ibl_targets)) {
/* FIXME: we could just remove the coarse ibl entries */
DEBUG_DECLARE(removed =)
fragment_remove_all_ibl_in_region(dcontext,
UNIVERSAL_REGION_BASE,
UNIVERSAL_REGION_END);
LOG(THREAD, LOG_FRAGMENT, 2, "\tremoved %d ibl entries\n", removed);
}
}
}
if (DYNAMO_OPTION(bb_ibl_targets)) {
/* FIXME: we could just remove the coarse ibl entries */
DEBUG_DECLARE(removed =)
fragment_remove_all_ibl_in_region(GLOBAL_DCONTEXT,
UNIVERSAL_REGION_BASE,
UNIVERSAL_REGION_END);
LOG(GLOBAL, LOG_FRAGMENT, 2, "\tremoved %d ibl entries\n", removed);
}
}
if (own_synch)
end_synch_with_all_threads(threads, num_threads, true/*resume*/);
KSTOP(coarse_freeze_all);
}
/* Removes dst's data and replaces it with src's data. Frees src.
* Assumes that src is thread-local and not reachable by any other thread,
* and that dst's lock is held.
*/
static void
coarse_replace_unit(dcontext_t *dcontext, coarse_info_t *dst, coarse_info_t *src)
{
/* Perhaps we should separately allocate the locks to avoid this copying
* for preservation? Or memcpy all but the lock fields? Or delete and
* re-init them? If we move to a model where the world isn't suspended
* we have to ensure no other thread is trying to lock.
*/
coarse_info_t *non_frozen;
mutex_t temp_lock, temp_incoming_lock;
DEBUG_DECLARE(const char *modname;)
ASSERT_OWN_MUTEX(true, &dst->lock);
mutex_lock(&dst->incoming_lock);
ASSERT(src->incoming == NULL); /* else we leak */
src->incoming = dst->incoming;
dst->incoming = NULL; /* do not free incoming */
mutex_unlock(&dst->incoming_lock);
non_frozen = dst->non_frozen;
coarse_unit_reset_free(dcontext, dst, true/*have locks*/,
false/*do not unlink*/, false/*keep primary*/);
temp_lock = dst->lock;
temp_incoming_lock = dst->incoming_lock;
DODEBUG({ modname = dst->module; });
memcpy(dst, src, sizeof(*dst));
dst->lock = temp_lock;
dst->incoming_lock = temp_incoming_lock;
dst->non_frozen = non_frozen;
DODEBUG({ dst->module = modname; });
ASSERT(dst->incoming == src->incoming);
/* update pointers from src to dst */
fcache_coarse_set_info(dcontext, dst);
patch_coarse_exit_prefix(dcontext, dst);
coarse_stubs_set_info(dst);
DODEBUG({
/* avoid asserts */
src->htable = NULL;
src->th_htable = NULL;
src->pclookup_htable = NULL;
src->cache = NULL;
src->incoming = NULL;
src->stubs = NULL;
src->cache_start_pc = NULL;
src->stubs_start_pc = NULL;
});
coarse_unit_free(dcontext, src);
}
/* In-place freezing replaces info with a frozen copy.
* Otherwise, a new copy is created for persisting, while the original
* copy is undisturbed and unfrozen.
* Only purpose of !in_place is to write out to disk a
* snapshot while letting coarse unit creation continue. Can
* write in_place out to disk as well, so we leave that up to
* caller.
* Caller must hold change_linking_lock.
* If in_place, caller is responsible for flushing the ibl tables (case 11057).
*/
coarse_info_t *
coarse_unit_freeze(dcontext_t *dcontext, coarse_info_t *info, bool in_place)
{
coarse_info_t *frozen = NULL;
coarse_info_t *res = NULL;
size_t frozen_stub_size, frozen_cache_size;
uint num_fragments, num_stubs;
coarse_freeze_info_t *freeze_info =
HEAP_TYPE_ALLOC(dcontext, coarse_freeze_info_t,
ACCT_MEM_MGT/*appropriate?*/, PROTECTED);
LOG(THREAD, LOG_CACHE, 2, "coarse_unit_freeze %s\n", info->module);
STATS_INC(coarse_units_frozen);
/* FIXME: Suspend world needed if not in-place? Even though unit lock
* is not held when changing unit links (e.g., unit flush holds only
* target unit lock when unlinking incoming), the change_linking_lock
* should give us guarantees.
* But if we don't suspend-all we'll have other issues:
* - fcache_coarse_init_frozen() will need to
* grab the shared cache lock, which is higher rank than coarse unit lock!
* - same issue w/ fcache_coarse_cache_delete (via coarse_unit_reset_free)
*/
/* FIXME: support single-unit freeze by having this routine itself
* do a synch-all?
*/
ASSERT(dynamo_all_threads_synched);
ASSERT(info != NULL);
ASSERT_OWN_RECURSIVE_LOCK(true, &change_linking_lock);
/* trigger lazy initialize to avoid deadlock on calling
* coarse_cti_is_intra_fragment() during shifting
*/
fragment_coarse_create_entry_pclookup_table(dcontext, info);
mutex_lock(&info->lock);
ASSERT(info->cache != NULL); /* don't freeze empty units */
ASSERT(!info->frozen); /* don't freeze already frozen units */
if (info->cache == NULL || info->frozen)
goto coarse_unit_freeze_exit;
/* invalid unit shouldn't get this far */
ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
if (TEST(PERSCACHE_CODE_INVALID, info->flags)) /* paranoid */
goto coarse_unit_freeze_exit;
memset(freeze_info, 0, sizeof(*freeze_info));
freeze_info->src_info = info;
/* Tasks:
* 1) Calculate final size of cache and stub space:
* Walk entrance stubs and count how many are intra-unit links
* that can be changed to direct jmps
* 2) Create single contiguous region to hold both cache and stubs,
* rounding up to a page boundary in the middle for +r->+rw
* 3) Copy each fragment and stub over
*
* FIXME case 9428: shrink the cache to take advantage of elided jmps!
* Requires a separate pass to touch up jmps to stubs/prefixes, or
* re-ordering w/ stubs on top and cache on bottom. That would also
* put a read-only page at the end, so no guard page needed -- unless we
* hook our own cache (case 9673) and are worried about brief periods of +w.
*/
frozen_stub_size = coarse_frozen_stub_size(dcontext, info, &num_fragments,
&num_stubs);
frozen_cache_size = coarse_frozen_cache_size(dcontext, info);
/* we need the stubs to start on a new page since will be +rw vs cache +r */
frozen_cache_size = ALIGN_FORWARD(frozen_cache_size, PAGE_SIZE);
freeze_info->cache_start_pc =
(cache_pc) heap_mmap(frozen_stub_size + frozen_cache_size);
/* FIXME: should show full non-frozen size as well */
LOG(THREAD, LOG_CACHE, 2,
"%d frozen stubs @ "SZFMT" bytes + %d fragments @ "SZFMT" bytes => "PFX"\n",
num_stubs, frozen_stub_size, num_fragments,
frozen_cache_size, freeze_info->cache_start_pc);
STATS_ADD(coarse_fragments_frozen, num_fragments);
/* We use raw pcs to build up our cache and stubs, and later we impose
* our regular data structures on them
*/
/* Whether freezing in-place or not, we create a new coarse_info_t.
* If in-place we delete the old one afterward.
*/
frozen = coarse_unit_create(info->base_pc, info->end_pc, &info->module_md5,
in_place && info->in_use);
freeze_info->dst_info = frozen;
frozen->frozen = true;
frozen->cache_start_pc = freeze_info->cache_start_pc;
frozen->mmap_size = frozen_stub_size + frozen_cache_size;
/* Our relative jmps require that we do not exceed 32-bit reachability */
IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(frozen->mmap_size)));
/* Same bounds, so same persistence privileges */
frozen->primary_for_module = info->primary_for_module;
freeze_info->stubs_start_pc =
coarse_stubs_create(frozen, freeze_info->cache_start_pc + frozen_cache_size,
frozen_stub_size);
ASSERT(freeze_info->stubs_start_pc != NULL);
ASSERT(ALIGNED(freeze_info->stubs_start_pc, coarse_stub_alignment(info)));
frozen->stubs_start_pc = freeze_info->stubs_start_pc;
ASSERT(frozen->fcache_return_prefix ==
freeze_info->cache_start_pc + frozen_cache_size);
#if 0
ASSERT(frozen->trace_head_return_prefix == frozen->fcache_return_prefix +
(info->trace_head_return_prefix - info->fcache_return_prefix));
ASSERT(frozen->ibl_ret_prefix == frozen->fcache_return_prefix +
(info->ibl_ret_prefix - info->fcache_return_prefix));
ASSERT(frozen->ibl_call_prefix == frozen->fcache_return_prefix +
(info->ibl_call_prefix - info->fcache_return_prefix));
ASSERT(frozen->ibl_jmp_prefix == frozen->fcache_return_prefix +
(info->ibl_jmp_prefix - info->fcache_return_prefix));
#endif
fragment_coarse_htable_create(frozen, num_fragments, num_stubs);
fcache_coarse_init_frozen(dcontext, frozen, freeze_info->cache_start_pc,
frozen_cache_size);
/* assumption: leave inter-unit links intact for in_place, but not (for
* persisting) otherwise
*/
freeze_info->unlink = !in_place;
freeze_info->cache_cur_pc = freeze_info->cache_start_pc;
freeze_info->stubs_cur_pc = freeze_info->stubs_start_pc;
fragment_coarse_unit_freeze(dcontext, freeze_info);
ASSERT(freeze_info->pending == NULL);
ASSERT(freeze_info->cache_cur_pc <=
freeze_info->cache_start_pc + frozen_cache_size);
ASSERT(freeze_info->stubs_cur_pc <=
freeze_info->stubs_start_pc + frozen_stub_size);
if (frozen->fcache_return_prefix + frozen_stub_size ==
freeze_info->stubs_cur_pc)
frozen->stubs_end_pc = freeze_info->stubs_cur_pc;
else {
/* FIXME case 9428: strange history here: I don't see a problem now,
* but leaving some release-build code just in case.
*/
ASSERT_NOT_REACHED();
coarse_stubs_set_end_pc(frozen, freeze_info->stubs_cur_pc);
}
frozen->cache_end_pc = freeze_info->cache_cur_pc;
LOG(THREAD, LOG_CACHE, 2, "frozen code stats for %s:\n %6d app code\n",
info->module, freeze_info->app_code_size);
LOG(THREAD, LOG_CACHE, 2, " %6d fallthrough\n", freeze_info->added_fallthrough);
LOG(THREAD, LOG_CACHE, 2, " %6d ind br mangle\n", freeze_info->added_indbr_mangle);
LOG(THREAD, LOG_CACHE, 2, " %6d indr br stubs\n", freeze_info->added_indbr_stub);
LOG(THREAD, LOG_CACHE, 2, " %6d jecxz mangle\n", freeze_info->added_jecxz_mangle);
LOG(THREAD, LOG_CACHE, 2, " -%6d = 5 x %d elisions\n",
freeze_info->num_elisions*5, freeze_info->num_elisions);
LOG(THREAD, LOG_CACHE, 2, "ctis: %5d cbr, %5d jmp, %5d call, %5d ind\n",
freeze_info->num_cbr, freeze_info->num_jmp, freeze_info->num_call,
freeze_info->num_indbr);
LOG(THREAD, LOG_CACHE, 2,
"frozen final size: stubs "SZFMT" bytes + cache "SZFMT" bytes\n",
freeze_info->stubs_cur_pc - freeze_info->stubs_start_pc,
freeze_info->cache_cur_pc - freeze_info->cache_start_pc);
/* FIXME case 9687: mark cache as read-only */
if (in_place) {
coarse_replace_unit(dcontext, info, frozen);
frozen = NULL;
mark_executable_area_coarse_frozen(info);
coarse_unit_shift_links(dcontext, info);
res = info;
} else {
/* we made separate copy that has no outgoing or incoming links */
res = frozen;
}
coarse_unit_freeze_exit:
HEAP_TYPE_FREE(dcontext, freeze_info, coarse_freeze_info_t,
ACCT_MEM_MGT/*appropriate?*/, PROTECTED);
mutex_unlock(&info->lock);
/* be sure to free to avoid missing entries if we add to info later */
fragment_coarse_free_entry_pclookup_table(dcontext, info);
DOLOG(3, LOG_CACHE, {
if (res != NULL) {
byte *pc = frozen->cache_start_pc;
LOG(THREAD, LOG_CACHE, 1, "frozen cache for %s:\n", info->module);
do {
app_pc tag = fragment_coarse_entry_pclookup(dcontext, frozen, pc);
if (tag != NULL)
LOG(THREAD, LOG_CACHE, 1, "tag "PFX":\n", tag);
pc = disassemble_with_bytes(dcontext, pc, THREAD);
} while (pc < frozen->cache_end_pc);
}
});
return res;
}
/* These decode-and-instr-using routines could go in arch/ as they assume that direct
* jump operands are 4 bytes and are at the end of the instruction.
*/
/* Transfers a coarse stub to a new location.
* If freeze_info->dst_info is non-NULL,
* shifts any unlinked stubs to point at the prefixes in freeze_info->dst_info.
* If freeze_info->unlink is true,
* points any linked stubs at freeze_info->dst_info->fcache_return_prefix if
* freeze_info->dst_info is non-NULL, else
* freeze_info->src_info->fcache_return_prefix. If trace_head, points at
* trace_head_return_prefix instead of fcache_return_prefix.
* replace_outgoing really only applies if in_place: should we replace
* outgoing link incoming entries (true), or add new ones (false)
*/
void
transfer_coarse_stub(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
cache_pc stub, bool trace_head, bool replace_outgoing)
{
cache_pc tgt = entrance_stub_jmp_target(stub);
cache_pc pc = freeze_info->stubs_cur_pc; /* target pc */
uint sz;
bool update_out = false;
/* Should not be targeting the cache, else our later shift will be wrong */
ASSERT(tgt < freeze_info->src_info->cache_start_pc ||
tgt >= freeze_info->src_info->cache_end_pc);
if (tgt == freeze_info->src_info->fcache_return_prefix) {
ASSERT(!trace_head);
if (freeze_info->dst_info != NULL)
tgt = freeze_info->dst_info->fcache_return_prefix;
LOG(THREAD, LOG_FRAGMENT, 4,
" transfer_coarse_stub "PFX": tgt is fcache_return_prefix\n", stub);
} else if (tgt == freeze_info->src_info->trace_head_return_prefix) {
ASSERT(trace_head);
if (freeze_info->dst_info != NULL)
tgt = freeze_info->dst_info->trace_head_return_prefix;
LOG(THREAD, LOG_FRAGMENT, 4,
" transfer_coarse_stub "PFX": tgt is trace_head_return_prefix\n", stub);
} else if (freeze_info->unlink) {
coarse_info_t *info = (freeze_info->dst_info != NULL) ?
freeze_info->dst_info : freeze_info->src_info;
if (trace_head) {
tgt = info->trace_head_return_prefix;
LOG(THREAD, LOG_FRAGMENT, 4,
" transfer_coarse_stub "PFX": unlinking as trace head\n", stub);
} else {
tgt = info->fcache_return_prefix;
LOG(THREAD, LOG_FRAGMENT, 4,
" transfer_coarse_stub "PFX": unlinking as non-trace head\n", stub);
}
} else
update_out = true;
sz = exit_stub_size(dcontext, tgt, FRAG_COARSE_GRAIN)
- (JMP_LONG_LENGTH - 1/*get opcode*/);
memcpy(pc, stub, sz);
pc += sz;
ASSERT(pc == entrance_stub_jmp(freeze_info->stubs_cur_pc) + 1/*skip opcode*/);
#ifdef X86
ASSERT(*(pc-1) == JMP_OPCODE);
#elif defined(ARM)
/* FIXME i#1551: NYI on ARM */
ASSERT_NOT_IMPLEMENTED(false);
#endif
/* if tgt unchanged we still need to re-relativize it */
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
pc = insert_relative_target(pc, tgt, NOT_HOT_PATCHABLE);
if (update_out) {
coarse_update_outgoing(dcontext, stub, freeze_info->stubs_cur_pc,
freeze_info->src_info, replace_outgoing);
}
pc = (cache_pc) ALIGN_FORWARD(pc, coarse_stub_alignment(freeze_info->src_info));
freeze_info->stubs_cur_pc = pc;
}
void
transfer_coarse_stub_fix_trace_head(dcontext_t *dcontext,
coarse_freeze_info_t *freeze_info,
cache_pc stub)
{
/* We don't know body pc at fragment exit processing time and so can
* add a stub and unlink it as a non-trace head if it was linked to
* a trace, so we fix it up later
*/
coarse_info_t *info = (freeze_info->dst_info != NULL) ?
freeze_info->dst_info : freeze_info->src_info;
ASSERT(freeze_info->unlink);
if (entrance_stub_jmp_target(stub) == info->fcache_return_prefix) {
cache_pc tgt = info->trace_head_return_prefix;
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
insert_relative_target(entrance_stub_jmp(stub) + 1/*skip opcode*/,
tgt, NOT_HOT_PATCHABLE);
LOG(THREAD, LOG_FRAGMENT, 4,
" fixing up stub "PFX" to be unlinked as a trace head\n", stub);
} else
ASSERT(entrance_stub_jmp_target(stub) == info->trace_head_return_prefix);
}
static void
push_pending_freeze(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
cache_pc exit_tgt, uint cti_len, cache_pc cti_pc,
cache_pc body_start_pc)
{
pending_freeze_t *pending;
cache_pc stub_target;
uint sz;
pending = HEAP_TYPE_ALLOC(dcontext, pending_freeze_t,
ACCT_MEM_MGT/*appropriate?*/, UNPROTECTED);
ASSERT(coarse_is_entrance_stub(exit_tgt));
pending->tag = entrance_stub_target_tag(exit_tgt, freeze_info->src_info);
stub_target = entrance_stub_jmp_target(exit_tgt);
if (entrance_stub_linked(exit_tgt, freeze_info->src_info) &&
get_fcache_coarse_info(stub_target) == freeze_info->src_info) {
/* Intra-unit non-trace-head target: eliminate stub */
pending->entrance_stub = false;
pending->cur_pc = stub_target;
/* trace heads are discovered in the htable walk, never pushed here */
pending->trace_head = false;
} else {
/* Leave stub */
pending->entrance_stub = true;
pending->cur_pc = exit_tgt;
DOCHECK(1, {
cache_pc body;
/* A rank order violation (grabbing src htable read lock while
* holding frozen htable write lock) prevents us from calling the
* exported interface fragment_coarse_lookup_in_unit, so instead we
* have a hack where we call the lower-level routine that is
* exported only for us.
*/
coarse_body_from_htable_entry(dcontext, freeze_info->src_info,
pending->tag, exit_tgt, NULL, &body);
ASSERT(body == NULL ||
coarse_is_trace_head(exit_tgt) ||
fragment_lookup_trace(dcontext, pending->tag) != NULL);
});
/* We do not look up body pc to see if a trace head stub linked to
* a trace; instead we fix up the unlinked target (for freeze_info->unlink)
* when we proactively add the stub when processing the head body
*/
pending->trace_head = coarse_is_trace_head(exit_tgt);
/* If target is trace head in same unit, we could add to pending,
* but don't have body pc handy, so we let htable walk find it
*/
}
ASSERT(cti_len > 4);
sz = cti_len - 4;
pending->link_cti_opnd = freeze_info->cache_cur_pc + sz;
memcpy(freeze_info->cache_cur_pc, cti_pc, sz);
if (body_start_pc == cti_pc && !DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr)) {
/* case 9677: unsafe to elide entire-bb-ubr */
pending->elide_ubr = false;
} else /* elide if possible */
pending->elide_ubr = true;
freeze_info->cache_cur_pc += cti_len;
pending->next = freeze_info->pending;
freeze_info->pending = pending;
}
static cache_pc
redirect_to_tgt_ibl_prefix(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
cache_pc tgt)
{
ASSERT(freeze_info != NULL && freeze_info->src_info != NULL &&
freeze_info->dst_info != NULL);
if (tgt == freeze_info->src_info->ibl_ret_prefix)
return freeze_info->dst_info->ibl_ret_prefix;
else if (tgt == freeze_info->src_info->ibl_call_prefix)
return freeze_info->dst_info->ibl_call_prefix;
else if (tgt == freeze_info->src_info->ibl_jmp_prefix)
return freeze_info->dst_info->ibl_jmp_prefix;
else
ASSERT_NOT_REACHED();
return tgt; /* best chance of continuing on */
}
/* Transfers a coarse fragment to a new location. Queues up all of its
* exit targets for transfer as well, scheduling ubr last to enable ubr elision.
*/
void
transfer_coarse_fragment(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
cache_pc body)
{
/* FIXME: for maximum code re-use, use decode_fragment instead of trying
* to be efficient?
* FIXME case 9428: 8-bit conversion
*/
cache_pc pc = body, next_pc = pc; /* source pcs */
app_pc tgt;
size_t sz;
bool intra_fragment = false;
instr_t *instr;
instr = instr_create(dcontext);
do {
instr_reset(dcontext, instr);
pc = next_pc;
ASSERT(pc - body <= MAX_FRAGMENT_SIZE);
next_pc = decode_cti(dcontext, pc, instr);
/* Case 8711: we can't distinguish exit ctis from others,
* so we must assume that any cti is an exit cti, although
* we do now support intra-fragment ctis (i#665).
* Assumption: coarse-grain bbs have 1 ind exit or 2 direct,
* and no code beyond the last exit!
*/
intra_fragment = false;
if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
if (instr_is_cti_short_rewrite(instr, pc)) {
/* Pull in the two short jmps for a "short-rewrite" instr.
* We must do this before asking whether it's an
* intra-fragment so we don't just look at the
* first part of the sequence.
*/
next_pc = remangle_short_rewrite(dcontext, instr, pc,
0/*same target*/);
}
if (coarse_cti_is_intra_fragment(dcontext, freeze_info->src_info,
instr, body))
intra_fragment = true;
}
} while (!instr_opcode_valid(instr) || !instr_is_cti(instr) || intra_fragment);
/* copy body of fragment, up to start of cti */
sz = pc - body;
memcpy(freeze_info->cache_cur_pc, body, sz);
freeze_info->cache_cur_pc += sz;
DODEBUG({ freeze_info->app_code_size += sz; });
/* Ensure we get proper target for short cti sequence */
if (instr_is_cti_short_rewrite(instr, pc)) {
/* We already remangled if a short-rewrite */
DODEBUG({
/* We mangled 2-byte jecxz/loop* into 9-byte sequence */
freeze_info->app_code_size -= 7;
freeze_info->added_jecxz_mangle += 7;
});
}
tgt = opnd_get_pc(instr_get_target(instr));
if (tgt == next_pc) {
ASSERT(instr_is_ubr(instr));
/* indirect exit stub */
ASSERT(coarse_is_indirect_stub(tgt));
/* elide the jmp to the stub */
pc += JMP_LONG_LENGTH /*ubr to stub*/;
sz = coarse_indirect_stub_size(freeze_info->src_info) - 4;
memcpy(freeze_info->cache_cur_pc, pc, sz);
freeze_info->cache_cur_pc += sz;
pc += sz;
tgt = PC_RELATIVE_TARGET(pc);
DODEBUG({
freeze_info->num_indbr++;
freeze_info->app_code_size -= 6; /* save ecx */
freeze_info->added_indbr_mangle += 6 /* save ecx */;
if (tgt == freeze_info->src_info->ibl_ret_prefix) {
/* ret imm goes from 3 bytes to 1+4=5 bytes
L3 c2 18 00 ret $0x0018 %esp (%esp) -> %esp
=>
L4 67 64 89 0e e8 0e addr16 mov %ecx -> %fs:0xee8
L4 59 pop %esp (%esp) -> %ecx %esp
L4 8d 64 24 18 lea 0x18(%esp) -> %esp
*/
/* guaranteed to be able to read 5 bytes back */
if (*(pc - 4) == 0x8d && *(pc - 3) == 0x6d && *(pc - 2) == 0x24) {
freeze_info->app_code_size -= 2;
freeze_info->added_indbr_mangle += 2;
}
} else if (tgt == freeze_info->src_info->ibl_call_prefix) {
/* change from call* to mov is no size diff */
freeze_info->added_indbr_mangle += 5 /* push immed */;
} else {
/* jmp*: change to mov is no size difference */
}
freeze_info->added_indbr_stub +=
coarse_indirect_stub_size(freeze_info->src_info) ;
});
tgt = redirect_to_tgt_ibl_prefix(dcontext, freeze_info, tgt);
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
freeze_info->cache_cur_pc =
insert_relative_target(freeze_info->cache_cur_pc, tgt, NOT_HOT_PATCHABLE);
} else {
/* FIXME: if we had profile info we could reverse the branch and
* make our cache trace-like
*/
DEBUG_DECLARE(bool is_cbr = false;)
if (instr_is_cbr(instr)) {
uint cbr_len;
/* push cbr target on todo-stack */
if (instr_is_cti_short_rewrite(instr, pc))
cbr_len = CBR_SHORT_REWRITE_LENGTH;
else
cbr_len = CBR_LONG_LENGTH;
push_pending_freeze(dcontext, freeze_info, tgt, cbr_len, pc, body);
ASSERT(pc + cbr_len == next_pc);
/* process ubr next */
instr_reset(dcontext, instr);
pc = next_pc;
next_pc = decode_cti(dcontext, pc, instr);
ASSERT(instr_opcode_valid(instr) && instr_is_ubr(instr));
tgt = opnd_get_pc(instr_get_target(instr));
DODEBUG({
freeze_info->num_cbr++;
/* FIXME: assumes 32-bit cbr! */
freeze_info->app_code_size += cbr_len;
freeze_info->added_fallthrough += 5;
is_cbr = true;
});
}
ASSERT(instr_is_ubr(instr));
/* push ubr last, so we can elide the jmp if we process it next */
push_pending_freeze(dcontext, freeze_info, tgt, JMP_LONG_LENGTH, pc, body);
ASSERT(pc + JMP_LONG_LENGTH == next_pc);
DODEBUG({
if (!is_cbr) {
if (pc >= body + 5 && *(pc - 5) == 0x68) {
/* FIXME: could be an app push immed followed by app jmp */
/* call => push immed: same size, but adding jmp */
freeze_info->num_call++;
freeze_info->added_fallthrough += 5; /* jmp */
} else {
/* FIXME: assumes 32-bit jmp! */
freeze_info->app_code_size += 5;
freeze_info->num_jmp++;
}
}
});
}
instr_destroy(dcontext, instr);
}
/* This routine walks info's cache and updates extra-cache jmp targets by cache_shift
* and jmps to stubs by stubs_shift.
* If !is_cache, assumes these are stubs and decodes and acts appropriately.
*/
static void
coarse_unit_shift_jmps_internal(dcontext_t *dcontext, coarse_info_t *info,
ssize_t cache_shift, ssize_t stubs_shift,
size_t old_mapsz,
cache_pc start, cache_pc end,
cache_pc bounds_start, cache_pc bounds_end,
bool is_cache)
{
/* We must patch up indirect and direct stub jmps to prefixes */
cache_pc pc = start;
cache_pc next_pc = pc;
app_pc tgt;
instr_t *instr;
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
ASSERT(info->frozen);
instr = instr_create(dcontext);
while (next_pc < end) {
instr_reset(dcontext, instr);
pc = next_pc;
next_pc = decode_cti(dcontext, pc, instr);
/* Case 8711: we can't distinguish exit ctis from others.
* Note that we don't need to distinguish intra-fragment ctis here
* b/c we want to shift them by the same amount (xref i#665).
*/
if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
if (instr_is_cti_short_rewrite(instr, pc))
next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
tgt = opnd_get_pc(instr_get_target(instr));
if (tgt < bounds_start || tgt >= bounds_end) {
ssize_t shift;
if (is_cache) {
/* break down into whether targeting stubs or not
* ok to use new prefix start, which is where old padding was
*/
if (tgt >= info->fcache_return_prefix &&
tgt < info->cache_start_pc + old_mapsz)
shift = stubs_shift;
else
shift = cache_shift;
} else {
/* Shifting jmps from stubs
* We started with [cache | padding | stubs | padding]
* We then allocate new memory and copy there [cache | stubs]
* Thus, the stubs have a double shift: once for padding bet
* cache and stubs, and once for shift of whole alloc.
* This doesn't work if stubs target cache, but we assert on
* that in transfer_coarse_stub().
*/
shift = cache_shift - stubs_shift;
}
LOG(THREAD, LOG_FRAGMENT, 4,
"\tshifting jmp @"PFX" "PFX" from "PFX" to "PFX"\n",
pc, next_pc - 4, tgt, tgt + shift);
insert_relative_target(next_pc - 4, tgt + shift, NOT_HOT_PATCHABLE);
if (!is_cache) {
/* we must update incoming after fixing target, since old_stub is
* inconsistent and we need a complete stub to dereference
*/
cache_pc old_stub, new_stub;
/* double-check post-shift: no prefix targets */
ASSERT(tgt + shift < bounds_start || tgt + shift >= bounds_end);
new_stub = (cache_pc) ALIGN_BACKWARD(pc, coarse_stub_alignment(info));
old_stub = new_stub + shift;
/* we can't assert that old_stub or new_stub are entrance_stubs
* since targets are currently inconsistent wrt info
*/
/* must update incoming stub for target */
coarse_update_outgoing(dcontext, old_stub, new_stub,
info, true/*replace*/);
}
}
if (!is_cache) {
/* for stubs, skip the padding (which we'll decode as garbage */
ASSERT(next_pc + IF_X64_ELSE(3, 1) ==
(cache_pc) ALIGN_FORWARD(next_pc, coarse_stub_alignment(info)));
next_pc = (cache_pc) ALIGN_FORWARD(next_pc, coarse_stub_alignment(info));
}
}
}
instr_destroy(dcontext, instr);
}
/* This routine walks info's cache and updates extra-cache jmp targets by cache_shift
* and jmps to stubs by stubs_shift.
* It also walks info's stubs and updates targets that are not prefixes:
* in other coarse units or in fine-grained fragment caches.
*/
static void
coarse_unit_shift_jmps(dcontext_t *dcontext, coarse_info_t *info,
ssize_t cache_shift, ssize_t stubs_shift, size_t old_mapsz)
{
LOG(THREAD, LOG_FRAGMENT, 4,
"shifting jmps for cache "PFX"-"PFX"\n",
info->cache_start_pc, info->cache_end_pc);
coarse_unit_shift_jmps_internal(dcontext, info,
cache_shift, stubs_shift, old_mapsz,
info->cache_start_pc, info->cache_end_pc,
info->cache_start_pc, info->cache_end_pc,
true/*cache*/);
LOG(THREAD, LOG_FRAGMENT, 4,
"shifting jmps for stubs "PFX"-"PFX"\n",
info->stubs_start_pc, info->stubs_end_pc);
coarse_unit_shift_jmps_internal(dcontext, info,
cache_shift, stubs_shift, old_mapsz,
info->stubs_start_pc, info->stubs_end_pc,
/* do not re-relativize prefix targets */
info->fcache_return_prefix, info->stubs_end_pc,
false/*stubs*/);
}
/***************************************************************************
* MERGING FROZEN UNITS
*/
/* Processes a stub in the original old source unit at stub whose
* targeting cti has been copied into the new being-built merged unit
* at dst_cache_pc and has length cti_len. Passing dst_cache_pc==NULL
* causes no cti patching to occur.
*/
static void
coarse_merge_process_stub(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
cache_pc old_stub, uint cti_len, cache_pc dst_cache_pc,
bool replace_outgoing)
{
app_pc old_stub_tgt;
cache_pc dst_body, dst_stub, patch_pc, src_body;
bool trace_head;
ASSERT(coarse_is_entrance_stub(old_stub));
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
ASSERT((dst_cache_pc == NULL && cti_len == 0) || cti_len > 4);
patch_pc = dst_cache_pc + cti_len - 4;
old_stub_tgt = entrance_stub_target_tag(old_stub, freeze_info->src_info);
fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
old_stub_tgt, &dst_stub, &dst_body);
/* We need to know for sure whether a trace head as we're not doing
* a pass through the htable like we do for regular freezing
*/
fragment_coarse_lookup_in_unit(dcontext, freeze_info->src_info, old_stub_tgt,
NULL, &src_body);
/* Consider both sources for headness */
trace_head =
coarse_is_trace_head_in_own_unit(dcontext, old_stub_tgt, old_stub,
src_body, true, freeze_info->src_info) ||
(dst_stub != NULL &&
coarse_is_trace_head_in_own_unit(dcontext, old_stub_tgt, dst_stub,
dst_body, true, freeze_info->dst_info));
/* Should only be adding w/ no source cti if a trace head or for the stub
* walk for the larger unit where we have a dup stub and aren't replacing */
ASSERT(dst_cache_pc != NULL || trace_head ||
(dst_body == NULL && dst_stub != NULL && !replace_outgoing));
if (dst_body != NULL && !trace_head) {
/* Directly link and do not copy the stub */
LOG(THREAD, LOG_FRAGMENT, 4,
"\ttarget "PFX" is in other cache @"PFX": directly linking\n",
old_stub_tgt, dst_body);
ASSERT(dst_stub == NULL);
ASSERT(dst_body >= freeze_info->dst_info->cache_start_pc &&
dst_body < freeze_info->dst_info->cache_end_pc);
if (dst_cache_pc != NULL)
insert_relative_target(patch_pc, dst_body, NOT_HOT_PATCHABLE);
if (!freeze_info->unlink && entrance_stub_linked(old_stub,
freeze_info->src_info)) {
/* ASSUMPTION: unlink == !in_place
* If in-place, we must update target incoming info, whether source is
* primary (being replaced) or secondary (probably being deleted since
* now in merge result, but we don't want to crash while unlinking it
* (case 10382)) source.
*/
coarse_remove_outgoing(dcontext, old_stub, freeze_info->src_info);
}
} else if (dst_stub != NULL) {
LOG(THREAD, LOG_FRAGMENT, 4,
"\ttarget "PFX" is already in stubs @"PFX"\n", old_stub_tgt, dst_stub);
ASSERT(dst_body == NULL || trace_head);
/* Stub already exists: point to it */
if (dst_cache_pc != NULL)
insert_relative_target(patch_pc, dst_stub, NOT_HOT_PATCHABLE);
/* Must remove incoming if one mergee had a cross link to the other */
if ((dst_body != NULL ||
/* If secondary merger was smaller and had a stub for the same target,
* we need to remove our outgoing since secondary added a new one.
* We want to do this only the 1st time we get here, and not if the
* primary merger added the stub, so we have the primary unlink
* the old stub (in else code below).
*/
replace_outgoing) &&
entrance_stub_linked(old_stub, freeze_info->src_info)) {
coarse_remove_outgoing(dcontext, old_stub, freeze_info->src_info);
}
} else {
/* Copy stub */
cache_pc stub_pc = freeze_info->stubs_cur_pc;
ASSERT(dst_body == NULL || trace_head);
LOG(THREAD, LOG_FRAGMENT, 4,
"\ttarget "PFX" is %s, adding stub @"PFX"\n",
old_stub_tgt, trace_head ? "trace head" : "not present", stub_pc);
transfer_coarse_stub(dcontext, freeze_info, old_stub, trace_head,
replace_outgoing);
if (replace_outgoing) {
/* Signal to later stubs that they don't need to remove the outgoing
* entry (as opposed to new stubs added by the secondary merger,
* for which we do need to remove).
* Assumption: if replace_outgoing then it's ok to unlink the old stub
* since it's going away anyway.
*/
unlink_entrance_stub(dcontext, old_stub,
trace_head ? FRAG_IS_TRACE_HEAD : 0,
freeze_info->src_info);
}
ASSERT(freeze_info->stubs_cur_pc == stub_pc +
coarse_stub_alignment(freeze_info->src_info));
fragment_coarse_th_add(dcontext, freeze_info->dst_info, old_stub_tgt,
stub_pc - (ptr_uint_t)
freeze_info->dst_info->stubs_start_pc);
if (dst_cache_pc != NULL)
insert_relative_target(patch_pc, stub_pc, NOT_HOT_PATCHABLE);
}
}
/* Assumption: cache has already been copied from src to dst.
* This routine walks the copied cache to find inter-unit links; it
* directly links them, eliminating their entrance stubs.
* replace_outgoing really only applies if in_place: should we replace
* outgoing link incoming entries (true), or add new ones (false)
*/
static void
coarse_merge_update_jmps(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
bool replace_outgoing)
{
/* Plan: cache has already been copied from src to dst, but we need to do
* inter-unit links. So we decode from the original cache to find the
* target stubs: if a target is present in dst, we do not copy the stub and
* we directly link; if not present, we copy the stub and re-relativize the
* jmp to the stub. We must also patch up indirect and direct stub jmps
* to prefixes.
*/
cache_pc pc = freeze_info->src_info->cache_start_pc;
cache_pc next_pc = pc;
cache_pc stop_pc = freeze_info->src_info->cache_end_pc;
app_pc tgt;
uint sz;
/* FIXME: share code w/ decode_fragment() and transfer_coarse_fragment() */
instr_t *instr;
/* Since mucking with caches, though if thread-private not necessary */
ASSERT(dynamo_all_threads_synched);
ASSERT(freeze_info->src_info->frozen);
LOG(THREAD, LOG_FRAGMENT, 4,
"coarse_merge_update_jmps %s "PFX" => "PFX"\n",
freeze_info->src_info->module, pc, freeze_info->cache_start_pc);
instr = instr_create(dcontext);
while (next_pc < stop_pc) {
instr_reset(dcontext, instr);
pc = next_pc;
next_pc = decode_cti(dcontext, pc, instr);
/* Case 8711: we can't distinguish exit ctis from others,
* so we must assume that any cti is an exit cti.
*/
/* We don't care about fragment boundaries so we can ignore elision.
* We only care about jmps to stubs.
*/
if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
/* Ensure we get proper target for short cti sequence */
if (instr_is_cti_short_rewrite(instr, pc))
next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
tgt = opnd_get_pc(instr_get_target(instr));
if (in_coarse_stub_prefixes(tgt)) {
/* We should not encounter prefix targets other than indirect while
* in the body of the cache (rest are from the stubs) */
ASSERT(coarse_is_indirect_stub
(next_pc - coarse_indirect_stub_size(freeze_info->src_info)));
/* indirect exit stub: need to update jmp to prefix */
ASSERT(instr_is_ubr(instr));
sz = JMP_LONG_LENGTH /*ubr to stub*/ - 4;
pc += sz;
tgt = PC_RELATIVE_TARGET(pc);
tgt = redirect_to_tgt_ibl_prefix(dcontext, freeze_info, tgt);
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
insert_relative_target(freeze_info->cache_start_pc +
(pc - freeze_info->src_info->cache_start_pc),
tgt, NOT_HOT_PATCHABLE);
next_pc = pc + 4;
} else if (tgt < freeze_info->src_info->cache_start_pc || tgt >= stop_pc) {
/* Must go through a stub */
cache_pc dst_cache_pc = freeze_info->cache_start_pc +
(pc - freeze_info->src_info->cache_start_pc);
ASSERT(tgt >= freeze_info->src_info->stubs_start_pc &&
tgt < freeze_info->src_info->stubs_end_pc);
if (instr_is_cbr(instr)) {
uint cbr_len;
if (instr_is_cti_short_rewrite(instr, pc))
cbr_len = CBR_SHORT_REWRITE_LENGTH;
else
cbr_len = CBR_LONG_LENGTH;
ASSERT(pc + cbr_len == next_pc);
coarse_merge_process_stub(dcontext, freeze_info, tgt, cbr_len,
dst_cache_pc, replace_outgoing);
/* If there is a ubr next (could be elided) we just hit
* it next time around loop */
} else {
ASSERT(instr_is_ubr(instr));
ASSERT(pc + JMP_LONG_LENGTH == next_pc);
coarse_merge_process_stub(dcontext, freeze_info, tgt,
JMP_LONG_LENGTH, dst_cache_pc,
replace_outgoing);
}
} else {
/* intra-cache target */
/* I would assert that a pclookup finds an entry but that hits
* a recursive lock on non-recursive freeze_info->src_info->lock */
}
}
}
instr_destroy(dcontext, instr);
/* Do the loop even w/o traces in debug for the assert */
if (!DYNAMO_OPTION(disable_traces) IF_DEBUG(|| true)) {
/* We can have trace heads with no intra-unit targeters (secondary trace
* heads!) so we must also walk the stubs. Rather than require an
* iterator or helper routine in fragment or link we directly
* walk here. */
for (pc = freeze_info->src_info->stubs_start_pc;
pc < freeze_info->src_info->stubs_end_pc;
pc += coarse_stub_alignment(freeze_info->src_info)) {
if (in_coarse_stub_prefixes(pc))
continue;
ASSERT(coarse_is_entrance_stub(pc));
if (entrance_stub_linked(pc, freeze_info->src_info)) {
cache_pc src_body;
/* for non-in-place merging we don't unlink stubs targeting
* the other mergee, so we must rule that out here.
* the only internally-untargeted stubs we need to add are
* those for our own bodies. */
fragment_coarse_lookup_in_unit
(dcontext, freeze_info->src_info,
entrance_stub_target_tag(pc, freeze_info->src_info),
NULL, &src_body);
if (src_body != NULL) {
ASSERT(!DYNAMO_OPTION(disable_traces));
coarse_merge_process_stub(dcontext, freeze_info, pc,
0, NULL, replace_outgoing);
}
}
}
}
}
/* Assumption: cache to be merged with has already been copied to dst.
* This routine walks the other src and copies over non-dup fragments,
* directly linking inter-unit links along the way.
* replace_outgoing really only applies if in_place: should we replace
* outgoing link incoming entries (true), or add new ones (false)
*/
static void
coarse_merge_without_dups(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
ssize_t cache_offs, bool replace_outgoing)
{
/* Plan: we need to append the non-dup portions of src to the already-copied
* other source, as well as fixing up inter-unit links: if a target is present in
* dst, we do not copy the stub and we directly link; if not present, we copy the
* stub and re-relativize the jmp to the stub. We must also patch up indirect
* and direct stub jmps to prefixes.
* Though we need to know fragment boundaries, note that walking the htable
* instead of the cache doesn't buy us much: due to elision we still have to do
* pclookups, so we go ahead and walk the cache as it is already laid out.
*/
cache_pc pc = freeze_info->src_info->cache_start_pc;
cache_pc src_body, next_pc = pc, fallthrough_body = NULL;
cache_pc dst_body = NULL, last_dst_body;
cache_pc stop_pc = freeze_info->src_info->cache_end_pc;
app_pc tag, fallthrough_tag = NULL, tgt = NULL;
/* FIXME: share code w/ decode_fragment() and transfer_coarse_fragment() */
instr_t *instr;
/* stored targets for fixup */
jmp_tgt_list_t *jmp_list = NULL;
bool intra_fragment = false;
/* Since mucking with caches, though if thread-private not necessary */
ASSERT(dynamo_all_threads_synched);
ASSERT(freeze_info->src_info->frozen);
LOG(THREAD, LOG_FRAGMENT, 4,
"coarse_merge_without_dups %s "PFX" => "PFX"\n",
freeze_info->src_info->module, pc, freeze_info->cache_cur_pc);
instr = instr_create(dcontext);
while (next_pc < stop_pc) {
last_dst_body = dst_body;
if (fallthrough_tag != NULL) {
/* still at dup fallthrough pc */
ASSERT(fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info,
next_pc) == fallthrough_tag);
tag = fallthrough_tag;
ASSERT(fallthrough_body != NULL);
dst_body = fallthrough_body;
/* do not go again through the fallthrough code below */
instr_reset(dcontext, instr);
} else {
tag = fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info,
next_pc);
}
/* We come back through the loop for fallthrough jmp of cbr */
ASSERT(tag != NULL || (instr_opcode_valid(instr) && instr_is_cbr(instr)));
if (tag != NULL && tag != fallthrough_tag) {
LOG(THREAD, LOG_FRAGMENT, 4,
"\tfragment entry point "PFX" = tag "PFX, next_pc, tag);
fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
tag, NULL, &dst_body);
if (dst_body == NULL) {
cache_pc src_stub = NULL;
fragment_coarse_add(dcontext, freeze_info->dst_info,
tag, freeze_info->cache_cur_pc -
(ptr_uint_t)freeze_info->cache_start_pc +
cache_offs);
LOG(THREAD, LOG_FRAGMENT, 4, " (new => "PFX")\n",
freeze_info->cache_cur_pc);
/* this may be a trace head, in which case we need to add its stub
* now in case there are no intra-unit targeters of it (which
* means it is probably a secondary trace head) */
fragment_coarse_lookup_in_unit(dcontext, freeze_info->src_info,
tag, &src_stub, NULL);
if (src_stub != NULL) {
ASSERT(!DYNAMO_OPTION(disable_traces));
coarse_merge_process_stub(dcontext, freeze_info, src_stub,
0, NULL, replace_outgoing);
}
} else { /* dup */
LOG(THREAD, LOG_FRAGMENT, 4, " (duplicate)\n");
/* if prev is cbr, this is a fall-through, which is handled below */
}
} /* else carry through dst_body from last iter */
src_body = next_pc;
fallthrough_tag = NULL;
fallthrough_body = NULL;
do {
ASSERT(next_pc < stop_pc);
if (next_pc >= stop_pc)
return; /* paranoid: avoid infinite loop */
pc = next_pc;
if (!intra_fragment &&
(next_pc != src_body ||
/* fall-through of cbr will be looked up pre-1st iter above */
(instr_opcode_valid(instr) && instr_is_cbr(instr)))) {
/* We assume at least one instr in each fragment, to avoid ambiguity */
ASSERT_NOT_IMPLEMENTED(!DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr));
if (next_pc == src_body) {
fallthrough_tag = tag;
fallthrough_body = dst_body;
} else {
fallthrough_tag =
fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info,
next_pc);
if (fallthrough_tag != NULL) {
fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
fallthrough_tag, NULL,
&fallthrough_body);
}
}
if (fallthrough_tag != NULL) {
/* We'd rather keep fall-through elision if we can */
LOG(THREAD, LOG_FRAGMENT, 4,
"\tfall-through tag "PFX" @"PFX, fallthrough_tag, next_pc);
if (fallthrough_body == NULL) {
/* Just keep going and process the fall-through's cti */
LOG(THREAD, LOG_FRAGMENT, 4, " (new => "PFX")\n",
freeze_info->cache_cur_pc + (next_pc - src_body));
if (dst_body != NULL) { /* prev is a dup */
ASSERT_NOT_TESTED();
src_body = next_pc;
tag = fallthrough_tag;
}
if (fallthrough_tag != tag) {
fragment_coarse_add(dcontext, freeze_info->dst_info,
fallthrough_tag,
freeze_info->cache_cur_pc +
(next_pc - src_body) -
(ptr_uint_t)freeze_info->cache_start_pc +
cache_offs);
DOCHECK(1, {
/* We should NOT need to add a stub like we might
* for the entry point add above: fall-through
* cannot be trace head! */
cache_pc src_stub = NULL;
fragment_coarse_lookup_in_unit(dcontext,
freeze_info->src_info,
fallthrough_tag,
&src_stub, NULL);
ASSERT(src_stub == NULL);
});
}
fallthrough_tag = NULL;
} else {
LOG(THREAD, LOG_FRAGMENT, 4, " (duplicate)\n");
break;
}
}
}
instr_reset(dcontext, instr);
next_pc = decode_cti(dcontext, pc, instr);
ASSERT(next_pc - src_body <= MAX_FRAGMENT_SIZE);
/* Case 8711: we can't distinguish exit ctis from others,
* so we must assume that any cti is an exit cti.
* Assumption: coarse-grain bbs have 1 ind exit or 2 direct,
* and no code beyond the last exit!
*/
intra_fragment = false;
if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
if (instr_is_cti_short_rewrite(instr, pc)) {
/* Pull in the two short jmps for a "short-rewrite" instr.
* We must do this before asking whether it's an
* intra-fragment so we don't just look at the
* first part of the sequence.
*/
next_pc = remangle_short_rewrite(dcontext, instr, pc,
0/*same target*/);
}
if (coarse_cti_is_intra_fragment(dcontext, freeze_info->src_info,
instr, src_body))
intra_fragment = true;
}
} while (!instr_opcode_valid(instr) || !instr_is_cti(instr) || intra_fragment);
if (dst_body == NULL) { /* not a dup */
/* copy body of fragment, including cti (if not ending @ fall-through) */
size_t sz = next_pc - src_body;
memcpy(freeze_info->cache_cur_pc, src_body, sz);
freeze_info->cache_cur_pc += sz;
}
if (fallthrough_tag != NULL) {
ASSERT(next_pc == pc); /* should have short-circuited */
/* add intra-cache jmp if elided but fall-through a dup */
ASSERT(fallthrough_body != NULL);
/* If start bb not a dup, or post-cbr, must un-elide */
if (dst_body == NULL || (next_pc == src_body && last_dst_body == NULL)) {
LOG(THREAD, LOG_FRAGMENT, 4,
"\tadding jmp @"PFX" to "PFX" for fall-through tag "PFX"\n",
freeze_info->cache_cur_pc, fallthrough_body, fallthrough_tag);
freeze_info->cache_cur_pc =
insert_relative_jump(freeze_info->cache_cur_pc, fallthrough_body,
NOT_HOT_PATCHABLE);
}
} else {
ASSERT(instr_opcode_valid(instr) && instr_is_cti(instr));
/* We already remangled if a short-rewrite so no extra work here */
tgt = opnd_get_pc(instr_get_target(instr));
if (in_coarse_stub_prefixes(tgt)) {
/* We should not encounter prefix targets other than indirect while
* in the body of the cache (rest are from the stubs) */
ASSERT(coarse_is_indirect_stub
(next_pc - coarse_indirect_stub_size(freeze_info->src_info)));
/* indirect exit stub: need to update jmp to prefix */
ASSERT(instr_is_ubr(instr));
if (dst_body == NULL) { /* not a dup */
tgt = PC_RELATIVE_TARGET(next_pc - 4);
tgt = redirect_to_tgt_ibl_prefix(dcontext, freeze_info, tgt);
ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
/* we've already copied the stub as part of the body */
ASSERT(coarse_is_indirect_stub
(freeze_info->cache_cur_pc -
coarse_indirect_stub_size(freeze_info->src_info)));
freeze_info->cache_cur_pc -= 4;
freeze_info->cache_cur_pc =
insert_relative_target(freeze_info->cache_cur_pc,
tgt, NOT_HOT_PATCHABLE);
}
} else if (tgt < freeze_info->src_info->cache_start_pc || tgt >= stop_pc) {
if (dst_body == NULL) { /* not a dup */
/* currently goes through a stub */
ASSERT(tgt >= freeze_info->src_info->stubs_start_pc &&
tgt < freeze_info->src_info->stubs_end_pc);
if (instr_is_cbr(instr)) {
uint cbr_len;
if (instr_is_cti_short_rewrite(instr, pc))
cbr_len = CBR_SHORT_REWRITE_LENGTH;
else
cbr_len = CBR_LONG_LENGTH;
ASSERT(pc + cbr_len == next_pc);
coarse_merge_process_stub(dcontext, freeze_info, tgt, cbr_len,
freeze_info->cache_cur_pc - cbr_len,
replace_outgoing);
/* If there is a ubr next (could be elided) we just hit
* it next time around loop */
} else {
ASSERT(instr_is_ubr(instr));
ASSERT(pc + JMP_LONG_LENGTH == next_pc);
coarse_merge_process_stub(dcontext, freeze_info, tgt,
JMP_LONG_LENGTH,
freeze_info->cache_cur_pc -
JMP_LONG_LENGTH,
replace_outgoing);
}
}
} else if (dst_body == NULL) { /* not a dup */
/* Intra-cache target, but we're moving things around and have to do
* a separate pass since don't know future locations. Since the
* layout is changing and later we'd need multiple lookups to find
* the corrsepondence between src and dst, we store the target tag in
* the jmp and replace it w/ the body in the later pass.
* We can't fit a 64-bit target, so we use offs from mod base.
* XXX: split pcaches up if app module is over 4GB.
*/
jmp_tgt_list_t *entry;
app_pc tgt_tag =
fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info, tgt);
ASSERT(tgt_tag != NULL);
LOG(THREAD, LOG_FRAGMENT, 4,
"\tintra-cache src "PFX"->"PFX" tag "PFX" dst pre-"PFX"\n",
pc, tgt, tgt_tag, freeze_info->cache_cur_pc);
entry = HEAP_TYPE_ALLOC(dcontext, jmp_tgt_list_t,
ACCT_VMAREAS, PROTECTED);
entry->tag = tgt_tag;
entry->jmp_end_pc = freeze_info->cache_cur_pc;
entry->next = jmp_list;
jmp_list = entry;
}
}
}
/* Second pass to update intra-cache targets.
* FIXME: combine w/ later coarse_unit_shift_jmps()
*/
while (jmp_list != NULL) {
jmp_tgt_list_t *next = jmp_list->next;
fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
jmp_list->tag, NULL, &dst_body);
ASSERT(dst_body != NULL);
LOG(THREAD, LOG_FRAGMENT, 4,
"\tintra-cache dst -"PFX"->"PFX" tag "PFX"\n",
jmp_list->jmp_end_pc, dst_body, tgt); /* tgt always set here */
/* FIXME: make 4 a named constant; used elsewhere as well */
insert_relative_target(jmp_list->jmp_end_pc - 4, dst_body, NOT_HOT_PATCHABLE);
HEAP_TYPE_FREE(dcontext, jmp_list, jmp_tgt_list_t, ACCT_VMAREAS, PROTECTED);
jmp_list = next;
}
instr_destroy(dcontext, instr);
}
/* Returns a new coarse_info_t (or if in_place returns info1) that combines
* info1 and info2. In in_place, info1 is replaced with the result and returned;
* else, a separate coarse_info_t is created and returned.
* If either of the units is live, then info1 must be live.
* If one of the two units covers a different code range, it must be info2,
* and it must be a subset of info1's range.
* If returns NULL, the merge failed; if in_place, info1 is unchanged on failure.
* If in_place, caller is responsible for flushing the ibl tables (case 11057).
*/
coarse_info_t *
coarse_unit_merge(dcontext_t *dcontext, coarse_info_t *info1, coarse_info_t *info2,
bool in_place)
{
coarse_info_t *merged;
coarse_info_t *res = NULL;
coarse_info_t *src_sm, *src_lg;
size_t cache1_size, cachelg_size, cache2_size, merged_cache_size;
size_t stubs1_size, stubs2_size;
coarse_freeze_info_t freeze_info;
LOG(THREAD, LOG_CACHE, 2, "coarse_unit_merge %s %s with %s\n",
info1->module, info1->persisted ? "persisted" : "non-persisted",
info2->persisted ? "persisted" : "non-persisted");
STATS_INC(coarse_units_merged);
ASSERT(info1 != NULL && info2 != NULL);
ASSERT(info1->base_pc <= info2->base_pc && info1->end_pc >= info2->end_pc);
if (info1->base_pc > info2->base_pc || info1->end_pc < info2->end_pc)
return NULL;
/* Currently we only do online merging where one unit is live */
ASSERT(!info1->persisted || !info2->persisted);
/* Much more efficient to merge smaller cache into larger */
if (fragment_coarse_num_entries(info1) > fragment_coarse_num_entries(info2)) {
src_lg = info1;
src_sm = info2;
} else {
src_lg = info2;
src_sm = info1;
}
/* Ensure the pclookup table is set up for src_sm, to avoid recursive
* lock issues
*/
if (src_sm->pclookup_htable == NULL) { /* read needs no lock */
fragment_coarse_entry_pclookup(dcontext, src_sm, NULL);
ASSERT(src_sm->pclookup_htable != NULL);
}
acquire_recursive_lock(&change_linking_lock);
#ifdef HOT_PATCHING_INTERFACE
/* we may call coarse_unit_calculate_persist_info() */
if (DYNAMO_OPTION(hot_patching))
read_lock(hotp_get_lock());
#endif
/* We can't grab both locks due to deadlock potential. Currently we are
* always fully synched, so we rely on that to synch with info2.
*/
ASSERT(dynamo_all_threads_synched);
mutex_lock(&info1->lock);
ASSERT(info1->cache != NULL && info2->cache != NULL); /* don't merge empty units */
ASSERT(info1->frozen && info2->frozen);
/* Tasks:
* 1) Merge the caches, eliminating duplicates. Various optimizations to
* preserve fall-through pairs if 2nd part is in other cache but 1st is
* not could be employed.
* 2) Turn cross-links into direct links and eliminate those entrance stubs.
* 3) Copy the rest of the entrance stubs.
* 4) Merge the RCT, RAC, and hotp fields, which may require calculating
* persist info if this unit has not been persisted; if neither unit
* has been persisted we do not need to do anything here.
*
* We will remove the extra post-stubs space since when persisting
* we only write up through stubs_end_pc and not the allocation end.
*/
/* Whether merging in-place or not, we create a new coarse_info_t.
* If in-place we delete the old one afterward.
*/
merged = coarse_unit_create(info1->base_pc, info1->end_pc, &info1->module_md5,
in_place && info1->in_use);
merged->frozen = true;
cache1_size = info1->cache_end_pc - info1->cache_start_pc;
cache2_size = info2->cache_end_pc - info2->cache_start_pc;
/* We shrink the cache to size below, after merging & removing dups */
merged_cache_size = cache1_size + cache2_size;
/* We need the stubs to start on a new page since will be +rw vs cache +r */
merged_cache_size = ALIGN_FORWARD(merged_cache_size, PAGE_SIZE);
/* We only need one set of prefixes */
stubs1_size = info1->stubs_end_pc - info1->fcache_return_prefix;
stubs2_size = info2->stubs_end_pc - info2->stubs_start_pc;
merged->mmap_size = merged_cache_size + stubs1_size + stubs2_size;
/* Our relative jmps require that we do not exceed 32-bit reachability */
IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(merged->mmap_size)));
merged->cache_start_pc = (cache_pc) heap_mmap(merged->mmap_size);
merged->cache_end_pc = merged->cache_start_pc + cache1_size + cache2_size;
merged->stubs_start_pc =
coarse_stubs_create(merged, merged->cache_start_pc + merged_cache_size,
stubs1_size + stubs2_size);
/* will be tightened up later */
merged->stubs_end_pc = merged->cache_start_pc + merged->mmap_size;
ASSERT(merged->stubs_start_pc != NULL);
ASSERT(ALIGNED(merged->stubs_start_pc, coarse_stub_alignment(info1)));
ASSERT(merged->fcache_return_prefix ==
merged->cache_start_pc + merged_cache_size);
ASSERT(merged->trace_head_return_prefix == merged->fcache_return_prefix +
(info1->trace_head_return_prefix - info1->fcache_return_prefix));
ASSERT(merged->ibl_ret_prefix == merged->fcache_return_prefix +
(info1->ibl_ret_prefix - info1->fcache_return_prefix));
ASSERT(merged->ibl_call_prefix == merged->fcache_return_prefix +
(info1->ibl_call_prefix - info1->fcache_return_prefix));
ASSERT(merged->ibl_jmp_prefix == merged->fcache_return_prefix +
(info1->ibl_jmp_prefix - info1->fcache_return_prefix));
/* Much more efficient to put the larger cache 1st, but we have to be sure
* to use the same order for both the htable and cache.
*/
/* Try to size the dst tables to avoid collision asserts.
* Put the larger unit's entries into the dst table up front.
* FIXME: do this earlier and if one is a subset of other then do
* a simpler merge?
*/
fragment_coarse_htable_merge(dcontext, merged, src_lg, src_sm,
false/*do not add src_sm yet*/,
false/*leave th htable empty*/);
/* Copy the 1st cache intact, and bring in the non-dup portions of the second
* (since know the offsets of the 1st); then, walk the 1st and patch up its
* inter-unit links while decoding from the original, now that the 2nd is in
* place.
*/
cachelg_size = (src_lg == info2) ? cache2_size : cache1_size;
memcpy(merged->cache_start_pc, src_lg->cache_start_pc, cachelg_size);
memset(&freeze_info, 0, sizeof(freeze_info));
freeze_info.dst_info = merged;
freeze_info.stubs_start_pc = merged->stubs_start_pc;
freeze_info.stubs_cur_pc = merged->stubs_start_pc;
/* Just like for freezing: leave inter-unit links intact for in_place.
* coarse_merge_process_stub() assumes that unlink == !in_place
*/
freeze_info.unlink = !in_place;
freeze_info.src_info = src_sm;
freeze_info.cache_start_pc = merged->cache_start_pc + cachelg_size;
freeze_info.cache_cur_pc = freeze_info.cache_start_pc;
coarse_merge_without_dups(dcontext, &freeze_info, cachelg_size,
/* replace for primary unit; add for secondary */
freeze_info.src_info == info1);
merged->cache_end_pc = freeze_info.cache_cur_pc;
freeze_info.src_info = src_lg;
freeze_info.cache_start_pc = merged->cache_start_pc;
freeze_info.cache_cur_pc = freeze_info.cache_start_pc;
coarse_merge_update_jmps(dcontext, &freeze_info,
/* replace for primary unit; add for secondary */
freeze_info.src_info == info1);
ASSERT((ptr_uint_t)(freeze_info.stubs_cur_pc - merged->fcache_return_prefix) <=
stubs1_size + stubs2_size);
/* We have extra space from extra in each stub region (from case 9428),
* from duplicate prefix space, and from eliminated inter-unit stubs, so
* we must set end pc.
*/
coarse_stubs_set_end_pc(merged, freeze_info.stubs_cur_pc);
LOG(THREAD, LOG_CACHE, 2, "merged size: stubs "SZFMT" => "SZFMT" bytes, "
"cache "SZFMT" ("SZFMT" align) => "SZFMT" ("SZFMT" align) bytes\n",
stubs1_size + stubs2_size, freeze_info.stubs_cur_pc - merged->stubs_start_pc,
cache1_size + cache2_size,
(info1->fcache_return_prefix - info1->cache_start_pc) +
(info2->fcache_return_prefix - info2->cache_start_pc),
merged->cache_end_pc - merged->cache_start_pc,
merged->fcache_return_prefix - merged->cache_start_pc);
if (merged_cache_size - (merged->cache_end_pc - merged->cache_start_pc) > 0) {
/* With duplicate elimination we often have a lot of empty space, so we
* re-allocate into a proper-fitting space
*/
size_t cachesz = merged->cache_end_pc - merged->cache_start_pc;
size_t cachesz_aligned = ALIGN_FORWARD(cachesz, PAGE_SIZE);
size_t stubsz = merged->stubs_end_pc - merged->fcache_return_prefix;
size_t newsz = cachesz_aligned + stubsz;
size_t old_mapsz = merged->mmap_size;
cache_pc newmap = (cache_pc) heap_mmap(newsz);
ssize_t cache_shift = merged->cache_start_pc - newmap;
/* stubs have moved too, so a relative shift not absolute */
ssize_t stubs_shift = cachesz_aligned -
(merged->fcache_return_prefix - merged->cache_start_pc);
LOG(THREAD, LOG_CACHE, 2,
"re-allocating merged unit: "SZFMT" @"PFX" "PFX" => "
SZFMT" @"PFX" "PFX" "SZFMT" "SZFMT"\n",
merged->mmap_size, merged->cache_start_pc, merged->fcache_return_prefix,
newsz, newmap, newmap + cachesz_aligned, cache_shift, stubs_shift);
memcpy(newmap, merged->cache_start_pc, cachesz);
memcpy(newmap + cachesz_aligned, merged->fcache_return_prefix, stubsz);
heap_munmap(merged->cache_start_pc, merged->mmap_size);
coarse_stubs_delete(merged);
merged->mmap_size = newsz;
/* Our relative jmps require that we do not exceed 32-bit reachability */
IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(merged->mmap_size)));
merged->cache_start_pc = newmap;
merged->cache_end_pc = merged->cache_start_pc + cachesz;
merged->stubs_start_pc =
coarse_stubs_create(merged, merged->cache_start_pc + cachesz_aligned,
stubsz);
ASSERT(merged->stubs_start_pc != NULL);
ASSERT(ALIGNED(merged->stubs_start_pc, coarse_stub_alignment(info1)));
ASSERT(merged->fcache_return_prefix == newmap + cachesz_aligned);
coarse_stubs_set_end_pc(merged, merged->cache_start_pc + newsz);
coarse_unit_shift_jmps(dcontext, merged, cache_shift, stubs_shift, old_mapsz);
}
/* Set cache bounds after we've potentially moved the initial cache */
fcache_coarse_init_frozen(dcontext, merged, merged->cache_start_pc,
merged->fcache_return_prefix - merged->cache_start_pc);
/* Currently we only do online merging where at least one unit is live,
* and we expect that to be info1
*/
ASSERT(!info1->persisted);
/* Store the source persisted size so we know whether we need to merge
* with that on-disk file
*/
if (info2->persisted)
merged->persisted_source_mmap_size = info2->mmap_size;
/* Merge the other fields */
coarse_unit_merge_persist_info(dcontext, merged, info1, info2);
DOLOG(5, LOG_CACHE, {
byte *pc = merged->cache_start_pc;
LOG(THREAD, LOG_CACHE, 1, "merged cache:\n");
do {
pc = disassemble_with_bytes(dcontext, pc, THREAD);
} while (pc < merged->cache_end_pc);
pc = merged->stubs_start_pc;
LOG(THREAD, LOG_CACHE, 1, "merged stubs:\n");
do {
if (((ptr_uint_t)pc) % coarse_stub_alignment(info1) ==
coarse_stub_alignment(info1)-1)
pc++;
pc = disassemble_with_bytes(dcontext, pc, THREAD);
} while (pc < merged->stubs_end_pc);
});
/* FIXME case 9687: mark cache as read-only */
if (in_place) {
coarse_incoming_t *e;
coarse_replace_unit(dcontext, info1, merged);
merged = NULL;
/* up to caller to call mark_executable_area_coarse_frozen() if necessary */
/* case 10877: must combine the incoming lists
* targets should be unique, so can just append
*/
mutex_lock(&info1->incoming_lock);
/* can't grab info2 lock, so just like for main lock we rely on synchall */
DODEBUG({
/* Make sure no inter-incoming left */
uint in1 = 0;
uint in2 = 0;
for (e = info1->incoming; e != NULL; e = e->next, in1++)
ASSERT(!e->coarse || get_stub_coarse_info(e->in.stub_pc) != info2);
for (e = info2->incoming; e != NULL; e = e->next, in2++)
ASSERT(!e->coarse || get_stub_coarse_info(e->in.stub_pc) != info1);
LOG(THREAD, LOG_CACHE, 1, "merging %d incoming into %d incoming\n",
in2, in1);
});
e = info1->incoming;
if (e == NULL) {
info1->incoming = info2->incoming;
} else {
while (e->next != NULL)
e = e->next;
e->next = info2->incoming;
}
mutex_unlock(&info1->incoming_lock);
info2->incoming = NULL; /* ensure not freed when info2 is freed */
coarse_unit_shift_links(dcontext, info1);
res = info1;
} else {
/* we made separate copy that has no outgoing or incoming links */
res = merged;
}
mutex_unlock(&info1->lock);
#ifdef HOT_PATCHING_INTERFACE
/* we may call coarse_unit_calculate_persist_info() */
if (DYNAMO_OPTION(hot_patching))
read_unlock(hotp_get_lock());
#endif
release_recursive_lock(&change_linking_lock);
return res;
}
/***************************************************************************
* PERSISTENT CODE CACHE
*/
#if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
extern bool seen_Borland_SEH;
#endif
/* get global or per-user directory name */
bool
perscache_dirname(char *directory /* OUT */, uint directory_len)
{
int retval;
bool param_ok = false;
/* Support specifying the pcache dir from either a config param (historical
* from ASLR piggyback) or runtime option, though config param gets precedence.
*/
const char *param_name = DYNAMO_OPTION(persist_per_user) ?
PARAM_STR(DYNAMORIO_VAR_PERSCACHE_ROOT) :
PARAM_STR(DYNAMORIO_VAR_PERSCACHE_SHARED);
retval = get_parameter(param_name, directory, directory_len);
if (IS_GET_PARAMETER_FAILURE(retval)) {
string_option_read_lock();
if (DYNAMO_OPTION(persist_per_user) && !IS_STRING_OPTION_EMPTY(persist_dir)) {
strncpy(directory, DYNAMO_OPTION(persist_dir), directory_len);
param_ok = true;
} else if (!IS_STRING_OPTION_EMPTY(persist_shared_dir)) {
strncpy(directory, DYNAMO_OPTION(persist_shared_dir), directory_len);
param_ok = true;
} else {
/* use log dir by default
* XXX: create subdir "logs/cache"? default is per-user so currently
* user dirs will be in logs/ which seems sufficient.
*/
uint len = directory_len;
create_log_dir(BASE_DIR);
if (get_log_dir(BASE_DIR, directory, &len) && len <= directory_len)
param_ok = true;
}
string_option_read_unlock();
} else
param_ok = true;
if (param_ok)
directory[directory_len - 1] = '\0';
return param_ok;
}
/* get global or per-user directory name */
static bool
get_persist_dir(char *directory /* OUT */,
uint directory_len,
bool create)
{
if (!perscache_dirname(directory, directory_len) ||
double_strchr(directory, DIRSEP, ALT_DIRSEP) == NULL) {
SYSLOG_INTERNAL_ERROR_ONCE("Persistent cache root dir is invalid. "
"Persistent cache will not operate.");
return false;
}
if (DYNAMO_OPTION(persist_per_user)) {
bool res = os_current_user_directory(directory, directory_len,
create);
/* null terminated */
if (!res) {
/* directory name may be set even on failure */
LOG(THREAD_GET, LOG_CACHE, 2, "\terror opening per-user dir %s\n", directory);
return false;
}
}
return true;
}
/* Checks for enough space on the volume where persisted caches are stored */
bool
coarse_unit_check_persist_space(file_t fd_in/*OPTIONAL*/, size_t size_needed)
{
bool room = false;
file_t fd = fd_in;
if (fd == INVALID_FILE) {
/* Use directory to get handle on proper volume */
char dir[MAXIMUM_PATH];
if (get_persist_dir(dir, BUFFER_SIZE_ELEMENTS(dir),
true /* note we MUST always create directory
* even if never persisting */
)) {
fd = os_open_directory(dir, 0);
} else
LOG(THREAD_GET, LOG_CACHE, 2, "\terror finding persist dir\n");
}
if (fd != INVALID_FILE) {
room = check_low_disk_threshold(fd, (uint64)size_needed);
if (fd_in == INVALID_FILE) {
/* FIXME: cache the handle, combine with -validate_owner_dir */
os_close(fd);
}
} else
LOG(THREAD_GET, LOG_CACHE, 2, "\terror opening persist dir\n");
return room;
}
/* If force_local, pretends module at pc has been exempted (so no effect
* unless -persist_check_exempted_options)
*/
static inline op_pcache_t
persist_get_options_level(app_pc pc, coarse_info_t *info, bool force_local)
{
if (!DYNAMO_OPTION(persist_check_options))
return OP_PCACHE_NOP;
else if (DYNAMO_OPTION(persist_check_local_options) ||
(DYNAMO_OPTION(persist_check_exempted_options) &&
(force_local ||
/* once loaded as local, must remain local, even if this
* process never hit the exemption */
(info != NULL && TEST(PERSCACHE_EXEMPTION_OPTIONS, info->flags)) ||
os_module_get_flag(pc, MODULE_WAS_EXEMPTED)) &&
/* don't use local if no such options: else when load will think
* local when really global */
has_pcache_dynamo_options(&dynamo_options, OP_PCACHE_LOCAL)))
return OP_PCACHE_LOCAL;
else
return OP_PCACHE_GLOBAL;
}
static const char *
persist_get_relevant_options(dcontext_t *dcontext, char *option_buf,
uint buf_len, op_pcache_t level)
{
if (level == OP_PCACHE_NOP)
return "";
get_pcache_dynamo_options_string(&dynamo_options, option_buf, buf_len, level);
option_buf[buf_len - 1] = '\0';
LOG(THREAD, LOG_CACHE, 2, "Pcache-affecting options = %s\n", option_buf);
return option_buf;
}
/* We identify persisted caches by mapping module info into a canonical name.
* There can be collisions (including for sub-module coarse units, such as
* separate +x module sections (xref case 9834 and case 9653); in addition, we
* can have different modules map to the same name), so caller must further
* verify matches.
* (Note that we use a different scheme than aslr's calculate_publish_name()
* as we are not dealing with file handles here but in-memory module images).
*/
static bool
get_persist_filename(char *filename /*OUT*/, uint filename_max /* max #chars */,
app_pc modbase, bool write, persisted_module_info_t *modinfo,
const char *option_string)
{
uint checksum, timestamp;
size_t size, code_size;
uint64 file_version;
const char *name;
uint hash;
char dir[MAXIMUM_PATH];
os_get_module_info_lock();
if (!os_get_module_info(modbase, &checksum, &timestamp, &size,
&name, &code_size, &file_version)) {
os_get_module_info_unlock();
return false;
}
if (name == NULL) {
/* theoretically possible but pathological, unless we came in late */
ASSERT_CURIOSITY(IF_WINDOWS_ELSE_0(!dr_early_injected));
LOG(GLOBAL, LOG_CACHE, 1, "\tmodule "PFX" has no name\n", modbase);
os_get_module_info_unlock();
return false;
}
/* Should not have path chars in the name */
ASSERT(get_short_name(name) == name && name[0] != DIRSEP);
name = get_short_name(name); /* paranoid */
/* Exclude list applies to both read and write */
if (!IS_STRING_OPTION_EMPTY(persist_exclude_list)) {
bool exclude;
string_option_read_lock();
exclude = check_filter(DYNAMO_OPTION(persist_exclude_list), name);
string_option_read_unlock();
if (exclude) {
LOG(GLOBAL, LOG_CACHE, 1, "\t%s is on exclude list\n", name);
DOSTATS({
if (write)
STATS_INC(coarse_units_persist_excluded);
else
STATS_INC(perscache_load_excluded);
});
os_get_module_info_unlock();
return false;
}
}
/* Prepend the perscache dir. We assume it has already been created.
* FIXME: cache this, or better, cache the dir handle and use an
* os_open that can take it in. Note that the directory handle
* doesn't help us in Linux - we can neither open files relative to it,
* nor there is any strong chown guarantee that we depend on.
*/
if (!get_persist_dir(dir, BUFFER_SIZE_ELEMENTS(dir), write)) {
os_get_module_info_unlock();
return false;
}
/* FIXME case 8494: version-independent names so we clobber files
* from old versions of modules and have less need of stale file
* cleanup? If so should add in (hash of) full path (w/ volume)
* to avoid name conflicts? But if try to share across machines
* we do not want to include path since can vary.
*/
/* should we go to a 64-bit hash? */
IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(size)));
hash = checksum ^ timestamp ^ (uint)size;
/* case 9799: make options part of namespace */
if (option_string != NULL) {
uint i;
ASSERT(DYNAMO_OPTION(persist_check_options));
for (i = 0; i < strlen(option_string); i++)
hash ^= option_string[i] << ((i % 4)*8);
}
LOG(GLOBAL, LOG_CACHE, 2, "\thash = 0x%08x^0x%08x^"PFX" ^ %s = "PFX"\n",
checksum, timestamp, size, option_string == NULL ? "" : option_string, hash);
ASSERT_CURIOSITY(hash != 0);
if (DYNAMO_OPTION(persist_per_app)) {
char *dirend = dir + strlen(dir);
/* FIXME case 9692: put tls offs instead of "dbg" here, and then
* sqlservr can have its own set if it ends up w/ separate tls offs
* (once we have non-per-app persisted files, that is).
*/
snprintf(dirend, BUFFER_SIZE_ELEMENTS(dir) - (dirend - dir), "%c%s%s",
DIRSEP, get_application_short_name(), IF_DEBUG_ELSE("-dbg", ""));
NULL_TERMINATE_BUFFER(dir);
LOG(GLOBAL, LOG_CACHE, 2, "\tper-app dir is %s\n", dir);
/* check for existence first so we can require new during creation */
if (!os_file_exists(dir, true/*is dir*/) && write) {
if (!os_create_dir(dir, CREATE_DIR_REQUIRE_NEW)) {
LOG(GLOBAL, LOG_CACHE, 2, "\terror creating per-app dir %s\n", dir);
os_get_module_info_unlock();
return false;
} else
LOG(GLOBAL, LOG_CACHE, 2, "\tcreated per-app dir %s\n", dir);
}
}
/* FIXME PR 214088/case 9653: should we put the section ordinal or vmarea range into
* the name to support simultaneous sub-module files? If sections are
* adjacent they'll be one vmarea, so this affects very few dlls. For now
* we only support one file per module. We could also support multiple
* ranges per file.
*/
snprintf(filename, filename_max, "%s%c%s%s-0x%08x.%s", dir, DIRSEP, name,
IF_DEBUG_ELSE("-dbg", ""), hash, PERSCACHE_FILE_SUFFIX);
filename[filename_max-1] = '\0';
os_get_module_info_unlock();
if (modinfo != NULL) {
modinfo->base = modbase;
modinfo->checksum = checksum;
modinfo->timestamp = timestamp;
modinfo->image_size = size;
modinfo->code_size = code_size;
modinfo->file_version = file_version;
}
return true;
}
#if defined(DEBUG) && defined(INTERNAL)
/* FIXME: share w/ aslr.c */
static void
print_module_digest(file_t f, module_digest_t *digest, const char *prefix)
{
LOG(f, LOG_CACHE, 1, "%s\n md5 short: ", prefix);
dump_buffer_as_bytes(f, digest->short_MD5, MD5_RAW_BYTES, DUMP_RAW);
LOG(f, LOG_CACHE, 1, "\n md5 long: ");
dump_buffer_as_bytes(f, digest->full_MD5, MD5_RAW_BYTES, DUMP_RAW);
LOG(f, LOG_CACHE, 1, "\n");
}
#endif
static void
persist_calculate_self_digest(module_digest_t *digest, coarse_persisted_info_t *pers,
app_pc map, uint validation_option)
{
struct MD5Context self_md5_cxt;
if (TEST(PERSCACHE_GENFILE_MD5_COMPLETE, validation_option)) {
MD5Init(&self_md5_cxt);
/* Even if generated w/ -persist_map_rw_separate but loaded w/o that
* option, the md5 should match since the memory layout is the same.
*/
MD5Update(&self_md5_cxt, map, pers->header_len + pers->data_len
- sizeof(persisted_footer_t));
MD5Final(digest->full_MD5, &self_md5_cxt);
}
if (TEST(PERSCACHE_GENFILE_MD5_SHORT, validation_option)) {
MD5Init(&self_md5_cxt);
MD5Update(&self_md5_cxt, (byte *) pers, pers->header_len);
MD5Final(digest->short_MD5, &self_md5_cxt);
}
}
static void
persist_calculate_module_digest(module_digest_t *digest, app_pc modbase, size_t modsize,
app_pc code_start, app_pc code_end,
uint validation_option)
{
size_t view_size = modsize;
if (TESTANY(PERSCACHE_MODULE_MD5_COMPLETE|PERSCACHE_MODULE_MD5_SHORT,
validation_option)) {
/* case 9717: need view size, not image size */
view_size = os_module_get_view_size(modbase);
}
if (TEST(PERSCACHE_MODULE_MD5_COMPLETE, validation_option)) {
/* We can't use a full md5 from module_calculate_digest() since .data
* and other sections change between persist and load times (this is
* in-memory image, not file). So we do md5 of code region. If we have
* hooks at persist time but not at load time we will cry foul;
* PERSCACHE_MODULE_MD5_AT_LOAD tries to get around this by using
* the load-time md5 when persisting.
*/
struct MD5Context code_md5_cxt;
MD5Init(&code_md5_cxt);
/* Code range should be within a single memory allocation so it should
* all be readable. Xref case 9653.
*/
code_end = MIN(code_end, modbase + view_size);
MD5Update(&code_md5_cxt, code_start, code_end - code_start);
MD5Final(digest->full_MD5, &code_md5_cxt);
}
if (TEST(PERSCACHE_MODULE_MD5_SHORT, validation_option)) {
/* Examine only the image header and the footer (if non-writable)
* FIXME: if view_size < modsize, better to skip the footer than have it
* cover a data section? Should be ok w/ PERSCACHE_MODULE_MD5_AT_LOAD.
*/
module_calculate_digest(digest, modbase, view_size,
false /* not full */, true /* yes short */,
DYNAMO_OPTION(persist_short_digest),
/* do not consider writable sections */
~((uint)OS_IMAGE_WRITE), OS_IMAGE_WRITE);
}
}
/* Compares all but the module base */
static bool
persist_modinfo_cmp(persisted_module_info_t *mi1, persisted_module_info_t *mi2)
{
bool match = true;
/* We'd like to know if we have an md5 mismatch */
ASSERT_CURIOSITY(module_digests_equal(&mi1->module_md5, &mi2->module_md5,
TEST(PERSCACHE_MODULE_MD5_SHORT,
DYNAMO_OPTION(persist_load_validation)),
TEST(PERSCACHE_MODULE_MD5_COMPLETE,
DYNAMO_OPTION(persist_load_validation)))
/* relocs => md5 diffs, until we handle relocs wrt md5 */
IF_WINDOWS(|| mi1->base != mi2->base)
|| check_filter("win32.partial_map.exe",
get_short_name(get_application_name())));
if (TESTALL(PERSCACHE_MODULE_MD5_SHORT|PERSCACHE_MODULE_MD5_COMPLETE,
DYNAMO_OPTION(persist_load_validation))) {
return (memcmp(&mi1->checksum, &mi2->checksum,
sizeof(*mi1)-offsetof(persisted_module_info_t, checksum)) == 0);
}
match = match && (memcmp(&mi1->checksum, &mi2->checksum,
offsetof(persisted_module_info_t, module_md5) -
offsetof(persisted_module_info_t, checksum)) == 0);
match = match && module_digests_equal(&mi1->module_md5, &mi2->module_md5,
TEST(PERSCACHE_MODULE_MD5_SHORT,
DYNAMO_OPTION(persist_load_validation)),
TEST(PERSCACHE_MODULE_MD5_COMPLETE,
DYNAMO_OPTION(persist_load_validation)));
return match;
}
#ifdef WINDOWS
static void
persist_record_base_mismatch(app_pc modbase)
{
/* The idea is that we shouldn't waste our time re-persisting modules
* whose base keeps mismatching due to ASLR (we don't support rebasing
* pcaches yet).
* To record whether to not persist, we can't use a VM_ flag b/c
* no simple way to tell vmareas.c why a load failed so we use a
* module flag
*/
if (!DYNAMO_OPTION(coarse_freeze_rebased_aslr) &&
os_module_has_dynamic_base(modbase))
os_module_set_flag(modbase, MODULE_DO_NOT_PERSIST);
}
#endif
/* key is meant to be a short string to help identify the purpose of this name.
* FIXME: right now up to caller to figure out if the name collided w/ an
* existing file; maybe this routine should do that and return a file handle?
* FIXME: combine w/ get_unique_logfile, which handles file creation race?
* As it is this is not mkstemp, and caller must use OS_OPEN_REQUIRE_NEW.
*/
static void
get_unique_name(const char *origname, const char *key,
char *filename /*OUT*/, uint filename_max /* max #chars */)
{
/* We need unique names for:
* 1) case 9696: a temp file to build our pcache in
* before renaming to the real thing
* 2) case 9701: to rename the existing file before we replace it, as for
* images or mmaps with file handles open we must rename before deleting.
*/
/* FIXME: should we use full 64-bit TSC instead of pseudo-random 32-bit?
* FIXME: if we make name w/ full path too long we'll truncate:
* could cache dir handle and use relative name only.
*/
/* update aslr_get_unique_wide_name() with any improvements here */
size_t timestamp = get_random_offset(UINT_MAX);
LOG_DECLARE(int trunc =) /* for DEBUG and INTERNAL */
snprintf(filename, filename_max,
"%s-"PIDFMT"-%010"SZFC"-%s", origname,
get_process_id(), timestamp, key);
ASSERT_CURIOSITY(trunc > 0 && trunc < (int)filename_max &&
"perscache new name truncated");
/* FIXME: case 10677 file name truncation */
filename[filename_max-1] = '\0';
}
/* Merges a given frozen unit with any new persisted cache file on disk.
* Caller must hold read lock hotp_get_lock(), if -hot_patching.
* If merge is successful, returns a new coarse_info_t, which caller is
* responsible for freeing; else returns NULL.
*/
static coarse_info_t *
coarse_unit_merge_with_disk(dcontext_t *dcontext, coarse_info_t *info,
const char *filename)
{
coarse_info_t *merge_with, *postmerge = NULL;
uint64 file_size;
size_t existing_size;
/* We may have already merged new code with an inuse persisted unit, so we
* check the stored size of that one if info is not itself persisted.
* FIXME: we could store the file handle: can we tell if two file handles
* refer to the same file?
*/
size_t inuse_size = (info->persisted) ? info->mmap_size :
info->persisted_source_mmap_size;
LOG(THREAD, LOG_CACHE, 2, "coarse_unit_merge_with_disk %s\n", info->module);
ASSERT(dynamo_all_threads_synched);
ASSERT(info != NULL && info->cache != NULL); /* don't merge empty units */
ASSERT(info->frozen);
#ifdef HOT_PATCHING_INTERFACE
ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
#endif
/* Strategy: check current pcache file size (not perfect but good enough):
* if different from source size, or source was not persisted, then
* load in and merge.
* FIXME case 10356: need a better check since can have false positive
* and false negatives by only looking at size.
* Could repeat, and could also check again after writing to tmp file but
* before renaming. FIXME: should we do those things to reduce the race
* window where we lose another process's appended code?
*/
if (!os_get_file_size(filename, &file_size)) {
LOG(THREAD, LOG_CACHE, 2, " no existing file %s to merge with\n", filename);
return postmerge;
}
ASSERT_TRUNCATE(existing_size, size_t, file_size);
existing_size = (size_t) file_size;
LOG(THREAD, LOG_CACHE, 2, " size of existing %s is "SZFMT" vs our "SZFMT"\n",
filename, existing_size, inuse_size);
if (existing_size == 0)
return postmerge;
/* Merge a non-persisted (and not merged with persisted) file w/ any on-disk file
* that has appeared since startup; or, our own, if we abandoned it but stayed
* coarse due to a reset or hotp flush.
*/
if ((!info->persisted && info->persisted_source_mmap_size == 0 &&
DYNAMO_OPTION(coarse_lone_merge)) ||
/* FIXME case 10356: need a better check since can have false positive
* and false negatives by only looking at size.
*/
(existing_size != inuse_size && DYNAMO_OPTION(coarse_disk_merge))) {
merge_with = coarse_unit_load(dcontext, info->base_pc, info->end_pc,
false/*not for execution*/);
/* We rely on coarse_unit_load to reject incompatible pcaches, whether for
* tls, trace support, or other reasons. We do need to check the region
* here. FIXME: once we support relocs we need to handle appropriately.
*/
if (merge_with != NULL) {
LOG(THREAD, LOG_CACHE, 2, " merging to-be-persisted %s with on-disk %s\n",
info->module, filename);
/* Case 8640: allow merging with a smaller on-disk file, to avoid
* being forever stuck at that size by prior cores with no IAT merging
* or other features */
if (merge_with->base_pc >= info->base_pc &&
merge_with->end_pc <= info->end_pc) {
/* If want in-place merging, need to arrange for ibl invalidation:
* case 11057 */
postmerge = coarse_unit_merge(dcontext, info, merge_with,
false/*!in-place*/);
ASSERT(postmerge != NULL);
/* if NULL info is still guaranteed to be unchanged so carry on */
DOSTATS({
if (postmerge == NULL)
STATS_INC(coarse_merge_disk_fail);
else
STATS_INC(coarse_merge_disk);
});
} else {
/* FIXME case 10357: w/o -unsafe_ignore_IAT_writes
* we're going to see a lot
* of this w/ process-shared pcaches. Should we abort
* the persist at this point to keep the small-region
* file on disk? Otherwise we're going to keep
* trading w/ the small-region process.
*/
LOG(THREAD, LOG_CACHE, 2,
" region mismatch: "PFX"-"PFX" on-disk vs "PFX"-"PFX" live\n",
merge_with->base_pc, merge_with->end_pc,
info->base_pc, info->end_pc);
STATS_INC(coarse_merge_disk_mismatch);
}
coarse_unit_reset_free(dcontext, merge_with, false/*no locks*/,
true/*need to unlink*/, false/*not in use anyway*/);
coarse_unit_free(dcontext, merge_with);
merge_with = NULL;
} else
STATS_INC(coarse_merge_disk_fail);
}
return postmerge;
}
/* Calculates information for persisting that we don't need for online-generated
* units, such as hotp points and RAC/RCT info.
* If just_live is true, ignores any currently in-use persisted info, assuming
* it will be merged in in a merge step by an in-use persisted unit.
* Caller must hold change_linking_lock, read lock hotp_get_lock(), and info's lock.
*/
static void
coarse_unit_calculate_persist_info(dcontext_t *dcontext, coarse_info_t *info)
{
#ifdef HOT_PATCHING_INTERFACE
int len;
#endif
/* we need real dcontext for rac_entries_in_region() */
ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
ASSERT_OWN_RECURSIVE_LOCK(true, &change_linking_lock);
#ifdef HOT_PATCHING_INTERFACE
ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
#endif
ASSERT_OWN_MUTEX(true, &info->lock);
LOG(THREAD, LOG_CACHE, 1, "coarse_unit_calculate_persist_info %s "PFX"-"PFX"\n",
info->module, info->base_pc, info->end_pc);
ASSERT(info->frozen && !info->persisted && !info->has_persist_info);
if (DYNAMO_OPTION(coarse_freeze_elide_ubr))
info->flags |= PERSCACHE_ELIDED_UBR;
#if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
if (seen_Borland_SEH)
info->flags |= PERSCACHE_SEEN_BORLAND_SEH;
#endif
if (!DYNAMO_OPTION(disable_traces))
info->flags |= PERSCACHE_SUPPORT_TRACES;
#ifdef RCT_IND_BRANCH
ASSERT(info->rct_table == NULL);
if ((TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
(DYNAMO_OPTION(persist_rct)
# if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
/* case 8648: we can have RCT entries that come from code patterns
* not derivable from relocation entries that we must persist.
*/
|| os_module_get_flag(info->base_pc, MODULE_HAS_BORLAND_SEH)
# endif
)) {
app_pc limit_start = info->base_pc;
app_pc limit_end = info->end_pc;
if (DYNAMO_OPTION(persist_rct) && DYNAMO_OPTION(persist_rct_entire)) {
limit_start = 0;
limit_end = (app_pc) POINTER_MAX;
}
info->flags |= PERSCACHE_SUPPORT_RCT;
/* We don't support pulling out just entries from this sub-module-region */
ASSERT(DYNAMO_OPTION(persist_rct_entire));
info->flags |= PERSCACHE_ENTIRE_MODULE_RCT;
/* Get a copy of the live + persisted tables merged together */
info->rct_table = rct_module_table_copy(dcontext, info->base_pc, RCT_RCT,
limit_start, limit_end);
}
#endif
#ifdef RETURN_AFTER_CALL
if (DYNAMO_OPTION(ret_after_call)) {
ASSERT(info->rac_table == NULL);
info->flags |= PERSCACHE_SUPPORT_RAC;
/* If we don't persist RAC we'll get violations when using! */
/* Get a copy of the live + persisted tables merged together,
* but only the entries in this region.
* To make sure we have entries for call sites in the region, not targets,
* we ignore target just over start and include just over end: thus the
* +1 on both sides.
*/
info->rac_table = rct_module_table_copy(dcontext, info->base_pc, RCT_RAC,
info->base_pc+1, info->end_pc+1);
}
#endif
#ifdef HOT_PATCHING_INTERFACE
if (DYNAMO_OPTION(hot_patching)) {
/* We expect 2 patch points per hotpatch on average; most have 1,
* some have up to 5. So we could hardcode a max length; instead
* we have a separate pass to get the size.
*/
/* FIXME: we could include the image entry point to avoid flushing the
* whole exe, but that only happens when we inject into secondary thread.
*/
/* FIXME: when merging live with in-use-persisted we don't need to
* re-calculate this and can just use the persisted array, since we would
* have flushed the unit on any new hotp
*/
ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
info->hotp_ppoint_vec_num =
hotp_num_matched_patch_points(info->base_pc, info->end_pc);
if (info->hotp_ppoint_vec_num > 0) {
info->hotp_ppoint_vec =
HEAP_ARRAY_ALLOC(dcontext, app_rva_t, info->hotp_ppoint_vec_num,
ACCT_HOT_PATCHING, PROTECTED);
len = hotp_get_matched_patch_points(info->base_pc, info->end_pc,
info->hotp_ppoint_vec,
info->hotp_ppoint_vec_num);
/* Should never have mismatch, as we're holding the hotp lock.
* Even if len < 0 the routine still fills up the array. */
ASSERT(len == (int)info->hotp_ppoint_vec_num);
if (len != (int)info->hotp_ppoint_vec_num) {
/* abort writing out hotp points */
info->hotp_ppoint_vec_num = 0;
}
LOG(THREAD, LOG_CACHE, 2, "hotp points for %s "PFX"-"PFX":\n",
info->module, info->base_pc, info->end_pc);
DODEBUG({
uint i;
for (i = 0; i < info->hotp_ppoint_vec_num; i++)
LOG(THREAD, LOG_CACHE, 2, "\t"PFX"\n", info->hotp_ppoint_vec[i]);
});
} else
ASSERT(info->hotp_ppoint_vec == NULL);
} else
ASSERT(info->hotp_ppoint_vec == NULL);
#endif
info->has_persist_info = true;
}
/* Merges the persist-calculated fields of info1 and info2 into dst,
* as well as non-persist-calculated fields like primary_for_module.
* Assumption: we've already checked for compatibility and here we just
* need to take the union (minus dups).
* Caller must hold info1's lock, and we must be dynamo_all_threads_synched.
*/
static void
coarse_unit_merge_persist_info(dcontext_t *dcontext, coarse_info_t *dst,
coarse_info_t *info1, coarse_info_t *info2)
{
ASSERT(dynamo_all_threads_synched);
LOG(THREAD, LOG_CACHE, 1, "coarse_unit_merge_persist_info %s "PFX"-"PFX"\n",
info1->module, info1->base_pc, info1->end_pc);
/* We can't grab both locks due to deadlock potential. Currently we are
* always fully synched, so we rely on that.
*/
ASSERT(dynamo_all_threads_synched);
ASSERT_OWN_MUTEX(true, &info1->lock);
/* We need to incorporate flags from persisted units (Borland, e.g.) */
dst->flags |= info1->flags;
dst->flags |= info2->flags;
/* Some flags are intersection rather than union (but those like
* PERSCACHE_SUPPORT_TRACES are weeded out at load time and should
* not differ here)
*/
if (!TEST(PERSCACHE_MAP_RW_SEPARATE, info1->flags) ||
!TEST(PERSCACHE_MAP_RW_SEPARATE, info2->flags))
dst->flags &= ~PERSCACHE_MAP_RW_SEPARATE;
/* Same bounds, so same persistence privileges */
dst->primary_for_module = info1->primary_for_module || info2->primary_for_module;
ASSERT(!info2->persisted || !info2->in_use ||
info2->has_persist_info); /* must have to use */
/* Everything else is only necessary if we've already calculated persist
* info; if we haven't, then if we do persist later we'll calculate it
* fresh, so no need to calculate and merge here
*/
if (!info1->has_persist_info && !info2->has_persist_info)
return;
if (!info1->has_persist_info)
coarse_unit_calculate_persist_info(dcontext, info1);
if (!info2->has_persist_info)
coarse_unit_calculate_persist_info(dcontext, info2);
ASSERT(info1->has_persist_info && info2->has_persist_info);
ASSERT(!dst->has_persist_info);
ASSERT((info1->flags &
(PERSCACHE_SUPPORT_TRACES|PERSCACHE_SUPPORT_RAC|PERSCACHE_SUPPORT_RCT)) ==
(info2->flags &
(PERSCACHE_SUPPORT_TRACES|PERSCACHE_SUPPORT_RAC|PERSCACHE_SUPPORT_RCT)));
#ifdef RCT_IND_BRANCH
ASSERT(dst->rct_table == NULL);
if (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) {
if (info2->persisted && info2->in_use && !info1->persisted) {
/* coarse_unit_calculate_persist_info already merged the new entries
* with the in-use persisted entries
*/
dst->rct_table = rct_table_copy(dcontext, info1->rct_table);
} else {
/* Note that we can't simply take one source if it has
* PERSCACHE_ENTIRE_MODULE_RCT as Borland can add entries: we must
* merge every time
*/
dst->rct_table = rct_table_merge(dcontext, info1->rct_table,
info2->rct_table);
}
}
#endif
#ifdef RETURN_AFTER_CALL
ASSERT(dst->rac_table == NULL);
if (DYNAMO_OPTION(ret_after_call)) {
if (info2->persisted && info2->in_use && !info1->persisted) {
/* coarse_unit_calculate_persist_info already merged the new entries
* with the in-use persisted entries
*/
dst->rac_table = rct_table_copy(dcontext, info1->rac_table);
} else {
dst->rac_table = rct_table_merge(dcontext, info1->rac_table,
info2->rac_table);
}
}
#endif
#ifdef HOT_PATCHING_INTERFACE
ASSERT(dst->hotp_ppoint_vec == NULL);
if (info2->persisted && info2->in_use && !info1->persisted) {
/* We would have flushed info2 if a new hotp overlapped it so just
* take its hotp list. info1 may have a subset.
*/
dst->hotp_ppoint_vec_num = info2->hotp_ppoint_vec_num;
if (dst->hotp_ppoint_vec_num > 0) {
dst->hotp_ppoint_vec =
HEAP_ARRAY_ALLOC(dcontext, app_rva_t, dst->hotp_ppoint_vec_num,
ACCT_HOT_PATCHING, PROTECTED);
memcpy(dst->hotp_ppoint_vec, info2->hotp_ppoint_vec,
dst->hotp_ppoint_vec_num*sizeof(app_rva_t));
}
} else {
/* We expect <= 5 entries in each array so quadratic and two passes
* shouldn't be a perf issue.
*/
ASSERT(dst->hotp_ppoint_vec_num == 0);
ASSERT(sizeof(app_rva_t) == sizeof(void *));
array_merge(dcontext, true /* intersect */,
(void **) info1->hotp_ppoint_vec, info1->hotp_ppoint_vec_num,
(void **) info2->hotp_ppoint_vec, info2->hotp_ppoint_vec_num,
(void ***) &dst->hotp_ppoint_vec, &dst->hotp_ppoint_vec_num
HEAPACCT(ACCT_HOT_PATCHING));
}
#endif
dst->has_persist_info = true;
}
static bool
write_persist_file(dcontext_t *dcontext, file_t fd, const void *buf, size_t count)
{
ASSERT(fd != INVALID_FILE && buf != NULL && count > 0);
if (os_write(fd, buf, count) != (ssize_t)count) {
LOG(THREAD, LOG_CACHE, 1, " unable to write "SZFMT" bytes to file\n", count);
SYSLOG_INTERNAL_WARNING_ONCE("unable to write persistent cache file");
STATS_INC(coarse_units_persist_error);
return false;
}
return true;
}
static bool
pad_persist_file(dcontext_t *dcontext, file_t fd, size_t bytes, coarse_info_t *info)
{
size_t towrite = bytes;
size_t thiswrite;
/* We don't have a handy page of zeros so we just write the cache start.
* FIXME: better to use file positions (just setting the size doesn't
* advance the pointer): write_file() has win32 support.
*/
ASSERT(fd != INVALID_FILE);
ASSERT(bytes < 64*1024); /* error */
while (towrite > 0) {
thiswrite = MIN(towrite, (ptr_uint_t)info->stubs_end_pc -
(ptr_uint_t)info->cache_start_pc);
if (!write_persist_file(dcontext, fd, info->cache_start_pc, thiswrite))
return false;
towrite -= thiswrite;
}
return true;
}
/* Fills in pers with data from info */
static void
coarse_unit_set_persist_data(dcontext_t *dcontext, coarse_info_t *info,
coarse_persisted_info_t *pers, app_pc modbase,
op_pcache_t option_level, const char *option_string)
{
size_t x_offs = 0; /* offs of start of +x section */
/* We do not need to freeze the world: merely the state of outgoing links. */
ASSERT_OWN_RECURSIVE_LOCK(true, &change_linking_lock);
#ifdef HOT_PATCHING_INTERFACE
ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
#endif
ASSERT_OWN_MUTEX(true, &info->lock);
/* Put the unit in the proper state for persisting */
coarse_unit_unlink_outgoing(dcontext, info);
/* Fill in the metadata */
pers->magic = PERSISTENT_CACHE_MAGIC;
pers->version = PERSISTENT_CACHE_VERSION;
pers->header_len = sizeof(*pers);
pers->data_len = 0;
/* Take flags from info */
pers->flags = info->flags;
/* FIXME: see above: should be set earlier */
pers->flags |= IF_X64_ELSE(PERSCACHE_X86_64, PERSCACHE_X86_32);
if (option_level == OP_PCACHE_LOCAL) {
ASSERT(option_string != NULL);
pers->flags |= PERSCACHE_EXEMPTION_OPTIONS;
}
#ifdef BUILD_NUMBER
pers->build_number = BUILD_NUMBER;
#else
pers->build_number = 0;
#endif
if (TEST(PERSCACHE_MODULE_MD5_AT_LOAD, DYNAMO_OPTION(persist_gen_validation))) {
ASSERT(!is_region_memset_to_char((byte *)&info->module_md5,
sizeof(info->module_md5), 0));
DOLOG(1, LOG_CACHE, {
print_module_digest(THREAD, &info->module_md5, "using md5 from load time: ");
});
memcpy(&pers->modinfo.module_md5, &info->module_md5,
sizeof(pers->modinfo.module_md5));
} else {
persist_calculate_module_digest(&pers->modinfo.module_md5, modbase,
(size_t) pers->modinfo.image_size,
info->base_pc, info->end_pc,
DYNAMO_OPTION(persist_gen_validation));
}
/* rest of modinfo filled in by get_persist_filename() */
ASSERT(pers->modinfo.base == modbase);
ASSERT(info->base_pc >= modbase);
ASSERT(info->end_pc > info->base_pc);
pers->start_offs = info->base_pc - modbase;
pers->end_offs = info->end_pc - modbase;
pers->tls_offs_base = os_tls_offset(0);
/* We need to go in forward order so we can tell the client the current offset */
x_offs += pers->header_len;
pers->option_string_len = (option_string == NULL || option_string[0] == '\0') ? 0 :
(ALIGN_FORWARD((strlen(option_string)+1/*include NULL*/)*sizeof(char),
OPTION_STRING_ALIGNMENT));
x_offs += pers->option_string_len;
#ifdef CLIENT_INTERFACE
pers->instrument_ro_len =
ALIGN_FORWARD(instrument_persist_ro_size(dcontext, info, x_offs),
CLIENT_ALIGNMENT);
#else
pers->instrument_ro_len = 0;
#endif
x_offs += pers->instrument_ro_len;
/* Add new data section here */
#ifdef HOT_PATCHING_INTERFACE
pers->hotp_patch_list_len = sizeof(app_rva_t) * info->hotp_ppoint_vec_num;
x_offs += pers->hotp_patch_list_len;
#endif
/* FIXME case 9581 NYI: need to add to coarse_unit_calculate_persist_info()
* and coarse_unit_merge_persist_info() as well.
*/
pers->reloc_len = 0;
x_offs += pers->reloc_len;
#ifdef RETURN_AFTER_CALL
pers->rac_htable_len = rct_table_persist_size(dcontext, info->rac_table);
#else
pers->rac_htable_len = 0;
#endif
x_offs += pers->rac_htable_len;
#ifdef RCT_IND_BRANCH
pers->rct_htable_len = rct_table_persist_size(dcontext, info->rct_table);
#else
pers->rct_htable_len = 0;
#endif
x_offs += pers->rct_htable_len;
pers->stub_htable_len =
fragment_coarse_htable_persist_size(dcontext, info, false/*stub table*/);
x_offs += pers->stub_htable_len;
pers->cache_htable_len =
fragment_coarse_htable_persist_size(dcontext, info, true/*cache table*/);
x_offs += pers->cache_htable_len;
/* end of read-only, start of +rx */
pers->data_len += (x_offs - pers->header_len);
/* We need to pad out the data before the +x boundary to get a new page */
pers->pad_len = ALIGN_FORWARD(x_offs, PAGE_SIZE) - x_offs;
pers->data_len += pers->pad_len;
#ifdef CLIENT_INTERFACE
pers->instrument_rx_len =
ALIGN_FORWARD(instrument_persist_rx_size(dcontext, info, x_offs + pers->pad_len),
CLIENT_ALIGNMENT);
#else
pers->instrument_rx_len = 0;
#endif
pers->data_len += pers->instrument_rx_len;
/* calculate cache_len so we can calculate padding. we need 2 different pads
* b/c of instrument_rx needing to know its offset.
*/
pers->cache_len = info->fcache_return_prefix - info->cache_start_pc;
if (DYNAMO_OPTION(persist_map_rw_separate)) {
/* We need the cache/stub split to be on a file view map boundary,
* yet have cache+stubs adjacent
*/
size_t rwx_offs = x_offs + pers->pad_len + pers->cache_len
IF_CLIENT_INTERFACE(+ pers->instrument_rx_len);
pers->view_pad_len = ALIGN_FORWARD(rwx_offs, MAP_FILE_VIEW_ALIGNMENT) - rwx_offs;
pers->flags |= PERSCACHE_MAP_RW_SEPARATE;
}
pers->data_len += pers->view_pad_len;
pers->post_cache_pad_len = info->fcache_return_prefix - info->cache_end_pc;
pers->data_len += pers->cache_len; /* includes post_cache_pad_len */
STATS_ADD(coarse_code_persisted, info->cache_end_pc - info->cache_start_pc);
pers->stubs_len = info->stubs_end_pc - info->stubs_start_pc;
pers->data_len += pers->stubs_len;
pers->ibl_jmp_prefix_len = info->stubs_start_pc - info->ibl_jmp_prefix;
pers->data_len += pers->ibl_jmp_prefix_len;
pers->ibl_call_prefix_len = info->ibl_jmp_prefix - info->ibl_call_prefix;
pers->data_len += pers->ibl_call_prefix_len;
pers->ibl_ret_prefix_len = info->ibl_call_prefix - info->ibl_ret_prefix;
pers->data_len += pers->ibl_ret_prefix_len;
pers->trace_head_return_prefix_len =
info->ibl_ret_prefix - info->trace_head_return_prefix;
pers->data_len += pers->trace_head_return_prefix_len;
pers->fcache_return_prefix_len =
info->trace_head_return_prefix - info->fcache_return_prefix;
pers->data_len += pers->fcache_return_prefix_len;
#ifdef CLIENT_INTERFACE
pers->instrument_rw_len =
ALIGN_FORWARD(instrument_persist_rw_size(dcontext, info,
pers->header_len + pers->data_len),
CLIENT_ALIGNMENT);
#else
pers->instrument_rw_len = 0;
#endif
pers->data_len += pers->instrument_rw_len;
/* We always write the footer regardless of whether we calculate the
* MD5. Otherwise we'd need a new flag; not really worth it.
* Plus, we have our magic field as an always-on extra check.
* Note that we do not set the total file size up front (which could make
* our writes faster), so we know if the footer is there then it was
* actually written.
*/
pers->data_len += sizeof(persisted_footer_t);
/* Our relative jmps require that we do not exceed 32-bit reachability */
IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(pers->data_len)));
LOG(THREAD, LOG_CACHE, 2, " header="SZFMT", data="SZFMT", pad="SZFMT"\n",
pers->header_len, pers->data_len, pers->pad_len+pers->view_pad_len);
}
/* Separated from coarse_unit_persist() to keep the stack space on the
* persist -> merge_with_disk -> load
* path to a minimum (case 10712).
*/
static bool
coarse_unit_persist_rename(dcontext_t *dcontext, const char *filename,
const char *tmpname)
{
bool success = false;
char rename[MAXIMUM_PATH];
if (os_rename_file(tmpname, filename, false/*do not replace*/)) {
success = true;
} else {
ASSERT_CURIOSITY(os_file_exists(filename, false/*!dir*/));
if (DYNAMO_OPTION(coarse_freeze_rename)) {
get_unique_name(filename, "todel", rename, BUFFER_SIZE_ELEMENTS(rename));
LOG(THREAD, LOG_CACHE, 1, " attempting to rename %s to %s\n",
filename, rename);
if (os_rename_file(filename, rename, false/*do not replace*/)) {
LOG(THREAD, LOG_CACHE, 1, " succeeded renaming %s to %s\n",
filename, rename);
STATS_INC(persist_rename_success);
if (DYNAMO_OPTION(coarse_freeze_clean)) {
/* Try to mark the renamed file for deletion. We have no handle
* open and are re-targeting based on name only but should have a
* unique name so we aren't worried about deleting the wrong file.
*/
if (os_delete_mapped_file(rename)) {
LOG(THREAD, LOG_CACHE, 1,
" succeeded marking for deletion %s\n", rename);
STATS_INC(persist_delete_success);
}
}
/* We could support os_delete_mapped_file() w/o renaming first but it
* only works for SEC_COMMIT with -no_persist_lock_file */
if (os_rename_file(tmpname, filename, false/*do not replace*/))
success = true;
else {
/* Race: someone beat us to the rename (or else no permissions).
* FIXME: should we try again? */
ASSERT_CURIOSITY(os_file_exists(filename, false/*!dir*/));
STATS_INC(persist_rename_race);
ASSERT(!success);
}
} else {
STATS_INC(persist_rename_fail);
ASSERT(!success);
}
}
}
return success;
}
#ifdef CLIENT_INTERFACE
bool
instrument_persist_section(dcontext_t *dcontext, file_t fd, coarse_info_t *info,
size_t len,
bool (*persist_func)(dcontext_t *, void *, file_t))
{
if (len > 0) {
/* keep aligned */
int64 post_pos, pre_pos = os_tell(fd);
if (!persist_func(dcontext, info, fd)) {
LOG(THREAD, LOG_CACHE, 1, " unable to write client data to file\n");
SYSLOG_INTERNAL_WARNING_ONCE("unable to write client data to pcache file");
STATS_INC(coarse_units_persist_error);
return false;
}
post_pos = os_tell(fd);
if (pre_pos == -1 || post_pos == -1) {
SYSLOG_INTERNAL_WARNING_ONCE("unable to tell pcache file position");
STATS_INC(coarse_units_persist_error);
return false;
}
/* pad out to the alignment we added to len */
ASSERT(len >= (size_t)(post_pos - pre_pos));
if (!pad_persist_file(dcontext, fd, len - (size_t)(post_pos - pre_pos), info))
return false; /* logs, stats in write_persist_file */
}
return true;
}
DR_API
app_pc
dr_persist_start(void *perscxt)
{
coarse_info_t *info = (coarse_info_t *) perscxt;
CLIENT_ASSERT(perscxt != NULL, "invalid arg: perscxt is NULL");
return info->base_pc;
}
DR_API
size_t
dr_persist_size(void *perscxt)
{
coarse_info_t *info = (coarse_info_t *) perscxt;
CLIENT_ASSERT(perscxt != NULL, "invalid arg: perscxt is NULL");
return info->end_pc - info->base_pc;
}
DR_API
bool
dr_fragment_persistable(void *drcontext, void *perscxt, void *tag_in)
{
dcontext_t *dcontext = (dcontext_t *) drcontext;
cache_pc res;
/* deliberately not calling dr_fragment_app_pc() b/c any pc there won't
* be coarse and thus not worth the extra overhead
*/
app_pc tag = (app_pc) tag_in;
if (perscxt != NULL) {
coarse_info_t *info = (coarse_info_t *) perscxt;
fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &res);
if (res != NULL)
return true;
if (info->non_frozen != NULL) {
ASSERT(info->frozen);
fragment_coarse_lookup_in_unit(dcontext, info->non_frozen, tag, NULL, &res);
}
} else {
res = fragment_coarse_lookup(dcontext, tag);
}
return (res != NULL);
}
#endif
/* Unlinks all inter-unit stubs. Can still use info afterward, as
* lazy linking should soon re-link them.
* Caller must hold change_linking_lock and read lock hotp_get_lock().
*/
bool
coarse_unit_persist(dcontext_t *dcontext, coarse_info_t *info)
{
coarse_persisted_info_t pers;
persisted_footer_t footer;
/* FIXME: this is a lot of stack space: make static and serialize? */
char filename[MAXIMUM_PATH];
char tmpname[MAXIMUM_PATH];
char option_buf[MAX_PCACHE_OPTIONS_STRING];
const char *option_string;
op_pcache_t option_level;
app_pc modbase;
bool success = false;
bool created_temp = false;
bool free_info = false;
file_t fd = INVALID_FILE;
/* we need real dcontext for rac_entries_in_region() */
ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
KSTART(persisted_generation);
LOG(THREAD, LOG_CACHE, 1, "coarse_unit_persist %s "PFX"-"PFX"\n",
info->module, info->base_pc, info->end_pc);
STATS_INC(coarse_units_persist_try);
ASSERT(info->frozen && !info->persisted);
/* invalid unit shouldn't get this far */
ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
if (TEST(PERSCACHE_CODE_INVALID, info->flags)) /* paranoid */
goto coarse_unit_persist_exit;
modbase = get_module_base(info->base_pc);
if (modbase == NULL) {
LOG(THREAD, LOG_CACHE, 1, " no module base for "PFX"\n", info->base_pc);
goto coarse_unit_persist_exit;
}
/* case 9653/10380: only one coarse unit in a module's +x region(s) is persisted */
if (!info->primary_for_module) {
/* the primary may have been flushed, so see if we can become primary */
coarse_unit_mark_primary(info);
if (!info->primary_for_module) {
LOG(THREAD, LOG_CACHE, 1,
" not primary unit for module %s: not persisting\n", info->module);
STATS_INC(coarse_units_persist_dup);
goto coarse_unit_persist_exit;
}
}
#ifdef WINDOWS
if (!DYNAMO_OPTION(coarse_freeze_rebased_aslr) &&
os_module_get_flag(modbase, MODULE_DO_NOT_PERSIST)) {
LOG(THREAD, LOG_CACHE, 1, " %s marked as do-not-persist\n", info->module);
goto coarse_unit_persist_exit;
}
#endif
/* case 9799: store pcache-affecting options */
option_level = persist_get_options_level(modbase, info, false/*use exemptions*/);
LOG(THREAD, LOG_CACHE, 2, " persisting option string at %d level\n", option_level);
option_string = persist_get_relevant_options(dcontext, option_buf,
BUFFER_SIZE_ELEMENTS(option_buf),
option_level);
/* get_persist_filename() fills in pers.modinfo */
memset(&pers, 0, sizeof(pers));
if (!get_persist_filename(filename, BUFFER_SIZE_ELEMENTS(filename), modbase,
true/*write*/, &pers.modinfo, option_string)) {
LOG(THREAD, LOG_CACHE, 1, " error calculating filename (or excluded)\n");
STATS_INC(coarse_units_persist_error);
goto coarse_unit_persist_exit;
}
LOG(THREAD, LOG_CACHE, 1, " persisted filename = %s\n", filename);
/* Open the file before unlinking since may be common point of failure.
* FIXME: better to cache base dir handle and pass just name here
* and to os_{rename,delete_mapped}_file? Would be win32-specific code.
*/
if (!DYNAMO_OPTION(coarse_freeze_rename) &&
!DYNAMO_OPTION(coarse_freeze_clobber) &&
os_file_exists(filename, false/*FIXME: would like to check all*/)) {
/* Check up front to avoid work */
LOG(THREAD, LOG_CACHE, 1, " will be unable to replace existing file %s\n",
filename);
STATS_INC(coarse_units_persist_error);
goto coarse_unit_persist_exit;
}
get_unique_name(filename, "tmp", tmpname, BUFFER_SIZE_ELEMENTS(tmpname));
fd = os_open(tmpname, OS_OPEN_WRITE |
/* We ask for read access to map it later for md5 */
OS_OPEN_READ |
/* Case 9964: we want to allow renaming while holding
* the file handle */
OS_SHARE_DELETE |
/* Renaming is the better option; and for SEC_COMMIT
* we can delete immediately, w/o the symlink risk case 9057;
* so we leave the CURIOSITY we get w/ the clobber option.
*/
(DYNAMO_OPTION(coarse_freeze_clobber) ? 0 : OS_OPEN_REQUIRE_NEW) |
/* case 10884: needed only for validate_owner_file */
OS_OPEN_FORCE_OWNER);
if (fd == INVALID_FILE) {
/* FIXME: it's possible but unlikely that our name collided: should we
* try again?
*/
LOG(THREAD, LOG_CACHE, 1, " unable to open temp file %s\n", tmpname);
STATS_INC(coarse_units_persist_error);
goto coarse_unit_persist_exit;
}
ASSERT_CURIOSITY((!DYNAMO_OPTION(validate_owner_file) ||
os_validate_user_owned(fd)) && "persisted while impersonating?");
created_temp = true;
/* FIXME case 9758: we could mmap the file: would that be more efficient?
* We could then more easily write each body section at the same
* time as we calculate its size for the header, instead of linear
* write order. We could also pass file offsets to out-of-order writes.
* Could we control write-through if we mmap (case 9696)?
* Note the prescribed order of unmap and then flush in case 9696.
*/
mutex_lock(&info->lock);
/* Get info that is not needed for online coarse units but is needed at persist
* time (RCT, RAC, and hotp info) now, so we can merge it more easily.
*/
if (!info->has_persist_info)
coarse_unit_calculate_persist_info(dcontext, info);
ASSERT(info->has_persist_info);
/* FIXME: the lock situation in all these routines is ugly: here we must
* release since coarse_unit_merge() will acquire. Should we not grab any
* info lock? We're already relying on dynamo_all_threads_synched.
*/
mutex_unlock(&info->lock);
/* Attempt to merge with any new on-disk file */
if (DYNAMO_OPTION(coarse_lone_merge) || DYNAMO_OPTION(coarse_disk_merge)) {
coarse_info_t *postmerge = coarse_unit_merge_with_disk(dcontext, info, filename);
if (postmerge != NULL) {
/* Just abandon the original info */
info = postmerge;
free_info = true;
}
}
mutex_lock(&info->lock);
/* Fill in the rest of pers */
coarse_unit_set_persist_data(dcontext, info, &pers, modbase,
option_level, option_string);
if (!coarse_unit_check_persist_space(fd, pers.header_len+pers.data_len)) {
/* We also check this prior to freezing/merging to avoid work on a
* perpetually-full volume */
LOG(THREAD, LOG_CACHE, 1,
" not enough disk space available for module %s: not persisting\n",
info->module);
STATS_INC(coarse_units_persist_nospace);
goto coarse_unit_persist_exit;
}
#ifdef CLIENT_INTERFACE
/* Give clients a chance to patch the cache mmap before we write it to
* the file. We do it here at a clean point between the size and persist
* calls, rather than in the middle of the persist (must be before persist_rw).
*/
if (!instrument_persist_patch(dcontext, info, info->cache_start_pc,
info->cache_end_pc - info->cache_start_pc)) {
LOG(THREAD, LOG_CACHE, 1, " client unable to patch module %s: not persisting\n",
info->module);
STATS_INC(coarse_units_persist_nopatch);
goto coarse_unit_persist_exit;
}
#endif
/* Write the headers */
if (!write_persist_file(dcontext, fd, &pers, pers.header_len))
goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
if (pers.option_string_len > 0) {
size_t len = strlen(option_string) + 1/*NULL*/;
ASSERT(len <= pers.option_string_len);
if (!write_persist_file(dcontext, fd, option_string, len))
goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
if (pers.option_string_len - len > 0) {
if (!pad_persist_file(dcontext, fd, pers.option_string_len - len, info))
goto coarse_unit_persist_exit; /* logs, stats in write_persist_file */
}
}
/* New data section goes here */
#ifdef CLIENT_INTERFACE
if (!instrument_persist_section(dcontext, fd, info, pers.instrument_ro_len,
instrument_persist_ro))
goto coarse_unit_persist_exit;
#endif
#ifdef HOT_PATCHING_INTERFACE
if (pers.hotp_patch_list_len > 0) {
if (!write_persist_file(dcontext, fd, info->hotp_ppoint_vec,
pers.hotp_patch_list_len))
goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
}
#endif
/* FIXME case 9581 NYI: write reloc section */
#ifdef RETURN_AFTER_CALL
if (pers.rac_htable_len > 0) {
if (!rct_table_persist(dcontext, info->rac_table, fd)) {
LOG(THREAD, LOG_CACHE, 1, " unable to write RAC htable to file\n");
SYSLOG_INTERNAL_WARNING_ONCE("unable to write RAC htable to pcache file");
STATS_INC(coarse_units_persist_error);
goto coarse_unit_persist_exit;
}
}
#else
ASSERT(pers.rac_htable_len == 0);
#endif
#ifdef RCT_IND_BRANCH
if (pers.rct_htable_len > 0) {
if (!rct_table_persist(dcontext, info->rct_table, fd)) {
LOG(THREAD, LOG_CACHE, 1, " unable to write RCT htable to file\n");
SYSLOG_INTERNAL_WARNING_ONCE("unable to write RCT htable to pcache file");
STATS_INC(coarse_units_persist_error);
goto coarse_unit_persist_exit;
}
}
#else
ASSERT(pers.rct_htable_len == 0);
#endif
/* For modularity we split out htable persistence. We could combine the
* tables with our cache+stubs mmap region at freeze time but there's no
* need as there are no cross-links so we keep them separate.
*/
if (!fragment_coarse_htable_persist(dcontext, info, true/*cache table*/, fd) ||
!fragment_coarse_htable_persist(dcontext, info, false/*stub table*/, fd)) {
LOG(THREAD, LOG_CACHE, 1, " unable to write htable(s) to file\n");
SYSLOG_INTERNAL_WARNING_ONCE("unable to write htable(s) to pcache file");
STATS_INC(coarse_units_persist_error);
goto coarse_unit_persist_exit;
}
/* If we used an image format, or did separate mappings, we could avoid
* storing padding on disk
*/
if (pers.pad_len > 0) {
if (!pad_persist_file(dcontext, fd, pers.pad_len, info))
goto coarse_unit_persist_exit; /* logs, stats in write_persist_file */
}
#ifdef CLIENT_INTERFACE
if (!instrument_persist_section(dcontext, fd, info, pers.instrument_rx_len,
instrument_persist_rx))
goto coarse_unit_persist_exit;
#endif
if (pers.view_pad_len > 0) {
if (!pad_persist_file(dcontext, fd, pers.view_pad_len, info))
goto coarse_unit_persist_exit; /* logs, stats in write_persist_file */
}
/* Write the cache + stubs. We omit the guard pages of course.
* FIXME: when freezing and persisting at once, more efficient to
* make our cache+stubs mmap backed by the file from the start.
*/
if (!write_persist_file(dcontext, fd, info->cache_start_pc,
info->stubs_end_pc - info->cache_start_pc))
goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
#ifdef CLIENT_INTERFACE
if (!instrument_persist_section(dcontext, fd, info, pers.instrument_rw_len,
instrument_persist_rw))
goto coarse_unit_persist_exit;
#endif
if (TESTANY(PERSCACHE_GENFILE_MD5_SHORT|PERSCACHE_GENFILE_MD5_COMPLETE,
DYNAMO_OPTION(persist_gen_validation))) {
byte *map = (byte *) &pers;
uint which = DYNAMO_OPTION(persist_gen_validation);
size_t sz = 0;
if (TEST(PERSCACHE_GENFILE_MD5_COMPLETE,
DYNAMO_OPTION(persist_gen_validation))) {
/* FIXME if mmap up front (case 9758) don't need this mmap */
sz = pers.header_len+pers.data_len-sizeof(persisted_footer_t);
map = map_file(fd, &sz, 0, NULL, MEMPROT_READ,
/* won't change so save pagefile by not asking for COW */
MAP_FILE_REACHABLE);
ASSERT(map != NULL);
if (map == NULL) {
/* give up */
which &= ~PERSCACHE_GENFILE_MD5_COMPLETE;
}
}
persist_calculate_self_digest(&footer.self_md5, &pers, map, which);
DOLOG(1, LOG_CACHE, {
print_module_digest(THREAD, &footer.self_md5, "self md5: ");
});
if (TEST(PERSCACHE_GENFILE_MD5_COMPLETE,
DYNAMO_OPTION(persist_gen_validation)) &&
map != (byte *) &pers) {
ASSERT(map != NULL); /* we cleared _COMPLETE so shouldn't get here */
unmap_file(map, sz);
}
} else {
memset(&footer, 0, sizeof(footer));
}
footer.magic = PERSISTENT_CACHE_MAGIC;
if (!write_persist_file(dcontext, fd, &footer, sizeof(footer)))
goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
os_flush(fd);
/* Temp file is all flushed and ready to go. Only now do we try to replace
* the original, to ensure we have a complete file (case 9696).
* We can't rename it if we have a handle with write privileges open,
* regardless of FILE_SHARE_* flags, so we close it and live with races.
* Nobody else should be creating a file of the same name (tmpname includes process id).
* Therefore no other process may have racily created an
* incomplete file overwriting the file we just produced and are
* about to rename.
*/
os_close(fd);
fd = INVALID_FILE;
success = coarse_unit_persist_rename(dcontext, filename, tmpname);
if (!success) {
LOG(THREAD, LOG_CACHE, 1, " unable to rename to %s\n", filename);
STATS_INC(persist_rename_tmp_fail);
goto coarse_unit_persist_exit;
}
/* success */
STATS_INC(coarse_units_persist);
ASSERT(success);
coarse_unit_persist_exit:
if (fd != INVALID_FILE)
os_close(fd);
if (!success && created_temp) {
if (!os_delete_mapped_file(tmpname)) {
LOG(THREAD, LOG_CACHE, 1,
" failed to delete on failure temp %s\n", tmpname);
STATS_INC(persist_delete_tmp_fail);
}
}
if (created_temp)
mutex_unlock(&info->lock);
if (free_info) {
/* if we dup-ed when merging with disk, free now */
coarse_unit_reset_free(dcontext, info, false/*no locks*/,
false/*no unlink*/, false/*not in use anyway*/);
coarse_unit_free(dcontext, info);
}
KSTOP(persisted_generation);
return success;
}
static bool
persist_check_option_compat(dcontext_t *dcontext, coarse_persisted_info_t *pers,
const char *option_string)
{
const char *pers_options;
ASSERT(option_string != NULL);
if (os_tls_offset(0) != pers->tls_offs_base) {
/* Bail out
* WARNING: if we ever change our -tls_align default from 1 we should
* consider implications on platform-independence of persisted caches.
* FIXME: should we have release-build notification so we can know
* when we have problems here?
*/
LOG(THREAD, LOG_CACHE, 1, " tls offset mismatch %d vs persisted %d\n",
os_tls_offset(0), pers->tls_offs_base);
STATS_INC(perscache_tls_mismatch);
SYSLOG_INTERNAL_WARNING_ONCE("persistent cache tls offset mismatch");
return false;
}
if (!TEST(PERSCACHE_SUPPORT_TRACES, pers->flags) && !DYNAMO_OPTION(disable_traces)) {
/* We bail out; the consequences of continuing are huge performance
* problems akin to -no_link_ibl.
*/
LOG(THREAD, LOG_CACHE, 1, " error: persisted cache has no trace support\n");
STATS_INC(perscache_trace_mismatch);
SYSLOG_INTERNAL_WARNING_ONCE("persistent cache trace support mismatch");
return false;
}
/* we don't bother to split the ifdefs */
#if defined(RCT_IND_BRANCH) || defined (RETURN_AFTER_CALL)
if ((!TEST(PERSCACHE_SUPPORT_RAC, pers->flags) &&
DYNAMO_OPTION(ret_after_call)) ||
(!TEST(PERSCACHE_SUPPORT_RCT, pers->flags) &&
(TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))))) {
LOG(THREAD, LOG_CACHE, 1, " error: persisted cache has no RAC/RCT support\n");
STATS_INC(perscache_rct_mismatch);
SYSLOG_INTERNAL_WARNING_ONCE("persistent cache RAC/RCT support mismatch");
return false;
}
#endif
if (DYNAMO_OPTION(persist_check_options)) {
/* Option string is 1st data section; we persisted a NULL so we can
* point directly at it. */
if (pers->option_string_len == 0)
pers_options = "";
else
pers_options = (const char *) (((byte *)pers) + pers->header_len);
LOG(THREAD, LOG_CACHE, 2, " checking pcache options \"%s\" vs current \"%s\"\n",
pers_options, option_string);
if (strcmp(option_string, pers_options) != 0) {
/* Incompatible options! In many cases the pcache may be safe to
* execute but we play it conservatively and don't delve deeper (case 9799).
* FIXME: put local options in name?!?
*/
LOG(THREAD, LOG_CACHE, 1, " error: options mismatch \"%s\" vs \"%s\"\n",
pers_options, option_string);
STATS_INC(perscache_options_mismatch);
SYSLOG_INTERNAL_WARNING_ONCE("persistent cache options mismatch");
return false;
}
}
return true;
}
#ifdef DEBUG
/* case 10712: this would be a lot of stack space so we use static buffers
* to avoid stack overflow
*/
DECLARE_NEVERPROT_VAR(static char pcache_dir_check_root[MAXIMUM_PATH], {0});
DECLARE_NEVERPROT_VAR(static char pcache_dir_check_temp[MAXIMUM_PATH], {0});
static void
pcache_dir_check_permissions(dcontext_t *dcontext, const char *filename)
{
/* Test that we cannot rename per-user directory before
* opening the file. Prevented by sharing/in use
* restrictions, not ACLs. xref case 10504
*/
/* `dirname filename` */
const char *file_parent = double_strrchr(filename, DIRSEP, ALT_DIRSEP);
size_t per_user_len;
mutex_lock(&pcache_dir_check_lock);
if (DYNAMO_OPTION(persist_per_app)) {
/* back up one level higher */
/* note that we only need to check whether the per-user directory
* cannot be renamed by an attacker, while the per-user\per-app
* directory should be created with correct ACLs
*/
snprintf(pcache_dir_check_temp, BUFFER_SIZE_ELEMENTS(pcache_dir_check_temp),
"%.*s", file_parent - filename, filename);
/* `dirname pcache_dir_check_temp` */
per_user_len =
double_strrchr(pcache_dir_check_temp, DIRSEP, ALT_DIRSEP) -
pcache_dir_check_temp;
} else {
per_user_len = file_parent - filename;
}
snprintf(pcache_dir_check_root, BUFFER_SIZE_ELEMENTS(pcache_dir_check_root), "%.*s",
per_user_len, filename);
snprintf(pcache_dir_check_temp, BUFFER_SIZE_ELEMENTS(pcache_dir_check_temp),
"%s-bumped", pcache_dir_check_root);
LOG(THREAD, LOG_CACHE, 3, " attempting rename %s -> %s\n",
pcache_dir_check_root, pcache_dir_check_temp);
ASSERT(!os_rename_file(pcache_dir_check_root, pcache_dir_check_temp,
false) &&
"directory can be bumped!");
mutex_unlock(&pcache_dir_check_lock);
}
#endif
static file_t
persist_get_name_and_open(dcontext_t *dcontext, app_pc modbase,
char *filename OUT, uint filename_sz,
char *option_buf OUT, uint option_buf_sz,
const char **option_string OUT,
op_pcache_t *option_level OUT,
persisted_module_info_t *modinfo OUT
_IF_DEBUG(app_pc start) _IF_DEBUG(app_pc end))
{
file_t fd = INVALID_FILE;
/* case 9799: pcache-affecting options are part of name.
* if -persist_check_exempted_options we must first try w/ the local
* options and then the global, hence the do-while loop.
*/
*option_level = persist_get_options_level(modbase, NULL, true/*force local*/);
do {
*option_string = persist_get_relevant_options(dcontext, option_buf,
option_buf_sz, *option_level);
memset(modinfo, 0, sizeof(*modinfo));
if (!get_persist_filename(filename, filename_sz, modbase,
false/*read*/, modinfo, *option_string)) {
LOG(THREAD, LOG_CACHE, 1,
" error computing name/excluded for "PFX"-"PFX"\n", start, end);
STATS_INC(perscache_load_noname);
return fd;
}
LOG(THREAD, LOG_CACHE, 1, " persisted filename = %s\n", filename);
if (DYNAMO_OPTION(validate_owner_dir)) {
ASSERT(DYNAMO_OPTION(persist_per_user));
if (perscache_user_directory == INVALID_FILE) {
/* there is no alternative location to test - currently
* only single per-user directories supported
*/
LOG(THREAD, LOG_CACHE, 1,
" directory is unsafe, cannot use persistent cache\n");
return fd;
}
DOCHECK(1, {
pcache_dir_check_permissions(dcontext, filename);
});
}
/* On win32 OS_EXECUTE is required to create a section w/ rwx
* permissions, which is in turn required to map a view w/ rwx
*/
fd = os_open(filename, OS_OPEN_READ | OS_EXECUTE |
/* Case 9964: we want to allow renaming while holding
* the file handle */
OS_SHARE_DELETE);
if (fd == INVALID_FILE && *option_level == OP_PCACHE_LOCAL) {
ASSERT(DYNAMO_OPTION(persist_check_exempted_options));
LOG(THREAD, LOG_CACHE, 1, " local-options file not found %s\n", filename);
*option_level = OP_PCACHE_GLOBAL;
/* try again */
} else
break;
} while (true);
if (fd == INVALID_FILE) {
LOG(THREAD, LOG_CACHE, 1, " error opening file %s\n", filename);
STATS_INC(perscache_load_nofile);
}
return fd;
}
/* It's up to the caller to do the work of mark_executable_area_coarse_frozen().
* Caller must hold read lock hotp_get_lock().
*/
coarse_info_t *
coarse_unit_load(dcontext_t *dcontext, app_pc start, app_pc end,
bool for_execution)
{
coarse_persisted_info_t *pers;
persisted_footer_t *footer;
coarse_info_t *info = NULL;
char option_buf[MAX_PCACHE_OPTIONS_STRING];
char filename[MAXIMUM_PATH];
const char *option_string;
op_pcache_t option_level;
file_t fd = INVALID_FILE;
byte *map = NULL, *map2 = NULL;
size_t map_size = 0, map2_size = 0;
uint64 file_size;
size_t stubs_and_prefixes_len;
byte *pc, *rx_pc, *rwx_pc;
persisted_module_info_t modinfo;
app_pc modbase = get_module_base(start);
bool success = false;
DEBUG_DECLARE(bool ok;)
KSTART(persisted_load);
DOLOG(1, LOG_CACHE, {
char modname[MAX_MODNAME_INTERNAL];
os_get_module_name_buf(modbase, modname, BUFFER_SIZE_ELEMENTS(modname));
LOG(THREAD, LOG_CACHE, 1, "coarse_unit_load %s "PFX"-"PFX"%s\n",
modname, start, end, for_execution ? "" : " (not for exec)");
});
DOSTATS({
if (for_execution)
STATS_INC(perscache_load_attempt);
else
STATS_INC(perscache_load_nox_attempt);
});
#ifdef HOT_PATCHING_INTERFACE
ASSERT_OWN_READWRITE_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
#endif
fd = persist_get_name_and_open(dcontext, modbase,
filename, BUFFER_SIZE_ELEMENTS(filename),
option_buf, BUFFER_SIZE_ELEMENTS(option_buf),
&option_string, &option_level, &modinfo
_IF_DEBUG(start) _IF_DEBUG(end));
if (fd == INVALID_FILE) {
/* stats done in persist_get_name_and_open */
goto coarse_unit_load_exit;
}
if (DYNAMO_OPTION(validate_owner_file)) {
if (!os_validate_user_owned(fd)) {
SYSLOG_INTERNAL_ERROR_ONCE("%s not owned by current process!"
" Persistent cache may be compromised, not using.",
filename);
/* note we don't attempt to overwrite here */
goto coarse_unit_load_exit;
}
} else {
ASSERT(!DYNAMO_OPTION(validate_owner_dir) ||
perscache_user_directory != INVALID_FILE);
/* note that if have a directory handle open, there is no need to
* validate file with DYNAMO_OPTION(validate_owner_file) here.
* of course it doesn't hurt to check both.
*/
DOCHECK(1, {
ASSERT_CURIOSITY((!DYNAMO_OPTION(validate_owner_file) ||
os_validate_user_owned(fd)) && "impostor not detected!");
});
}
if (!os_get_file_size_by_handle(fd, &file_size)) {
LOG(THREAD, LOG_CACHE, 1, " error obtaining file size for %s\n", filename);
goto coarse_unit_load_exit;
}
ASSERT_TRUNCATE(map_size, size_t, file_size);
map_size = (size_t) file_size;
LOG(THREAD, LOG_CACHE, 1, " size of %s is "SZFMT"\n", filename, map_size);
/* FIXME case 9642: control where in address space we map the file:
* right after vmheap? Randomized?
*/
map = map_file(fd, &map_size, 0, NULL,
/* Ask for max, then restrict pieces */
MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC,
/* case 9599: asking for COW commits pagefile space
* up front, so we map two separate views later: see below
*/
MAP_FILE_COPY_ON_WRITE/*writes should not change file*/ |
MAP_FILE_REACHABLE);
/* case 9925: if we keep the file handle open we can prevent writes
* to the file while it's mapped in, but it prevents our rename replacement
* scheme (case 9701/9720) so we have it under option control.
*/
if (!DYNAMO_OPTION(persist_lock_file)) {
os_close(fd);
fd = INVALID_FILE;
}
if (map == NULL) {
LOG(THREAD, LOG_CACHE, 1, " error mapping file %s\n", filename);
goto coarse_unit_load_exit;
}
pers = (coarse_persisted_info_t *) map;
ASSERT(pers->header_len + pers->data_len <= map_size &&
ALIGN_FORWARD(pers->header_len + pers->data_len, PAGE_SIZE) ==
ALIGN_FORWARD(map_size, PAGE_SIZE));
footer = (persisted_footer_t *)
(map + pers->header_len + pers->data_len - sizeof(persisted_footer_t));
if (pers->magic != PERSISTENT_CACHE_MAGIC ||
/* case 10160: don't crash trying to read footer */
pers->header_len + pers->data_len > map_size ||
ALIGN_FORWARD(pers->header_len + pers->data_len, PAGE_SIZE) !=
ALIGN_FORWARD(map_size, PAGE_SIZE) ||
footer->magic != PERSISTENT_CACHE_MAGIC ||
TEST(PERSCACHE_CODE_INVALID, pers->flags)) {
LOG(THREAD, LOG_CACHE, 1, " invalid persisted file %s\n", filename);
/* This flag shouldn't be set for on-disk units */
ASSERT(!TEST(PERSCACHE_CODE_INVALID, pers->flags));
STATS_INC(perscache_bad_file);
/* We didn't have a footer prior to version 4 */
ASSERT_CURIOSITY_ONCE(pers->version < 4 && "persistent cache file corrupt");
goto coarse_unit_load_exit;
}
if (pers->version != PERSISTENT_CACHE_VERSION) {
/* FIXME case 9655: should we clobber such a file when we persist? */
LOG(THREAD, LOG_CACHE, 1, " invalid persisted file version %d for %s\n",
pers->version, filename);
STATS_INC(perscache_version_mismatch);
goto coarse_unit_load_exit;
}
if (!TEST(IF_X64_ELSE(PERSCACHE_X86_64, PERSCACHE_X86_32), pers->flags)) {
LOG(THREAD, LOG_CACHE, 1, " invalid architecture: not %s %s\n",
IF_X64_ELSE("AMD64", "IA-32"), filename);
STATS_INC(perscache_version_mismatch);
SYSLOG_INTERNAL_WARNING_ONCE("persistent cache architecture mismatch");
goto coarse_unit_load_exit;
}
/* Check file's md5 */
if (TESTANY(PERSCACHE_GENFILE_MD5_SHORT|PERSCACHE_GENFILE_MD5_COMPLETE,
DYNAMO_OPTION(persist_load_validation))) {
module_digest_t self_md5;
persist_calculate_self_digest(&self_md5, pers, map,
DYNAMO_OPTION(persist_load_validation));
DOLOG(1, LOG_CACHE, {
print_module_digest(THREAD, &footer->self_md5, "md5 stored in file: ");
print_module_digest(THREAD, &footer->self_md5, "md5 calculated: ");
});
if ((TEST(PERSCACHE_GENFILE_MD5_SHORT,
DYNAMO_OPTION(persist_load_validation)) &&
!md5_digests_equal(self_md5.short_MD5, footer->self_md5.short_MD5)) ||
(TEST(PERSCACHE_GENFILE_MD5_COMPLETE,
DYNAMO_OPTION(persist_load_validation)) &&
!md5_digests_equal(self_md5.full_MD5, footer->self_md5.full_MD5))) {
LOG(THREAD, LOG_CACHE, 1, " file header md5 mismatch\n");
STATS_INC(perscache_md5_mismatch);
ASSERT_CURIOSITY_ONCE(false && "persistent cache md5 mismatch");
goto coarse_unit_load_exit;
}
}
/* Consistency with original module */
persist_calculate_module_digest(&modinfo.module_md5, modbase,
(size_t) modinfo.image_size,
modbase + pers->start_offs,
modbase + pers->end_offs,
DYNAMO_OPTION(persist_load_validation));
/* Compare the module digest and module fields (except base) all at once */
if (!persist_modinfo_cmp(&modinfo, &pers->modinfo)) {
LOG(THREAD, LOG_CACHE, 1, " module info mismatch\n");
DOLOG(1, LOG_CACHE, {
LOG(THREAD, LOG_CACHE, 1, "modinfo stored in file: ");
dump_buffer_as_bytes(THREAD, &pers->modinfo, sizeof(pers->modinfo),
DUMP_RAW|DUMP_DWORD);
LOG(THREAD, LOG_CACHE, 1, "\nmodinfo in memory: ");
dump_buffer_as_bytes(THREAD, &modinfo, sizeof(modinfo),
DUMP_RAW|DUMP_DWORD);
LOG(THREAD, LOG_CACHE, 1, "\n");
});
SYSLOG_INTERNAL_WARNING_ONCE("persistent cache module mismatch");
#ifdef WINDOWS
if (modbase != pers->modinfo.base)
persist_record_base_mismatch(modbase);
#endif
STATS_INC(perscache_modinfo_mismatch);
goto coarse_unit_load_exit;
}
/* modinfo cmp checks all but base, which is separate for reloc support */
if (modbase != pers->modinfo.base) {
#ifdef UNIX
/* for linux, we can trust lack of textrel flag as guaranteeing no text relocs */
if (DYNAMO_OPTION(persist_trust_textrel) &&
/* XXX: may not be at_map if re-add coarse post-load (e.g., IAT or other
* special cases): how know?
*/
!module_has_text_relocs(modbase, dynamo_initialized/*at_map*/)) {
LOG(THREAD, LOG_CACHE, 1, " module base mismatch "PFX" vs persisted "PFX
", but no text relocs so ok\n", modbase, pers->modinfo.base);
} else {
#endif
/* FIXME case 9581/9649: Bail out for now since relocs NYI
* Once we do support them, make sure to do the right thing when merging:
* current code will always apply relocs before merging.
*/
LOG(THREAD, LOG_CACHE, 1, " module base mismatch "PFX" vs persisted "PFX"\n",
modbase, pers->modinfo.base);
#ifdef WINDOWS
persist_record_base_mismatch(modbase);
#endif
STATS_INC(perscache_base_mismatch);
goto coarse_unit_load_exit;
#ifdef UNIX
}
#endif
}
if (modbase + pers->start_offs < start ||
modbase + pers->end_offs > end) {
/* case 9653: only load a file that covers a subset of the vmarea region */
LOG(THREAD, LOG_CACHE, 1,
" region mismatch "PFX"-"PFX" vs persisted "PFX"-"PFX"\n",
start, end, modbase + pers->start_offs, modbase + pers->end_offs);
STATS_INC(perscache_region_mismatch);
goto coarse_unit_load_exit;
}
if (!persist_check_option_compat(dcontext, pers, option_string)) {
goto coarse_unit_load_exit;
}
stubs_and_prefixes_len =
(pers->stubs_len + pers->ibl_jmp_prefix_len + pers->ibl_call_prefix_len
+ pers->ibl_ret_prefix_len + pers->trace_head_return_prefix_len
+ pers->fcache_return_prefix_len);
if (TEST(PERSCACHE_MAP_RW_SEPARATE, pers->flags) &&
DYNAMO_OPTION(persist_map_rw_separate)) {
size_t ro_size;
map2_size = stubs_and_prefixes_len + sizeof(persisted_footer_t);
ro_size = (size_t)file_size/*un-aligned*/ - map2_size;
ASSERT(ro_size == ALIGN_FORWARD(pers->header_len + pers->data_len
- stubs_and_prefixes_len
- pers->view_pad_len
- sizeof(persisted_footer_t),
MAP_FILE_VIEW_ALIGNMENT));
LOG(THREAD, LOG_CACHE, 2,
" attempting double mapping: size "PIFX" and "PIFX"\n", ro_size, map2_size);
if (!DYNAMO_OPTION(persist_lock_file)) {
fd = os_open(filename, OS_OPEN_READ|OS_EXECUTE|OS_SHARE_DELETE);
/* FIXME: case 10547 what do we know about this second
* file that we just opened? Nothing guarantees it is
* still the same one we used earlier.
*/
if (fd != INVALID_FILE && DYNAMO_OPTION(validate_owner_file)) {
if (!os_validate_user_owned(fd)) {
os_close(fd);
fd = INVALID_FILE;
SYSLOG_INTERNAL_ERROR_ONCE("%s not owned by current process!"
" Persistent cache may be compromised, "
"not using.",
filename);
}
}
}
ASSERT(fd != INVALID_FILE);
if (fd != INVALID_FILE) {
unmap_file(map, map_size);
pers = NULL;
map_size = ro_size;
/* we know the whole thing fit @map, so try there */
map = map_file(fd, &map_size, 0, map,
/* Ask for max, then restrict pieces */
MEMPROT_READ|MEMPROT_EXEC,
/* no COW to avoid pagefile cost */
MAP_FILE_REACHABLE);
map2 = map_file(fd, &map2_size, map_size, map + map_size,
/* Ask for max, then restrict pieces */
MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC,
MAP_FILE_COPY_ON_WRITE/*writes should not change file*/ |
MAP_FILE_REACHABLE);
/* FIXME: try again if racy alloc and they both don't fit */
if (!DYNAMO_OPTION(persist_lock_file)) {
os_close(fd);
fd = INVALID_FILE;
}
if (map == NULL || map2 != map + ro_size) {
SYSLOG_INTERNAL_ERROR_ONCE("double perscache mapping failed");
LOG(THREAD, LOG_CACHE, 1,
" error: 2nd map "PFX" not adjacent to 1st "PFX"\n", map, map2);
STATS_INC(perscache_maps_not_adjacent);
goto coarse_unit_load_exit;
}
/* the two views are adjacent so most code can treat it as one view */
LOG(THREAD, LOG_CACHE, 1,
" mapped view1 @"PFX" and view2 @"PFX"\n", map, map2);
pers = (coarse_persisted_info_t *) map;
}
}
/* We assume that once info!=NULL we have been successful, though we do
* abort for hotp or client conflicts below */
info = coarse_unit_create(modbase + pers->start_offs, modbase + pers->end_offs,
&modinfo.module_md5, for_execution);
info->frozen = true;
info->persisted = true;
info->has_persist_info = true;
info->persist_base = pers->modinfo.base;
info->mod_shift = (pers->modinfo.base - modbase);
info->mmap_pc = map;
if (map2 != NULL) {
info->mmap_ro_size = map_size;
info->mmap_size = map_size + map2_size;
ASSERT(map2 == info->mmap_pc + info->mmap_ro_size);
} else
info->mmap_size = map_size;
ASSERT(ALIGN_FORWARD(info->mmap_size, PAGE_SIZE) ==
ALIGN_FORWARD(pers->header_len + pers->data_len, PAGE_SIZE));
if (DYNAMO_OPTION(persist_lock_file))
info->fd = fd;
info->flags = pers->flags;
#if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
/* FIXME: provide win32/ interface for setting the flag? */
if (TEST(PERSCACHE_SEEN_BORLAND_SEH, pers->flags) && !seen_Borland_SEH) {
SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
seen_Borland_SEH = true;
SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
}
#endif
ASSERT(option_level != OP_PCACHE_LOCAL ||
TEST(PERSCACHE_EXEMPTION_OPTIONS, info->flags));
/* Process data sections (other than option string) in reverse order */
pc = map + pers->header_len + pers->data_len; /* end of file */
pc -= sizeof(persisted_footer_t);
pc -= pers->instrument_rw_len;
#ifdef CLIENT_INTERFACE
if (pers->instrument_rw_len > 0) {
if (!instrument_resurrect_rw(GLOBAL_DCONTEXT, info, pc))
goto coarse_unit_load_exit;
}
#endif
info->stubs_end_pc = pc;
pc -= stubs_and_prefixes_len;
info->fcache_return_prefix = pc;
rwx_pc = info->fcache_return_prefix;
ASSERT(ALIGNED(rwx_pc, PAGE_SIZE));
/* To patch up the prefixes, it's just as easy to re-emit them all as
* most are just plain jmps anyway
*/
info->stubs_start_pc = coarse_stubs_create(info, info->fcache_return_prefix,
stubs_and_prefixes_len);
/* We could omit these from the file and assume their sizes */
DOCHECK(1, {
byte *check = info->fcache_return_prefix;
check += pers->fcache_return_prefix_len;
ASSERT(check == info->trace_head_return_prefix);
check += pers->trace_head_return_prefix_len;
ASSERT(check == info->ibl_ret_prefix);
check += pers->ibl_ret_prefix_len;
ASSERT(check == info->ibl_call_prefix);
check += pers->ibl_call_prefix_len;
ASSERT(check == info->ibl_jmp_prefix);
check += pers->ibl_jmp_prefix_len;
ASSERT(check == info->stubs_start_pc);
check += pers->stubs_len;
ASSERT(check == info->stubs_end_pc);
});
if (DYNAMO_OPTION(persist_protect_stubs)) {
/* Mark prefixes + stubs read-only. Note that we can't map in as +rx, since
* we must have +w to get COW (unless we switch to MEM_IMAGE). So we have
* no way to avoid the pagefile cost, except the mysterious case 10074,
* which we take advantage of here by touching every page (yes it works
* even after the prefix write above), paying a one-time working set cost.
*/
if (DYNAMO_OPTION(persist_touch_stubs)) {
app_pc touch_pc = rwx_pc;
volatile byte touch_value;
for (; touch_pc < (app_pc) (map + pers->header_len + pers->data_len);
touch_pc += PAGE_SIZE) {
touch_value = *touch_pc;
STATS_INC(pcache_stub_touched);
}
}
DEBUG_DECLARE(ok =)
set_protection(rwx_pc, map + pers->header_len + pers->data_len - rwx_pc,
MEMPROT_READ|MEMPROT_EXEC);
ASSERT(ok);
info->stubs_readonly = true;
} else {
/* FIXME case 9650: we could mark the prefixes as read-only now, if we put them
* on their own page
*/
}
info->cache_end_pc = pc - pers->post_cache_pad_len;
pc -= pers->cache_len;
info->cache_start_pc = pc;
pc -= pers->instrument_rx_len;
rx_pc = pc;
ASSERT(ALIGNED(rx_pc, PAGE_SIZE));
fcache_coarse_init_frozen(dcontext, info, info->cache_start_pc,
info->fcache_return_prefix - info->cache_start_pc);
pc -= pers->view_pad_len; /* just skip it */
#ifdef CLIENT_INTERFACE
if (pers->instrument_rx_len > 0) {
if (!instrument_resurrect_rx(GLOBAL_DCONTEXT, info, pc))
goto coarse_unit_load_exit;
}
#endif
pc -= pers->pad_len; /* just skip it */
pc -= pers->stub_htable_len;
fragment_coarse_htable_resurrect(GLOBAL_DCONTEXT, info, false/*stub table*/, pc);
ASSERT(fragment_coarse_htable_persist_size(dcontext, info, false/*stub*/) ==
pers->stub_htable_len);
pc -= pers->cache_htable_len;
fragment_coarse_htable_resurrect(GLOBAL_DCONTEXT, info, true/*cache table*/, pc);
ASSERT(fragment_coarse_htable_persist_size(dcontext, info, true/*cache*/) ==
pers->cache_htable_len);
ASSERT(offsetof(coarse_persisted_info_t, cache_htable_len) < pers->header_len);
/* From here on out, check offsets so will work w/ earlier-versioned file
*/
if (offsetof(coarse_persisted_info_t, rct_htable_len) < pers->header_len) {
pc -= pers->rct_htable_len;
#ifdef RCT_IND_BRANCH
if (pers->rct_htable_len > 0) {
/* We may be merging with the on-disk file, so load it in
* regardless of whether we care in this process instance
* Use GLOBAL_DCONTEXT in case we do use.
*/
info->rct_table = rct_table_resurrect(GLOBAL_DCONTEXT, pc, RCT_RCT);
ASSERT(info->rct_table != NULL);
if (for_execution &&
(TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
(DYNAMO_OPTION(use_persisted_rct) ||
TEST(PERSCACHE_SEEN_BORLAND_SEH, pers->flags)) &&
/* Case 9834: avoid double-add from earlier entire-module resurrect */
!os_module_get_flag(info->base_pc, MODULE_RCT_LOADED)) {
/* load into the official module table */
DEBUG_DECLARE(bool used =)
rct_module_table_set(GLOBAL_DCONTEXT, modbase,
info->rct_table, RCT_RCT);
ASSERT(used);
# ifdef WINDOWS
if (TEST(PERSCACHE_ENTIRE_MODULE_RCT, pers->flags)) {
/* mark as not needing to be analyzed */
os_module_set_flag(info->base_pc, MODULE_RCT_LOADED);
}
# endif
}
}
#endif
}
if (offsetof(coarse_persisted_info_t, rac_htable_len) < pers->header_len) {
pc -= pers->rac_htable_len;
#ifdef RETURN_AFTER_CALL
if (pers->rac_htable_len > 0) {
/* We may be merging with the on-disk file, so load it in
* regardless of whether we care in this process instance.
* Use GLOBAL_DCONTEXT in case we do use.
*/
info->rac_table = rct_table_resurrect(GLOBAL_DCONTEXT, pc, RCT_RAC);
ASSERT(info->rac_table != NULL);
if (for_execution && DYNAMO_OPTION(ret_after_call)) {
/* load into the official module table */
DEBUG_DECLARE(bool used =)
rct_module_table_set(GLOBAL_DCONTEXT, modbase,
info->rac_table, RCT_RAC);
ASSERT(used);
}
}
#endif
}
/* FIXME case 9581 NYI: reloc section */
if (offsetof(coarse_persisted_info_t, reloc_len) < pers->header_len) {
pc -= pers->reloc_len;
}
#ifdef HOT_PATCHING_INTERFACE
if (offsetof(coarse_persisted_info_t, hotp_patch_list_len) < pers->header_len) {
/* Case 9995: store list of active patch points in perscache file and don't
* abort if those are the only matching points at this time.
* FIXME if we had an efficient way to see if an app pc is present
* in the pcache (case 9969) we could be more precise.
*/
pc -= pers->hotp_patch_list_len;
IF_X64(ASSERT_TRUNCATE(info->hotp_ppoint_vec_num, uint,
pers->hotp_patch_list_len / sizeof(app_rva_t)));
info->hotp_ppoint_vec_num = (uint) pers->hotp_patch_list_len / sizeof(app_rva_t);
if (info->hotp_ppoint_vec_num > 0)
info->hotp_ppoint_vec = (app_rva_t *) pc;
} else
info->hotp_ppoint_vec_num = 0;
ASSERT(info->hotp_ppoint_vec == NULL || info->hotp_ppoint_vec_num > 0);
if (DYNAMO_OPTION(hot_patching) &&
hotp_point_not_on_list(info->base_pc, info->end_pc,
true/*own hotp lock*/,
info->hotp_ppoint_vec, info->hotp_ppoint_vec_num)) {
LOG(THREAD, LOG_CACHE, 1,
" error: hotp match prevents using persistence\n");
STATS_INC(perscache_hotp_conflict);
/* FIXME: we could duplicate the calculation of info->hotp_ppoint_vec
* to detect this error before allocating info. Instead we have to
* free what we've created. A bonus of waiting to detect the error
* is that we've already loaded the RCT entries, which the error does
* not invalidate.
*/
goto coarse_unit_load_exit;
}
#endif
/* we go ahead and check offset here, but we inserted fields above to keep
* in a sane order and bumped the version. currently no backcompat anyway.
*/
if (offsetof(coarse_persisted_info_t, instrument_ro_len) < pers->header_len) {
pc -= pers->instrument_ro_len;
#ifdef CLIENT_INTERFACE
if (pers->instrument_ro_len > 0) {
if (!instrument_resurrect_ro(GLOBAL_DCONTEXT, info, pc))
goto coarse_unit_load_exit;
}
#endif
}
ASSERT(pc - map >= (int)pers->header_len);
DEBUG_DECLARE(ok =)
set_protection(map, rx_pc - map, MEMPROT_READ);
ASSERT(ok);
DEBUG_DECLARE(ok =)
set_protection(rx_pc, rwx_pc - rx_pc, MEMPROT_READ|MEMPROT_EXEC);
ASSERT(ok);
/* FIXME case 9648: don't forget to append a guard page -- but we
* should only need it if we fill up the full allocation region,
* so on win32 we will get a free guard page whenever we're not
* end-aligned to 64KB.
*/
RSTATS_INC(perscache_loaded);
success = true;
coarse_unit_load_exit:
if (!success) {
if (info != NULL) {
/* XXX: see hotp notes above: better to determine invalidation
* prior to creating info. Note that RCT entries are not reset here.
*/
coarse_unit_reset_free_internal(dcontext, info, false/*no locks*/,
false/*not linked yet*/,
true/*give up primary*/,
/* case 11064: avoid rank order */
false/*local, no info lock needed*/);
coarse_unit_free(dcontext, info);
/* reset_free unmaps map and map2 for us */
info = NULL;
} else {
if (map != NULL)
unmap_file(map, map_size);
if (map2 != NULL)
unmap_file(map2, map2_size);
if (fd != INVALID_FILE)
os_close(fd);
}
}
KSTOP(persisted_load);
return info;
}
bool
exists_coarse_ibl_pending_table(dcontext_t *dcontext, /* in case per-thread someday */
coarse_info_t *info, ibl_branch_type_t branch_type)
{
#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
bool exists = false;
if (info != NULL) {
if (branch_type == IBL_RETURN)
exists = rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RAC);
else if (branch_type == IBL_INDCALL)
exists = rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RCT);
else {
ASSERT(branch_type == IBL_INDJMP);
exists = rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RCT)
|| rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RAC);
}
return (exists && !TEST(COARSE_FILL_IBL_MASK(branch_type),
info->ibl_pending_used));
}
#endif
return false;
}
/* If pc is in a module, marks that module as exempted (case 9799) */
void
mark_module_exempted(app_pc pc)
{
if (DYNAMO_OPTION(persist_check_options) &&
/* if persist_check_local_options is on no reason to track exemptions */
!DYNAMO_OPTION(persist_check_local_options) &&
DYNAMO_OPTION(persist_check_exempted_options) &&
module_info_exists(pc) &&
!os_module_get_flag(pc, MODULE_WAS_EXEMPTED)) {
LOG(GLOBAL, LOG_VMAREAS, 1, "marking module @"PFX" as exempted\n");
os_module_set_flag(pc, MODULE_WAS_EXEMPTED);
}
}