diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 8956403a7a..4f7f26d14a 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -80,6 +80,7 @@ #include "accel/tcg/internal.h" #include "ts.h" #include "latx-smc.h" +#include "smc_reload.h" #endif #ifdef CONFIG_LATX_FAST_JMPCACHE #include "exec/fasttb.h" @@ -564,7 +565,7 @@ static bool is_shadow_page_shmm(target_ulong address) return spd && spd->is_shmm; } -static bool is_shadow_page_not_shmm(target_ulong address) +bool is_shadow_page_not_shmm(target_ulong address) { ShadowPageDesc *spd = page_get_target_data(address); return spd && !(spd->is_shmm); @@ -1682,6 +1683,10 @@ static void gen_aot_and_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) void tb_flush(CPUState *cpu) { + if (option_smc_reload) { + reload_tree_clear(); + } + if (tcg_enabled()) { unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count); @@ -3583,10 +3588,60 @@ bool pageflags_set_clear(target_ulong start, target_ulong last, return inval_tb; } +static void add_smc_node(tb_page_addr_t start, tb_page_addr_t end) +{ + if (!option_smc_reload || !option_aot) { + return; + } + if (segment_tree_lookup2(start, end)) { + return; + } + + TranslationBlock *tb; + PageForEachNext next_tb; + int ir1_size; + int curr_page_tb_num; + + for (target_ulong curr_page = start; curr_page < end; curr_page += TARGET_PAGE_SIZE) { + curr_page_tb_num = 0; + ir1_size = 0; + PAGE_FOR_EACH_TB(curr_page, curr_page + TARGET_PAGE_SIZE, unused, tb, next_tb) { + if ((tb->pc & TARGET_PAGE_MASK) == curr_page) { + curr_page_tb_num++; + ir1_size += tb->size; + } + } + if (curr_page_tb_num) { + reload_info *reload_node = (reload_info *)malloc(sizeof(reload_info)); + reload_node->tb_num = curr_page_tb_num; + reload_node->page_addr = curr_page; + assert(reload_node); + reload_node->tb_vector = + (TranslationBlock **)malloc(curr_page_tb_num * sizeof(TranslationBlock *)); + assert(reload_node->tb_vector); + reload_node->ir1_code_buffer = malloc(ir1_size); + assert(reload_node->ir1_code_buffer); + int tb_id = 0; + ir1_size = 0; + PAGE_FOR_EACH_TB(curr_page, curr_page + TARGET_PAGE_SIZE, unused, tb, next_tb) { + if ((tb->pc & TARGET_PAGE_MASK) == curr_page) { + memcpy(reload_node->ir1_code_buffer + ir1_size, + (void *)(uintptr_t)tb->pc, tb->size); + reload_node->tb_vector[tb_id++] = tb; + ir1_size += tb->size; + } + } + reload_tree_insert(reload_node); + } + } +} + /* Modify the flags of a page and invalidate the code if necessary. + Save SMC TB to reload_tree if necessary. The flag PAGE_WRITE_ORG is positioned automatically depending on PAGE_WRITE. The mmap_lock should already be held. */ -void page_set_flags(target_ulong start, target_ulong end, int flags) +void page_set_flags_tb_reload(target_ulong start, target_ulong end, + int flags, bool tb_reload) { #ifdef CONFIG_LATX_PERF latx_timer_start(TIMER_PAGE_FLAGS); @@ -3657,6 +3712,9 @@ void page_set_flags(target_ulong start, target_ulong end, int flags) } if (inval_tb) { + if (tb_reload) { + add_smc_node(start, end); + } tb_invalidate_phys_range(start, end); } #ifdef CONFIG_LATX_PERF @@ -3664,6 +3722,14 @@ void page_set_flags(target_ulong start, target_ulong end, int flags) #endif } +/* Modify the flags of a page and invalidate the code if necessary. + The flag PAGE_WRITE_ORG is positioned automatically depending + on PAGE_WRITE. The mmap_lock should already be held. */ +void page_set_flags(target_ulong start, target_ulong end, int flags) +{ + page_set_flags_tb_reload(start, end, flags, false); +} + typedef struct TargetPageDataNode { IntervalTreeNode itree; void *target_data; diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 6dd2fc37e0..78c7873def 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -291,6 +291,8 @@ target_ulong get_first_page(target_ulong start, target_ulong end); int page_get_flags(target_ulong address); void page_clear_overflow(target_ulong start, target_ulong end); void page_set_flags(target_ulong start, target_ulong end, int flags); +void page_set_flags_tb_reload(target_ulong start, target_ulong end, + int flags, bool tb_reload); bool pageflags_set_clear(target_ulong start, target_ulong last, int set_flags, int clear_flags); int page_get_page_state(target_ulong address); diff --git a/include/exec/translate-all.h b/include/exec/translate-all.h index 962d23a79a..489fc1aeb7 100644 --- a/include/exec/translate-all.h +++ b/include/exec/translate-all.h @@ -23,6 +23,7 @@ extern void *l1_map[]; extern IntervalTreeRoot pageflags_root; /* translate-all.c */ +bool is_shadow_page_not_shmm(target_ulong address); struct page_collection *page_collection_lock(tb_page_addr_t start, tb_page_addr_t end); void page_collection_unlock(struct page_collection *set); diff --git a/linux-user/main.c b/linux-user/main.c index e397b76516..6f6ef5cf98 100644 --- a/linux-user/main.c +++ b/linux-user/main.c @@ -726,6 +726,14 @@ static void handle_arg_latx_monitor_shared_mem(const char *arg) option_monitor_shared_mem = strtol(arg, NULL, 0); } +static void handle_arg_smc_reload(const char *arg) +{ + option_smc_reload = strtol(arg, NULL, 0); + if (option_smc_reload) { + option_jr_ra = 0; + } +} + #ifdef CONFIG_LATX_AOT static void handle_arg_latx_aot(const char *arg) { @@ -833,6 +841,8 @@ static const struct qemu_argument arg_table[] = { "", "enable get real self maps"}, {"latx-monitor-shared-mem", "LATX_MONITOR_SHARED_MEM", true, handle_arg_latx_monitor_shared_mem, "", "monitor shared memory, retranslate self modifying page"}, + {"smc_reload", "LATX_SMC_RELOAD", true, handle_arg_smc_reload, + "", "reload SMC TB"}, #ifdef CONFIG_LATX_AOT {"latx-aot", "LATX_AOT", true, handle_arg_latx_aot, "", "enable aot"}, diff --git a/linux-user/mmap.c b/linux-user/mmap.c index c68e7aabae..281d95a21e 100644 --- a/linux-user/mmap.c +++ b/linux-user/mmap.c @@ -173,8 +173,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) prot_tmp = page_get_flags(addr); if ((prot_tmp & PAGE_EXEC) && !(prot_tmp & PAGE_WRITE) && (prot_tmp & PAGE_WRITE_ORG) && (target_prot & PAGE_WRITE)) { - page_set_flags(addr, addr + TARGET_PAGE_SIZE, - prot_tmp | PAGE_WRITE); + page_set_flags_tb_reload(addr, addr + TARGET_PAGE_SIZE, + prot_tmp | PAGE_WRITE, true); } prot1 |= prot_tmp; } @@ -184,8 +184,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) prot_tmp = page_get_flags(addr); if ((prot_tmp & PAGE_EXEC) && !(prot_tmp & PAGE_WRITE) && (prot_tmp & PAGE_WRITE_ORG) && (target_prot & PAGE_WRITE)) { - page_set_flags(addr, addr + TARGET_PAGE_SIZE, - prot_tmp | PAGE_WRITE); + page_set_flags_tb_reload(addr, addr + TARGET_PAGE_SIZE, + prot_tmp | PAGE_WRITE, true); } prot1 |= prot_tmp; } @@ -222,8 +222,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) prot_tmp = page_get_flags(addr); if ((prot_tmp & PAGE_EXEC) && !(prot_tmp & PAGE_WRITE) && (prot_tmp & PAGE_WRITE_ORG) && (target_prot & PAGE_WRITE)) { - page_set_flags(addr, addr + TARGET_PAGE_SIZE, - prot_tmp | PAGE_WRITE); + page_set_flags_tb_reload(addr, addr + TARGET_PAGE_SIZE, + prot_tmp | PAGE_WRITE, true); } prot1 |= prot_tmp; } @@ -258,7 +258,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) } } - page_set_flags(start, start + len, page_flags); + page_set_flags_tb_reload(start, start + len, page_flags, true); + if(target_prot == PROT_NONE) { #ifdef CONFIG_LATX_AOT if (option_aot && segment_tree_lookup2(start, start + len)) { diff --git a/target/i386/latx/include/latx-options.h b/target/i386/latx/include/latx-options.h index bbe3756c17..d68753b0fb 100644 --- a/target/i386/latx/include/latx-options.h +++ b/target/i386/latx/include/latx-options.h @@ -44,6 +44,7 @@ extern int option_tunnel_lib; extern uint64_t option_end_trace_addr; extern uint64_t option_begin_trace_addr; extern int option_aot; +extern int option_smc_reload; extern int option_aot_wine; extern int option_load_aot; extern int option_debug_aot; diff --git a/target/i386/latx/include/smc_reload.h b/target/i386/latx/include/smc_reload.h new file mode 100644 index 0000000000..c8a7781630 --- /dev/null +++ b/target/i386/latx/include/smc_reload.h @@ -0,0 +1,21 @@ +#ifndef __SMC_RELOAD_H_ +#define __SMC_RELOAD_H_ + +#include "qemu-def.h" + +typedef struct reoload_info { + int tb_num; + target_ulong page_addr; + TranslationBlock **tb_vector; + void *ir1_code_buffer; +} reload_info; + +void reload_tree_init(void); +void reload_tree_insert(reload_info *reload_node); +reload_info *reload_tree_lookup(target_ulong page_addr); +void reload_tree_remove(reload_info *reload_node); +gint get_reload_node_num(void); +void reload_tree_foreach(void); +void reload_tree_clear(void); + +#endif diff --git a/target/i386/latx/latx-options.c b/target/i386/latx/latx-options.c index e52ae462df..fd5ee5c31e 100644 --- a/target/i386/latx/latx-options.c +++ b/target/i386/latx/latx-options.c @@ -62,6 +62,7 @@ int option_debug_lative; int option_aot; int option_load_aot; int option_aot_wine; +int option_smc_reload; int option_debug_aot; int option_imm_reg; int option_imm_rip; @@ -238,6 +239,7 @@ void options_init(void) option_aot_wine = 0; option_debug_aot = 0; #endif + option_smc_reload = 0; #ifdef CONFIG_LATX_IMM_REG option_imm_reg = 0; option_imm_rip = 0; diff --git a/target/i386/latx/sbt/aot_recover_tb.c b/target/i386/latx/sbt/aot_recover_tb.c index 9e287e14f4..a7243348ea 100644 --- a/target/i386/latx/sbt/aot_recover_tb.c +++ b/target/i386/latx/sbt/aot_recover_tb.c @@ -8,8 +8,10 @@ #include "lsenv.h" #include "accel/tcg/internal.h" #include "include/exec/cpu-all.h" +#include "include/exec/translate-all.h" #include "aot_page.h" #include "aot_smc.h" +#include "smc_reload.h" #ifdef CONFIG_LATX_TU #include "tu.h" #endif @@ -383,6 +385,55 @@ inline int load_page(target_ulong pc, uint32_t cflags, seg_info *info) return 1; } +static int smc_page_reload(target_ulong page_addr, uint32_t cflags) +{ + if (!option_smc_reload) { + return 0; + } + reload_info *reload_node = reload_tree_lookup(page_addr); + if (!reload_node) { + return 0; + } + + int p_flags = page_get_flags(page_addr); + if (!(p_flags & PAGE_READ)) { + return 0; + } + /* + * remove write prot of the page in case of paralell code modification, + * but the paired page cross 16K boundary is still not protected. + */ + if ((p_flags & PAGE_WRITE) && !is_shadow_page_not_shmm(page_addr)) { + mprotect((void*)(page_addr & qemu_host_page_mask), qemu_host_page_size, PROT_READ); + } + + tcg_ctx->tb_cflags = cflags; + + int ir1_offset = 0; + for (int tb_id = 0; tb_id < reload_node->tb_num; tb_id++) { + TranslationBlock *tb = reload_node->tb_vector[tb_id]; + if (!memcmp((void *)(uintptr_t)tb->pc, + reload_node->ir1_code_buffer + ir1_offset, tb->size)) { + tb->cflags = tb->cflags & ~CF_INVALID; + if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { + tb_reset_jump(tb, 0); + } + if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { + tb_reset_jump(tb, 1); + } + tb->jmp_list_head = (uintptr_t)NULL; + tb->jmp_list_next[0] = (uintptr_t)NULL; + tb->jmp_list_next[1] = (uintptr_t)NULL; + tb->jmp_dest[0] = (uintptr_t)NULL; + tb->jmp_dest[1] = (uintptr_t)NULL; + aot_tb_register(tb); + } + ir1_offset += tb->size; + } + reload_tree_remove(reload_node); + return 1; +} + int load_aot(target_ulong pc, uint32_t cflags) { if (in_pre_translate) { @@ -391,6 +442,9 @@ int load_aot(target_ulong pc, uint32_t cflags) seg_info *info = segment_tree_lookup(pc); if (info == NULL || info->buffer == NULL) { + if (smc_page_reload(pc & TARGET_PAGE_MASK, cflags)) { + return 1; + } page_set_page_state(pc, PAGE_NOINFO); return 0; } diff --git a/target/i386/latx/sbt/meson.build b/target/i386/latx/sbt/meson.build index 7d2924cbc0..0f2c5b2c72 100644 --- a/target/i386/latx/sbt/meson.build +++ b/target/i386/latx/sbt/meson.build @@ -8,4 +8,5 @@ i386_ss.add(when: 'CONFIG_LATX', if_true: files( 'aot_lib.c', 'aot_page.c', 'aot_smc.c', + 'smc_reload.c', )) diff --git a/target/i386/latx/sbt/smc_reload.c b/target/i386/latx/sbt/smc_reload.c new file mode 100644 index 0000000000..abaf523d0e --- /dev/null +++ b/target/i386/latx/sbt/smc_reload.c @@ -0,0 +1,100 @@ +#include "smc_reload.h" +#include "latx-options.h" +#include + +static GTree *reload_tree = NULL; + +static gint compare_page_addr(gconstpointer a, gconstpointer b, gpointer user_data) +{ + target_ulong addr_a = *((target_ulong *)a); + target_ulong addr_b = *((target_ulong *)b); + + if (addr_a < addr_b) return -1; + if (addr_a > addr_b) return 1; + return 0; +} + +static void free_reload_info(gpointer data) +{ + reload_info *info = (reload_info *)data; + if (info) { + if (info->tb_vector) { + g_free(info->tb_vector); + } + if (info->ir1_code_buffer) { + g_free(info->ir1_code_buffer); + } + g_free(info); } +} + +void reload_tree_init(void) +{ + if (reload_tree != NULL) { + reload_tree_clear(); + } + reload_tree = g_tree_new_full(compare_page_addr, NULL, NULL, free_reload_info); +} + +void reload_tree_insert(reload_info *reload_node) +{ + if (reload_tree == NULL) { + reload_tree_init(); + } + + if (reload_node == NULL) { + return; + } + + g_tree_insert(reload_tree, &reload_node->page_addr, reload_node); +} + +reload_info *reload_tree_lookup(target_ulong page_addr) +{ + if (reload_tree == NULL) { + return NULL; + } + + return (reload_info *)g_tree_lookup(reload_tree, &page_addr); +} + +void reload_tree_remove(reload_info *reload_node) +{ + if (reload_tree == NULL || reload_node == NULL) { + return; + } + + g_tree_remove(reload_tree, &reload_node->page_addr); +} + +gint get_reload_node_num(void) +{ + if (reload_tree == NULL) { + return 0; + } + + return g_tree_nnodes(reload_tree); +} + +static gboolean foreach_print_callback(gpointer key, gpointer value, gpointer data) +{ + reload_info *info = (reload_info *)value; + fprintf(stderr, "Page addr: 0x%lx, TB num: %d\n", (uint64_t)info->page_addr, info->tb_num); + return false; +} + +void reload_tree_foreach(void) +{ + if (reload_tree == NULL) { + return; + } + + g_tree_foreach(reload_tree, foreach_print_callback, NULL); +} + +void reload_tree_clear(void) +{ + if (reload_tree != NULL) { + g_tree_destroy(reload_tree); + reload_tree = NULL; + } +}