LATX, opt: SMC TB reload

accel/tcg/translate-all.c

@@ -80,6 +80,7 @@
 #include "accel/tcg/internal.h"
 #include "ts.h"
 #include "latx-smc.h"
+#include "smc_reload.h"
 #endif
 #ifdef CONFIG_LATX_FAST_JMPCACHE
 #include "exec/fasttb.h"
@@ -564,7 +565,7 @@ static bool is_shadow_page_shmm(target_ulong address)
     return spd && spd->is_shmm;
 }
 
-static bool is_shadow_page_not_shmm(target_ulong address)
+bool is_shadow_page_not_shmm(target_ulong address)
 {
     ShadowPageDesc *spd = page_get_target_data(address);
     return spd && !(spd->is_shmm);
@@ -1682,6 +1683,10 @@ static void gen_aot_and_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
 
 void tb_flush(CPUState *cpu)
 {
+    if (option_smc_reload) {
+        reload_tree_clear();
+    }
+
     if (tcg_enabled()) {
         unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
 
@@ -3583,10 +3588,60 @@ bool pageflags_set_clear(target_ulong start, target_ulong last,
     return inval_tb;
 }
 
+static void add_smc_node(tb_page_addr_t start, tb_page_addr_t end)
+{
+    if (!option_smc_reload || !option_aot) {
+        return;
+    }
+    if (segment_tree_lookup2(start, end)) {
+        return;
+    }
+
+    TranslationBlock *tb;
+    PageForEachNext next_tb;
+    int ir1_size;
+    int curr_page_tb_num;
+
+    for (target_ulong curr_page = start; curr_page < end; curr_page += TARGET_PAGE_SIZE) {
+        curr_page_tb_num = 0;
+        ir1_size = 0;
+        PAGE_FOR_EACH_TB(curr_page, curr_page + TARGET_PAGE_SIZE, unused, tb, next_tb) {
+            if ((tb->pc & TARGET_PAGE_MASK) == curr_page) {
+                curr_page_tb_num++;
+                ir1_size += tb->size;
+            }
+        }
+        if (curr_page_tb_num) {
+            reload_info *reload_node = (reload_info *)malloc(sizeof(reload_info));
+            reload_node->tb_num = curr_page_tb_num;
+            reload_node->page_addr = curr_page;
+            assert(reload_node);
+            reload_node->tb_vector =
+                (TranslationBlock **)malloc(curr_page_tb_num * sizeof(TranslationBlock *));
+            assert(reload_node->tb_vector);
+            reload_node->ir1_code_buffer = malloc(ir1_size);
+            assert(reload_node->ir1_code_buffer);
+            int tb_id = 0;
+            ir1_size = 0;
+            PAGE_FOR_EACH_TB(curr_page, curr_page + TARGET_PAGE_SIZE, unused, tb, next_tb) {
+                if ((tb->pc & TARGET_PAGE_MASK) == curr_page) {
+                    memcpy(reload_node->ir1_code_buffer + ir1_size,
+                            (void *)(uintptr_t)tb->pc, tb->size);
+                    reload_node->tb_vector[tb_id++] = tb;
+                    ir1_size += tb->size;
+                }
+            }
+            reload_tree_insert(reload_node);
+        }
+    }
+}
+
 /* Modify the flags of a page and invalidate the code if necessary.
+   Save SMC TB to reload_tree if necessary.
    The flag PAGE_WRITE_ORG is positioned automatically depending
    on PAGE_WRITE.  The mmap_lock should already be held.  */
-void page_set_flags(target_ulong start, target_ulong end, int flags)
+void page_set_flags_tb_reload(target_ulong start, target_ulong end,
+        int flags, bool tb_reload)
 {
 #ifdef CONFIG_LATX_PERF
     latx_timer_start(TIMER_PAGE_FLAGS);
@@ -3657,13 +3712,24 @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
     }
 
     if (inval_tb) {
+        if (tb_reload) {
+            add_smc_node(start, end);
+        }
         tb_invalidate_phys_range(start, end);
     }
 #ifdef CONFIG_LATX_PERF
     latx_timer_stop(TIMER_PAGE_FLAGS);
 #endif
 }
 
+/* Modify the flags of a page and invalidate the code if necessary.
+   The flag PAGE_WRITE_ORG is positioned automatically depending
+   on PAGE_WRITE.  The mmap_lock should already be held.  */
+void page_set_flags(target_ulong start, target_ulong end, int flags)
+{
+    page_set_flags_tb_reload(start, end, flags, false);
+}
+
 typedef struct TargetPageDataNode {
     IntervalTreeNode itree;
     void *target_data;

include/exec/cpu-all.h

@@ -291,6 +291,8 @@ target_ulong get_first_page(target_ulong start, target_ulong end);
 int page_get_flags(target_ulong address);
 void page_clear_overflow(target_ulong start, target_ulong end);
 void page_set_flags(target_ulong start, target_ulong end, int flags);
+void page_set_flags_tb_reload(target_ulong start, target_ulong end,
+        int flags, bool tb_reload);
 bool pageflags_set_clear(target_ulong start, target_ulong last,
                                 int set_flags, int clear_flags);
 int page_get_page_state(target_ulong address);

include/exec/translate-all.h

@@ -23,6 +23,7 @@
 extern void *l1_map[];
 extern IntervalTreeRoot pageflags_root;
 /* translate-all.c */
+bool is_shadow_page_not_shmm(target_ulong address);
 struct page_collection *page_collection_lock(tb_page_addr_t start,
                                              tb_page_addr_t end);
 void page_collection_unlock(struct page_collection *set);

linux-user/main.c

@@ -726,6 +726,14 @@ static void handle_arg_latx_monitor_shared_mem(const char *arg)
     option_monitor_shared_mem = strtol(arg, NULL, 0);
 }
 
+static void handle_arg_smc_reload(const char *arg)
+{
+    option_smc_reload = strtol(arg, NULL, 0);
+    if (option_smc_reload) {
+        option_jr_ra = 0;
+    }
+}
+
 #ifdef CONFIG_LATX_AOT
 static void handle_arg_latx_aot(const char *arg)
 {
@@ -833,6 +841,8 @@ static const struct qemu_argument arg_table[] = {
     "",           "enable get real self maps"},
     {"latx-monitor-shared-mem",    "LATX_MONITOR_SHARED_MEM",     true,  handle_arg_latx_monitor_shared_mem,
     "",           "monitor shared memory, retranslate self modifying page"},
+    {"smc_reload",    "LATX_SMC_RELOAD",     true,  handle_arg_smc_reload,
+    "",           "reload SMC TB"},
 #ifdef CONFIG_LATX_AOT
     {"latx-aot",    "LATX_AOT",     true,  handle_arg_latx_aot,
     "",           "enable aot"},

linux-user/mmap.c

@@ -173,8 +173,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot)
             prot_tmp = page_get_flags(addr);
             if ((prot_tmp & PAGE_EXEC) && !(prot_tmp & PAGE_WRITE) &&
                 (prot_tmp & PAGE_WRITE_ORG) && (target_prot & PAGE_WRITE)) {
-                page_set_flags(addr, addr + TARGET_PAGE_SIZE,
-                                prot_tmp | PAGE_WRITE);
+                page_set_flags_tb_reload(addr, addr + TARGET_PAGE_SIZE,
+                                prot_tmp | PAGE_WRITE, true);
             }
             prot1 |= prot_tmp;
         }
@@ -184,8 +184,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot)
                 prot_tmp = page_get_flags(addr);
                 if ((prot_tmp & PAGE_EXEC) && !(prot_tmp & PAGE_WRITE) &&
                     (prot_tmp & PAGE_WRITE_ORG) && (target_prot & PAGE_WRITE)) {
-                    page_set_flags(addr, addr + TARGET_PAGE_SIZE,
-                                    prot_tmp | PAGE_WRITE);
+                    page_set_flags_tb_reload(addr, addr + TARGET_PAGE_SIZE,
+                                    prot_tmp | PAGE_WRITE, true);
                 }
                 prot1 |= prot_tmp;
             }
@@ -222,8 +222,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot)
             prot_tmp = page_get_flags(addr);
             if ((prot_tmp & PAGE_EXEC) && !(prot_tmp & PAGE_WRITE) &&
                 (prot_tmp & PAGE_WRITE_ORG) && (target_prot & PAGE_WRITE)) {
-                page_set_flags(addr, addr + TARGET_PAGE_SIZE,
-                                prot_tmp | PAGE_WRITE);
+                page_set_flags_tb_reload(addr, addr + TARGET_PAGE_SIZE,
+                                prot_tmp | PAGE_WRITE, true);
             }
             prot1 |= prot_tmp;
         }
@@ -258,7 +258,8 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot)
         }
     }
 
-    page_set_flags(start, start + len, page_flags);
+    page_set_flags_tb_reload(start, start + len, page_flags, true);
+
     if(target_prot == PROT_NONE) {
 #ifdef CONFIG_LATX_AOT
         if (option_aot && segment_tree_lookup2(start, start + len)) {

target/i386/latx/include/latx-options.h

@@ -44,6 +44,7 @@ extern int option_tunnel_lib;
 extern uint64_t option_end_trace_addr;
 extern uint64_t option_begin_trace_addr;
 extern int option_aot;
+extern int option_smc_reload;
 extern int option_aot_wine;
 extern int option_load_aot;
 extern int option_debug_aot;

target/i386/latx/include/smc_reload.h

@@ -0,0 +1,21 @@
+#ifndef __SMC_RELOAD_H_
+#define __SMC_RELOAD_H_
+
+#include "qemu-def.h"
+
+typedef struct reoload_info {
+    int tb_num;
+    target_ulong page_addr;
+    TranslationBlock **tb_vector;
+    void *ir1_code_buffer;
+} reload_info;
+
+void reload_tree_init(void);
+void reload_tree_insert(reload_info *reload_node);
+reload_info *reload_tree_lookup(target_ulong page_addr);
+void reload_tree_remove(reload_info *reload_node);
+gint get_reload_node_num(void);
+void reload_tree_foreach(void);
+void reload_tree_clear(void);
+
+#endif

target/i386/latx/latx-options.c

@@ -62,6 +62,7 @@ int option_debug_lative;
 int option_aot;
 int option_load_aot;
 int option_aot_wine;
+int option_smc_reload;
 int option_debug_aot;
 int option_imm_reg;
 int option_imm_rip;
@@ -238,6 +239,7 @@ void options_init(void)
     option_aot_wine = 0;
     option_debug_aot = 0;
 #endif
+    option_smc_reload = 0;
 #ifdef CONFIG_LATX_IMM_REG
     option_imm_reg = 0;
     option_imm_rip = 0;

target/i386/latx/sbt/aot_recover_tb.c

@@ -8,8 +8,10 @@
 #include "lsenv.h"
 #include "accel/tcg/internal.h"
 #include "include/exec/cpu-all.h"
+#include "include/exec/translate-all.h"
 #include "aot_page.h"
 #include "aot_smc.h"
+#include "smc_reload.h"
 #ifdef CONFIG_LATX_TU
 #include "tu.h"
 #endif
@@ -383,6 +385,55 @@ inline int load_page(target_ulong pc, uint32_t cflags, seg_info *info)
     return 1;
 }
 
+static int smc_page_reload(target_ulong page_addr, uint32_t cflags)
+{
+    if (!option_smc_reload) {
+        return 0;
+    }
+    reload_info *reload_node = reload_tree_lookup(page_addr);
+    if (!reload_node) {
+        return 0;
+    }
+
+    int p_flags = page_get_flags(page_addr);
+    if (!(p_flags & PAGE_READ)) {
+        return 0;
+    }
+    /*
+     * remove write prot of the page in case of paralell code modification,
+     * but the paired page cross 16K boundary is still not protected.
+     */
+    if ((p_flags & PAGE_WRITE) && !is_shadow_page_not_shmm(page_addr)) {
+        mprotect((void*)(page_addr & qemu_host_page_mask), qemu_host_page_size, PROT_READ);
+    }
+
+    tcg_ctx->tb_cflags = cflags;
+
+    int ir1_offset = 0;
+    for (int tb_id = 0; tb_id < reload_node->tb_num; tb_id++) {
+        TranslationBlock *tb = reload_node->tb_vector[tb_id];
+        if (!memcmp((void *)(uintptr_t)tb->pc,
+                    reload_node->ir1_code_buffer + ir1_offset, tb->size)) {
+            tb->cflags = tb->cflags & ~CF_INVALID;
+            if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+                tb_reset_jump(tb, 0);
+            }
+            if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
+                tb_reset_jump(tb, 1);
+            }
+            tb->jmp_list_head = (uintptr_t)NULL;
+            tb->jmp_list_next[0] = (uintptr_t)NULL;
+            tb->jmp_list_next[1] = (uintptr_t)NULL;
+            tb->jmp_dest[0] = (uintptr_t)NULL;
+            tb->jmp_dest[1] = (uintptr_t)NULL;
+            aot_tb_register(tb);
+        }
+        ir1_offset += tb->size;
+    }
+    reload_tree_remove(reload_node);
+    return 1;
+}
+
 int load_aot(target_ulong pc, uint32_t cflags)
 {
     if (in_pre_translate) {
@@ -391,6 +442,9 @@ int load_aot(target_ulong pc, uint32_t cflags)
 
     seg_info *info = segment_tree_lookup(pc);
     if (info == NULL || info->buffer == NULL) {
+        if (smc_page_reload(pc & TARGET_PAGE_MASK, cflags)) {
+            return 1;
+        }
         page_set_page_state(pc, PAGE_NOINFO);
         return 0;
     }

target/i386/latx/sbt/meson.build

@@ -8,4 +8,5 @@ i386_ss.add(when: 'CONFIG_LATX', if_true: files(
   'aot_lib.c',
   'aot_page.c',
   'aot_smc.c',
+  'smc_reload.c',
 ))