LATX: extend EFLAGS reduction across TU calls and native patterns

linux-user/main.c

@@ -659,6 +659,19 @@ static void handle_arg_latx_prlimit(const char *arg)
     option_prlimit = strtol(arg, NULL, 0);
 }
 
+static void handle_arg_latx_eflags_cross(const char *arg)
+{
+    int value;
+
+    if (qemu_strtoi(arg, NULL, 0, &value) || value < 0 || value > 1) {
+        fprintf(stderr,
+                "LATX_EFLAGS_CROSS must be exactly 0 or 1 (got '%s')\n",
+                arg);
+        exit(EXIT_FAILURE);
+    }
+    option_eflags_cross = value;
+}
+
 #ifdef CONFIG_LATX_AVX_OPT
 static void handle_arg_latx_avx_cpuid(const char *arg)
 {
@@ -920,6 +933,9 @@ static const struct qemu_argument arg_table[] = {
     "",           "enable rounding opt"},
     {"latx-cvt-opt",    "LATX_CVT_OPT",     true,  handle_arg_latx_cvt_opt,
     "",           "enable cvt opt"},
+    {"latx-eflags-cross", "LATX_EFLAGS_CROSS", true,
+    handle_arg_latx_eflags_cross, "0|1",
+    "enable TU EFLAGS analysis across direct CALL and RET"},
 #if defined(CONFIG_LATX_AVX_OPT)
     {"latx-avx-cpuid",    "LATX_AVX_CPUID",     true,  handle_arg_latx_avx_cpuid,
     "",           "enable avx cpuid"},

target/i386/latx/include/flag-reduction.h

@@ -40,11 +40,14 @@ typedef struct {
 #define __ALL_EFLAGS (__OF | __SF | __ZF | __AF | __PF | __CF)
 
 void flag_gen(IR1_INST *pir1);
+uint8 flag_reduction_get_arch_use(IR1_INST *pir1);
 
 #ifdef CONFIG_LATX_FLAG_REDUCTION
 uint8 pending_use_of_succ(void *tb, int indirect_depth, int max_depth);
 void flag_reduction(IR1_INST *pir1, uint8 *pending_use);
 uint8 flag_reduction_check(TranslationBlock *tb);
+void flag_reduction_get_tb_summary(TranslationBlock *tb,
+        uint8 *use_before_def, uint8 *must_def);
 
 #define DEF_FLAG_RDTN(_prex) \
         uint8 _prex##_pending_use = __ALL_EFLAGS;

target/i386/latx/include/insts-pattern.h

@@ -163,4 +163,13 @@ bool insts_pattern_scan_jcc_end(TranslationBlock *tb, IR1_INST *ir1, int index,
 
 bool try_translate_instptn(IR1_INST *pir1);
 
+static inline uint8_t instptn_replaced_eflag_use(IR1_INST *ir1)
+{
+#ifdef CONFIG_LATX_INSTS_PATTERN
+    return ir1->instptn.replaced_eflag_use;
+#else
+    return 0;
+#endif
+}
+
 #endif

target/i386/latx/include/ir1.h

@@ -143,6 +143,8 @@ typedef struct IR1_INST {
 #ifdef CONFIG_LATX_INSTS_PATTERN
     struct {
         int opc;
+        /* EFLAGS reads replaced by the native pattern translation. */
+        uint8_t replaced_eflag_use;
         struct IR1_INST * next; /* index of IR1 list */
     } instptn;
 #endif

target/i386/latx/include/latx-config.h

@@ -44,6 +44,7 @@ void latx_handle_args(char *filename);
 
 #ifdef CONFIG_LATX_TU
 void target_disasm(struct TranslationBlock *tb, int max_insns);
+extern __thread bool suppress_disasm_side_effects;
 #endif
 
 #ifdef CONFIG_LATX_LARGE_CC

target/i386/latx/include/latx-options.h

@@ -27,6 +27,9 @@ extern int option_instptn;
 #ifdef CONFIG_LATX_FLAG_REDUCTION
 extern int option_flag_reduction;
 #endif
+
+/* Proof-driven TU EFLAGS analysis across direct CALL/RET. */
+extern int option_eflags_cross;
 #ifdef CONFIG_LATX_TU
 extern int option_tu_link;
 #endif
@@ -132,6 +135,7 @@ extern unsigned long long counter_mips_tr;
     ENVFUN(LATX_PRLIMIT, handle_arg_latx_prlimit) \
     ENVFUN(LATX_ROUNDING_OPT, handle_arg_latx_rounding) \
     ENVFUN(LATX_CVT_OPT, handle_arg_latx_cvt_opt) \
+    ENVFUN(LATX_EFLAGS_CROSS, handle_arg_latx_eflags_cross) \
     ENVFUN(LATX_FPUTAG, handle_arg_latx_fputag) \
     ENVFUN(SAVE_XMM, handle_arg_save_xmm) \
     ENVFUN(LATX_JRRA, handle_arg_latx_jrra) \

target/i386/latx/ir1/ir1.c

@@ -390,6 +390,7 @@ ADDRX ir1_disasm(IR1_INST *ir1, uint8_t *addr, ADDRX t_pc, int ir1_num, void *pi
     }
 #ifdef CONFIG_LATX_INSTS_PATTERN
     ir1->instptn.opc  = INSTPTN_OPC_NONE;
+    ir1->instptn.replaced_eflag_use = 0;
     ir1->instptn.next = NULL;
 #endif
     return (ADDRX)(ir1->info->address + ir1->info->size);

target/i386/latx/latx-config.c

@@ -27,15 +27,16 @@
 void target_disasm(struct TranslationBlock *tb, int max_insns)
 {
     /* max_insns = 100; */
-    counter_tb_tr += 1;
-
-    trace_xtm_tr_tb((void *)tb, (void *)tb->tc.ptr,
-                    (void *)(unsigned long long)tb->pc);
-
-    if (option_dump) {
-        qemu_log("=====================================\n");
-        qemu_log("|| TB translation : %14p ||\n", tb);
-        qemu_log("=====================================\n");
+    if (!suppress_disasm_side_effects) {
+        counter_tb_tr += 1;
+        trace_xtm_tr_tb((void *)tb, (void *)tb->tc.ptr,
+                        (void *)(unsigned long long)tb->pc);
+
+        if (option_dump) {
+            qemu_log("=====================================\n");
+            qemu_log("|| TB translation : %14p ||\n", tb);
+            qemu_log("=====================================\n");
+        }
     }
 
     /* target => IR1
@@ -62,14 +63,18 @@ void target_disasm(struct TranslationBlock *tb, int max_insns)
 #endif
 
 #ifdef CONFIG_LATX_DEBUG
-    counter_ir1_tr += tb->icount;
+    if (!suppress_disasm_side_effects) {
+        counter_ir1_tr += tb->icount;
+    }
 #endif
 
     /* tr_disasm(tb, max_insns); */
 
 #ifdef CONFIG_LATX_PROFILER
-    qatomic_set(&prof->tr_disasm_time,
-                prof->tr_disasm_time + profile_getclock() - ti);
+    if (!suppress_disasm_side_effects) {
+        qatomic_set(&prof->tr_disasm_time,
+                    prof->tr_disasm_time + profile_getclock() - ti);
+    }
 #endif
 
     /* ir1_optimization(tb); */

target/i386/latx/latx-options.c

@@ -31,6 +31,7 @@ int option_avx_cpuid = 0;
 #ifdef CONFIG_LATX_FLAG_REDUCTION
 int option_flag_reduction = 1;
 #endif
+int option_eflags_cross;
 
 int option_lative = 0;
 #if defined(CONFIG_LATX_JRRA_STACK) && defined(CONFIG_LATX_LSFPU)
@@ -236,6 +237,7 @@ void options_init(void)
     option_latx_disassemble_trace_cmp = 0;
     option_enable_lasx = 1;
     option_vpaes = 0;
+    option_eflags_cross = 0;
 
     counter_tb_exec = 0;
     counter_tb_tr = 0;

target/i386/latx/optimization/flag-reduction.c

@@ -8,6 +8,7 @@
 #include "ir1.h"
 #include "translate.h"
 #include "flag-reduction.h"
+#include "insts-pattern.h"
 
 /**
  * @brief ir1 opcode (x86) per instruction eflags using table
@@ -293,6 +294,19 @@ static const IR1_EFLAG_USEDEF *ir1_opcode_to_eflag_usedef(IR1_INST *ir1)
     return ir1_opcode_eflag_usedef + (ir1_opcode(ir1) - dt_X86_INS_INVALID);
 }
 
+uint8 flag_reduction_get_arch_use(IR1_INST *ir1)
+{
+    return ir1_opcode_to_eflag_usedef(ir1)->use & __ALL_EFLAGS;
+}
+
+static inline IR1_EFLAG_USEDEF ir1_effective_eflag_usedef(IR1_INST *ir1)
+{
+    IR1_EFLAG_USEDEF usedef = *ir1_opcode_to_eflag_usedef(ir1);
+
+    usedef.use &= ~instptn_replaced_eflag_use(ir1);
+    return usedef;
+}
+
 #ifdef CONFIG_LATX_FLAG_REDUCTION
 
 static inline uint32_t rotate_shift_get_masked_imm(IR1_OPND *d, IR1_OPND *s)
@@ -362,6 +376,87 @@ static inline bool cmp_scas_need_zf(IR1_INST *pir1)
             ir1_opcode(pir1) == dt_X86_INS_SCASQ);
 }
 
+/*
+ * Some instructions only update EFLAGS when their runtime count is non-zero.
+ * Such an update is not a must-def and therefore cannot stop EFLAGS liveness
+ * propagated from a successor TB.
+ */
+static bool eflags_def_may_be_skipped(IR1_INST *pir1)
+{
+    IR1_OPCODE op = ir1_opcode(pir1);
+
+    if ((op == dt_X86_INS_CMPSB || op == dt_X86_INS_CMPSW ||
+         op == dt_X86_INS_CMPSD || op == dt_X86_INS_CMPSQ ||
+         op == dt_X86_INS_SCASB || op == dt_X86_INS_SCASW ||
+         op == dt_X86_INS_SCASD || op == dt_X86_INS_SCASQ) &&
+        (ir1_prefix(pir1) == dt_X86_PREFIX_REPE ||
+         ir1_prefix(pir1) == dt_X86_PREFIX_REPNE)) {
+        /* RCX may be zero, in which case no comparison takes place. */
+        return true;
+    }
+
+    if (op == dt_X86_INS_RCL || op == dt_X86_INS_RCR) {
+        /* The effective count also depends on the operand width. */
+        return true;
+    }
+
+    if (op == dt_X86_INS_ROL || op == dt_X86_INS_ROR ||
+        op == dt_X86_INS_SAL || op == dt_X86_INS_SAR ||
+        op == dt_X86_INS_SHL || op == dt_X86_INS_SHR ||
+        op == dt_X86_INS_SHLD || op == dt_X86_INS_SHRD) {
+        int opnd_num = ir1_get_opnd_num(pir1);
+
+        /* A one-operand form has an implicit count of one. */
+        if (opnd_num == 1) {
+            return false;
+        }
+
+        IR1_OPND *count = ir1_get_opnd(pir1, opnd_num - 1);
+        if (!ir1_opnd_is_imm(count)) {
+            return true;
+        }
+
+        IR1_OPND *dest = ir1_get_opnd(pir1, 0);
+        return rotate_shift_get_masked_imm(dest, count) == 0;
+    }
+
+    return false;
+}
+
+/*
+ * Summarize a straight-line TB using the same use/def table as the existing
+ * flag-reduction pass.
+ *
+ * use_before_def: flags read before this TB certainly overwrites them.
+ * must_def:       flags certainly overwritten on every normal execution.
+ */
+void flag_reduction_get_tb_summary(TranslationBlock *tb,
+        uint8 *use_before_def, uint8 *must_def)
+{
+    uint8 preserve = __ALL_EFLAGS;
+    uint8 use = __NONE;
+
+    for (int i = 0; i < tb_ir1_num(tb); i++) {
+        IR1_INST *pir1 = tb_ir1_inst(tb, i);
+        IR1_EFLAG_USEDEF usedef = ir1_effective_eflag_usedef(pir1);
+        uint8 curr_use = usedef.use & __ALL_EFLAGS;
+        uint8 curr_def = (usedef.def | usedef.undef) & __ALL_EFLAGS;
+
+        if (cmp_scas_need_zf(pir1)) {
+            curr_use |= __ZF;
+        }
+
+        use |= curr_use & preserve;
+        if (eflags_def_may_be_skipped(pir1)) {
+            curr_def = __NONE;
+        }
+        preserve &= ~curr_def;
+    }
+
+    *use_before_def = use;
+    *must_def = __ALL_EFLAGS & ~preserve;
+}
+
 /**
  * @brief Find if we have enough information by scan TB
  *
@@ -383,16 +478,16 @@ static bool flag_reduction_pass1(void *tb)
     /* scanning if this insts will def ALL_EFLAGS */
     for (int i = tb_ir1_num(ptb) - 1; i >= 0; --i) {
         pir1 = tb_ir1_inst(ptb, i);
-        const IR1_EFLAG_USEDEF *usedef = ir1_opcode_to_eflag_usedef(pir1);
+        IR1_EFLAG_USEDEF usedef = ir1_effective_eflag_usedef(pir1);
 
         /*
          * NOTE: if you find some insts will use ALL_EFLAGS
          * you can add this case:
-         * if (usedef->use == __ALL_EFLAGS) return false;
+         * if (usedef.use == __ALL_EFLAGS) return false;
          */
-        if (usedef->use != __NONE) {
+        if (usedef.use != __NONE) {
             goto _false_path;
-        } else if (usedef->def == __NONE) {
+        } else if (usedef.def == __NONE) {
             /* curr_inst not def any flags */
             continue;
         } else {
@@ -442,7 +537,7 @@ void flag_reduction(IR1_INST *pir1, uint8 *pending_use)
      *   - flag use:     current inst will use flags
      *   - flag undef:   current inst mark undef flags
      */
-    IR1_EFLAG_USEDEF curr_usedef = *ir1_opcode_to_eflag_usedef(pir1);
+    IR1_EFLAG_USEDEF curr_usedef = ir1_effective_eflag_usedef(pir1);
 
 #ifndef CONFIG_LATX_RADICAL_EFLAGS
     current_def = curr_usedef.def;