From 308fad2fe457e5b1b41ca00a3603ec641a2408b8 Mon Sep 17 00:00:00 2001 From: Meng-Hung Chen Date: Tue, 29 Apr 2025 13:15:42 +0800 Subject: [PATCH] Fix T2C in system simulation - Add MMU-related code in memory operation - Fix indirect branch handling --- .github/workflows/main.yml | 8 +- src/jit.h | 7 +- src/system.c | 12 +- src/system.h | 7 ++ src/t2c.c | 106 ++++++++++------ src/t2c_template.c | 251 ++++++++++++++++++++++++++++++------- 6 files changed, 290 insertions(+), 101 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4f9c309..9ccf501a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -279,9 +279,9 @@ jobs: env: CC: ${{ steps.install_cc.outputs.cc }} run: | - make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_T2C=0 ENABLE_MOP_FUSION=0 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL + make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL bash -c "${BOOT_LINUX_TEST}" - make ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_T2C=0 ENABLE_MOP_FUSION=0 clean + make ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 clean if: ${{ always() }} - name: Architecture test env: @@ -484,9 +484,9 @@ jobs: env: CC: ${{ steps.install_cc.outputs.cc }} run: | - make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_T2C=0 ENABLE_MOP_FUSION=0 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL + make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL bash -c "${BOOT_LINUX_TEST}" - make ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_T2C=0 ENABLE_MOP_FUSION=0 clean + make ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 clean if: ${{ always() }} - name: Architecture test env: diff --git a/src/jit.h b/src/jit.h index 4bbafa2f..0f1ecdf5 100644 --- a/src/jit.h +++ b/src/jit.h @@ -71,12 +71,13 @@ typedef void (*exec_t2c_func_t)(riscv_t *); #define N_JIT_CACHE_ENTRIES (1 << 12) struct jit_cache { - uint64_t pc; /* program counter, easy to build LLVM IR with 64-bit width */ - void *entry; /* entry of JIT-ed code */ + uint64_t key; /* program counter, composed to satp if it's in system + simulation */ + void *entry; /* entry of JIT-ed code */ }; struct jit_cache *jit_cache_init(); void jit_cache_exit(struct jit_cache *cache); -void jit_cache_update(struct jit_cache *cache, uint32_t pc, void *entry); +void jit_cache_update(struct jit_cache *cache, uint64_t key, void *entry); void jit_cache_clear(struct jit_cache *cache); #endif diff --git a/src/system.c b/src/system.c index 1895740c..a87b5548 100644 --- a/src/system.c +++ b/src/system.c @@ -216,7 +216,7 @@ static uint32_t mmu_ifetch(riscv_t *rv, const uint32_t vaddr) return memory_ifetch(ppn | offset); } -static uint32_t mmu_read_w(riscv_t *rv, const uint32_t vaddr) +uint32_t mmu_read_w(riscv_t *rv, const uint32_t vaddr) { uint32_t addr = rv->io.mem_translate(rv, vaddr, R); @@ -235,7 +235,7 @@ static uint32_t mmu_read_w(riscv_t *rv, const uint32_t vaddr) __UNREACHABLE; } -static uint16_t mmu_read_s(riscv_t *rv, const uint32_t vaddr) +uint16_t mmu_read_s(riscv_t *rv, const uint32_t vaddr) { uint32_t addr = rv->io.mem_translate(rv, vaddr, R); @@ -247,7 +247,7 @@ static uint16_t mmu_read_s(riscv_t *rv, const uint32_t vaddr) return memory_read_s(addr); } -static uint8_t mmu_read_b(riscv_t *rv, const uint32_t vaddr) +uint8_t mmu_read_b(riscv_t *rv, const uint32_t vaddr) { uint32_t addr = rv->io.mem_translate(rv, vaddr, R); @@ -266,7 +266,7 @@ static uint8_t mmu_read_b(riscv_t *rv, const uint32_t vaddr) __UNREACHABLE; } -static void mmu_write_w(riscv_t *rv, const uint32_t vaddr, const uint32_t val) +void mmu_write_w(riscv_t *rv, const uint32_t vaddr, const uint32_t val) { uint32_t addr = rv->io.mem_translate(rv, vaddr, W); @@ -285,7 +285,7 @@ static void mmu_write_w(riscv_t *rv, const uint32_t vaddr, const uint32_t val) #endif } -static void mmu_write_s(riscv_t *rv, const uint32_t vaddr, const uint16_t val) +void mmu_write_s(riscv_t *rv, const uint32_t vaddr, const uint16_t val) { uint32_t addr = rv->io.mem_translate(rv, vaddr, W); @@ -300,7 +300,7 @@ static void mmu_write_s(riscv_t *rv, const uint32_t vaddr, const uint16_t val) memory_write_s(addr, (uint8_t *) &val); } -static void mmu_write_b(riscv_t *rv, const uint32_t vaddr, const uint8_t val) +void mmu_write_b(riscv_t *rv, const uint32_t vaddr, const uint8_t val) { uint32_t addr = rv->io.mem_translate(rv, vaddr, W); diff --git a/src/system.h b/src/system.h index 872c4201..af94f612 100644 --- a/src/system.h +++ b/src/system.h @@ -173,3 +173,10 @@ uint32_t *mmu_walk(riscv_t *rv, const uint32_t addr, uint32_t *level); offset = level == 1 ? vaddr & MASK((RV_PG_SHIFT + 10)) \ : vaddr & MASK(RV_PG_SHIFT); \ } while (0) + +uint8_t mmu_read_b(riscv_t *rv, const uint32_t vaddr); +uint16_t mmu_read_s(riscv_t *rv, const uint32_t vaddr); +uint32_t mmu_read_w(riscv_t *rv, const uint32_t vaddr); +void mmu_write_b(riscv_t *rv, const uint32_t vaddr, const uint8_t val); +void mmu_write_s(riscv_t *rv, const uint32_t vaddr, const uint16_t val); +void mmu_write_w(riscv_t *rv, const uint32_t vaddr, const uint32_t val); diff --git a/src/t2c.c b/src/t2c.c index b4b407e6..343b85e6 100644 --- a/src/t2c.c +++ b/src/t2c.c @@ -49,21 +49,21 @@ FORCE_INLINE LLVMBasicBlockRef t2c_block_map_search(struct LLVM_block_map *map, return NULL; } -#define T2C_OP(inst, code) \ - static void t2c_##inst( \ - LLVMBuilderRef *builder UNUSED, LLVMTypeRef *param_types UNUSED, \ - LLVMValueRef start UNUSED, LLVMBasicBlockRef *entry UNUSED, \ - LLVMBuilderRef *taken_builder UNUSED, \ - LLVMBuilderRef *untaken_builder UNUSED, riscv_t *rv UNUSED, \ - uint64_t mem_base UNUSED, rv_insn_t *ir UNUSED) \ - { \ - LLVMValueRef timer_ptr = t2c_gen_timer_addr(start, builder, ir); \ - LLVMValueRef timer = \ - LLVMBuildLoad2(*builder, LLVMInt64Type(), timer_ptr, ""); \ - timer = LLVMBuildAdd(*builder, timer, \ - LLVMConstInt(LLVMInt64Type(), 1, false), ""); \ - LLVMBuildStore(*builder, timer, timer_ptr); \ - code; \ +#define T2C_OP(inst, code) \ + static void t2c_##inst( \ + LLVMBuilderRef *builder UNUSED, LLVMTypeRef *param_types UNUSED, \ + LLVMValueRef start UNUSED, LLVMBasicBlockRef *entry UNUSED, \ + LLVMBuilderRef *taken_builder UNUSED, \ + LLVMBuilderRef *untaken_builder UNUSED, riscv_t *rv UNUSED, \ + uint64_t mem_base UNUSED, block_t *block UNUSED, rv_insn_t *ir UNUSED) \ + { \ + LLVMValueRef timer_ptr = t2c_gen_timer_addr(start, builder, ir); \ + LLVMValueRef timer = \ + LLVMBuildLoad2(*builder, LLVMInt64Type(), timer_ptr, ""); \ + timer = LLVMBuildAdd(*builder, timer, \ + LLVMConstInt(LLVMInt64Type(), 1, false), ""); \ + LLVMBuildStore(*builder, timer, timer_ptr); \ + code; \ } #define T2C_LLVM_GEN_ADDR(reg, rv_member, ir_member) \ @@ -190,6 +190,7 @@ typedef void (*t2c_codegen_block_func_t)(LLVMBuilderRef *builder UNUSED, LLVMBuilderRef *untaken_builder UNUSED, riscv_t *rv UNUSED, uint64_t mem_base UNUSED, + block_t *block UNUSED, rv_insn_t *ir UNUSED); static void t2c_trace_ebb(LLVMBuilderRef *builder, @@ -197,10 +198,12 @@ static void t2c_trace_ebb(LLVMBuilderRef *builder, LLVMValueRef start, LLVMBasicBlockRef *entry, riscv_t *rv, - rv_insn_t *ir, + block_t *block, set_t *set, struct LLVM_block_map *map) { + rv_insn_t *ir = block->ir_head; + if (set_has(set, ir->pc)) return; set_add(set, ir->pc); @@ -210,7 +213,7 @@ static void t2c_trace_ebb(LLVMBuilderRef *builder, while (1) { ((t2c_codegen_block_func_t) dispatch_table[ir->opcode])( builder, param_types, start, entry, &tk, &utk, rv, - (uint64_t) ((memory_t *) PRIV(rv)->mem)->mem_base, ir); + (uint64_t) ((memory_t *) PRIV(rv)->mem)->mem_base, block, ir); if (!ir->next) break; ir = ir->next; @@ -222,15 +225,21 @@ static void t2c_trace_ebb(LLVMBuilderRef *builder, LLVMBuildBr(utk, t2c_block_map_search(map, ir->branch_untaken->pc)); else { - LLVMBasicBlockRef untaken_entry = - LLVMAppendBasicBlock(start, - "untaken_" - "entry"); - LLVMBuilderRef untaken_builder = LLVMCreateBuilder(); - LLVMPositionBuilderAtEnd(untaken_builder, untaken_entry); - LLVMBuildBr(utk, untaken_entry); - t2c_trace_ebb(&untaken_builder, param_types, start, - &untaken_entry, rv, ir->branch_untaken, set, map); + block_t *blk = + cache_get(rv->block_cache, ir->branch_untaken->pc, false); + if (blk && blk->translatable +#if RV32_HAS(SYSTEM) + && blk->satp == block->satp +#endif + ) { + LLVMBasicBlockRef untaken_entry = + LLVMAppendBasicBlock(start, "untaken_entry"); + LLVMBuilderRef untaken_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(untaken_builder, untaken_entry); + LLVMBuildBr(utk, untaken_entry); + t2c_trace_ebb(&untaken_builder, param_types, start, + &untaken_entry, rv, blk, set, map); + } } } if (ir->branch_taken) { @@ -238,14 +247,21 @@ static void t2c_trace_ebb(LLVMBuilderRef *builder, LLVMBuildBr(tk, t2c_block_map_search(map, ir->branch_taken->pc)); else { - LLVMBasicBlockRef taken_entry = LLVMAppendBasicBlock(start, - "taken_" - "entry"); - LLVMBuilderRef taken_builder = LLVMCreateBuilder(); - LLVMPositionBuilderAtEnd(taken_builder, taken_entry); - LLVMBuildBr(tk, taken_entry); - t2c_trace_ebb(&taken_builder, param_types, start, &taken_entry, - rv, ir->branch_taken, set, map); + block_t *blk = + cache_get(rv->block_cache, ir->branch_taken->pc, false); + if (blk && blk->translatable +#if RV32_HAS(SYSTEM) + && blk->satp == block->satp +#endif + ) { + LLVMBasicBlockRef taken_entry = + LLVMAppendBasicBlock(start, "taken_entry"); + LLVMBuilderRef taken_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(taken_builder, taken_entry); + LLVMBuildBr(tk, taken_entry); + t2c_trace_ebb(&taken_builder, param_types, start, + &taken_entry, rv, blk, set, map); + } } } } @@ -254,6 +270,9 @@ static void t2c_trace_ebb(LLVMBuilderRef *builder, void t2c_compile(riscv_t *rv, block_t *block) { LLVMModuleRef module = LLVMModuleCreateWithName("my_module"); + /* FIXME: riscv_t structure would change according to different + * configuration. The linked block might jump to the wrong function pointer. + */ LLVMTypeRef io_members[] = { LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), @@ -291,8 +310,7 @@ void t2c_compile(riscv_t *rv, block_t *block) struct LLVM_block_map map; map.count = 0; /* Translate custon IR into LLVM IR */ - t2c_trace_ebb(&builder, param_types, start, &entry, rv, block->ir_head, - &set, &map); + t2c_trace_ebb(&builder, param_types, start, &entry, rv, block, &set, &map); /* Offload LLVM IR to LLVM backend */ char *error = NULL, *triple = LLVMGetDefaultTargetTriple(); LLVMExecutionEngineRef engine; @@ -319,7 +337,15 @@ void t2c_compile(riscv_t *rv, block_t *block) /* Return the function pointer of T2C generated machine code */ block->func = (exec_t2c_func_t) LLVMGetPointerToGlobal(engine, start); - jit_cache_update(rv->jit_cache, block->pc_start, block->func); + +#if RV32_HAS(SYSTEM) + uint64_t key = (uint64_t) block->pc_start | ((uint64_t) block->satp << 32); +#else + uint64_t key = (uint64_t) block->pc_start; +#endif + + jit_cache_update(rv->jit_cache, key, block->func); + block->hot2 = true; } @@ -333,11 +359,11 @@ void jit_cache_exit(struct jit_cache *cache) free(cache); } -void jit_cache_update(struct jit_cache *cache, uint32_t pc, void *entry) +void jit_cache_update(struct jit_cache *cache, uint64_t key, void *entry) { - uint32_t pos = pc & (N_JIT_CACHE_ENTRIES - 1); + uint32_t pos = key & (N_JIT_CACHE_ENTRIES - 1); - cache[pos].pc = pc; + cache[pos].key = key; cache[pos].entry = entry; } diff --git a/src/t2c_template.c b/src/t2c_template.c index 01c9d8f3..8ec3e51b 100644 --- a/src/t2c_template.c +++ b/src/t2c_template.c @@ -18,14 +18,30 @@ T2C_OP(auipc, { t2c_gen_rd_addr(start, builder, ir)); }) +/* Query the block by pc and return if it is valid. */ +static bool t2c_check_valid_blk(riscv_t *rv, block_t *block UNUSED, uint32_t pc) +{ + block_t *blk = cache_get(rv->block_cache, pc, false); + if (!blk || !blk->translatable) + return false; + +#if RV32_HAS(SYSTEM) + if (blk->satp != block->satp) + return false; +#endif + + return true; +} + T2C_OP(jal, { if (ir->rd) T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 4, t2c_gen_rd_addr(start, builder, ir)); - if (ir->branch_taken) + if (ir->branch_taken && + t2c_check_valid_blk(rv, block, ir->branch_taken->pc)) { *taken_builder = *builder; - else { + } else { T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + ir->imm, t2c_gen_PC_addr(start, builder, ir)); LLVMBuildRetVoid(*builder); @@ -36,6 +52,7 @@ FORCE_INLINE void t2c_jit_cache_helper(LLVMBuilderRef *builder, LLVMValueRef start, LLVMValueRef addr, riscv_t *rv, + block_t *block UNUSED, rv_insn_t *ir) { LLVMBasicBlockRef true_path = LLVMAppendBasicBlock(start, ""); @@ -56,17 +73,27 @@ FORCE_INLINE void t2c_jit_cache_helper(LLVMBuilderRef *builder, *builder, addr, LLVMConstInt(LLVMInt32Type(), N_JIT_CACHE_ENTRIES - 1, false), ""); - /* get jit_cache_t::pc */ + /* get jit_cache_t::key */ LLVMValueRef cast = LLVMBuildIntCast2(*builder, hash, LLVMInt64Type(), false, ""); LLVMValueRef element_ptr = LLVMBuildInBoundsGEP2( *builder, t2c_jit_cache_struct_type, base, &cast, 1, ""); LLVMValueRef pc_ptr = LLVMBuildStructGEP2( *builder, t2c_jit_cache_struct_type, element_ptr, 0, ""); - LLVMValueRef pc = LLVMBuildLoad2(*builder, LLVMInt32Type(), pc_ptr, ""); /* compare with calculated destination */ - LLVMValueRef cmp = LLVMBuildICmp(*builder, LLVMIntEQ, pc, addr, ""); + +#if RV32_HAS(SYSTEM) + LLVMValueRef pc = LLVMBuildLoad2(*builder, LLVMInt64Type(), pc_ptr, ""); + LLVMValueRef key = T2C_LLVM_GEN_ALU64_IMM( + Add, LLVMBuildIntCast2(*builder, addr, LLVMInt64Type(), false, ""), + (uint64_t) block->satp << 32); +#else + LLVMValueRef pc = LLVMBuildLoad2(*builder, LLVMInt32Type(), pc_ptr, ""); + LLVMValueRef key = addr; +#endif + + LLVMValueRef cmp = LLVMBuildICmp(*builder, LLVMIntEQ, pc, key, ""); LLVMBuildCondBr(*builder, cmp, true_path, false_path); @@ -101,7 +128,7 @@ T2C_OP(jalr, { T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 4, t2c_gen_rd_addr(start, builder, ir)); - t2c_jit_cache_helper(builder, start, val_rs1, rv, ir); + t2c_jit_cache_helper(builder, start, val_rs1, rv, block, ir); }) #define BRANCH_FUNC(type, cond) \ @@ -115,18 +142,20 @@ T2C_OP(jalr, { LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); \ LLVMBuilderRef builder2 = LLVMCreateBuilder(); \ LLVMPositionBuilderAtEnd(builder2, taken); \ - if (ir->branch_taken) \ + if (ir->branch_taken && \ + t2c_check_valid_blk(rv, block, ir->branch_taken->pc)) { \ *taken_builder = builder2; \ - else { \ + } else { \ T2C_LLVM_GEN_STORE_IMM32(builder2, ir->pc + ir->imm, addr_PC); \ LLVMBuildRetVoid(builder2); \ } \ LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); \ LLVMBuilderRef builder3 = LLVMCreateBuilder(); \ LLVMPositionBuilderAtEnd(builder3, untaken); \ - if (ir->branch_untaken) \ + if (ir->branch_untaken && \ + t2c_check_valid_blk(rv, block, ir->branch_untaken->pc)) { \ *untaken_builder = builder3; \ - else { \ + } else { \ T2C_LLVM_GEN_STORE_IMM32(builder3, ir->pc + 4, addr_PC); \ LLVMBuildRetVoid(builder3); \ } \ @@ -140,62 +169,185 @@ BRANCH_FUNC(bge, SGE) BRANCH_FUNC(bltu, ULT) BRANCH_FUNC(bgeu, UGE) +#if RV32_HAS(SYSTEM) + +#include "system.h" + +#define t2c_mmu_wrapper(opcode) t2c_mmu_wrapper_##opcode + +#define T2C_MMU_LOAD(opcode, fn, bits, is_signed) \ + static void t2c_mmu_wrapper_##opcode(LLVMBuilderRef *builder, \ + LLVMValueRef start, rv_insn_t *ir) \ + { \ + LLVMValueRef val_rs1 = \ + LLVMBuildLoad2(*builder, LLVMInt32Type(), \ + t2c_gen_rs1_addr(start, builder, ir), ""); \ + LLVMValueRef vaddr = T2C_LLVM_GEN_ALU32_IMM(Add, val_rs1, ir->imm); \ + vaddr = LLVMBuildZExt(*builder, vaddr, LLVMInt64Type(), ""); \ + LLVMTypeRef param_types[] = {LLVMPointerType(LLVMInt64Type(), 0), \ + LLVMInt64Type()}; \ + LLVMTypeRef mmu_fn_type = \ + LLVMFunctionType(LLVMInt##bits##Type(), param_types, 2, 0); \ + LLVMValueRef mmu_fn_addr = \ + LLVMConstInt(LLVMInt64Type(), (uintptr_t) fn, false); \ + LLVMValueRef mmu_fn_ptr = LLVMBuildIntToPtr( \ + *builder, mmu_fn_addr, LLVMPointerType(mmu_fn_type, 0), ""); \ + LLVMValueRef params[] = {LLVMGetParam(start, 0), vaddr}; \ + LLVMValueRef ret = \ + LLVMBuildCall2(*builder, mmu_fn_type, mmu_fn_ptr, params, 2, ""); \ + ret = \ + LLVMBuildIntCast2(*builder, ret, LLVMInt32Type(), is_signed, ""); \ + LLVMBuildStore(*builder, ret, t2c_gen_rd_addr(start, builder, ir)); \ + } + +#define T2C_MMU_STORE(opcode, fn) \ + static void t2c_mmu_wrapper_##opcode(LLVMBuilderRef *builder, \ + LLVMValueRef start, rv_insn_t *ir) \ + { \ + LLVMValueRef val_rs1 = \ + LLVMBuildLoad2(*builder, LLVMInt32Type(), \ + t2c_gen_rs1_addr(start, builder, ir), ""); \ + LLVMValueRef vaddr = T2C_LLVM_GEN_ALU32_IMM(Add, val_rs1, ir->imm); \ + vaddr = LLVMBuildZExt(*builder, vaddr, LLVMInt64Type(), ""); \ + LLVMTypeRef param_types[] = {LLVMPointerType(LLVMInt64Type(), 0), \ + LLVMInt64Type(), LLVMInt64Type()}; \ + LLVMTypeRef mmu_fn_type = \ + LLVMFunctionType(LLVMVoidType(), param_types, 3, 0); \ + LLVMValueRef mmu_fn_addr = \ + LLVMConstInt(LLVMInt64Type(), (uintptr_t) fn, false); \ + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, \ + t2c_gen_rs2_addr(start, builder, ir)); \ + val_rs2 = \ + LLVMBuildIntCast2(*builder, val_rs2, LLVMInt64Type(), true, ""); \ + LLVMValueRef mmu_fn_ptr = LLVMBuildIntToPtr( \ + *builder, mmu_fn_addr, LLVMPointerType(mmu_fn_type, 0), ""); \ + LLVMValueRef params[] = {LLVMGetParam(start, 0), vaddr, val_rs2}; \ + LLVMBuildCall2(*builder, mmu_fn_type, mmu_fn_ptr, params, 3, ""); \ + } + +T2C_MMU_LOAD(lb, mmu_read_b, 8, true); +T2C_MMU_LOAD(lbu, mmu_read_b, 8, false); +T2C_MMU_LOAD(lh, mmu_read_s, 16, true); +T2C_MMU_LOAD(lhu, mmu_read_s, 16, false); +T2C_MMU_LOAD(lw, mmu_read_w, 32, true); + +T2C_MMU_STORE(sb, mmu_write_b); +T2C_MMU_STORE(sh, mmu_write_s); +T2C_MMU_STORE(sw, mmu_write_w); + +#endif + T2C_OP(lb, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - LLVMValueRef res = LLVMBuildSExt( - *builder, LLVMBuildLoad2(*builder, LLVMInt8Type(), mem_loc, "res"), - LLVMInt32Type(), "sext8to32"); - LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(lb)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildSExt( + *builder, + LLVMBuildLoad2(*builder, LLVMInt8Type(), mem_loc, "res"), + LLVMInt32Type(), "sext8to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + }); }) T2C_OP(lh, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - LLVMValueRef res = LLVMBuildSExt( - *builder, LLVMBuildLoad2(*builder, LLVMInt16Type(), mem_loc, "res"), - LLVMInt32Type(), "sext16to32"); - LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(lh)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildSExt( + *builder, + LLVMBuildLoad2(*builder, LLVMInt16Type(), mem_loc, "res"), + LLVMInt32Type(), "sext16to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + }); }) T2C_OP(lw, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - LLVMValueRef res = - LLVMBuildLoad2(*builder, LLVMInt32Type(), mem_loc, "res"); - LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(lw)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = + LLVMBuildLoad2(*builder, LLVMInt32Type(), mem_loc, "res"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + }); }) T2C_OP(lbu, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - LLVMValueRef res = LLVMBuildZExt( - *builder, LLVMBuildLoad2(*builder, LLVMInt8Type(), mem_loc, "res"), - LLVMInt32Type(), "zext8to32"); - LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(lbu)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildZExt( + *builder, + LLVMBuildLoad2(*builder, LLVMInt8Type(), mem_loc, "res"), + LLVMInt32Type(), "zext8to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + }); }) T2C_OP(lhu, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - LLVMValueRef res = LLVMBuildZExt( - *builder, LLVMBuildLoad2(*builder, LLVMInt16Type(), mem_loc, "res"), - LLVMInt32Type(), "zext16to32"); - LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(lhu)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildZExt( + *builder, + LLVMBuildLoad2(*builder, LLVMInt16Type(), mem_loc, "res"), + LLVMInt32Type(), "zext16to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); + }); }) T2C_OP(sb, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - T2C_LLVM_GEN_LOAD_VMREG(rs2, 8, t2c_gen_rs2_addr(start, builder, ir)); - LLVMBuildStore(*builder, val_rs2, mem_loc); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(sb)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 8, + t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); + }); }) T2C_OP(sh, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - T2C_LLVM_GEN_LOAD_VMREG(rs2, 16, t2c_gen_rs2_addr(start, builder, ir)); - LLVMBuildStore(*builder, val_rs2, mem_loc); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(sh)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 16, + t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); + }); }) T2C_OP(sw, { - LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); - T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); - LLVMBuildStore(*builder, val_rs2, mem_loc); + IIF(RV32_HAS(SYSTEM)) + ( + { t2c_mmu_wrapper(sw)(builder, start, ir); }, + { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, + t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); + }); }) T2C_OP(addi, { @@ -734,7 +886,7 @@ T2C_OP(clwsp, { T2C_OP(cjr, { T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); - t2c_jit_cache_helper(builder, start, val_rs1, rv, ir); + t2c_jit_cache_helper(builder, start, val_rs1, rv, block, ir); }) T2C_OP(cmv, { @@ -756,7 +908,7 @@ T2C_OP(cjalr, { T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 2, t2c_gen_ra_addr(start, builder, ir)); - t2c_jit_cache_helper(builder, start, val_rs1, rv, ir); + t2c_jit_cache_helper(builder, start, val_rs1, rv, block, ir); }) T2C_OP(cadd, { @@ -920,15 +1072,18 @@ T2C_OP(fuse5, { switch (fuse[i].opcode) { case rv_insn_slli: t2c_slli(builder, param_types, start, entry, taken_builder, - untaken_builder, rv, mem_base, (rv_insn_t *) (&fuse[i])); + untaken_builder, rv, mem_base, block, + (rv_insn_t *) (&fuse[i])); break; case rv_insn_srli: t2c_srli(builder, param_types, start, entry, taken_builder, - untaken_builder, rv, mem_base, (rv_insn_t *) (&fuse[i])); + untaken_builder, rv, mem_base, block, + (rv_insn_t *) (&fuse[i])); break; case rv_insn_srai: t2c_srai(builder, param_types, start, entry, taken_builder, - untaken_builder, rv, mem_base, (rv_insn_t *) (&fuse[i])); + untaken_builder, rv, mem_base, block, + (rv_insn_t *) (&fuse[i])); break; default: __UNREACHABLE;