diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index a372a63..c32d78b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -15,7 +15,7 @@ jobs: matrix: type: [ Debug, Release ] # macos-13 is intel, macos-14 is arm, blaze/ubuntu-22.04 is arm - os: [ ubuntu-latest, windows-latest, macos-13, macos-14, blaze/ubuntu-22.04 ] + os: [ ubuntu-latest, windows-latest, macos-15-intel, macos-14, blaze/ubuntu-22.04 ] name: ${{ matrix.os }} (${{ (matrix.os == 'macos-14' || matrix.os == 'blaze/ubuntu-22.04') && 'arm64' || 'x64' }}, ${{ matrix.type }}) steps: - name: Checkout @@ -50,16 +50,34 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_MAKE_PROGRAM=ninja -G Ninja -S . -B cmake-build - cmake --build cmake-build --config ${{ matrix.type }} --target N64Recomp -j $(nproc) + cmake --build cmake-build --config ${{ matrix.type }} -j $(nproc) + - name: Upload Build Artifacts (Unix) + if: runner.os != 'Windows' + uses: actions/upload-artifact@v4 + with: + name: N64Recomp-${{ runner.os }}-${{ runner.arch }}-${{ matrix.type }} + path: | + ./cmake-build/LiveRecompTest + ./cmake-build/N64Recomp + ./cmake-build/OfflineModRecomp + ./cmake-build/RecompModTool + ./cmake-build/RSPRecomp - name: Build N64Recomp (Windows) if: runner.os == 'Windows' run: |- # enable ccache set $env:PATH="$env:USERPROFILE/.cargo/bin;$env:PATH" - $cpuCores = (Get-CimInstance -ClassName Win32_Processor).NumberOfLogicalProcessors - # remove LLVM from PATH so it doesn't overshadow the one provided by VS - $env:PATH = ($env:PATH -split ';' | Where-Object { $_ -ne 'C:\Program Files\LLVM\bin' }) -join ';' - cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_MAKE_PROGRAM=ninja -G Ninja -S . -B cmake-build - cmake --build cmake-build --config ${{ matrix.type }} --target N64Recomp -j $cpuCores + cmake --build cmake-build --config ${{ matrix.type }} + - name: Upload Build Artifacts (Windows) + if: runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: N64Recomp-${{ runner.os }}-${{ runner.arch }}-${{ matrix.type }} + path: | + ./cmake-build/LiveRecompTest.exe + ./cmake-build/N64Recomp.exe + ./cmake-build/OfflineModRecomp.exe + ./cmake-build/RecompModTool.exe + ./cmake-build/RSPRecomp.exe diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fc7581..55f10a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,7 @@ add_library(N64RecompElf) target_sources(N64RecompElf PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/elf.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/mdebug.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/symbol_lists.cpp ) @@ -165,6 +166,17 @@ target_sources(OfflineModRecomp PRIVATE target_link_libraries(OfflineModRecomp fmt rabbitizer tomlplusplus::tomlplusplus N64Recomp) +# Mod combiner +project(RecompModMerger) +add_executable(RecompModMerger) + +target_sources(RecompModMerger PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/config.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/RecompModMerger/main.cpp +) + +target_link_libraries(RecompModMerger N64Recomp) + # Live recompiler project(LiveRecomp) add_library(LiveRecomp) diff --git a/LiveRecomp/live_generator.cpp b/LiveRecomp/live_generator.cpp index cb733f1..61e1ff5 100644 --- a/LiveRecomp/live_generator.cpp +++ b/LiveRecomp/live_generator.cpp @@ -238,15 +238,27 @@ constexpr int get_fpr_double_context_offset(int fpr_index) { return offsetof(recomp_context, f0.d) + sizeof(recomp_context::f0) * fpr_index; } -constexpr int get_fpr_u32l_context_offset(int fpr_index) { +constexpr bool is_fpr_u32l(N64Recomp::Operand operand) { + return + operand == N64Recomp::Operand::FdU32L || + operand == N64Recomp::Operand::FsU32L || + operand == N64Recomp::Operand::FtU32L; + return false; +} + +constexpr void get_fpr_u32l_context_offset(int fpr_index, sljit_compiler* compiler, int odd_float_address_register, sljit_sw& out, sljit_sw& outw) { if (fpr_index & 1) { - // TODO implement odd floats. - assert(false); - return -1; - // return fmt::format("ctx->f_odd[({} - 1) * 2]", fpr_index); + assert(compiler != nullptr); + // Load ctx->f_odd into the address register. + sljit_emit_op1(compiler, SLJIT_MOV_P, odd_float_address_register, 0, SLJIT_MEM1(Registers::ctx), offsetof(recomp_context, f_odd)); + // sljit_emit_op0(compiler, SLJIT_BREAKPOINT); + out = SLJIT_MEM1(odd_float_address_register); + // Set a memory offset of ((fpr_index - 1) * 2) * sizeof(*f_odd). + outw = ((fpr_index - 1) * 2) * sizeof(*recomp_context::f_odd); } else { - return offsetof(recomp_context, f0.u32l) + sizeof(recomp_context::f0) * fpr_index; + out = SLJIT_MEM1(Registers::ctx); + outw = offsetof(recomp_context, f0.u32l) + sizeof(recomp_context::f0) * fpr_index; } } @@ -265,7 +277,10 @@ void get_gpr_values(int gpr, sljit_sw& out, sljit_sw& outw) { } } -bool get_operand_values(N64Recomp::Operand operand, const N64Recomp::InstructionContext& context, sljit_sw& out, sljit_sw& outw) { +bool get_operand_values(N64Recomp::Operand operand, const N64Recomp::InstructionContext& context, sljit_sw& out, sljit_sw& outw, + sljit_compiler* compiler, int odd_float_address_register +) +{ using namespace N64Recomp; switch (operand) { @@ -303,16 +318,13 @@ bool get_operand_values(N64Recomp::Operand operand, const N64Recomp::Instruction outw = get_fpr_double_context_offset(context.ft); break; case Operand::FdU32L: - out = SLJIT_MEM1(Registers::ctx); - outw = get_fpr_u32l_context_offset(context.fd); + get_fpr_u32l_context_offset(context.fd, compiler, odd_float_address_register, out, outw); break; case Operand::FsU32L: - out = SLJIT_MEM1(Registers::ctx); - outw = get_fpr_u32l_context_offset(context.fs); + get_fpr_u32l_context_offset(context.fs, compiler, odd_float_address_register, out, outw); break; case Operand::FtU32L: - out = SLJIT_MEM1(Registers::ctx); - outw = get_fpr_u32l_context_offset(context.ft); + get_fpr_u32l_context_offset(context.ft, compiler, odd_float_address_register, out, outw); break; case Operand::FdU32H: assert(false); @@ -389,16 +401,30 @@ void N64Recomp::LiveGenerator::process_binary_op(const BinaryOp& op, const Instr if (outputs_to_zero(op.output, ctx)) { return; } - + + // Float u32l input operands are not allowed in a binary operation. + if (is_fpr_u32l(op.operands.operands[0]) || is_fpr_u32l(op.operands.operands[1])) { + assert(false); + errored = true; + return; + } + + // A float u32l output operand is only allowed for lwc1, which has an op type of LW. + if (is_fpr_u32l(op.output) && op.type != BinaryOpType::LW) { + assert(false); + errored = true; + return; + } + sljit_sw dst; sljit_sw dstw; sljit_sw src1; sljit_sw src1w; sljit_sw src2; sljit_sw src2w; - bool output_good = get_operand_values(op.output, ctx, dst, dstw); - bool input0_good = get_operand_values(op.operands.operands[0], ctx, src1, src1w); - bool input1_good = get_operand_values(op.operands.operands[1], ctx, src2, src2w); + bool output_good = get_operand_values(op.output, ctx, dst, dstw, compiler, Registers::arithmetic_temp2); + bool input0_good = get_operand_values(op.operands.operands[0], ctx, src1, src1w, nullptr, 0); + bool input1_good = get_operand_values(op.operands.operands[1], ctx, src2, src2w, nullptr, 0); if (!output_good || !input0_good || !input1_good) { assert(false); @@ -748,6 +774,10 @@ void N64Recomp::LiveGenerator::process_binary_op(const BinaryOp& op, const Instr case BinaryOpType::LessEqDouble: do_float_compare_op(SLJIT_F_LESS_EQUAL, SLJIT_SET_F_LESS_EQUAL, true); break; + case BinaryOpType::False: + // Load 0 into condition destination + sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, SLJIT_IMM, 0); + break; // Loads case BinaryOpType::LD: @@ -797,6 +827,7 @@ void N64Recomp::LiveGenerator::process_binary_op(const BinaryOp& op, const Instr } } +// TODO these four operations should use banker's rounding, but roundeven is C23 so it's unavailable here. int32_t do_round_w_s(float num) { return lroundf(num); } @@ -865,12 +896,19 @@ void N64Recomp::LiveGenerator::process_unary_op(const UnaryOp& op, const Instruc return; } + // A unary op may have a float u32l as the source or destination, but not both. + if (is_fpr_u32l(op.input) && is_fpr_u32l(op.output)) { + assert(false); + errored = true; + return; + } + sljit_sw dst; sljit_sw dstw; sljit_sw src; sljit_sw srcw; - bool output_good = get_operand_values(op.output, ctx, dst, dstw); - bool input_good = get_operand_values(op.input, ctx, src, srcw); + bool output_good = get_operand_values(op.output, ctx, dst, dstw, compiler, Registers::arithmetic_temp3); + bool input_good = get_operand_values(op.input, ctx, src, srcw, compiler, Registers::arithmetic_temp3); if (!output_good || !input_good) { assert(false); @@ -1088,11 +1126,23 @@ void N64Recomp::LiveGenerator::process_unary_op(const UnaryOp& op, const Instruc emit_l_from_d_func(do_floor_l_d); break; case UnaryOpType::None: - jit_op = SLJIT_MOV; + // Only write 32 bits to the output is a fpr u32l operand. + if (is_fpr_u32l(op.output)) { + jit_op = SLJIT_MOV32; + } + else { + jit_op = SLJIT_MOV; + } break; case UnaryOpType::ToS32: case UnaryOpType::ToInt32: - jit_op = SLJIT_MOV_S32; + // sljit won't emit a sign extension with SLJIT_MOV_32 if the destination is memory, + // so emit an explicit move into a register and set that register as the new src. + sljit_emit_op1(compiler, SLJIT_MOV_S32, Registers::arithmetic_temp1, 0, src, srcw); + // Replace the original input with the temporary register. + src = Registers::arithmetic_temp1; + srcw = 0; + jit_op = SLJIT_MOV; break; // Unary ops that can't be used as a standalone operation case UnaryOpType::ToU32: @@ -1121,7 +1171,7 @@ void N64Recomp::LiveGenerator::process_store_op(const StoreOp& op, const Instruc sljit_sw srcw; sljit_sw imm = (sljit_sw)(int16_t)ctx.imm16; - get_operand_values(op.value_input, ctx, src, srcw); + get_operand_values(op.value_input, ctx, src, srcw, compiler, Registers::arithmetic_temp2); // Only LO16 relocs are valid on stores. if (ctx.reloc_type != RelocType::R_MIPS_NONE && ctx.reloc_type != RelocType::R_MIPS_LO16) { @@ -1259,6 +1309,17 @@ void N64Recomp::LiveGenerator::emit_function_start(const std::string& function_n // sljit_emit_op0(compiler, SLJIT_BREAKPOINT); sljit_emit_enter(compiler, 0, SLJIT_ARGS2V(P, P), 4 | SLJIT_ENTER_FLOAT(1), 5 | SLJIT_ENTER_FLOAT(0), 0); sljit_emit_op2(compiler, SLJIT_SUB, Registers::rdram, 0, Registers::rdram, 0, SLJIT_IMM, rdram_offset); + + // Check if this function's entry is hooked and emit the hook call if so. + auto find_hook_it = inputs.entry_func_hooks.find(func_index); + if (find_hook_it != inputs.entry_func_hooks.end()) { + // Load rdram and ctx into R0 and R1. + sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, Registers::rdram, 0, SLJIT_IMM, rdram_offset); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, Registers::ctx, 0); + // Load the hook's index into R2. + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, find_hook_it->second); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3V(P, P, W), SLJIT_IMM, sljit_sw(inputs.run_hook)); + } } void N64Recomp::LiveGenerator::emit_function_end() const { @@ -1438,6 +1499,13 @@ void N64Recomp::LiveGenerator::emit_branch_condition(const ConditionalBranchOp& return; } + // Branch conditions do not allow float u32l operands. + if (is_fpr_u32l(op.operands.operands[0]) || is_fpr_u32l(op.operands.operands[1])) { + assert(false); + errored = true; + return; + } + sljit_s32 condition_type; bool cmp_signed = op.operands.operand_operations[0] == UnaryOpType::ToS64; // Comparisons need to be inverted to account for the fact that the generator is expected to generate a code block that only runs if @@ -1491,8 +1559,8 @@ void N64Recomp::LiveGenerator::emit_branch_condition(const ConditionalBranchOp& sljit_sw src2; sljit_sw src2w; - get_operand_values(op.operands.operands[0], ctx, src1, src1w); - get_operand_values(op.operands.operands[1], ctx, src2, src2w); + get_operand_values(op.operands.operands[0], ctx, src1, src1w, nullptr, 0); + get_operand_values(op.operands.operands[1], ctx, src2, src2w, nullptr, 0); // Relocations aren't valid on conditional branches. if(ctx.reloc_type != RelocType::R_MIPS_NONE) { @@ -1545,8 +1613,14 @@ void N64Recomp::LiveGenerator::emit_switch(const Context& recompiler_context, co // Get the relocated address of the jump table. uint32_t section_offset = jtbl.vram - jtbl_section.ram_addr; + // Get the section index to use for relocation at runtime. + uint16_t reloc_section_index = jtbl.section_index; + if (!inputs.original_section_indices.empty()) { + reloc_section_index = inputs.original_section_indices[reloc_section_index]; + } + // Populate the necessary fields of the dummy context and load the relocated address into temp2. - dummy_context.reloc_section_index = jtbl.section_index; + dummy_context.reloc_section_index = reloc_section_index; dummy_context.reloc_target_section_offset = section_offset; load_relocated_address(dummy_context, Registers::arithmetic_temp2); @@ -1590,7 +1664,19 @@ void N64Recomp::LiveGenerator::emit_switch_close() const { // Nothing to do here, the jump table is built in emit_switch. } -void N64Recomp::LiveGenerator::emit_return(const Context& context) const { +void N64Recomp::LiveGenerator::emit_return(const Context& context, size_t func_index) const { + (void)context; + + // Check if this function's return is hooked and emit the hook call if so. + auto find_hook_it = inputs.return_func_hooks.find(func_index); + if (find_hook_it != inputs.return_func_hooks.end()) { + // Load rdram and ctx into R0 and R1. + sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, Registers::rdram, 0, SLJIT_IMM, rdram_offset); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, Registers::ctx, 0); + // Load the return hook's index into R2. + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, find_hook_it->second); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3V(P, P, W), SLJIT_IMM, sljit_sw(inputs.run_hook)); + } sljit_emit_return_void(compiler); } @@ -1642,8 +1728,11 @@ void N64Recomp::LiveGenerator::emit_cop1_cs_read(int reg) const { // Call get_cop1_cs. sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS0(32), SLJIT_IMM, sljit_sw(get_cop1_cs)); - // Store the result in the output register. - sljit_emit_op1(compiler, SLJIT_MOV_S32, dst, dstw, SLJIT_RETURN_REG, 0); + // Sign extend the result into a temp register. + sljit_emit_op1(compiler, SLJIT_MOV_S32, Registers::arithmetic_temp1, 0, SLJIT_RETURN_REG, 0); + + // Move the sign extended result into the destination. + sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, Registers::arithmetic_temp1, 0); } } @@ -1863,3 +1952,29 @@ bool N64Recomp::recompile_function_live(LiveGenerator& generator, const Context& return recompile_function_custom(generator, context, function_index, output_file, static_funcs_out, tag_reference_relocs); } +N64Recomp::ShimFunction::ShimFunction(recomp_func_ext_t* to_shim, uintptr_t value) { + sljit_compiler* compiler = sljit_create_compiler(nullptr); + + // Create the function. + sljit_label* func_label = sljit_emit_label(compiler); + sljit_emit_enter(compiler, 0, SLJIT_ARGS2V(P_R, P_R), 3, 0, 0); + + // Move the provided value into the third argument. + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, sljit_sw(value)); + + // Tail call the provided function. + sljit_emit_icall(compiler, SLJIT_CALL | SLJIT_CALL_RETURN, SLJIT_ARGS3V(P, P, W), SLJIT_IMM, sljit_sw(to_shim)); + + // Generate the function's code and get the address to the function. + code = sljit_generate_code(compiler, 0, nullptr); + func = reinterpret_cast(sljit_get_label_addr(func_label)); + + // Cleanup. + sljit_free_compiler(compiler); +} + +N64Recomp::ShimFunction::~ShimFunction() { + sljit_free_code(code, nullptr); + code = nullptr; + func = nullptr; +} diff --git a/RSPRecomp/src/rsp_recomp.cpp b/RSPRecomp/src/rsp_recomp.cpp index 02f99ab..518477d 100644 --- a/RSPRecomp/src/rsp_recomp.cpp +++ b/RSPRecomp/src/rsp_recomp.cpp @@ -1054,7 +1054,7 @@ void create_function(const std::string& function_name, std::ofstream& output_fil fmt::print(output_file, " r1 = 0xFC0;\n"); } else { fmt::print(output_file, - "RspExitReason {}(uint8_t* rdram) {{\n" + "RspExitReason {}(uint8_t* rdram, [[maybe_unused]] uint32_t ucode_addr) {{\n" " uint32_t r1 = 0, r2 = 0, r3 = 0, r4 = 0, r5 = 0, r6 = 0, r7 = 0;\n" " uint32_t r8 = 0, r9 = 0, r10 = 0, r11 = 0, r12 = 0, r13 = 0, r14 = 0, r15 = 0;\n" " uint32_t r16 = 0, r17 = 0, r18 = 0, r19 = 0, r20 = 0, r21 = 0, r22 = 0, r23 = 0;\n" diff --git a/RecompModMerger/main.cpp b/RecompModMerger/main.cpp new file mode 100644 index 0000000..17db691 --- /dev/null +++ b/RecompModMerger/main.cpp @@ -0,0 +1,306 @@ +#include +#include + +#include "recompiler/context.h" + +template +bool read_file(const std::filesystem::path& p, std::vector& out) { + static_assert(sizeof(T) == 1); + std::vector ret{}; + + std::ifstream input_file{p, std::ios::binary}; + if (!input_file.good()) { + return false; + } + + input_file.seekg(0, std::ios::end); + ret.resize(input_file.tellg()); + input_file.seekg(0, std::ios::beg); + + input_file.read(reinterpret_cast(ret.data()), ret.size()); + + out = std::move(ret); + + return true; +} + +bool write_file(const std::filesystem::path& p, std::span in) { + std::ofstream out{ p, std::ios::binary }; + if (!out.good()) { + return false; + } + + out.write(in.data(), in.size()); + return true; +} + +std::span reinterpret_span_u8(std::span s) { + return std::span(reinterpret_cast(s.data()), s.size()); +} + +std::span reinterpret_span_char(std::span s) { + return std::span(reinterpret_cast(s.data()), s.size()); +} + +bool copy_into_context(N64Recomp::Context& out, const N64Recomp::Context& in) { + size_t rom_offset = out.rom.size(); + size_t section_offset = out.sections.size(); + size_t function_offset = out.functions.size(); + size_t event_offset = out.event_symbols.size(); + + // Append the input rom to the end of the output rom. + out.rom.insert(out.rom.end(), in.rom.begin(), in.rom.end()); + + // Merge dependencies from the input. Copy new ones and remap existing ones. + std::vector new_dependency_indices(in.dependencies.size()); + for (size_t dep_index = 0; dep_index < in.dependencies.size(); dep_index++) { + const std::string& dep = in.dependencies[dep_index]; + auto find_dep_it = out.dependencies_by_name.find(dep); + if (find_dep_it != out.dependencies_by_name.end()) { + new_dependency_indices[dep_index] = find_dep_it->second; + } + else { + out.dependencies_by_name[dep] = out.dependencies.size(); + new_dependency_indices[dep_index] = out.dependencies.size(); + out.dependencies.emplace_back(dep); + } + } + + // Merge imports from the input. Copy new ones and remap existing ones. + std::vector new_import_indices(in.import_symbols.size()); + for (size_t import_index = 0; import_index < in.import_symbols.size(); import_index++) { + const N64Recomp::ImportSymbol& sym = in.import_symbols[import_index]; + size_t dependency_index = new_dependency_indices[sym.dependency_index]; + + size_t original_import_index = (size_t)-1; + + // Check if any import symbols have the same dependency index and symbol name. + for (size_t i = 0; i < out.import_symbols.size(); i++) { + const N64Recomp::ImportSymbol& sym_out = out.import_symbols[i]; + if (sym_out.dependency_index == dependency_index && sym_out.base.name == sym.base.name) { + original_import_index = i; + break; + } + } + + if (original_import_index != (size_t)-1) { + new_import_indices[import_index] = original_import_index; + } + else { + new_import_indices[import_index] = out.import_symbols.size(); + N64Recomp::ImportSymbol new_sym{}; + new_sym.dependency_index = dependency_index; + new_sym.base.name = sym.base.name; + out.import_symbols.emplace_back(std::move(new_sym)); + } + } + + // Merge dependency events from the input. Copy new ones and remap existing ones. + std::vector new_dependency_event_indices(in.dependency_events.size()); + for (size_t dependency_event_index = 0; dependency_event_index < in.dependency_events.size(); dependency_event_index++) { + const N64Recomp::DependencyEvent& event = in.dependency_events[dependency_event_index]; + size_t dependency_index = new_dependency_indices[event.dependency_index]; + + size_t original_event_index = (size_t)-1; + + // Check if any dependency events have the same dependency index and event name. + for (size_t i = 0; i < out.dependency_events.size(); i++) { + const N64Recomp::DependencyEvent& event_out = out.dependency_events[i]; + if (event_out.dependency_index == dependency_index && event_out.event_name == event.event_name) { + original_event_index = i; + break; + } + } + + if (original_event_index != (size_t)-1) { + new_dependency_event_indices[dependency_event_index] = original_event_index; + } + else { + new_dependency_event_indices[dependency_event_index] = out.dependency_events.size(); + out.dependency_events.emplace_back(N64Recomp::DependencyEvent{ .dependency_index = dependency_index, .event_name = event.event_name }); + } + } + + // Copy every section from the input. + for (size_t section_index = 0; section_index < in.sections.size(); section_index++) { + const N64Recomp::Section& section = in.sections[section_index]; + + size_t out_section_index = section_offset + section_index; + N64Recomp::Section& section_out = out.sections.emplace_back(section); + section_out.rom_addr += rom_offset; + section_out.name = ""; + + // Adjust the section index of all the section's relocs. + for (N64Recomp::Reloc& reloc : section_out.relocs) { + if (reloc.target_section == N64Recomp::SectionAbsolute) { + printf("Internal error: reloc in section %zu references an absolute symbol and should have been relocated already. Please report this issue.\n", + section_index); + // Nothing to do for absolute relocs. + } + else if (reloc.target_section == N64Recomp::SectionImport) { + // symbol_index indexes context.import_symbols + reloc.symbol_index = new_import_indices[reloc.symbol_index]; + } + else if (reloc.target_section == N64Recomp::SectionEvent) { + // symbol_index indexes context.event_symbols + reloc.symbol_index += event_offset; + } + else if (reloc.reference_symbol) { + // symbol_index indexes context.reference_symbols + // Nothing to do here, reference section indices will remain unchanged. + } + else { + reloc.target_section += section_offset; + } + } + } + + out.section_functions.resize(out.sections.size()); + + // Copy every function from the input. + for (size_t func_index = 0; func_index < in.functions.size(); func_index++) { + const N64Recomp::Function& func = in.functions[func_index]; + + size_t out_func_index = function_offset + func_index; + N64Recomp::Function& function_out = out.functions.emplace_back(func); + + function_out.section_index += section_offset; + function_out.rom += rom_offset; + // functions_by_name unused + out.functions_by_vram[function_out.vram].push_back(out_func_index); + + out.section_functions[function_out.section_index].push_back(out_func_index); + } + + // Copy replacements from the input. + for (size_t replacement_index = 0; replacement_index < in.replacements.size(); replacement_index++) { + const N64Recomp::FunctionReplacement& replacement = in.replacements[replacement_index]; + N64Recomp::FunctionReplacement& replacement_out = out.replacements.emplace_back(replacement); + replacement_out.func_index += function_offset; + } + + // Copy hooks from the input. + for (size_t hook_index = 0; hook_index < in.hooks.size(); hook_index++) { + const N64Recomp::FunctionHook& hook = in.hooks[hook_index]; + N64Recomp::FunctionHook& hook_out = out.hooks.emplace_back(hook); + hook_out.func_index += function_offset; + } + + // Copy callbacks from the input. + for (size_t callback_index = 0; callback_index < in.callbacks.size(); callback_index++) { + const N64Recomp::Callback& callback = in.callbacks[callback_index]; + N64Recomp::Callback callback_out = out.callbacks.emplace_back(callback); + callback_out.dependency_event_index = new_dependency_event_indices[callback_out.dependency_event_index]; + } + + // Copy exports from the input. + for (size_t exported_func : in.exported_funcs) { + out.exported_funcs.push_back(exported_func + function_offset); + } + + // Copy events from the input. + for (const N64Recomp::EventSymbol& event_sym : in.event_symbols) { + out.event_symbols.emplace_back(event_sym); + } + + return true; +} + +int main(int argc, const char** argv) { + if (argc != 8) { + printf("Usage: %s \n", argv[0]); + return EXIT_SUCCESS; + } + + const char* function_symbol_toml_path = argv[1]; + const char* sym_file_path_1 = argv[2]; + const char* binary_path_1 = argv[3]; + const char* sym_file_path_2 = argv[4]; + const char* binary_path_2 = argv[5]; + const char* output_sym_path = argv[6]; + const char* output_binary_path = argv[7]; + + // Load the symbol and binary files. + std::vector sym_file_1; + if (!read_file(sym_file_path_1, sym_file_1)) { + fprintf(stderr, "Error reading file %s\n", sym_file_path_1); + return EXIT_FAILURE; + } + + std::vector binary_1; + if (!read_file(binary_path_1, binary_1)) { + fprintf(stderr, "Error reading file %s\n", binary_path_1); + return EXIT_FAILURE; + } + + std::vector sym_file_2; + if (!read_file(sym_file_path_2, sym_file_2)) { + fprintf(stderr, "Error reading file %s\n", sym_file_path_2); + return EXIT_FAILURE; + } + + std::vector binary_2; + if (!read_file(binary_path_2, binary_2)) { + fprintf(stderr, "Error reading file %s\n", binary_path_2); + return EXIT_FAILURE; + } + + N64Recomp::ModSymbolsError err; + + // Parse the symbol toml. + std::vector dummy_rom{}; + N64Recomp::Context reference_context{}; + if (!N64Recomp::Context::from_symbol_file(function_symbol_toml_path, std::move(dummy_rom), reference_context, false)) { + fprintf(stderr, "Failed to load provided function reference symbol file\n"); + return EXIT_FAILURE; + } + + // Build a reference section lookup of rom address. + std::unordered_map sections_by_rom{}; + for (size_t section_index = 0; section_index < reference_context.sections.size(); section_index++) { + sections_by_rom[reference_context.sections[section_index].rom_addr] = section_index; + } + + // Parse the two contexts. + N64Recomp::Context context1{}; + err = N64Recomp::parse_mod_symbols(sym_file_1, binary_1, sections_by_rom, context1); + if (err != N64Recomp::ModSymbolsError::Good) { + fprintf(stderr, "Error parsing mod symbols %s\n", sym_file_path_1); + return EXIT_FAILURE; + } + context1.rom = std::move(binary_1); + + N64Recomp::Context context2{}; + err = N64Recomp::parse_mod_symbols(sym_file_2, binary_2, sections_by_rom, context2); + if (err != N64Recomp::ModSymbolsError::Good) { + fprintf(stderr, "Error parsing mod symbols %s\n", sym_file_path_2); + return EXIT_FAILURE; + } + context2.rom = std::move(binary_2); + + N64Recomp::Context merged{}; + merged.import_reference_context(reference_context); + + if (!copy_into_context(merged, context1)) { + fprintf(stderr, "Failed to merge first mod into output\n"); + return EXIT_FAILURE; + } + if (!copy_into_context(merged, context2)) { + fprintf(stderr, "Failed to merge second mod into output\n"); + return EXIT_FAILURE; + } + + std::vector syms_out = N64Recomp::symbols_to_bin_v1(merged); + + if (!write_file(output_sym_path, reinterpret_span_char(syms_out))) { + fprintf(stderr, "Failed to write symbol file to %s\n", output_sym_path); + return EXIT_FAILURE; + } + + if (!write_file(output_binary_path, reinterpret_span_char(std::span{ merged.rom }))) { + fprintf(stderr, "Failed to write binary file to %s\n", output_binary_path); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/RecompModTool/main.cpp b/RecompModTool/main.cpp index 9fbb7d1..0768f16 100644 --- a/RecompModTool/main.cpp +++ b/RecompModTool/main.cpp @@ -20,17 +20,24 @@ constexpr std::string_view symbol_filename = "mod_syms.bin"; constexpr std::string_view binary_filename = "mod_binary.bin"; -constexpr std::string_view manifest_filename = "manifest.json"; +constexpr std::string_view manifest_filename = "mod.json"; struct ModManifest { std::string mod_id; std::string version_string; + std::string display_name; + std::string description; + std::string short_description; std::vector authors; std::string game_id; std::string minimum_recomp_version; std::unordered_map> native_libraries; + bool custom_gamemode = false; + std::vector config_options; std::vector dependencies; std::vector full_dependency_strings; + std::vector optional_dependencies; + std::vector full_optional_dependency_strings; }; struct ModInputs { @@ -162,7 +169,7 @@ static T read_toml_value(const toml::table& data, std::string_view key, bool req if (value_node == nullptr) { if (required) { - throw toml::parse_error(("Missing required field " + std::string{key}).c_str(), data.source()); + throw toml::parse_error(("Missing required field \"" + std::string{key} + "\"").c_str(), data.source()); } else { return T{}; @@ -174,7 +181,7 @@ static T read_toml_value(const toml::table& data, std::string_view key, bool req return opt.value(); } else { - throw toml::parse_error(("Incorrect type for field " + std::string{key}).c_str(), data.source()); + throw toml::parse_error(("Incorrect type for field \"" + std::string{key} + "\"").c_str(), data.source()); } } @@ -215,11 +222,19 @@ static std::vector get_toml_path_array(const toml::array& return ret; } +bool validate_config_option(const toml::table& option) { + // TODO config option validation. + return true; +} + ModManifest parse_mod_config_manifest(const std::filesystem::path& basedir, const toml::table& manifest_table) { ModManifest ret; // Mod ID ret.mod_id = read_toml_value(manifest_table, "id", true); + if (!N64Recomp::validate_mod_id(ret.mod_id)) { + throw toml::parse_error("Invalid mod id", manifest_table["id"].node()->source()); + } // Mod version ret.version_string = read_toml_value(manifest_table, "version", true); @@ -228,6 +243,15 @@ ModManifest parse_mod_config_manifest(const std::filesystem::path& basedir, cons throw toml::parse_error("Invalid mod version", manifest_table["version"].node()->source()); } + // Display name + ret.display_name = read_toml_value(manifest_table, "display_name", true); + + // Description (optional) + ret.description = read_toml_value(manifest_table, "description", false); + + // Short description (optional) + ret.short_description = read_toml_value(manifest_table, "short_description", false); + // Authors const toml::array& authors_array = read_toml_array(manifest_table, "authors", true); authors_array.for_each([&ret](auto&& el) { @@ -302,6 +326,51 @@ ModManifest parse_mod_config_manifest(const std::filesystem::path& basedir, cons }); } + // Optional dependency list (optional) + const toml::array& optional_dependency_array = read_toml_array(manifest_table, "optional_dependencies", false); + if (!optional_dependency_array.empty()) { + // Reserve room for all the dependencies. + ret.dependencies.reserve(optional_dependency_array.size()); + optional_dependency_array.for_each([&ret](const auto& el) { + if constexpr (toml::is_string) { + size_t dependency_id_length; + bool dependency_version_has_label; + if (!validate_dependency_string(el.template ref(), dependency_id_length, dependency_version_has_label)) { + throw toml::parse_error("Invalid optional dependency entry", el.source()); + } + if (dependency_version_has_label) { + throw toml::parse_error("Dependency versions may not have labels", el.source()); + } + std::string dependency_id = el.template ref().substr(0, dependency_id_length); + ret.optional_dependencies.emplace_back(dependency_id); + ret.full_optional_dependency_strings.emplace_back(el.template ref()); + } + else { + throw toml::parse_error("Invalid type for optional dependency entry", el.source()); + } + }); + } + + // Config schema (optional) + const toml::array& config_options_array = read_toml_array(manifest_table, "config_options", false); + if (!config_options_array.empty()) { + ret.config_options.reserve(config_options_array.size()); + config_options_array.for_each([&ret](const auto& el) { + if constexpr (toml::is_table) { + if (!validate_config_option(el)) { + throw toml::parse_error("Invalid config option", el.source()); + } + ret.config_options.emplace_back(el); + } + else { + throw toml::parse_error("Invalid type for config option", el.source()); + } + }); + } + + // Custom gamemode (optional) + ret.custom_gamemode = read_toml_value(manifest_table, "custom_gamemode", false); + return ret; } @@ -426,57 +495,69 @@ bool parse_callback_name(std::string_view data, std::string& dependency_name, st return true; } -void print_vector_elements(std::ostream& output_file, const std::vector& vec, bool compact) { - char separator = compact ? ' ' : '\n'; - for (size_t i = 0; i < vec.size(); i++) { - const std::string& val = vec[i]; - fmt::print(output_file, "{}\"{}\"{}{}", - compact ? "" : " ", val, i == vec.size() - 1 ? "" : ",", separator); +toml::array string_vector_to_toml(const std::vector& input) { + toml::array ret{}; + for (const std::string& str : input) { + ret.emplace_back(str); } + return ret; } void write_manifest(const std::filesystem::path& path, const ModManifest& manifest) { - std::ofstream output_file(path); + toml::table output_data{}; - fmt::print(output_file, - "{{\n" - " \"game_id\": \"{}\",\n" - " \"id\": \"{}\",\n" - " \"version\": \"{}\",\n" - " \"authors\": [\n", - manifest.game_id, manifest.mod_id, manifest.version_string); + output_data.emplace("game_id", manifest.game_id); + output_data.emplace("id", manifest.mod_id); + output_data.emplace("version", manifest.version_string); + output_data.emplace("display_name", manifest.display_name); + + if (!manifest.description.empty()) { + output_data.emplace("description", manifest.description); + } + + if (!manifest.short_description.empty()) { + output_data.emplace("short_description", manifest.short_description); + } - print_vector_elements(output_file, manifest.authors, false); + output_data.emplace("authors", string_vector_to_toml(manifest.authors)); - fmt::print(output_file, - " ],\n" - " \"minimum_recomp_version\": \"{}\"", - manifest.minimum_recomp_version); + output_data.emplace("minimum_recomp_version", manifest.minimum_recomp_version); if (!manifest.native_libraries.empty()) { - fmt::print(output_file, ",\n" - " \"native_libraries\": {{\n"); + toml::table libraries_table{}; + size_t library_index = 0; for (const auto& [library, funcs] : manifest.native_libraries) { - fmt::print(output_file, " \"{}\": [ ", - library); - print_vector_elements(output_file, funcs, true); - fmt::print(output_file, "]{}\n", - library_index == manifest.native_libraries.size() - 1 ? "" : ","); - library_index++; + libraries_table.emplace(library, string_vector_to_toml(funcs)); } - fmt::print(output_file, " }}"); + + output_data.emplace("native_libraries", std::move(libraries_table)); } if (!manifest.full_dependency_strings.empty()) { - fmt::print(output_file, ",\n" - " \"dependencies\": [\n"); - print_vector_elements(output_file, manifest.full_dependency_strings, false); - fmt::print(output_file, " ]"); + output_data.emplace("dependencies", string_vector_to_toml(manifest.full_dependency_strings)); } + if (!manifest.full_optional_dependency_strings.empty()) { + output_data.emplace("optional_dependencies", string_vector_to_toml(manifest.full_optional_dependency_strings)); + } - fmt::print(output_file, "\n}}\n"); + if (!manifest.config_options.empty()) { + toml::array options_array{}; + for (const auto& option : manifest.config_options) { + options_array.emplace_back(option); + } + output_data.emplace("config_schema", toml::table{{"options", std::move(options_array)}}); + } + + if (manifest.custom_gamemode) { + output_data.emplace("custom_gamemode", manifest.custom_gamemode); + } + + toml::json_formatter formatter{output_data, toml::format_flags::indentation | toml::format_flags::indentation}; + std::ofstream output_file(path); + + output_file << formatter << std::endl; } N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bool& good) { @@ -573,6 +654,8 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo bool event_section = cur_section.name == N64Recomp::EventSectionName; bool import_section = cur_section.name.starts_with(N64Recomp::ImportSectionPrefix); bool callback_section = cur_section.name.starts_with(N64Recomp::CallbackSectionPrefix); + bool hook_section = cur_section.name.starts_with(N64Recomp::HookSectionPrefix); + bool hook_return_section = cur_section.name.starts_with(N64Recomp::HookReturnSectionPrefix); // Add the functions from the current input section to the current output section. auto& section_out = ret.sections[output_section_index]; @@ -638,6 +721,42 @@ N64Recomp::Context build_mod_context(const N64Recomp::Context& input_context, bo ); } + if (hook_section || hook_return_section) { + // Get the name of the hooked function. + size_t section_prefix_length = hook_section ? N64Recomp::HookSectionPrefix.size() : N64Recomp::HookReturnSectionPrefix.size(); + std::string hooked_function_name = cur_section.name.substr(section_prefix_length); + + // Find the corresponding symbol in the reference symbols. + N64Recomp::SymbolReference cur_reference; + bool original_func_exists = input_context.find_regular_reference_symbol(hooked_function_name, cur_reference); + + // Check that the function being patched exists in the original reference symbols. + if (!original_func_exists) { + fmt::print(stderr, "Function {} hooks a function ({}) that doesn't exist in the original ROM.\n", cur_func.name, hooked_function_name); + return {}; + } + + // Check that the reference symbol is actually a function. + const auto& reference_symbol = input_context.get_reference_symbol(cur_reference); + if (!reference_symbol.is_function) { + fmt::print(stderr, "Function {0} hooks {1}, but {1} was a variable in the original ROM.\n", cur_func.name, hooked_function_name); + return {}; + } + + uint32_t reference_section_vram = input_context.get_reference_section_vram(reference_symbol.section_index); + uint32_t reference_section_rom = input_context.get_reference_section_rom(reference_symbol.section_index); + + // Add a replacement for this function to the output context. + ret.hooks.emplace_back( + N64Recomp::FunctionHook { + .func_index = (uint32_t)output_func_index, + .original_section_vrom = reference_section_rom, + .original_vram = reference_section_vram + reference_symbol.section_offset, + .flags = hook_return_section ? N64Recomp::HookFlags::AtReturn : N64Recomp::HookFlags{} + } + ); + } + std::string name_out; if (export_section) { @@ -894,11 +1013,11 @@ bool create_mod_zip(const std::filesystem::path& output_dir, const ModConfig& co #ifdef _WIN32 std::filesystem::path temp_zip_path = output_path; temp_zip_path.replace_extension(".zip"); - std::string command_string = fmt::format("powershell -command Compress-Archive -Force -CompressionLevel Optimal -DestinationPath \"{}\" -Path \"{}\",\"{}\",\"{}\"", + std::string command_string = fmt::format("powershell -command Compress-Archive -Force -CompressionLevel Optimal -DestinationPath '{}' -Path '{}','{}','{}'", temp_zip_path.string(), (output_dir / symbol_filename).string(), (output_dir / binary_filename).string(), (output_dir / manifest_filename).string()); for (const auto& cur_file : config.inputs.additional_files) { - command_string += fmt::format(",\"{}\"", cur_file.string()); + command_string += fmt::format(",'{}'", cur_file.string()); } STARTUPINFOA si{}; @@ -1058,8 +1177,9 @@ int main(int argc, const char** argv) { } } - // Copy the dependencies from the config into the context. + // Copy the dependencies and optional dependencies from the config into the context. context.add_dependencies(config.manifest.dependencies); + context.add_dependencies(config.manifest.optional_dependencies); N64Recomp::ElfParsingConfig elf_config { .bss_section_suffix = {}, @@ -1069,7 +1189,8 @@ int main(int argc, const char** argv) { .entrypoint_address = 0, .use_absolute_symbols = false, .unpaired_lo16_warnings = false, - .all_sections_relocatable = true + .all_sections_relocatable = true, + .use_mdebug = false, }; bool dummy_found_entrypoint; N64Recomp::DataSymbolMap dummy_syms_map; diff --git a/include/recomp.h b/include/recomp.h index 578d0f9..73f3e73 100644 --- a/include/recomp.h +++ b/include/recomp.h @@ -439,7 +439,11 @@ gpr cop0_status_read(recomp_context* ctx); void switch_error(const char* func, uint32_t vram, uint32_t jtbl); void do_break(uint32_t vram); +// The function signature for all recompiler output functions. typedef void (recomp_func_t)(uint8_t* rdram, recomp_context* ctx); +// The function signature for special functions that need a third argument. +// These get called via generated shims to allow providing some information about the caller, such as mod id. +typedef void (recomp_func_ext_t)(uint8_t* rdram, recomp_context* ctx, uintptr_t arg); recomp_func_t* get_function(int32_t vram); diff --git a/include/recompiler/context.h b/include/recompiler/context.h index 7dd70c2..e7b4431 100644 --- a/include/recompiler/context.h +++ b/include/recompiler/context.h @@ -85,6 +85,8 @@ namespace N64Recomp { constexpr std::string_view EventSectionName = ".recomp_event"; constexpr std::string_view ImportSectionPrefix = ".recomp_import."; constexpr std::string_view CallbackSectionPrefix = ".recomp_callback."; + constexpr std::string_view HookSectionPrefix = ".recomp_hook."; + constexpr std::string_view HookReturnSectionPrefix = ".recomp_hook_return."; // Special dependency names. constexpr std::string_view DependencySelf = "."; @@ -102,6 +104,8 @@ namespace N64Recomp { bool executable = false; bool relocatable = false; // TODO is this needed? relocs being non-empty should be an equivalent check. bool has_mips32_relocs = false; + bool fixed_address = false; // Only used in mods, indicates that the section shouldn't be relocated or placed into mod memory. + bool globally_loaded = false; // Only used in mods, indicates that the section's functions should be globally loaded. Does not actually load the section's contents into ram. std::optional got_ram_addr = std::nullopt; }; @@ -125,11 +129,19 @@ namespace N64Recomp { std::unordered_map manually_sized_funcs; // The section names that were specified as relocatable std::unordered_set relocatable_sections; + // Symbols to ignore. + std::unordered_set ignored_syms; + // Manual mappings of mdebug file records to elf sections. + std::unordered_map mdebug_text_map; + std::unordered_map mdebug_data_map; + std::unordered_map mdebug_rodata_map; + std::unordered_map mdebug_bss_map; bool has_entrypoint; int32_t entrypoint_address; bool use_absolute_symbols; bool unpaired_lo16_warnings; bool all_sections_relocatable; + bool use_mdebug; }; struct DataSymbol { @@ -183,6 +195,19 @@ namespace N64Recomp { ReplacementFlags flags; }; + enum class HookFlags : uint32_t { + AtReturn = 1 << 0, + }; + inline HookFlags operator&(HookFlags lhs, HookFlags rhs) { return HookFlags(uint32_t(lhs) & uint32_t(rhs)); } + inline HookFlags operator|(HookFlags lhs, HookFlags rhs) { return HookFlags(uint32_t(lhs) | uint32_t(rhs)); } + + struct FunctionHook { + uint32_t func_index; + uint32_t original_section_vrom; + uint32_t original_vram; + HookFlags flags; + }; + class Context { private: //// Reference symbols (used for populating relocations for patches) @@ -208,6 +233,8 @@ namespace N64Recomp { std::vector rom; // Whether reference symbols should be validated when emitting function calls during recompilation. bool skip_validating_reference_symbols = true; + // Whether all function calls (excluding reference symbols) should go through lookup. + bool use_lookup_for_all_function_calls = false; //// Only used by the CLI, TODO move this to a struct in the internal headers. // A mapping of function name to index in the functions vector @@ -216,6 +243,8 @@ namespace N64Recomp { //// Mod dependencies and their symbols //// Imported values + // Dependency names. + std::vector dependencies; // Mapping of dependency name to dependency index. std::unordered_map dependencies_by_name; // List of symbols imported from dependencies. @@ -236,6 +265,8 @@ namespace N64Recomp { std::vector callbacks; // List of symbols from events, which contains the names of events that this context provides. std::vector event_symbols; + // List of hooks, which contains the original function to hook and the function index to call at the hook. + std::vector hooks; // Causes functions to print their name to the console the first time they're called. bool trace_mode; @@ -257,6 +288,7 @@ namespace N64Recomp { size_t dependency_index = dependencies_by_name.size(); + dependencies.emplace_back(id); dependencies_by_name.emplace(id, dependency_index); dependency_events_by_name.resize(dependencies_by_name.size()); dependency_imports_by_name.resize(dependencies_by_name.size()); @@ -276,6 +308,7 @@ namespace N64Recomp { for (const std::string& dep : new_dependencies) { size_t dependency_index = dependencies_by_name.size(); + dependencies.emplace_back(dep); dependencies_by_name.emplace(dep, dependency_index); } @@ -367,7 +400,7 @@ namespace N64Recomp { return reference_symbols[symbol_index]; } - size_t num_regular_reference_symbols() { + size_t num_regular_reference_symbols() const { return reference_symbols.size(); } @@ -539,6 +572,10 @@ namespace N64Recomp { } } + size_t num_reference_sections() const { + return reference_sections.size(); + } + void copy_reference_sections_from(const Context& rhs) { reference_sections = rhs.reference_sections; } @@ -546,6 +583,10 @@ namespace N64Recomp { void set_all_reference_sections_relocatable() { all_reference_sections_relocatable = true; } + + void add_reference_section(const ReferenceSection& sec) { + reference_sections.emplace_back(sec); + } }; class Generator; @@ -562,6 +603,22 @@ namespace N64Recomp { ModSymbolsError parse_mod_symbols(std::span data, std::span binary, const std::unordered_map& sections_by_vrom, Context& context_out); std::vector symbols_to_bin_v1(const Context& mod_context); + + inline bool is_manual_patch_symbol(uint32_t vram) { + // Zero-sized symbols between 0x8F000000 and 0x90000000 are manually specified symbols for use with patches. + // TODO make this configurable or come up with a more sensible solution for dealing with manual symbols for patches. + return vram >= 0x8F000000 && vram < 0x90000000; + } + + // Locale-independent ASCII-only version of isalpha. + inline bool isalpha_nolocale(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + // Locale-independent ASCII-only version of isalnum. + inline bool isalnum_nolocale(char c) { + return isalpha_nolocale(c) || (c >= '0' && c <= '9'); + } inline bool validate_mod_id(std::string_view str) { // Disallow empty ids. @@ -578,13 +635,13 @@ namespace N64Recomp { // so this is just to prevent "weird" mod ids. // Check the first character, which must be alphabetical or an underscore. - if (!isalpha(str[0]) && str[0] != '_') { + if (!isalpha_nolocale(str[0]) && str[0] != '_') { return false; } // Check the remaining characters, which can be alphanumeric or underscore. for (char c : str.substr(1)) { - if (!isalnum(c) && c != '_') { + if (!isalnum_nolocale(c) && c != '_') { return false; } } diff --git a/include/recompiler/generator.h b/include/recompiler/generator.h index 4d36f6c..145bcd0 100644 --- a/include/recompiler/generator.h +++ b/include/recompiler/generator.h @@ -48,7 +48,7 @@ namespace N64Recomp { virtual void emit_case(int case_index, const std::string& target_label) const = 0; virtual void emit_switch_error(uint32_t instr_vram, uint32_t jtbl_vram) const = 0; virtual void emit_switch_close() const = 0; - virtual void emit_return(const Context& context) const = 0; + virtual void emit_return(const Context& context, size_t func_index) const = 0; virtual void emit_check_fr(int fpr) const = 0; virtual void emit_check_nan(int fpr, bool is_double) const = 0; virtual void emit_cop0_status_read(int reg) const = 0; @@ -85,7 +85,7 @@ namespace N64Recomp { void emit_case(int case_index, const std::string& target_label) const final; void emit_switch_error(uint32_t instr_vram, uint32_t jtbl_vram) const final; void emit_switch_close() const final; - void emit_return(const Context& context) const final; + void emit_return(const Context& context, size_t func_index) const final; void emit_check_fr(int fpr) const final; void emit_check_nan(int fpr, bool is_double) const final; void emit_cop0_status_read(int reg) const final; diff --git a/include/recompiler/live_recompiler.h b/include/recompiler/live_recompiler.h index cc7670f..d155b83 100644 --- a/include/recompiler/live_recompiler.h +++ b/include/recompiler/live_recompiler.h @@ -78,6 +78,14 @@ namespace N64Recomp { void (*trigger_event)(uint8_t* rdram, recomp_context* ctx, uint32_t event_index); int32_t *reference_section_addresses; int32_t *local_section_addresses; + void (*run_hook)(uint8_t* rdram, recomp_context* ctx, size_t hook_table_index); + // Maps function index in recompiler context to function's entry hook slot. + std::unordered_map entry_func_hooks; + // Maps function index in recompiler context to function's return hook slot. + std::unordered_map return_func_hooks; + // Maps section index in the generated code to original section index. Used by regenerated + // code to relocate using the corresponding original section's address. + std::vector original_section_indices; }; class LiveGenerator final : public Generator { public: @@ -109,7 +117,7 @@ namespace N64Recomp { void emit_case(int case_index, const std::string& target_label) const final; void emit_switch_error(uint32_t instr_vram, uint32_t jtbl_vram) const final; void emit_switch_close() const final; - void emit_return(const Context& context) const final; + void emit_return(const Context& context, size_t func_index) const final; void emit_check_fr(int fpr) const final; void emit_check_nan(int fpr, bool is_double) const final; void emit_cop0_status_read(int reg) const final; @@ -136,6 +144,16 @@ namespace N64Recomp { void live_recompiler_init(); bool recompile_function_live(LiveGenerator& generator, const Context& context, size_t function_index, std::ostream& output_file, std::span> static_funcs_out, bool tag_reference_relocs); + + class ShimFunction { + private: + void* code; + recomp_func_t* func; + public: + ShimFunction(recomp_func_ext_t* to_shim, uintptr_t value); + ~ShimFunction(); + recomp_func_t* get_func() { return func; } + }; } #endif \ No newline at end of file diff --git a/lib/fmt b/lib/fmt index 8e72804..407c905 160000 --- a/lib/fmt +++ b/lib/fmt @@ -1 +1 @@ -Subproject commit 8e728044f673774160f43b44a07c6b185352310f +Subproject commit 407c905e45ad75fc29bf0f9bb7c5c2fd3475976f diff --git a/src/cgenerator.cpp b/src/cgenerator.cpp index 1ac63a8..94f85c5 100644 --- a/src/cgenerator.cpp +++ b/src/cgenerator.cpp @@ -301,6 +301,7 @@ void N64Recomp::CGenerator::get_operand_string(Operand operand, UnaryOpType oper case UnaryOpType::TruncateLFromD: operand_string = "TRUNC_L_D(" + operand_string + ")"; break; + // TODO these four operations should use banker's rounding, but roundeven is C23 so it's unavailable here. case UnaryOpType::RoundWFromS: operand_string = "lroundf(" + operand_string + ")"; break; @@ -350,7 +351,6 @@ void N64Recomp::CGenerator::get_binary_expr_string(BinaryOpType type, const Bina thread_local std::string input_b{}; thread_local std::string func_string{}; thread_local std::string infix_string{}; - bool is_infix; get_operand_string(operands.operands[0], operands.operand_operations[0], ctx, input_a); get_operand_string(operands.operands[1], operands.operand_operations[1], ctx, input_b); get_notation(type, func_string, infix_string); @@ -393,6 +393,7 @@ void N64Recomp::CGenerator::get_binary_expr_string(BinaryOpType type, const Bina } void N64Recomp::CGenerator::emit_function_start(const std::string& function_name, size_t func_index) const { + (void)func_index; fmt::print(output_file, "RECOMP_FUNC void {}(uint8_t* rdram, recomp_context* ctx) {{\n" // these variables shouldn't need to be preserved across function boundaries, so make them local for more efficient output @@ -476,7 +477,8 @@ void N64Recomp::CGenerator::emit_switch_error(uint32_t instr_vram, uint32_t jtbl fmt::print(output_file, "default: switch_error(__func__, 0x{:08X}, 0x{:08X});\n", instr_vram, jtbl_vram); } -void N64Recomp::CGenerator::emit_return(const Context& context) const { +void N64Recomp::CGenerator::emit_return(const Context& context, size_t func_index) const { + (void)func_index; if (context.trace_mode) { fmt::print(output_file, "TRACE_RETURN()\n "); } @@ -575,7 +577,6 @@ void N64Recomp::CGenerator::process_unary_op(const UnaryOp& op, const Instructio // TODO these thread locals probably don't actually help right now, so figure out a better way to prevent allocations. thread_local std::string output{}; thread_local std::string input{}; - bool is_infix; get_operand_string(op.output, UnaryOpType::None, ctx, output); get_operand_string(op.input, op.operation, ctx, input); fmt::print(output_file, "{} = {};\n", output, input); @@ -587,7 +588,6 @@ void N64Recomp::CGenerator::process_store_op(const StoreOp& op, const Instructio thread_local std::string base_str{}; thread_local std::string imm_str{}; thread_local std::string value_input{}; - bool is_infix; get_operand_string(Operand::Base, UnaryOpType::None, ctx, base_str); get_operand_string(Operand::ImmS16, UnaryOpType::None, ctx, imm_str); get_operand_string(op.value_input, UnaryOpType::None, ctx, value_input); diff --git a/src/config.cpp b/src/config.cpp index 477ab88..5df8873 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -201,8 +201,8 @@ std::vector get_instruction_patches(const toml::tab return ret; } -std::vector get_function_hooks(const toml::table* patches_data) { - std::vector ret; +std::vector get_function_hooks(const toml::table* patches_data) { + std::vector ret; // Check if the function hook array exists. const toml::node_view func_hook_data = (*patches_data)["hook"]; @@ -230,7 +230,7 @@ std::vector get_function_hooks(const toml::table* patch throw toml::parse_error("before_vram is not word-aligned", el.source()); } - ret.push_back(N64Recomp::FunctionHook{ + ret.push_back(N64Recomp::FunctionTextHook{ .func_name = func_name.value(), .before_vram = before_vram.has_value() ? (int32_t)before_vram.value() : 0, .text = text.value(), @@ -245,6 +245,46 @@ std::vector get_function_hooks(const toml::table* patch return ret; } +void get_mdebug_mappings(const toml::array* mdebug_mappings_array, + std::unordered_map& mdebug_text_map, + std::unordered_map& mdebug_data_map, + std::unordered_map& mdebug_rodata_map, + std::unordered_map& mdebug_bss_map +) { + mdebug_mappings_array->for_each([&mdebug_text_map, &mdebug_data_map, &mdebug_rodata_map, &mdebug_bss_map](auto&& el) { + if constexpr (toml::is_table) { + std::optional filename = el["filename"].template value(); + std::optional input_section = el["input_section"].template value(); + std::optional output_section = el["output_section"].template value(); + + if (filename.has_value() && input_section.has_value() && output_section.has_value()) { + const std::string& input_section_val = input_section.value(); + if (input_section_val == ".text") { + mdebug_text_map.emplace(filename.value(), output_section.value()); + } + else if (input_section_val == ".data") { + mdebug_data_map.emplace(filename.value(), output_section.value()); + } + else if (input_section_val == ".rodata") { + mdebug_rodata_map.emplace(filename.value(), output_section.value()); + } + else if (input_section_val == ".bss") { + mdebug_bss_map.emplace(filename.value(), output_section.value()); + } + else { + throw toml::parse_error("Invalid input section in mdebug file mapping entry", el.source()); + } + } + else { + throw toml::parse_error("Mdebug file mappings entry is missing required value(s)", el.source()); + } + } + else { + throw toml::parse_error("Invalid mdebug file mappings entry", el.source()); + } + }); +} + N64Recomp::Config::Config(const char* path) { // Start this config out as bad so that it has to finish parsing without errors to be good. entrypoint = 0; @@ -368,6 +408,22 @@ N64Recomp::Config::Config(const char* path) { unpaired_lo16_warnings = true; } + // Control whether the recompiler should look for and parse mdebug (optional, defaults to false) + std::optional use_mdebug_opt = input_data["use_mdebug"].value(); + if (use_mdebug_opt.has_value()) { + use_mdebug = use_mdebug_opt.value(); + } + else { + use_mdebug = false; + } + + // Symbols to ignore when parsing mdebug (option, defaults to empty) + toml::node_view mdebug_mappings_data = input_data["mdebug_file_mappings"]; + if (mdebug_mappings_data.is_array()) { + get_mdebug_mappings(mdebug_mappings_data.as_array(), + mdebug_text_map, mdebug_data_map, mdebug_rodata_map, mdebug_bss_map); + } + std::optional recomp_include_opt = input_data["recomp_include"].value(); if (recomp_include_opt.has_value()) { recomp_include = recomp_include_opt.value(); @@ -609,7 +665,7 @@ bool N64Recomp::Context::from_symbol_file(const std::filesystem::path& symbol_fi RelocType reloc_type = reloc_type_from_name(type_string.value()); - if (reloc_type != RelocType::R_MIPS_HI16 && reloc_type != RelocType::R_MIPS_LO16 && reloc_type != RelocType::R_MIPS_32) { + if (reloc_type != RelocType::R_MIPS_HI16 && reloc_type != RelocType::R_MIPS_LO16 && reloc_type != RelocType::R_MIPS_26 && reloc_type != RelocType::R_MIPS_32) { throw toml::parse_error("Invalid reloc entry type", reloc_el.source()); } diff --git a/src/config.h b/src/config.h index 0f01a33..7d6db23 100644 --- a/src/config.h +++ b/src/config.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace N64Recomp { struct InstructionPatch { @@ -12,7 +13,7 @@ namespace N64Recomp { uint32_t value; }; - struct FunctionHook { + struct FunctionTextHook { std::string func_name; int32_t before_vram; std::string text; @@ -42,6 +43,7 @@ namespace N64Recomp { bool single_file_output; bool use_absolute_symbols; bool unpaired_lo16_warnings; + bool use_mdebug; bool trace_mode; bool allow_exports; bool strict_patch_mode; @@ -57,11 +59,16 @@ namespace N64Recomp { std::vector ignored_funcs; std::vector renamed_funcs; std::vector instruction_patches; - std::vector function_hooks; + std::vector function_hooks; std::vector manual_func_sizes; std::vector manual_functions; std::string bss_section_suffix; std::string recomp_include; + // Manual mappings of mdebug file records to elf sections. + std::unordered_map mdebug_text_map; + std::unordered_map mdebug_data_map; + std::unordered_map mdebug_rodata_map; + std::unordered_map mdebug_bss_map; Config(const char* path); bool good() { return !bad; } diff --git a/src/elf.cpp b/src/elf.cpp index 68f6cc7..f0ab22f 100644 --- a/src/elf.cpp +++ b/src/elf.cpp @@ -6,6 +6,8 @@ #include "recompiler/context.h" #include "elfio/elfio.hpp" +#include "mdebug.h" + bool read_symbols(N64Recomp::Context& context, const ELFIO::elfio& elf_file, ELFIO::section* symtab_section, const N64Recomp::ElfParsingConfig& elf_config, bool dumping_context, std::unordered_map>& data_syms) { bool found_entrypoint_func = false; ELFIO::symbol_section_accessor symbols{ elf_file, symtab_section }; @@ -104,10 +106,10 @@ bool read_symbols(N64Recomp::Context& context, const ELFIO::elfio& elf_file, ELF if (section_index < context.sections.size()) { auto section_offset = value - elf_file.sections[section_index]->get_address(); - const uint32_t* words = reinterpret_cast(elf_file.sections[section_index]->get_data() + section_offset); uint32_t vram = static_cast(value); uint32_t num_instructions = type == ELFIO::STT_FUNC ? size / 4 : 0; uint32_t rom_address = static_cast(section_offset + section.rom_addr); + const uint32_t* words = reinterpret_cast(context.rom.data() + rom_address); section.function_addrs.push_back(vram); context.functions_by_vram[vram].push_back(context.functions.size()); @@ -126,9 +128,9 @@ bool read_symbols(N64Recomp::Context& context, const ELFIO::elfio& elf_file, ELF if (bind == ELFIO::STB_LOCAL) { name = fmt::format("{}_{:08X}", name, rom_address); } - + if (num_instructions > 0) { - context.section_functions[section_index].push_back(context.functions.size()); + context.section_functions[section_index].push_back(context.functions.size()); recorded_symbol = true; } context.functions_by_name[name] = context.functions.size(); @@ -178,10 +180,13 @@ bool read_symbols(N64Recomp::Context& context, const ELFIO::elfio& elf_file, ELF fmt::print("Symbol \"{}\" not in a valid section ({})\n", name, section_index); } - // Move this symbol into the corresponding non-bss section if it's in a bss section. + // Move this symbol into the corresponding non-bss section if it's in a bss section and the paired section is relocatable. auto find_bss_it = bss_section_to_target_section.find(target_section_index); if (find_bss_it != bss_section_to_target_section.end()) { - target_section_index = find_bss_it->second; + uint16_t new_target_section_index = find_bss_it->second; + if (new_target_section_index < context.sections.size() && context.sections[new_target_section_index].relocatable) { + target_section_index = find_bss_it->second; + } } data_syms[target_section_index].emplace_back( @@ -215,7 +220,7 @@ std::optional get_segment(const std::vector& segments, ELF return std::nullopt; } -ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfParsingConfig& elf_config, const ELFIO::elfio& elf_file) { +ELFIO::section* read_sections(N64Recomp::Context& context, ELFIO::section*& mdebug_section_out, const N64Recomp::ElfParsingConfig& elf_config, const ELFIO::elfio& elf_file) { ELFIO::section* symtab_section = nullptr; std::vector segments{}; segments.resize(elf_file.segments.size()); @@ -247,7 +252,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP ELFIO::Elf_Xword flags = section->get_flags(); ELFIO::Elf_Xword section_size = section->get_size(); - // Check if this section will end up in the ROM. It must not be a nobits (NOLOAD) type, must have the alloc flag set and must have a nonzero size. + // Check if this section will end up in the ROM. It must not be a nobits (NOLOAD) type, must have the alloc flag set and must have a nonzero size. if (type != ELFIO::SHT_NOBITS && (flags & ELFIO::SHF_ALLOC) && section_size != 0) { std::optional segment_index = get_segment(segments, section_size, section->get_offset()); if (!segment_index.has_value()) { @@ -283,10 +288,15 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP symtab_section = section.get(); } + // Check if this section is an mdebug section and record it if so. Note we expect just one mdebug section + if (type == 0x70000005/* SHT_MIPS_DEBUG */) { + mdebug_section_out = section.get(); + } + if (elf_config.all_sections_relocatable || elf_config.relocatable_sections.contains(section_name)) { section_out.relocatable = true; } - + // Check if this section is a reloc section if (type == ELFIO::SHT_REL) { // If it is, determine the name of the section it relocates @@ -294,7 +304,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP fmt::print(stderr, "Could not determine corresponding section for reloc section {}\n", section_name.c_str()); return nullptr; } - + // FIXME This should be using SH_INFO to create a reloc section to target section mapping instead of using the name. std::string reloc_target_section = section_name.substr(strlen(".rel")); @@ -310,8 +320,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP if (type == ELFIO::SHT_NOBITS && section_name.ends_with(elf_config.bss_section_suffix)) { std::string bss_target_section = section_name.substr(0, section_name.size() - elf_config.bss_section_suffix.size()); - // If this bss section is for a section that has been marked as relocatable, record it in the reloc section lookup - if (elf_config.all_sections_relocatable || elf_config.relocatable_sections.contains(bss_target_section)) { + if (!bss_target_section.empty()) { bss_sections_by_name[bss_target_section] = section.get(); } } @@ -368,8 +377,8 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP ELFIO::relocation_section_accessor rel_accessor{ elf_file, reloc_find->second }; // Allocate space for the relocs in this section section_out.relocs.resize(rel_accessor.get_entries_num()); - // Track whether the previous reloc was a HI16 and its previous full_immediate - bool prev_hi = false; + // Track consecutive identical HI16 relocs to handle the GNU extension to the o32 ABI. + int prev_hi_count = 0; // Track whether the previous reloc was a LO16 bool prev_lo = false; uint32_t prev_hi_immediate = 0; @@ -430,7 +439,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP rel_symbol_name); return nullptr; } - + reloc_out.reference_symbol = true; // Replace the reloc's symbol index with the index into the reference symbol array. rel_section_vram = 0; @@ -469,7 +478,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP uint32_t rel_immediate = reloc_rom_word & 0xFFFF; uint32_t full_immediate = (prev_hi_immediate << 16) + (int16_t)rel_immediate; reloc_out.target_section_offset = full_immediate + rel_symbol_offset - rel_section_vram; - if (prev_hi) { + if (prev_hi_count != 0) { if (prev_hi_symbol != rel_symbol) { fmt::print(stderr, "Paired HI16 and LO16 relocations have different symbols\n" " LO16 reloc index {} in section {} referencing symbol {} with offset 0x{:08X}\n", @@ -477,8 +486,12 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP return nullptr; } - // Set the previous HI16 relocs' relocated address. - section_out.relocs[i - 1].target_section_offset = reloc_out.target_section_offset; + // Set the previous HI16 relocs' relocated addresses. + for (size_t paired_index = i - prev_hi_count; paired_index < i; paired_index++) { + uint32_t hi_immediate = section_out.relocs[paired_index].target_section_offset; + uint32_t paired_full_immediate = hi_immediate + (int16_t)rel_immediate; + section_out.relocs[paired_index].target_section_offset = paired_full_immediate + rel_symbol_offset - rel_section_vram; + } } else { // Orphaned LO16 reloc warnings. @@ -502,7 +515,8 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP } prev_lo = true; } else { - if (prev_hi) { + // Allow a HI16 to follow another HI16 for the GNU ABI extension. + if (reloc_out.type != N64Recomp::RelocType::R_MIPS_HI16 && prev_hi_count != 0) { // This is an invalid elf as the MIPS System V ABI documentation states: // "Each relocation type of R_MIPS_HI16 must have an associated R_MIPS_LO16 entry // immediately following it in the list of relocations." @@ -515,11 +529,26 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP if (reloc_out.type == N64Recomp::RelocType::R_MIPS_HI16) { uint32_t rel_immediate = reloc_rom_word & 0xFFFF; - prev_hi = true; - prev_hi_immediate = rel_immediate; - prev_hi_symbol = rel_symbol; + // First HI16, store its immediate. + if (prev_hi_count == 0) { + prev_hi_immediate = rel_immediate; + prev_hi_symbol = rel_symbol; + } + // HI16 that follows another HI16, ensure they reference the same symbol. + else { + if (prev_hi_symbol != rel_symbol) { + fmt::print(stderr, "HI16 reloc (index {} symbol {} offset 0x{:08X}) follows another HI16 reloc with a different symbol (index {} symbol {} offset 0x{:08X}) in section {}\n", + i, rel_symbol, section_out.relocs[i].address, + i - 1, prev_hi_symbol, section_out.relocs[i - 1].address, + section_out.name); + return nullptr; + } + } + // Populate the reloc temporarily, the full offset will be calculated upon pairing. + reloc_out.target_section_offset = rel_immediate << 16; + prev_hi_count++; } else { - prev_hi = false; + prev_hi_count = 0; } if (reloc_out.type == N64Recomp::RelocType::R_MIPS_32) { @@ -553,11 +582,41 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP // Sort this section's relocs by address, which allows for binary searching and more efficient iteration during recompilation. // This is safe to do as the entire full_immediate in present in relocs due to the pairing that was done earlier, so the HI16 does not // need to directly preceed the matching LO16 anymore. - std::sort(section_out.relocs.begin(), section_out.relocs.end(), + std::sort(section_out.relocs.begin(), section_out.relocs.end(), [](const N64Recomp::Reloc& a, const N64Recomp::Reloc& b) { return a.address < b.address; } ); + + // Patch the ROM word for HI16 and LO16 reference symbol relocs to non-relocatable sections. + for (size_t i = 0; i < section_out.relocs.size(); i++) { + auto& reloc = section_out.relocs[i]; + if (reloc.reference_symbol && (reloc.type == N64Recomp::RelocType::R_MIPS_HI16 || reloc.type == N64Recomp::RelocType::R_MIPS_LO16)) { + bool target_section_relocatable = context.is_reference_section_relocatable(reloc.target_section); + if (!target_section_relocatable) { + uint32_t reloc_rom_addr = reloc.address - section_out.ram_addr + section_out.rom_addr; + uint32_t reloc_rom_word = byteswap(*reinterpret_cast(context.rom.data() + reloc_rom_addr)); + + uint32_t ref_section_vram = context.get_reference_section_vram(reloc.target_section); + uint32_t full_immediate = reloc.target_section_offset + ref_section_vram; + + uint32_t imm; + + if (reloc.type == N64Recomp::RelocType::R_MIPS_HI16) { + imm = (full_immediate >> 16) + ((full_immediate >> 15) & 1); + } + else { + imm = full_immediate & 0xFFFF; + } + + *reinterpret_cast(context.rom.data() + reloc_rom_addr) = byteswap(reloc_rom_word | imm); + // Remove the reloc by setting it to a type of NONE. + reloc.type = N64Recomp::RelocType::R_MIPS_NONE; + reloc.reference_symbol = false; + reloc.symbol_index = (uint32_t)-1; + } + } + } } } @@ -587,14 +646,15 @@ bool N64Recomp::Context::from_elf_file(const std::filesystem::path& elf_file_pat } if (elf_file.get_encoding() != ELFIO::ELFDATA2MSB) { - fmt::print("Incorrect endianness\n"); + fmt::print("Incorrect elf endianness\n"); return false; } setup_context_for_elf(out, elf_file); // Read all of the sections in the elf and look for the symbol table section - ELFIO::section* symtab_section = read_sections(out, elf_config, elf_file); + ELFIO::section* mdebug_section = nullptr; + ELFIO::section* symtab_section = read_sections(out, mdebug_section, elf_config, elf_file); // If no symbol table was found then exit if (symtab_section == nullptr) { @@ -604,5 +664,17 @@ bool N64Recomp::Context::from_elf_file(const std::filesystem::path& elf_file_pat // Read all of the symbols in the elf and look for the entrypoint function found_entrypoint_out = read_symbols(out, elf_file, symtab_section, elf_config, for_dumping_context, data_syms_out); + // Process an mdebug section for static symbols. The presence of an mdebug section in the input is optional. + if (elf_config.use_mdebug) { + if (mdebug_section == nullptr) { + fmt::print("\"use_mdebug\" set to true in config, but no mdebug section is present in the elf!\n"); + return false; + } + if (!N64Recomp::MDebug::parse_mdebug(elf_config, mdebug_section->get_data(), static_cast(mdebug_section->get_offset()), out, data_syms_out)) { + fmt::print("Failed to parse mdebug section\n"); + return false; + } + } + return true; } diff --git a/src/main.cpp b/src/main.cpp index 8a8fe91..f79580f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -199,7 +199,7 @@ void dump_context(const N64Recomp::Context& context, const std::unordered_map(reloc.type)], reloc.address, reloc.target_section_offset + section.ram_addr); } @@ -272,22 +272,26 @@ int main(int argc, char** argv) { std::exit(EXIT_FAILURE); }; - bool dumping_context; + bool dumping_context = false; - if (argc >= 3) { - std::string arg2 = argv[2]; - if (arg2 == "--dump-context") { - dumping_context = true; - } else { - fmt::print("Usage: {} [--dump-context]\n", argv[0]); - std::exit(EXIT_SUCCESS); - } - } else { - dumping_context = false; + if (argc < 2) { + fmt::print("Usage: {} [--dump-context]\n", argv[0]); + return EXIT_SUCCESS; } const char* config_path = argv[1]; + for (size_t i = 2; i < argc; i++) { + std::string_view cur_arg = argv[i]; + if (cur_arg == "--dump-context") { + dumping_context = true; + } + else { + fmt::print("Unknown argument \"{}\"\n", cur_arg); + return EXIT_FAILURE; + } + } + N64Recomp::Config config{ config_path }; if (!config.good()) { exit_failure(fmt::format("Failed to load config file: {}\n", config_path)); @@ -310,6 +314,9 @@ int main(int argc, char** argv) { std::unordered_set relocatable_sections{}; relocatable_sections.insert(relocatable_sections_ordered.begin(), relocatable_sections_ordered.end()); + std::unordered_set ignored_syms_set{}; + ignored_syms_set.insert(config.ignored_funcs.begin(), config.ignored_funcs.end()); + N64Recomp::Context context{}; if (!config.elf_path.empty() && !config.symbols_file_path.empty()) { @@ -347,11 +354,17 @@ int main(int argc, char** argv) { N64Recomp::ElfParsingConfig elf_config { .bss_section_suffix = config.bss_section_suffix, .relocatable_sections = std::move(relocatable_sections), + .ignored_syms = std::move(ignored_syms_set), + .mdebug_text_map = config.mdebug_text_map, + .mdebug_data_map = config.mdebug_data_map, + .mdebug_rodata_map = config.mdebug_rodata_map, + .mdebug_bss_map = config.mdebug_bss_map, .has_entrypoint = config.has_entrypoint, .entrypoint_address = config.entrypoint, .use_absolute_symbols = config.use_absolute_symbols, .unpaired_lo16_warnings = config.unpaired_lo16_warnings, .all_sections_relocatable = false, + .use_mdebug = config.use_mdebug, }; for (const auto& func_size : config.manual_func_sizes) { @@ -359,7 +372,9 @@ int main(int argc, char** argv) { } bool found_entrypoint_func; - N64Recomp::Context::from_elf_file(config.elf_path, context, elf_config, dumping_context, data_syms, found_entrypoint_func); + if (!N64Recomp::Context::from_elf_file(config.elf_path, context, elf_config, dumping_context, data_syms, found_entrypoint_func)) { + exit_failure("Failed to parse elf\n"); + } // Add any manual functions add_manual_functions(context, config.manual_functions); @@ -536,7 +551,7 @@ int main(int argc, char** argv) { } // Apply any function hooks. - for (const N64Recomp::FunctionHook& patch : config.function_hooks) { + for (const N64Recomp::FunctionTextHook& patch : config.function_hooks) { // Check if the specified function exists. auto func_find = context.functions_by_name.find(patch.func_name); if (func_find == context.functions_by_name.end()) { @@ -646,6 +661,11 @@ int main(int argc, char** argv) { continue; } + // Ignore R_MIPS_NONE relocs, which get produced during symbol parsing for non-relocatable reference sections. + if (reloc.type == N64Recomp::RelocType::R_MIPS_NONE) { + continue; + } + // Check if the reloc points to the event section. if (reloc.target_section == event_section_index) { // It does, so find the function it's pointing at. @@ -774,7 +794,7 @@ int main(int argc, char** argv) { // Search for the closest function size_t closest_func_index = 0; - while (section_funcs[closest_func_index] < static_func_addr && closest_func_index < section_funcs.size()) { + while (closest_func_index < section_funcs.size() && section_funcs[closest_func_index] < static_func_addr) { closest_func_index++; } @@ -866,13 +886,6 @@ int main(int argc, char** argv) { ); } - fmt::print(func_header_file, - "\n" - "#ifdef __cplusplus\n" - "}}\n" - "#endif\n" - ); - { std::ofstream overlay_file(config.output_func_path / "recomp_overlays.inl"); std::string section_load_table = "static SectionTableEntry section_table[] = {\n"; @@ -891,6 +904,7 @@ int main(int argc, char** argv) { for (size_t section_index = 0; section_index < context.sections.size(); section_index++) { const auto& section = context.sections[section_index]; const auto& section_funcs = context.section_functions[section_index]; + const auto& section_relocs = section.relocs; if (section.has_mips32_relocs || !section_funcs.empty()) { std::string_view section_name_trimmed{ section.name }; @@ -904,21 +918,66 @@ int main(int argc, char** argv) { } std::string section_funcs_array_name = fmt::format("section_{}_{}_funcs", section_index, section_name_trimmed); + std::string section_relocs_array_name = section_relocs.empty() ? "nullptr" : fmt::format("section_{}_{}_relocs", section_index, section_name_trimmed); + std::string section_relocs_array_size = section_relocs.empty() ? "0" : fmt::format("ARRLEN({})", section_relocs_array_name); - section_load_table += fmt::format(" {{ .rom_addr = 0x{0:08X}, .ram_addr = 0x{1:08X}, .size = 0x{2:08X}, .funcs = {3}, .num_funcs = ARRLEN({3}), .index = {4} }},\n", - section.rom_addr, section.ram_addr, section.size, section_funcs_array_name, section_index); + // Write the section's table entry. + section_load_table += fmt::format(" {{ .rom_addr = 0x{0:08X}, .ram_addr = 0x{1:08X}, .size = 0x{2:08X}, .funcs = {3}, .num_funcs = ARRLEN({3}), .relocs = {4}, .num_relocs = {5}, .index = {6} }},\n", + section.rom_addr, section.ram_addr, section.size, section_funcs_array_name, + section_relocs_array_name, section_relocs_array_size, section_index); + // Write the section's functions. fmt::print(overlay_file, "static FuncEntry {}[] = {{\n", section_funcs_array_name); for (size_t func_index : section_funcs) { const auto& func = context.functions[func_index]; + size_t func_size = func.reimplemented ? 0 : func.words.size() * sizeof(func.words[0]); if (func.reimplemented || (!func.name.empty() && !func.ignored && func.words.size() != 0)) { - fmt::print(overlay_file, " {{ .func = {}, .offset = 0x{:08x} }},\n", func.name, func.rom - section.rom_addr); + fmt::print(overlay_file, " {{ .func = {}, .offset = 0x{:08X}, .rom_size = 0x{:08X} }},\n", + func.name, func.rom - section.rom_addr, func_size); } } fmt::print(overlay_file, "}};\n"); + + // Write the section's relocations. + if (!section_relocs.empty()) { + // Determine if reference symbols are being used. + bool reference_symbol_mode = !config.func_reference_syms_file_path.empty(); + + fmt::print(overlay_file, "static RelocEntry {}[] = {{\n", section_relocs_array_name); + + for (const N64Recomp::Reloc& reloc : section_relocs) { + bool emit_reloc = false; + uint16_t target_section = reloc.target_section; + // In reference symbol mode, only emit relocations into the table that point to + // non-absolute reference symbols, events, or manual patch symbols. + if (reference_symbol_mode) { + bool manual_patch_symbol = N64Recomp::is_manual_patch_symbol(reloc.target_section_offset); + bool is_absolute = reloc.target_section == N64Recomp::SectionAbsolute; + emit_reloc = (reloc.reference_symbol && !is_absolute) || target_section == N64Recomp::SectionEvent || manual_patch_symbol; + } + // Otherwise, emit all relocs. + else { + emit_reloc = true; + } + if (emit_reloc) { + uint32_t target_section_offset; + if (reloc.target_section == N64Recomp::SectionEvent) { + target_section_offset = reloc.symbol_index; + } + else { + target_section_offset = reloc.target_section_offset; + } + fmt::print(overlay_file, " {{ .offset = 0x{:08X}, .target_section_offset = 0x{:08X}, .target_section = {}, .type = {} }}, \n", + reloc.address - section.ram_addr, target_section_offset, reloc.target_section, reloc_names[static_cast(reloc.type)] ); + } + } + + fmt::print(overlay_file, "}};\n"); + } + written_sections++; } } @@ -982,9 +1041,45 @@ int main(int argc, char** argv) { // Add a dummy element at the end to ensure the array has a valid length because C doesn't allow zero-size arrays. fmt::print(overlay_file, " NULL\n"); fmt::print(overlay_file, "}};\n"); + + // Collect manual patch symbols. + std::vector> manual_patch_syms{}; + + for (const auto& func : context.functions) { + if (func.words.empty() && N64Recomp::is_manual_patch_symbol(func.vram)) { + manual_patch_syms.emplace_back(func.vram, func.name); + } + } + + // Sort the manual patch symbols by vram. + std::sort(manual_patch_syms.begin(), manual_patch_syms.end(), [](const auto& lhs, const auto& rhs) { + return lhs.first < rhs.first; + }); + + // Emit the manual patch symbols. + fmt::print(overlay_file, + "\n" + "static const ManualPatchSymbol manual_patch_symbols[] = {{\n" + ); + for (const auto& manual_patch_sym_entry : manual_patch_syms) { + fmt::print(overlay_file, " {{ 0x{:08X}, {} }},\n", manual_patch_sym_entry.first, manual_patch_sym_entry.second); + + fmt::print(func_header_file, + "void {}(uint8_t* rdram, recomp_context* ctx);\n", manual_patch_sym_entry.second); + } + // Add a dummy element at the end to ensure the array has a valid length because C doesn't allow zero-size arrays. + fmt::print(overlay_file, " {{ 0, NULL }}\n"); + fmt::print(overlay_file, "}};\n"); } } + fmt::print(func_header_file, + "\n" + "#ifdef __cplusplus\n" + "}}\n" + "#endif\n" + ); + if (!config.output_binary_path.empty()) { std::ofstream output_binary{config.output_binary_path, std::ios::binary}; output_binary.write(reinterpret_cast(context.rom.data()), context.rom.size()); diff --git a/src/mdebug.cpp b/src/mdebug.cpp new file mode 100644 index 0000000..bc357ca --- /dev/null +++ b/src/mdebug.cpp @@ -0,0 +1,670 @@ +#include +#include + +#include "fmt/format.h" + +#include "mdebug.h" + +struct MDebugSymbol { + std::string name; + uint32_t address; + uint32_t size; + bool is_func; + bool is_static; + bool is_bss; + bool is_rodata; + bool ignored; +}; + +struct MDebugFile { + std::string filename; + std::vector symbols; +}; + +class MDebugInfo { +public: + MDebugInfo(const N64Recomp::ElfParsingConfig& config, const char* mdebug_section, uint32_t mdebug_offset) { + using namespace N64Recomp; + good_ = false; + // Read, byteswap and relocate the symbolic header. Relocation here means convert file-relative offsets to section-relative offsets. + + MDebug::HDRR hdrr; + std::memcpy(&hdrr, mdebug_section, sizeof(MDebug::HDRR)); + hdrr.swap(); + hdrr.relocate(mdebug_offset); + + // Check the magic value and version number are what we expect. + + if (hdrr.magic != MDebug::MAGIC || hdrr.vstamp != 0) { + fmt::print(stderr, "Warning: Found an mdebug section with bad magic value or version (magic={} version={}). Skipping.\n", hdrr.magic, hdrr.vstamp); + return; + } + + // Read the various records that are relevant for collecting static symbols and where they are declared. + + std::vector fdrs = hdrr.read_fdrs(mdebug_section); + std::vector all_auxs = hdrr.read_auxs(mdebug_section); + std::vector all_pdrs = hdrr.read_pdrs(mdebug_section); + std::vector all_symrs = hdrr.read_symrs(mdebug_section); + + // For each file descriptor + for (size_t fdr_index = 0; fdr_index < fdrs.size(); fdr_index++) { + MDebug::FDR& fdr = fdrs[fdr_index]; + MDebugSymbol pending_sym{}; + bool pending_sym_ready = false; + + auto flush_pending_sym = [&]() { + if (pending_sym_ready) { + // Handle ignored symbols. + pending_sym.ignored = config.ignored_syms.contains(pending_sym.name); + // Add the symbol. + add_symbol(fdr_index, std::move(pending_sym)); + } + pending_sym_ready = false; + }; + + const char* fdr_name = fdr.get_name(mdebug_section + hdrr.cbSsOffset); + add_file(fdr_name); + + // For every symbol record in the file descriptor record. + for (auto symr : fdr.get_symrs(all_symrs)) { + MDebug::ST type = symr.get_st(); + MDebug::SC storage_class = symr.get_sc(); + + switch (type) { + case MDebug::ST_PROC: + case MDebug::ST_STATICPROC: + flush_pending_sym(); + if (symr.value != 0) { + pending_sym.name = fdr.get_string(mdebug_section + hdrr.cbSsOffset, symr.iss); + pending_sym.address = symr.value; + pending_sym.size = 0; + pending_sym.is_func = true; + pending_sym.is_static = (type == MDebug::ST_STATICPROC); + pending_sym.is_bss = false; + pending_sym.is_rodata = false; + pending_sym_ready = true; + } + break; + case MDebug::ST_END: + if (pending_sym.is_func) { + pending_sym.size = symr.value; + } + flush_pending_sym(); + break; + case MDebug::ST_GLOBAL: + case MDebug::ST_STATIC: + flush_pending_sym(); + if (symr.value != 0) { + pending_sym.name = fdr.get_string(mdebug_section + hdrr.cbSsOffset, symr.iss); + pending_sym.address = symr.value; + pending_sym.size = 0; + pending_sym.is_func = false; + pending_sym.is_static = (type == MDebug::ST_STATIC); + pending_sym.is_bss = (storage_class == MDebug::SC_BSS); + pending_sym.is_rodata = (storage_class == MDebug::SC_RDATA); + pending_sym_ready = true; + } + break; + default: + flush_pending_sym(); + break; + } + } + flush_pending_sym(); + } + good_ = true; + } + + bool is_identifier_char(char c) { + if (c >= 'a' && c <= 'z') { + return true; + } + if (c >= 'A' && c <= 'Z') { + return true; + } + if (c == '_') { + return true; + } + if (c >= '0' && c <= '9') { + return true; + } + return false; + } + + std::string sanitize_section_name(std::string section_name) { + // Skip periods at the start of the section name. + size_t start_pos = 0; + while (section_name[start_pos] == '.' && start_pos < section_name.size()) { + start_pos++; + } + + std::string ret = section_name.substr(start_pos); + for (size_t char_index = 0; char_index < ret.size(); char_index++) { + if (!is_identifier_char(ret[char_index])) { + ret[char_index] = '_'; + } + } + return ret; + } + + bool populate_context(const N64Recomp::ElfParsingConfig& elf_config, N64Recomp::Context& context, N64Recomp::DataSymbolMap& data_syms) { + size_t num_files = files_.size(); + std::vector file_text_sections{}; + std::vector file_data_sections{}; + std::vector file_rodata_sections{}; + std::vector file_bss_sections{}; + file_text_sections.resize(num_files, (uint16_t)-1); + file_data_sections.resize(num_files, (uint16_t)-1); + file_rodata_sections.resize(num_files, (uint16_t)-1); + file_bss_sections.resize(num_files, (uint16_t)-1); + std::unordered_map> mdebug_symbol_names{}; // Maps symbol name to list of files that have a symbol of that name. + + // Build a lookup of section name to elf section index. + std::unordered_map elf_sections_by_name{}; + for (uint16_t section_index = 0; section_index < context.sections.size(); section_index++) { + const N64Recomp::Section& section = context.sections[section_index]; + elf_sections_by_name.emplace(section.name, section_index); + } + + // First pass to collect symbol names and map mdebug files to elf sections. + for (size_t file_index = 0; file_index < num_files; file_index++) { + const MDebugFile& file = files_[file_index]; + + if (file.symbols.empty()) { + continue; + } + + bool has_funcs = false; + bool has_data = false; + bool has_rodata = false; + bool has_bss = false; + // Find the section that this file's .text was placed into by looking up global functions. + int file_text_section = -1; + uint32_t min_data_address = 0x0; + uint32_t max_data_address = 0x0; + uint32_t min_rodata_address = 0x0; + uint32_t max_rodata_address = 0x0; + uint32_t min_bss_address = 0x0; + uint32_t max_bss_address = 0x0; + for (const auto& sym : file.symbols) { + if (sym.ignored) { + continue; + } + mdebug_symbol_names[sym.name].emplace_back(file_index); + if (sym.is_func) { + has_funcs = true; + if (!sym.is_static) { + auto find_it = context.functions_by_name.find(sym.name); + if (find_it != context.functions_by_name.end()) { + const auto& found_sym = context.functions[find_it->second]; + file_text_section = found_sym.section_index; + } + } + } + else { + if (sym.address != 0) { + // .bss + if (sym.is_bss) { + has_bss = true; + if (min_bss_address == 0) { + min_bss_address = sym.address; + } + else { + min_bss_address = std::min(min_bss_address, sym.address); + } + max_bss_address = std::max(max_bss_address, sym.address + sym.size); + } + // .rodata + else if (sym.is_rodata) { + has_rodata = true; + if (min_rodata_address == 0) { + min_rodata_address = sym.address; + } + else { + min_rodata_address = std::min(min_rodata_address, sym.address); + } + max_rodata_address = std::max(max_rodata_address, sym.address + sym.size); + } + // .data + else { + has_data = true; + if (min_data_address == 0) { + min_data_address = sym.address; + } + else { + min_data_address = std::min(min_data_address, sym.address); + } + max_data_address = std::max(max_data_address, sym.address + sym.size); + } + } + } + } + + if (!has_funcs) { + continue; + } + + // Manual file text section mapping. + { + auto find_text_mapping_it = elf_config.mdebug_text_map.find(file.filename); + if (find_text_mapping_it != elf_config.mdebug_text_map.end()) { + auto find_text_section_it = elf_sections_by_name.find(find_text_mapping_it->second); + if (find_text_section_it == elf_sections_by_name.end()) { + printf(".text section for mdebug source file \"%s\" is mapped to section \"%s\", which doesn't exist in the elf\n", file.filename.c_str(), find_text_mapping_it->second.c_str()); + return false; + } + file_text_section = find_text_section_it->second; + } + } + + if (file_text_section == -1) { + printf("Couldn't determine elf section of mdebug info for file %s\n", file.filename.c_str()); + return false; + } + + file_text_sections[file_index] = file_text_section; + const N64Recomp::Section& text_section = context.sections[file_text_section]; + + if (has_data) { + uint16_t file_data_section; + + // Manual file data section mapping. + auto find_data_mapping_it = elf_config.mdebug_data_map.find(file.filename); + if (find_data_mapping_it != elf_config.mdebug_data_map.end()) { + auto find_data_section_it = elf_sections_by_name.find(find_data_mapping_it->second); + if (find_data_section_it == elf_sections_by_name.end()) { + printf(".data section for mdebug source file \"%s\" is mapped to section \"%s\", which doesn't exist in the elf\n", file.filename.c_str(), find_data_mapping_it->second.c_str()); + return false; + } + file_data_section = find_data_section_it->second; + } + // Automatic mapping, attempt to use the same section that .text was placed in. + else { + if (min_data_address < text_section.ram_addr || max_data_address > text_section.ram_addr + text_section.size) { + printf("File %s has static data in mdebug which did not overlap with section %s\n", file.filename.c_str(), text_section.name.c_str()); + return false; + } + file_data_section = file_text_section; + } + + file_data_sections[file_index] = file_data_section; + } + + if (has_rodata) { + uint16_t file_rodata_section;; + + // Manual file rodata section mapping. + auto find_rodata_mapping_it = elf_config.mdebug_data_map.find(file.filename); + if (find_rodata_mapping_it != elf_config.mdebug_data_map.end()) { + auto find_rodata_section_it = elf_sections_by_name.find(find_rodata_mapping_it->second); + if (find_rodata_section_it == elf_sections_by_name.end()) { + printf(".rodata section for mdebug source file \"%s\" is mapped to section \"%s\", which doesn't exist in the elf\n", file.filename.c_str(), find_rodata_mapping_it->second.c_str()); + return false; + } + file_rodata_section = find_rodata_section_it->second; + } + // Automatic mapping, attempt to use the same section that .text was placed in. + else { + if (min_rodata_address < text_section.ram_addr || max_rodata_address > text_section.ram_addr + text_section.size) { + printf("File %s has static rodata in mdebug which did not overlap with section %s\n", file.filename.c_str(), text_section.name.c_str()); + return false; + } + file_rodata_section = file_text_section; + } + + file_rodata_sections[file_index] = file_rodata_section; + } + + if (has_bss) { + uint16_t file_bss_section; + + // Manual file bss section mapping. + auto find_bss_mapping_it = elf_config.mdebug_data_map.find(file.filename); + if (find_bss_mapping_it != elf_config.mdebug_data_map.end()) { + auto find_bss_section_it = elf_sections_by_name.find(find_bss_mapping_it->second); + if (find_bss_section_it == elf_sections_by_name.end()) { + printf(".bss section for mdebug source file \"%s\" is mapped to section \"%s\", which doesn't exist in the elf\n", file.filename.c_str(), find_bss_mapping_it->second.c_str()); + return false; + } + file_bss_section = find_bss_section_it->second; + } + // Automatic mapping, attempt to use the corresponding bss section for the section .text was placed in. + else { + if (text_section.bss_section_index == (uint16_t)-1) { + printf("File %s has static bss in mdebug but no paired bss section. Use the \"bss_section_suffix\" option to pair bss sections.\n", file.filename.c_str()); + return false; + } + const N64Recomp::Section& bss_section = context.sections[text_section.bss_section_index]; + if (min_bss_address < bss_section.ram_addr || max_bss_address > bss_section.ram_addr + bss_section.size) { + printf("File %s has static bss in mdebug which did not overlap with bss section %s\n", file.filename.c_str(), bss_section.name.c_str()); + return false; + } + file_bss_section = text_section.bss_section_index; + } + + file_bss_sections[file_index] = file_bss_section; + } + } + + // Maps symbol name to list of sections that will receive a symbol with that name. + std::unordered_map> symbol_name_to_sections; + // Elf section of each mdebug symbol, indexed by file index and then symbol index. + std::vector> file_symbol_sections; + file_symbol_sections.resize(num_files); + + // Second pass to assign symbols to elf sections. This allows the third pass to rename symbols if necessary. + // This has to be done after the first pass as we don't know section indices while processing symbols at that point. + for (size_t file_index = 0; file_index < num_files; file_index++) { + const MDebugFile& file = files_[file_index]; + file_symbol_sections[file_index].resize(file.symbols.size()); + for (size_t sym_index = 0; sym_index < file.symbols.size(); sym_index++) { + const MDebugSymbol& sym = file.symbols[sym_index]; + if (sym.ignored) { + continue; + } + if (sym.is_static) { + uint16_t sym_section; + // Static .text + if (sym.is_func) { + sym_section = file_text_sections[file_index]; + } + // Static .bss + else if (sym.is_bss) { + sym_section = file_text_sections[file_index]; + } + // Static .data/.rodata + else { + sym_section = file_text_sections[file_index]; + } + symbol_name_to_sections[sym.name].emplace_back(sym_section); + file_symbol_sections[file_index][sym_index] = sym_section; + } + } + } + + // Mapping of datasym name to section. + std::unordered_map datasyms_by_name{}; + for (const auto &[section_index, section_datasyms] : data_syms) { + for (const auto& datasym : section_datasyms) { + datasyms_by_name.emplace(datasym.name, section_index); + } + } + + // Third pass to populate the context and data symbol map, renaming symbols as needed to avoid conflicts. + for (size_t file_index = 0; file_index < num_files; file_index++) { + const MDebugFile& file = files_[file_index]; + for (size_t sym_index = 0; sym_index < file.symbols.size(); sym_index++) { + const MDebugSymbol& sym = file.symbols[sym_index]; + if (sym.ignored) { + continue; + } + + uint16_t sym_section = file_symbol_sections[file_index][sym_index]; + + // Skip symbols with no section. This should only apply to static data/bss symbols in a file that had no functions. + if (sym_section == (uint16_t)-1) { + continue; + } + + if (sym.is_static) { + bool already_exists = false; + bool already_exists_in_section = false; + + // Check if the symbol name exists in the base symbol list already. + auto find_in_context_it = context.functions_by_name.find(sym.name); + if (find_in_context_it != context.functions_by_name.end()) { + already_exists = true; + uint16_t found_section_index = context.functions[find_in_context_it->second].section_index; + if (sym_section == found_section_index) { + already_exists_in_section = true; + } + } + + // Check if the symbol name exists in the data syms already. + auto find_in_datasyms_it = datasyms_by_name.find(sym.name); + if (find_in_datasyms_it != datasyms_by_name.end()) { + already_exists = true; + uint16_t found_section_index = find_in_datasyms_it->second; + if (sym_section == found_section_index) { + already_exists_in_section = true; + } + } + + // Check if the symbol name exists in the mdebug symbols. + auto find_in_mdebug_it = symbol_name_to_sections.find(sym.name); + if (find_in_mdebug_it != symbol_name_to_sections.end()) { + const std::vector& section_list = find_in_mdebug_it->second; + size_t count_in_section = std::count(section_list.begin(), section_list.end(), sym_section); + // The count will always be at least one because of this symbol itself, so check that it's greater than one for duplicates. + if (count_in_section > 1) { + already_exists_in_section = true; + } + // If the symbol name shows up in multiple sections, rename it. + else if (section_list.size() > 1) { + already_exists = true; + } + } + + std::string sym_output_name = sym.name; + if (already_exists_in_section) { + sym_output_name += fmt::format("_{}_{:08X}", sanitize_section_name(context.sections[sym_section].name), sym.address); + printf("Renamed static symbol \"%s\" to \"%s\"\n", sym.name.c_str(), sym_output_name.c_str()); + } + else if (already_exists) { + sym_output_name += "_" + sanitize_section_name(context.sections[sym_section].name); + printf("Renamed static symbol \"%s\" to \"%s\"\n", sym.name.c_str(), sym_output_name.c_str()); + } + + // Emit the symbol. + if (sym.is_func) { + uint32_t section_vram = context.sections[sym_section].ram_addr; + uint32_t section_offset = sym.address - section_vram; + uint32_t rom_address = static_cast(section_offset + context.sections[sym_section].rom_addr); + const uint32_t* words = reinterpret_cast(context.rom.data() + rom_address); + uint32_t num_instructions = sym.size / sizeof(uint32_t); + + std::vector insn_words(num_instructions); + insn_words.assign(words, words + num_instructions); + + context.functions_by_vram[sym.address].push_back(context.functions.size()); + context.section_functions[sym_section].push_back(context.functions.size()); + context.functions.emplace_back(N64Recomp::Function{ + sym.address, + rom_address, + std::move(insn_words), + std::move(sym_output_name), + sym_section, + // TODO read these from elf config. + false, // ignored + false, // reimplemented + false, // stubbed + }); + } + else { + data_syms[sym_section].emplace_back(N64Recomp::DataSymbol { + sym.address, + std::move(sym_output_name) + }); + } + } + } + } + + return true; + } + void print() { + printf("Mdebug Info\n"); + for (const auto& file : files_) { + printf(" File %s\n", file.filename.c_str()); + for (const auto& symbol : file.symbols) { + printf(" %s @ 0x%08X (size 0x%08X)\n", symbol.name.c_str(), symbol.address, symbol.size); + } + } + } + bool good() { + return good_; + } +private: + void add_file(std::string&& filename) { + files_.emplace_back(MDebugFile{ + .filename = std::move(filename), + .symbols = {} + }); + } + void add_symbol(size_t file_index, MDebugSymbol&& sym) { + symbols_by_name_.emplace(sym.name, std::make_pair(file_index, files_[file_index].symbols.size())); + files_[file_index].symbols.emplace_back(std::move(sym)); + } + std::vector files_; + // Map of symbol name to file index, symbol index. Multimap because multiple symbols may have the same name due to statics. + std::unordered_multimap> symbols_by_name_; + bool good_ = false; +}; + +#if 0 +bool get_func(const char *mdata, const MDebug::HDRR& hdrr, const MDebug::FDR& fdr, const MDebug::PDR& pdr, const std::vector& all_auxs, std::span all_symrs, N64Recomp::Function& func_out) { + func_out = {}; + + std::pair sym_bounds = pdr.sym_bounds(all_symrs, all_auxs); + std::span fdr_symrs = fdr.get_symrs(all_symrs); + + for (uint32_t i = sym_bounds.first; i < sym_bounds.second; i++) { + const MDebug::SYMR& symr = fdr_symrs[i]; + MDebug::ST type = symr.get_st(); + + if (type == MDebug::ST_PROC || type == MDebug::ST_STATICPROC) { + const char* name = fdr.get_string(mdata + hdrr.cbSsOffset, symr.iss); + printf(" %s\n", name); + } + else if (type == MDebug::ST_END) { + printf(" %08lX\n", symr.value); + } + } + + fflush(stdout); + return true; +} +void read_mdebug(N64Recomp::Context& context, ELFIO::section* mdebug_section, std::unordered_map>& data_syms) { + if (mdebug_section == nullptr) { + return; + } + + ELFIO::Elf64_Off base_offset = mdebug_section->get_offset(); + const char *mdata = mdebug_section->get_data(); + + // Read, byteswap and relocate the symbolic header. Relocation here means convert file-relative offsets to section-relative offsets. + + MDebug::HDRR hdrr; + std::memcpy(&hdrr, mdata, sizeof(MDebug::HDRR)); + hdrr.swap(); + hdrr.relocate(base_offset); + + // Check the magic value and version number are what we expect. + + if (hdrr.magic != MDebug::MAGIC || hdrr.vstamp != 0) { + fmt::print(stderr, "Warning: Found an mdebug section with bad magic value or version (magic={} version={}). Skipping.\n", hdrr.magic, hdrr.vstamp); + return; + } + + // Read the various records that are relevant for collecting static symbols and where they are declared. + + std::vector fdrs = hdrr.read_fdrs(mdata); + std::vector all_auxs = hdrr.read_auxs(mdata); + std::vector all_pdrs = hdrr.read_pdrs(mdata); + std::vector all_symrs = hdrr.read_symrs(mdata); + + // For each file descriptor + for (MDebug::FDR& fdr : fdrs) { + const char* fdr_name = fdr.get_name(mdata + hdrr.cbSsOffset); + + printf("%s @ 0x%08X:\n", fdr_name, fdr.adr); + printf(" Procedures:\n"); + + bool at_function = false; + + + // For every symbol record in the file descriptor record. + for (auto symr : fdr.get_symrs(all_symrs)) { + MDebug::ST type = symr.get_st(); + if (type == MDebug::ST_PROC || type == MDebug::ST_STATICPROC) { + at_function = true; + const char* name = fdr.get_string(mdata + hdrr.cbSsOffset, symr.iss); + printf(" %s @ 0x%08X\n", name, symr.value); + auto find_it = context.functions_by_name.find(name); + if (find_it != context.functions_by_name.end()) { + printf(" in map: 0x%08lX\n", context.functions[find_it->second].vram); + } + } + else if (type == MDebug::ST_END) { + printf(" size: 0x%08lX\n", symr.value); + } + } + + printf(" Globals:\n"); + + for (auto symr : fdr.get_symrs(all_symrs)) { + MDebug::ST type = symr.get_st(); + if (type == MDebug::ST_GLOBAL) { + const char* name = fdr.get_string(mdata + hdrr.cbSsOffset, symr.iss); + printf(" %s @ 0x%08X\n", name, symr.value); + } + } + + // // Consider only procedures and symbols defined in this file + // std::span auxs = fdr.get_auxs(all_auxs); + // std::span pdrs = fdr.get_pdrs(all_pdrs); + // std::span symrs = fdr.get_symrs(all_symrs); + + // std::vector> bounds(pdrs.size()); + + // // For each procedure record, determine which symbols belong to it + // for (MDebug::PDR& pdr : pdrs) { + // auto res = pdr.sym_bounds(symrs, auxs); + // bounds.push_back(res); + // } + + // // For each symbol defined in this file + // for (uint32_t isym = 0; isym < symrs.size(); isym++) { + // MDebug::SYMR& symr = symrs[isym]; + + // // // Skip non-statics + // // if (symr.get_st() != MDebug::ST_STATIC && symr.get_st() != MDebug::ST_STATICPROC) { + // // continue; + // // } + + // // // Find the name of the PDR, if any, that contains this symbol. We computed the (ordered) + // // // symbol bounds above, bsearch for the PDR. + // // auto it = std::lower_bound(bounds.begin(), bounds.end(), isym, + // // [](const auto& bound, uint32_t x) { + // // return bound.second <= x; + // // } + // // ); + // // const char* pdr_name = nullptr; + // // if (it != bounds.end() && it->first <= isym && isym < it->second) { + // // // The procedure name is the name of the first symbol + // // pdr_name = fdr.get_string(mdata + hdrr.cbSsOffset, symrs[it->first].iss); + // // } + + // // Get the name of the static symbol + // const char* sym_name = fdr.get_string(mdata + hdrr.cbSsOffset, symr.iss); + + // // // Present info (TODO: plug into everything else) + // // std::printf("0x%08X %s (in %s, in %s(0x%08X))\n", symr.value, sym_name, pdr_name, fdr_name, fdr.adr); + + // // Present info (TODO: plug into everything else) + // std::printf("0x%08X (0x%08X) %s (in %s(0x%08X))\n", symr.value, symr.bits, sym_name, fdr_name, fdr.adr); + // } + } +} +#endif + +bool N64Recomp::MDebug::parse_mdebug(const N64Recomp::ElfParsingConfig& elf_config, const char* mdebug_section, uint32_t mdebug_offset, N64Recomp::Context& context, N64Recomp::DataSymbolMap& data_syms) { + MDebugInfo mdebug_info{ elf_config, mdebug_section, mdebug_offset }; + + if (!mdebug_info.populate_context(elf_config, context, data_syms)) { + return false; + } + + return true; +} diff --git a/src/mdebug.h b/src/mdebug.h new file mode 100644 index 0000000..54105f6 --- /dev/null +++ b/src/mdebug.h @@ -0,0 +1,411 @@ +#ifndef __RECOMP_MDEBUG_H__ +#define __RECOMP_MDEBUG_H__ + +#include +#include +#include +#include +#include + +#include "recompiler/context.h" + +namespace N64Recomp +{ + namespace MDebug { + + #ifdef _MSC_VER + inline uint32_t bswap32(uint32_t val) { + return _byteswap_ulong(val); + } + inline uint16_t bswap16(uint16_t val) { + return _byteswap_ushort(val); + } + #else + constexpr uint32_t bswap32(uint32_t val) { + return __builtin_bswap32(val); + } + constexpr uint16_t bswap16(uint16_t val) { + return __builtin_bswap16(val); + } + #endif + + /* MIPS Symbol Table Debugging Format */ + + enum SC { + SC_NIL = 0, + SC_TEXT = 1, /* .text symbol */ + SC_DATA = 2, /* .data symbol */ + SC_BSS = 3, /* .bss symbol */ + SC_REGISTER = 4, /* value of symbol is register number */ + SC_ABS = 5, /* value of symbol is absolute */ + SC_UNDEFINED = 6, /* value of symbol is undefined */ + SC_CDBLOCAL = 7, /* variable value is in se->va.?? */ + SC_BITS = 8, /* variable is a bit field */ + SC_CDBSYSTEM = 9, /* variable value is in cdb address space */ + SC_REGIMAGE = 10, /* register value is saved on stack */ + SC_INFO = 11, /* symbol contains debugger information */ + SC_USERSTRUCT = 12, /* address in struct user for current process */ + SC_SDATA = 13, /* load time only small data */ + SC_SBSS = 14, /* load time only small common */ + SC_RDATA = 15, /* load time only read-only data */ + SC_VAR = 16, /* var parameter (FORTRAN, Pascal) */ + SC_COMMON = 17, /* common variable */ + SC_SCOMMON = 18, /* small common */ + SC_VARREGISTER = 19, /* var parameter in a register */ + SC_VARIANT = 20, /* variant record */ + SC_SUNDEFINED = 21, /* small undefined (external) data */ + SC_INIT = 22, /* .init section symbol */ + SC_BASEDVAR = 23, /* FORTRAN or PL/1 ptr based var */ + SC_XDATA = 24, /* exception handling data */ + SC_PDATA = 25, /* procedure section */ + SC_FINI = 26, /* .fini section */ + SC_RCONST = 27, /* .rconst section */ + SC_MAX = 32 + }; + + enum ST { + ST_NIL = 0, + ST_GLOBAL = 1, /* external symbol */ + ST_STATIC = 2, /* static symbol */ + ST_PARAM = 3, /* procedure argument */ + ST_LOCAL = 4, /* local variable */ + ST_LABEL = 5, /* label */ + ST_PROC = 6, /* procedure */ + ST_BLOCK = 7, /* beginning of block */ + ST_END = 8, /* end of something */ + ST_MEMBER = 9, /* member of struct/union/enum/.. */ + ST_TYPEDEF = 10, /* type definition */ + ST_FILE = 11, /* filename */ + ST_REGRELOC = 12, /* register relocation */ + ST_FORWARD = 13, /* forwarding address */ + ST_STATICPROC = 14, /* load time only static procedures */ + /* (CONSTANT and STAPARAM are in different orders between different sources...) */ + ST_CONSTANT = 15, /* constant */ + ST_STAPARAM = 16, /* FORTRAN static parameters */ + ST_STRUCT = 26, /* structure */ + ST_UNION = 27, /* union */ + ST_ENUM = 28, /* enum */ + ST_INDIRECT = 34 + }; + + struct SYMR { + int32_t iss; /* index into String Space of name */ + int32_t value; /* symbol value; can be an address, size or frame offset depending on symbol type */ + uint32_t bits; /* Bitfield: */ + #if 0 + ST st : 6; /* symbol type */ + SC sc : 5; /* storage class - text, data, etc */ + int32_t _reserved : 1; /* reserved bit */ + int32_t index : 20; /* index into sym/aux table */ + #endif + + inline ST get_st() const { + return (ST)(bits >> 26 & 0b111111); + } + + inline SC get_sc() const { + return (SC)(bits >> 21 & 0b11111); + } + + inline int32_t get_index() const { + return bits & 0b11111111111111111111; + } + + void swap() { + iss = bswap32(iss); + value = bswap32(value); + bits = bswap32(bits); + } + }; + + union AUX { + uint32_t any_; + uint32_t ti; /* type information record */ + uint32_t rndx; /* relative index into symbol table */ + uint32_t dnLow; /* low dimension of array */ + uint32_t dnHigh; /* high dimension of array */ + uint32_t isym; /* symbol table index (end of proc) */ + uint32_t iss; /* index into string space (not used) */ + uint32_t width; /* width for non-default sized struct fields */ + uint32_t count; /* count of ranges for variant arm */ + + void swap() { + any_ = bswap32(any_); + } + }; + + struct PDR { + uint32_t addr; /* memory address of start of procedure */ + uint32_t isym; /* start of local symbol entries */ + uint32_t iline; /* start of line number entries */ + uint32_t regmask; /* save register mask */ + uint32_t regoffset; /* save register offset */ + uint32_t iopt; /* start of optimization symbol entries */ + uint32_t fregmask; /* save floating point register mask */ + uint32_t fregoffset; /* save floating point register offset */ + uint32_t frameoffset; /* frame size */ + uint16_t framereg; /* frame pointer register */ + uint16_t pcreg; /* offset or reg of return pc */ + int32_t lnLow; /* lowest line in the procedure */ + int32_t lnHigh; /* highest line in the procedure */ + int32_t cbLineOffset; /* byte offset for this procedure from the fd base */ + + void swap() { + addr = bswap32(addr); + isym = bswap32(isym); + iline = bswap32(iline); + regmask = bswap32(regmask); + regoffset = bswap32(regoffset); + iopt = bswap32(iopt); + fregmask = bswap32(fregmask); + fregoffset = bswap32(fregoffset); + frameoffset = bswap32(frameoffset); + framereg = bswap16(framereg); + pcreg = bswap16(pcreg); + lnLow = bswap32(lnLow); + lnHigh = bswap32(lnHigh); + cbLineOffset = bswap32(cbLineOffset); + } + + inline std::pair sym_bounds(std::span symrs, std::span auxs) const { + const SYMR& first = symrs[isym]; + const ST first_st = first.get_st(); + // The first symbol is the symbol of the procedure itself. The procedure name is the name of this symbol. + assert(first_st == ST_PROC || first_st == ST_STATICPROC); + + const AUX& aux = auxs[first.get_index()]; + const SYMR& last = symrs[aux.isym - 1]; + const ST last_st = last.get_st(); + // The last symbol is the END marker, pointed to by the first AUX of the stPROC symbol. + assert(last_st == ST_END); + + // Return the symbol bounds + return std::make_pair(isym, aux.isym); + } + }; + + enum LANG { + LANG_C = 0, + LANG_PASCAL = 1, + LANG_FORTRAN = 2, + LANG_ASM = 3, + LANG_MACHINE = 4, + LANG_NIL = 5, + LANG_ADA = 6, + LANG_PL1 = 7, + LANG_COBOL = 8, + LANG_STDC = 9, + LANG_CPLUSPLUSV2 = 10, + LANG_MAX = 11 + }; + + struct FDR { + uint32_t adr; /* memory address of beginning of file */ + int32_t rss; /* file name (of source, if known) */ + int32_t issBase; /* file's string space */ + int32_t cbSs; /* number of bytes in the ss */ + int32_t isymBase; /* beginning of symbols */ + int32_t csym; /* count file's of symbols */ + int32_t ilineBase; /* file's line symbols */ + int32_t cline; /* count of file's line symbols */ + int32_t ioptBase; /* file's optimization entries */ + int32_t copt; /* count of file's optimization entries */ + uint16_t ipdFirst; /* start of procedures for this file */ + uint16_t cpd; /* count of procedures for this file */ + int32_t iauxBase; /* file's auxiliary entries */ + int32_t caux; /* count of file's auxiliary entries */ + int32_t rfdBase; /* index into the file indirect table */ + int32_t crfd; /* count file indirect entries */ + uint32_t bits; /* Bitfield: */ + #if 0 + LANG lang : 5; /* language for this file */ + uint32_t fMerge : 1; /* whether this file can be merged */ + uint32_t fReadin : 1; /* true if it was read in (not just created) */ + uint32_t fBigEndian : 1; /* true if AUXU's are big endian */ + uint32_t glevel : 2; /* level this file was compiled with */ + uint32_t _reserved : 20; /* reserved bits */ + #endif + int32_t cbLineOffset; /* byte offset from header for this file ln's */ + int32_t cbLine; /* size of lines for this file */ + + inline LANG get_lang() const { + return (LANG)(bits >> 27 & 0b11111); + } + + inline uint32_t get_fMerge() const { + return bits >> 26 & 1; + } + + inline uint32_t get_fReadin() const { + return bits >> 25 & 1; + } + + inline uint32_t get_fBigEndian() const { + return bits >> 24 & 1; + } + + inline uint32_t get_glevel() const { + return bits >> 22 & 0b11; + } + + void swap() { + adr = bswap32(adr); + rss = bswap32(rss); + issBase = bswap32(issBase); + cbSs = bswap32(cbSs); + isymBase = bswap32(isymBase); + csym = bswap32(csym); + ilineBase = bswap32(ilineBase); + cline = bswap32(cline); + ioptBase = bswap32(ioptBase); + copt = bswap32(copt); + ipdFirst = bswap16(ipdFirst); + cpd = bswap16(cpd); + iauxBase = bswap32(iauxBase); + caux = bswap32(caux); + rfdBase = bswap32(rfdBase); + crfd = bswap32(crfd); + bits = bswap32(bits); + cbLineOffset = bswap32(cbLineOffset); + cbLine = bswap32(cbLine); + } + + inline std::span get_auxs(const std::vector& all_auxs) const { + return std::span(all_auxs).subspan(iauxBase, caux); + } + + inline std::span get_pdrs(const std::vector& all_pdrs) const { + return std::span(all_pdrs).subspan(ipdFirst, cpd); + } + + inline std::span get_symrs(std::span all_symrs) const { + return std::span(all_symrs).subspan(isymBase, csym); + } + + inline const char* get_string(const char* data, size_t index) const { + return data + issBase + index; + } + + inline const char* get_name(const char* data) const { + return get_string(data, rss); + } + }; + + static const uint16_t MAGIC = 0x7009; + + /** + * mdebug sections always start with a Symbolic Header (HDRR) containing + * file-relative (not section-relative) offsets for where to find the rest + * of the data. + */ + struct HDRR { + uint16_t magic; /* 0x7009 */ + uint16_t vstamp; /* version stamp */ + int32_t ilineMax; /* number of line number entries */ + int32_t cbLine; /* number of bytes for line number entries */ + int32_t cbLineOffset; /* offset to start of line number entries */ + int32_t idnMax; /* max index into dense number table */ + int32_t cbDnOffset; /* offset to start dense number table */ + int32_t ipdMax; /* number of procedures */ + int32_t cbPdOffset; /* offset to procedure descriptor table */ + int32_t isymMax; /* number of local symbols */ + int32_t cbSymOffset; /* offset to start of local symbols */ + int32_t ioptMax; /* max index into optimization symbol entries */ + int32_t cbOptOffset; /* offset to optimization symbol entries */ + int32_t iauxMax; /* number of auxillary symbol entries */ + int32_t cbAuxOffset; /* offset to start of auxillary symbol entries */ + int32_t issMax; /* max index into local strings */ + int32_t cbSsOffset; /* offset to start of local strings */ + int32_t issExtMax; /* max index into external strings */ + int32_t cbSsExtOffset; /* offset to start of external strings */ + int32_t ifdMax; /* number of file descriptor entries */ + int32_t cbFdOffset; /* offset to file descriptor table */ + int32_t crfd; /* number of relative file descriptor entries */ + int32_t cbRfdOffset; /* offset to relative file descriptor table */ + int32_t iextMax; /* max index into external symbols */ + int32_t cbExtOffset; /* offset to start of external symbol entries */ + + void swap() { + magic = bswap16(magic); + vstamp = bswap16(vstamp); + ilineMax = bswap32(ilineMax); + cbLine = bswap32(cbLine); + cbLineOffset = bswap32(cbLineOffset); + idnMax = bswap32(idnMax); + cbDnOffset = bswap32(cbDnOffset); + ipdMax = bswap32(ipdMax); + cbPdOffset = bswap32(cbPdOffset); + isymMax = bswap32(isymMax); + cbSymOffset = bswap32(cbSymOffset); + ioptMax = bswap32(ioptMax); + cbOptOffset = bswap32(cbOptOffset); + iauxMax = bswap32(iauxMax); + cbAuxOffset = bswap32(cbAuxOffset); + issMax = bswap32(issMax); + cbSsOffset = bswap32(cbSsOffset); + issExtMax = bswap32(issExtMax); + cbSsExtOffset = bswap32(cbSsExtOffset); + ifdMax = bswap32(ifdMax); + cbFdOffset = bswap32(cbFdOffset); + crfd = bswap32(crfd); + cbRfdOffset = bswap32(cbRfdOffset); + iextMax = bswap32(iextMax); + cbExtOffset = bswap32(cbExtOffset); + } + + void relocate(uint32_t offset) { + cbLineOffset -= offset; + cbDnOffset -= offset; + cbPdOffset -= offset; + cbSymOffset -= offset; + cbOptOffset -= offset; + cbAuxOffset -= offset; + cbSsOffset -= offset; + cbSsExtOffset -= offset; + cbFdOffset -= offset; + cbRfdOffset -= offset; + cbExtOffset -= offset; + } + + inline std::vector read_fdrs(const char* data) { + std::vector fdrs(ifdMax); + const FDR* p = reinterpret_cast(data + cbFdOffset); + fdrs.assign(p, p + ifdMax); + for (FDR& fdr : fdrs) + fdr.swap(); + return fdrs; + } + + inline std::vector read_auxs(const char* data) { + std::vector auxs(iauxMax); + const AUX* p = reinterpret_cast(data + cbAuxOffset); + auxs.assign(p, p + iauxMax); + for (AUX& aux : auxs) + aux.swap(); + return auxs; + } + + inline std::vector read_pdrs(const char* data) { + std::vector pdrs(ipdMax); + const PDR* p = reinterpret_cast(data + cbPdOffset); + pdrs.assign(p, p + ipdMax); + for (PDR& pdr : pdrs) + pdr.swap(); + return pdrs; + } + + inline std::vector read_symrs(const char* data) { + std::vector symrs(isymMax); + const SYMR* p = reinterpret_cast(data + cbSymOffset); + symrs.assign(p, p + isymMax); + for (SYMR& symr : symrs) + symr.swap(); + return symrs; + } + + }; + bool parse_mdebug(const N64Recomp::ElfParsingConfig& elf_config, const char* mdebug_section, uint32_t mdebug_offset, N64Recomp::Context& context, N64Recomp::DataSymbolMap& data_syms); + } +} + +#endif diff --git a/src/mod_symbols.cpp b/src/mod_symbols.cpp index fcfdead..7318a07 100644 --- a/src/mod_symbols.cpp +++ b/src/mod_symbols.cpp @@ -16,9 +16,15 @@ struct FileSubHeaderV1 { uint32_t num_exports; uint32_t num_callbacks; uint32_t num_provided_events; + uint32_t num_hooks; uint32_t string_data_size; }; +enum class SectionFlags : uint32_t { + FixedAddress = 1 << 0, + GloballyLoaded = 1 << 1, +}; + struct SectionHeaderV1 { uint32_t flags; uint32_t file_offset; @@ -89,6 +95,13 @@ struct EventV1 { uint32_t name_size; }; +struct HookV1 { + uint32_t func_index; + uint32_t original_section_vrom; + uint32_t original_vram; + uint32_t flags; // end +}; + template const T* reinterpret_data(std::span data, size_t& offset, size_t count = 1) { if (offset + (sizeof(T) * count) > data.size()) { @@ -126,6 +139,7 @@ bool parse_v1(std::span data, const std::unordered_mapnum_exports; size_t num_callbacks = subheader->num_callbacks; size_t num_provided_events = subheader->num_provided_events; + size_t num_hooks = subheader->num_hooks; size_t string_data_size = subheader->string_data_size; if (string_data_size & 0b11) { @@ -147,6 +161,7 @@ bool parse_v1(std::span data, const std::unordered_map(data, offset); @@ -162,10 +177,22 @@ bool parse_v1(std::span data, const std::unordered_mapbss_size; cur_section.name = "mod_section_" + std::to_string(section_index); cur_section.relocatable = true; + cur_section.fixed_address = (section_header->flags & static_cast(SectionFlags::FixedAddress)) != 0; + cur_section.globally_loaded = (section_header->flags & static_cast(SectionFlags::GloballyLoaded)) != 0; + + if (cur_section.fixed_address && !cur_section.globally_loaded) { + printf("Fixed address sections that aren't globally loaded aren't currently supported\n"); + return false; + } + + if (cur_section.globally_loaded && !cur_section.fixed_address) { + printf("A globally loaded section must have a fixed address\n"); + return false; + } + uint32_t num_funcs = section_header->num_funcs; uint32_t num_relocs = section_header->num_relocs; - const FuncV1* funcs = reinterpret_data(data, offset, num_funcs); if (funcs == nullptr) { printf("Failed to read funcs (count: %d)\n", num_funcs); @@ -434,6 +461,22 @@ bool parse_v1(std::span data, const std::unordered_map(data, offset, num_hooks); + if (hooks == nullptr) { + printf("Failed to read hooks (count: %zu)\n", num_hooks); + return false; + } + + for (size_t hook_index = 0; hook_index < num_hooks; hook_index++) { + const HookV1& hook_in = hooks[hook_index]; + N64Recomp::FunctionHook& hook_out = mod_context.hooks.emplace_back(); + + hook_out.func_index = hook_in.func_index; + hook_out.original_section_vrom = hook_in.original_section_vrom; + hook_out.original_vram = hook_in.original_vram; + hook_out.flags = static_cast(hook_in.flags); + } + return offset == data.size(); } @@ -512,6 +555,7 @@ std::vector N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont size_t num_events = context.event_symbols.size(); size_t num_callbacks = context.callbacks.size(); size_t num_provided_events = context.event_symbols.size(); + size_t num_hooks = context.hooks.size(); FileSubHeaderV1 sub_header { .num_sections = static_cast(context.sections.size()), @@ -522,6 +566,7 @@ std::vector N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont .num_exports = static_cast(num_exported_funcs), .num_callbacks = static_cast(num_callbacks), .num_provided_events = static_cast(num_provided_events), + .num_hooks = static_cast(num_hooks), .string_data_size = 0, }; @@ -603,7 +648,15 @@ std::vector N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont for (size_t section_index = 0; section_index < context.sections.size(); section_index++) { const Section& cur_section = context.sections[section_index]; + uint32_t section_flags = 0; + if (cur_section.fixed_address) { + section_flags |= static_cast(SectionFlags::FixedAddress); + } + if (cur_section.globally_loaded) { + section_flags |= static_cast(SectionFlags::GloballyLoaded); + } SectionHeaderV1 section_out { + .flags = section_flags, .file_offset = cur_section.rom_addr, .vram = cur_section.ram_addr, .rom_size = cur_section.size, @@ -757,5 +810,22 @@ std::vector N64Recomp::symbols_to_bin_v1(const N64Recomp::Context& cont vec_put(ret, &event_out); } + // Write the hooks. + for (const FunctionHook& cur_hook : context.hooks) { + uint32_t flags = 0; + if ((cur_hook.flags & HookFlags::AtReturn) == HookFlags::AtReturn) { + flags |= 0x1; + } + + HookV1 hook_out { + .func_index = cur_hook.func_index, + .original_section_vrom = cur_hook.original_section_vrom, + .original_vram = cur_hook.original_vram, + .flags = flags + }; + + vec_put(ret, &hook_out); + } + return ret; } diff --git a/src/operations.cpp b/src/operations.cpp index 70201d3..ca2f095 100644 --- a/src/operations.cpp +++ b/src/operations.cpp @@ -9,6 +9,8 @@ namespace N64Recomp { { InstrId::cpu_mflo, { UnaryOpType::None, Operand::Rd, Operand::Lo } }, { InstrId::cpu_mtc1, { UnaryOpType::None, Operand::FsU32L, Operand::Rt } }, { InstrId::cpu_mfc1, { UnaryOpType::ToInt32, Operand::Rt, Operand::FsU32L } }, + { InstrId::cpu_dmtc1, { UnaryOpType::None, Operand::FsU64, Operand::Rt } }, + { InstrId::cpu_dmfc1, { UnaryOpType::None, Operand::Rt, Operand::FsU64 } }, // Float operations { InstrId::cpu_mov_s, { UnaryOpType::None, Operand::Fd, Operand::Fs, true } }, { InstrId::cpu_mov_d, { UnaryOpType::None, Operand::FdDouble, Operand::FsDouble, true } }, @@ -98,34 +100,46 @@ namespace N64Recomp { { InstrId::cpu_mul_d, { BinaryOpType::MulDouble, Operand::FdDouble, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true, true } }, { InstrId::cpu_div_s, { BinaryOpType::DivFloat, Operand::Fd, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true, true } }, { InstrId::cpu_div_d, { BinaryOpType::DivDouble, Operand::FdDouble, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true, true } }, - // Float comparisons TODO remaining operations and investigate ordered/unordered and default values - { InstrId::cpu_c_lt_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_nge_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_olt_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_ult_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_lt_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_nge_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_olt_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_ult_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - - { InstrId::cpu_c_le_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_ngt_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_ole_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_ule_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_le_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_ngt_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_ole_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_ule_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - - { InstrId::cpu_c_eq_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_ueq_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_ngl_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, - { InstrId::cpu_c_seq_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + // Float comparisons TODO investigate ordered/unordered and default values + // Single Ordered + { InstrId::cpu_c_f_s, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_eq_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_olt_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ole_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_sf_s, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_seq_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_lt_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_le_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + // Single Unordered + { InstrId::cpu_c_un_s, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ueq_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ult_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ule_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ngle_s, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ngl_s, { BinaryOpType::EqualFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_nge_s, { BinaryOpType::LessFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + { InstrId::cpu_c_ngt_s, { BinaryOpType::LessEqFloat, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Fs, Operand::Ft }}, true } }, + // Double Ordered + { InstrId::cpu_c_f_d, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, { InstrId::cpu_c_eq_d, { BinaryOpType::EqualDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_ueq_d, { BinaryOpType::EqualDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, - { InstrId::cpu_c_ngl_d, { BinaryOpType::EqualDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_olt_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ole_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + /* TODO rename to c_sf_d when fixed in rabbitizer */ + { InstrId::cpu_c_df_d, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, /* TODO rename to c_seq_d when fixed in rabbitizer */ { InstrId::cpu_c_deq_d, { BinaryOpType::EqualDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_lt_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_le_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + // Double Unordered + { InstrId::cpu_c_un_d, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ueq_d, { BinaryOpType::EqualDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ult_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ule_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ngle_d, { BinaryOpType::False, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ngl_d, { BinaryOpType::EqualDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_nge_d, { BinaryOpType::LessDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + { InstrId::cpu_c_ngt_d, { BinaryOpType::LessEqDouble, Operand::Cop1cs, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::FsDouble, Operand::FtDouble }}, true } }, + // Loads { InstrId::cpu_ld, { BinaryOpType::LD, Operand::Rt, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Base, Operand::ImmS16 }}} }, { InstrId::cpu_lw, { BinaryOpType::LW, Operand::Rt, {{ UnaryOpType::None, UnaryOpType::None }, { Operand::Base, Operand::ImmS16 }}} }, diff --git a/src/recompilation.cpp b/src/recompilation.cpp index 5761b79..c228433 100644 --- a/src/recompilation.cpp +++ b/src/recompilation.cpp @@ -22,6 +22,11 @@ enum class JalResolutionResult { }; JalResolutionResult resolve_jal(const N64Recomp::Context& context, size_t cur_section_index, uint32_t target_func_vram, size_t& matched_function_index) { + // Skip resolution if all function calls should use lookup and just return Ambiguous. + if (context.use_lookup_for_all_function_calls) { + return JalResolutionResult::Ambiguous; + } + // Look for symbols with the target vram address const N64Recomp::Section& cur_section = context.sections[cur_section_index]; const auto matching_funcs_find = context.functions_by_vram.find(target_func_vram); @@ -41,9 +46,7 @@ JalResolutionResult resolve_jal(const N64Recomp::Context& context, size_t cur_se // Zero-sized symbol handling. unless there's only one matching target. if (target_func.words.empty()) { - // Allow zero-sized symbols between 0x8F000000 and 0x90000000 for use with patches. - // TODO make this configurable or come up with a more sensible solution for dealing with manual symbols for patches. - if (target_func.vram < 0x8F000000 || target_func.vram > 0x90000000) { + if (!N64Recomp::is_manual_patch_symbol(target_func.vram)) { continue; } } @@ -109,7 +112,7 @@ std::string_view ctx_gpr_prefix(int reg) { } template -bool process_instruction(GeneratorType& generator, const N64Recomp::Context& context, const N64Recomp::Function& func, const N64Recomp::FunctionStats& stats, const std::unordered_set& jtbl_lw_instructions, size_t instr_index, const std::vector& instructions, std::ostream& output_file, bool indent, bool emit_link_branch, int link_branch_index, size_t reloc_index, bool& needs_link_branch, bool& is_branch_likely, bool tag_reference_relocs, std::span> static_funcs_out) { +bool process_instruction(GeneratorType& generator, const N64Recomp::Context& context, const N64Recomp::Function& func, size_t func_index, const N64Recomp::FunctionStats& stats, const std::unordered_set& jtbl_lw_instructions, size_t instr_index, const std::vector& instructions, std::ostream& output_file, bool indent, bool emit_link_branch, int link_branch_index, size_t reloc_index, bool& needs_link_branch, bool& is_branch_likely, bool tag_reference_relocs, std::span> static_funcs_out) { using namespace N64Recomp; const auto& section = context.sections[func.section_index]; @@ -149,6 +152,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con } N64Recomp::RelocType reloc_type = N64Recomp::RelocType::R_MIPS_NONE; + bool has_reloc = false; uint32_t reloc_section = 0; uint32_t reloc_target_section_offset = 0; size_t reloc_reference_symbol = (size_t)-1; @@ -159,6 +163,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con // Check if this instruction has a reloc. if (section.relocs.size() > 0 && section.relocs[reloc_index].address == instr_vram) { + has_reloc = true; // Get the reloc data for this instruction const auto& reloc = section.relocs[reloc_index]; reloc_section = reloc.target_section; @@ -181,19 +186,10 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con reloc_reference_symbol = reloc.symbol_index; // Don't try to relocate special section symbols. if (context.is_regular_reference_section(reloc.target_section) || reloc_section == N64Recomp::SectionAbsolute) { + // TODO this may not be needed anymore as HI16/LO16 relocs to non-relocatable sections is handled directly in elf parsing. bool ref_section_relocatable = context.is_reference_section_relocatable(reloc.target_section); // Resolve HI16 and LO16 reference symbol relocs to non-relocatable sections by patching the instruction immediate. if (!ref_section_relocatable && (reloc_type == N64Recomp::RelocType::R_MIPS_HI16 || reloc_type == N64Recomp::RelocType::R_MIPS_LO16)) { - uint32_t ref_section_vram = context.get_reference_section_vram(reloc.target_section); - uint32_t full_immediate = reloc.target_section_offset + ref_section_vram; - - if (reloc_type == N64Recomp::RelocType::R_MIPS_HI16) { - imm = (full_immediate >> 16) + ((full_immediate >> 15) & 1); - } - else if (reloc_type == N64Recomp::RelocType::R_MIPS_LO16) { - imm = full_immediate & 0xFFFF; - } - // The reloc has been processed, so set it to none to prevent it getting processed a second time during instruction code generation. reloc_type = N64Recomp::RelocType::R_MIPS_NONE; reloc_reference_symbol = (size_t)-1; @@ -219,7 +215,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con if (reloc_index + 1 < section.relocs.size() && next_vram > section.relocs[reloc_index].address) { next_reloc_index++; } - if (!process_instruction(generator, context, func, stats, jtbl_lw_instructions, instr_index + 1, instructions, output_file, use_indent, false, link_branch_index, next_reloc_index, dummy_needs_link_branch, dummy_is_branch_likely, tag_reference_relocs, static_funcs_out)) { + if (!process_instruction(generator, context, func, func_index, stats, jtbl_lw_instructions, instr_index + 1, instructions, output_file, use_indent, false, link_branch_index, next_reloc_index, dummy_needs_link_branch, dummy_is_branch_likely, tag_reference_relocs, static_funcs_out)) { return false; } } @@ -238,7 +234,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con return false; } print_indent(); - generator.emit_return(context); + generator.emit_return(context, func_index); print_link_branch(); return true; }; @@ -263,7 +259,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con return true; }; - auto print_func_call_by_address = [&generator, reloc_target_section_offset, reloc_section, reloc_reference_symbol, reloc_type, &context, &func, &static_funcs_out, &needs_link_branch, &print_indent, &process_delay_slot, &print_link_branch] + auto print_func_call_by_address = [&generator, reloc_target_section_offset, has_reloc, reloc_section, reloc_reference_symbol, reloc_type, &context, &func, &static_funcs_out, &needs_link_branch, &print_indent, &process_delay_slot, &print_link_branch] (uint32_t target_func_vram, bool tail_call = false, bool indent = false) { bool call_by_lookup = false; @@ -300,7 +296,12 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con } } else { - JalResolutionResult jal_result = resolve_jal(context, func.section_index, target_func_vram, matched_func_index); + uint32_t target_section = func.section_index; + // If this instruction has a reloc and the target section is a normal section, use the section of the reloc when searching for a matching target function. + if (has_reloc && reloc_section < 65500) { + target_section = reloc_section; + } + JalResolutionResult jal_result = resolve_jal(context, target_section, target_func_vram, matched_func_index); switch (jal_result) { case JalResolutionResult::NoMatch: @@ -316,7 +317,10 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con call_by_name = true; break; case JalResolutionResult::Ambiguous: - fmt::print(stderr, "[Info] Ambiguous jal target 0x{:08X} in function {}, falling back to function lookup\n", target_func_vram, func.name); + // Print a warning if lookup isn't forced for all non-reloc function calls. + if (!context.use_lookup_for_all_function_calls) { + fmt::print(stderr, "[Info] Ambiguous jal target 0x{:08X} in function {}, falling back to function lookup\n", target_func_vram, func.name); + } // Relocation isn't necessary for jumps inside a relocatable section, as this code path will never run if the target vram // is in the current function's section (see the branch for `in_current_section` above). // If a game ever needs to jump between multiple relocatable sections, relocation will be necessary here. @@ -363,7 +367,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con return false; } print_indent(); - generator.emit_return(context); + generator.emit_return(context, func_index); // TODO check if this branch close should exist. // print_indent(); // generator.emit_branch_close(); @@ -512,7 +516,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con return false; } print_indent(); - generator.emit_return(context); + generator.emit_return(context, func_index); } else { fmt::print(stderr, "Unhandled branch in {} at 0x{:08X} to 0x{:08X}\n", func.name, instr_vram, branch_target); @@ -552,7 +556,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con fmt::print("[Info] Indirect tail call in {}\n", func.name); print_func_call_by_register(rs); print_indent(); - generator.emit_return(context); + generator.emit_return(context, func_index); break; } break; @@ -561,7 +565,7 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con generator.emit_syscall(instr_vram); // syscalls don't link, so treat it like a tail call print_indent(); - generator.emit_return(context); + generator.emit_return(context, func_index); break; case InstrId::cpu_break: print_indent(); @@ -840,7 +844,7 @@ bool recompile_function_impl(GeneratorType& generator, const N64Recomp::Context& } // Process the current instruction and check for errors - if (process_instruction(generator, context, func, stats, jtbl_lw_instructions, instr_index, instructions, output_file, false, needs_link_branch, num_link_branches, reloc_index, needs_link_branch, is_branch_likely, tag_reference_relocs, static_funcs_out) == false) { + if (process_instruction(generator, context, func, func_index, stats, jtbl_lw_instructions, instr_index, instructions, output_file, false, needs_link_branch, num_link_branches, reloc_index, needs_link_branch, is_branch_likely, tag_reference_relocs, static_funcs_out) == false) { fmt::print(stderr, "Error in recompiling {}, clearing output file\n", func.name); output_file.clear(); return false; diff --git a/src/symbol_lists.cpp b/src/symbol_lists.cpp index cbe5ff5..9c8f27d 100644 --- a/src/symbol_lists.cpp +++ b/src/symbol_lists.cpp @@ -58,6 +58,7 @@ const std::unordered_set N64Recomp::reimplemented_funcs { // Parallel interface (cartridge, DMA, etc.) functions "osCartRomInit", "osCreatePiManager", + "osPiReadIo", "osPiStartDma", "osEPiStartDma", "osPiGetStatus", @@ -93,6 +94,7 @@ const std::unordered_set N64Recomp::reimplemented_funcs { "osSetEventMesg", // Timer functions "osGetTime", + "osSetTime", "osSetTimer", "osStopTimer", // Voice functions @@ -113,6 +115,7 @@ const std::unordered_set N64Recomp::reimplemented_funcs { "osVirtualToPhysical", // Coprocessor 0/1 functions "osGetCount", + "osSetCount", "__osSetFpcCsr", // Cache funcs "osInvalDCache", @@ -268,7 +271,6 @@ const std::unordered_set N64Recomp::ignored_funcs { "__osDevMgrMain", "osPiGetCmdQueue", "osPiGetStatus", - "osPiReadIo", "osPiStartDma", "osPiWriteIo", "osEPiGetDeviceType", @@ -329,6 +331,7 @@ const std::unordered_set N64Recomp::ignored_funcs { "osSetTimer", "osStopTimer", "osGetTime", + "osSetTime", "__osInsertTimer", "__osTimerInterrupt", "__osTimerServicesInit", @@ -391,6 +394,7 @@ const std::unordered_set N64Recomp::ignored_funcs { // Coprocessor 0/1 functions "__osSetCount", "osGetCount", + "osSetCount", "__osSetSR", "__osGetSR", "__osSetCause", @@ -657,4 +661,8 @@ const std::unordered_set N64Recomp::renamed_funcs { "div64_32", "__moddi3", "_matherr", + + // File IO + "open", + "close", };