etc/pineapple-src
2
1
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity

early-access version 2621

Browse source
This commit is contained in:
pineappleEA 2022-03-29 23:49:35 +02:00
parent 8775571f5d
commit e2ba36a19f
10 changed files with 816 additions and 995 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -1,7 +1,7 @@
yuzu emulator early access yuzu emulator early access
============= =============
This is the source code for early-access 2620. This is the source code for early-access 2621.
## Legal Notice ## Legal Notice

1
externals/dynarmic/src/dynarmic/CMakeLists.txt vendored
View file

@ -285,6 +285,7 @@ if (ARCHITECTURE STREQUAL "x86_64")
backend/x64/emit_x64_data_processing.cpp backend/x64/emit_x64_data_processing.cpp
backend/x64/emit_x64_floating_point.cpp backend/x64/emit_x64_floating_point.cpp
backend/x64/emit_x64_memory.h backend/x64/emit_x64_memory.h
backend/x64/emit_x64_memory.cpp.inc
backend/x64/emit_x64_packed.cpp backend/x64/emit_x64_packed.cpp
backend/x64/emit_x64_saturation.cpp backend/x64/emit_x64_saturation.cpp
backend/x64/emit_x64_sm4.cpp backend/x64/emit_x64_sm4.cpp

19
externals/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h vendored
View file

@ -71,9 +71,12 @@ protected:
std::array<FastDispatchEntry, fast_dispatch_table_size> fast_dispatch_table; std::array<FastDispatchEntry, fast_dispatch_table_size> fast_dispatch_table;
void ClearFastDispatchTable(); void ClearFastDispatchTable();
std::map<std::tuple<size_t, int, int>, void (*)()> read_fallbacks; void (*memory_read_128)() = nullptr; // Dummy
std::map<std::tuple<size_t, int, int>, void (*)()> write_fallbacks; void (*memory_write_128)() = nullptr; // Dummy
std::map<std::tuple<size_t, int, int>, void (*)()> exclusive_write_fallbacks;
std::map<std::tuple<bool, size_t, int, int>, void (*)()> read_fallbacks;
std::map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks;
std::map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks;
void GenFastmemFallbacks(); void GenFastmemFallbacks();
const void* terminal_handler_pop_rsb_hint; const void* terminal_handler_pop_rsb_hint;
@ -99,7 +102,7 @@ protected:
u64 resume_rip; u64 resume_rip;
u64 callback; u64 callback;
DoNotFastmemMarker marker; DoNotFastmemMarker marker;
bool compile; bool recompile;
}; };
tsl::robin_map<u64, FastmemPatchInfo> fastmem_patch_info; tsl::robin_map<u64, FastmemPatchInfo> fastmem_patch_info;
std::set<DoNotFastmemMarker> do_not_fastmem; std::set<DoNotFastmemMarker> do_not_fastmem;
@ -112,13 +115,13 @@ protected:
template<std::size_t bitsize, auto callback> template<std::size_t bitsize, auto callback>
void EmitMemoryWrite(A32EmitContext& ctx, IR::Inst* inst); void EmitMemoryWrite(A32EmitContext& ctx, IR::Inst* inst);
template<std::size_t bitsize, auto callback> template<std::size_t bitsize, auto callback>
void ExclusiveReadMemory(A32EmitContext& ctx, IR::Inst* inst); void EmitExclusiveReadMemory(A32EmitContext& ctx, IR::Inst* inst);
template<std::size_t bitsize, auto callback> template<std::size_t bitsize, auto callback>
void ExclusiveWriteMemory(A32EmitContext& ctx, IR::Inst* inst); void EmitExclusiveWriteMemory(A32EmitContext& ctx, IR::Inst* inst);
template<std::size_t bitsize, auto callback> template<std::size_t bitsize, auto callback>
void ExclusiveReadMemoryInline(A32EmitContext& ctx, IR::Inst* inst); void EmitExclusiveReadMemoryInline(A32EmitContext& ctx, IR::Inst* inst);
template<std::size_t bitsize, auto callback> template<std::size_t bitsize, auto callback>
void ExclusiveWriteMemoryInline(A32EmitContext& ctx, IR::Inst* inst); void EmitExclusiveWriteMemoryInline(A32EmitContext& ctx, IR::Inst* inst);
// Terminal instruction emitters // Terminal instruction emitters
void EmitSetUpperLocationDescriptor(IR::LocationDescriptor new_location, IR::LocationDescriptor old_location); void EmitSetUpperLocationDescriptor(IR::LocationDescriptor new_location, IR::LocationDescriptor old_location);

506
externals/dynarmic/src/dynarmic/backend/x64/a32_emit_x64_memory.cpp vendored
View file

@ -47,222 +47,93 @@ void A32EmitX64::GenFastmemFallbacks() {
{64, Devirtualize<&A32::UserCallbacks::MemoryWriteExclusive64>(conf.callbacks)}, {64, Devirtualize<&A32::UserCallbacks::MemoryWriteExclusive64>(conf.callbacks)},
}}; }};
for (int vaddr_idx : idxes) { for (bool ordered : {false, true}) {
for (int value_idx : idxes) { for (int vaddr_idx : idxes) {
for (const auto& [bitsize, callback] : read_callbacks) { for (int value_idx : idxes) {
code.align(); for (const auto& [bitsize, callback] : read_callbacks) {
read_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>(); code.align();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx)); read_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
if (vaddr_idx != code.ABI_PARAM2.getIdx()) { ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx));
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
callback.EmitCall(code);
if (value_idx != code.ABI_RETURN.getIdx()) {
code.mov(Xbyak::Reg64{value_idx}, code.ABI_RETURN);
}
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx));
code.ZeroExtendFrom(bitsize, Xbyak::Reg64{value_idx});
code.ret();
PerfMapRegister(read_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a32_read_fallback_{}", bitsize));
}
for (const auto& [bitsize, callback] : write_callbacks) {
code.align();
write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStack(code);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3);
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) { if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx}); code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
} }
if (ordered) {
code.mfence();
}
callback.EmitCall(code);
if (value_idx != code.ABI_RETURN.getIdx()) {
code.mov(Xbyak::Reg64{value_idx}, code.ABI_RETURN);
}
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx));
code.ZeroExtendFrom(bitsize, Xbyak::Reg64{value_idx});
code.ret();
PerfMapRegister(read_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a32_read_fallback_{}", bitsize));
} }
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
callback.EmitCall(code);
ABI_PopCallerSaveRegistersAndAdjustStack(code);
code.ret();
PerfMapRegister(write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a32_write_fallback_{}", bitsize));
}
for (const auto& [bitsize, callback] : exclusive_write_callbacks) { for (const auto& [bitsize, callback] : write_callbacks) {
code.align(); code.align();
exclusive_write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>(); write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX); ABI_PushCallerSaveRegistersAndAdjustStack(code);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) { if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3); code.xchg(code.ABI_PARAM2, code.ABI_PARAM3);
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) { } else if (vaddr_idx == code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx}); code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
} }
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
callback.EmitCall(code);
if (ordered) {
code.mfence();
}
ABI_PopCallerSaveRegistersAndAdjustStack(code);
code.ret();
PerfMapRegister(write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a32_write_fallback_{}", bitsize));
}
for (const auto& [bitsize, callback] : exclusive_write_callbacks) {
code.align();
exclusive_write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3);
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
}
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
code.mov(code.ABI_PARAM4, rax);
code.ZeroExtendFrom(bitsize, code.ABI_PARAM4);
callback.EmitCall(code);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
code.ret();
PerfMapRegister(exclusive_write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a32_exclusive_write_fallback_{}", bitsize));
} }
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
code.mov(code.ABI_PARAM4, rax);
code.ZeroExtendFrom(bitsize, code.ABI_PARAM4);
callback.EmitCall(code);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
code.ret();
PerfMapRegister(exclusive_write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a32_exclusive_write_fallback_{}", bitsize));
} }
} }
} }
} }
std::optional<A32EmitX64::DoNotFastmemMarker> A32EmitX64::ShouldFastmem(A32EmitContext& ctx, IR::Inst* inst) const { #define Axx A32
if (!conf.fastmem_pointer || !exception_handler.SupportsFastmem()) { #include "emit_x64_memory.cpp.inc"
return std::nullopt; #undef Axx
}
const auto marker = std::make_tuple(ctx.Location(), ctx.GetInstOffset(inst));
if (do_not_fastmem.count(marker) > 0) {
return std::nullopt;
}
return marker;
}
FakeCall A32EmitX64::FastmemCallback(u64 rip_) {
const auto iter = fastmem_patch_info.find(rip_);
if (iter == fastmem_patch_info.end()) {
fmt::print("dynarmic: Segfault happened within JITted code at rip = {:016x}\n", rip_);
fmt::print("Segfault wasn't at a fastmem patch location!\n");
fmt::print("Now dumping code.......\n\n");
Common::DumpDisassembledX64((void*)(rip_ & ~u64(0xFFF)), 0x1000);
ASSERT_FALSE("iter != fastmem_patch_info.end()");
}
if (iter->second.compile) {
const auto marker = iter->second.marker;
do_not_fastmem.emplace(marker);
InvalidateBasicBlocks({std::get<0>(marker)});
}
return FakeCall{
.call_rip = iter->second.callback,
.ret_rip = iter->second.resume_rip,
};
}
template<std::size_t bitsize, auto callback>
void A32EmitX64::EmitMemoryRead(A32EmitContext& ctx, IR::Inst* inst) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (!conf.page_table && !fastmem_marker) {
// Neither fastmem nor page table: Use callbacks
ctx.reg_alloc.HostCall(inst, {}, args[0]);
Devirtualize<callback>(conf.callbacks).EmitCall(code);
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
return;
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const Xbyak::Reg64 value = ctx.reg_alloc.ScratchGpr();
const auto wrapped_fn = read_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value.getIdx())];
if (fastmem_marker) {
// Use fastmem
const auto src_ptr = r13 + vaddr;
const auto location = code.getCurr();
EmitReadMemoryMov<bitsize>(code, value.getIdx(), src_ptr);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_fastmem_failure,
});
ctx.reg_alloc.DefineValue(inst, value);
return;
}
// Use page table
ASSERT(conf.page_table);
Xbyak::Label abort, end;
const auto src_ptr = EmitVAddrLookup(code, ctx, bitsize, abort, vaddr);
EmitReadMemoryMov<bitsize>(code, value.getIdx(), src_ptr);
code.L(end);
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
ctx.reg_alloc.DefineValue(inst, value);
}
template<std::size_t bitsize, auto callback>
void A32EmitX64::EmitMemoryWrite(A32EmitContext& ctx, IR::Inst* inst) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (!conf.page_table && !fastmem_marker) {
// Neither fastmem nor page table: Use callbacks
ctx.reg_alloc.HostCall(nullptr, {}, args[0], args[1]);
Devirtualize<callback>(conf.callbacks).EmitCall(code);
return;
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const Xbyak::Reg64 value = ctx.reg_alloc.UseGpr(args[1]);
const auto wrapped_fn = write_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value.getIdx())];
if (fastmem_marker) {
// Use fastmem
const auto dest_ptr = r13 + vaddr;
const auto location = code.getCurr();
EmitWriteMemoryMov<bitsize>(code, dest_ptr, value.getIdx());
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_fastmem_failure,
});
return;
}
// Use page table
ASSERT(conf.page_table);
Xbyak::Label abort, end;
const auto dest_ptr = EmitVAddrLookup(code, ctx, bitsize, abort, vaddr);
EmitWriteMemoryMov<bitsize>(code, dest_ptr, value.getIdx());
code.L(end);
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
void A32EmitX64::EmitA32ReadMemory8(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ReadMemory8(A32EmitContext& ctx, IR::Inst* inst) {
EmitMemoryRead<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst); EmitMemoryRead<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst);
@ -296,268 +167,71 @@ void A32EmitX64::EmitA32WriteMemory64(A32EmitContext& ctx, IR::Inst* inst) {
EmitMemoryWrite<64, &A32::UserCallbacks::MemoryWrite64>(ctx, inst); EmitMemoryWrite<64, &A32::UserCallbacks::MemoryWrite64>(ctx, inst);
} }
template<size_t bitsize, auto callback>
void A32EmitX64::ExclusiveReadMemory(A32EmitContext& ctx, IR::Inst* inst) {
using T = mp::unsigned_integer_of_size<bitsize>;
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A32::UserConfig& conf, u32 vaddr) -> T {
return conf.global_monitor->ReadAndMark<T>(conf.processor_id, vaddr, [&]() -> T {
return (conf.callbacks->*callback)(vaddr);
});
});
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
}
template<size_t bitsize, auto callback>
void A32EmitX64::ExclusiveWriteMemory(A32EmitContext& ctx, IR::Inst* inst) {
using T = mp::unsigned_integer_of_size<bitsize>;
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(inst, {}, args[0], args[1]);
Xbyak::Label end;
code.mov(code.ABI_RETURN, u32(1));
code.cmp(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(0));
code.je(end);
code.mov(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(0));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A32::UserConfig& conf, u32 vaddr, T value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr,
[&](T expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
})
? 0
: 1;
});
code.L(end);
}
template<std::size_t bitsize, auto callback>
void A32EmitX64::ExclusiveReadMemoryInline(A32EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
ExclusiveReadMemory<bitsize, callback>(ctx, inst);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const Xbyak::Reg64 value = ctx.reg_alloc.ScratchGpr();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
const Xbyak::Reg64 tmp2 = ctx.reg_alloc.ScratchGpr();
const auto wrapped_fn = read_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value.getIdx())];
EmitExclusiveLock(code, conf, tmp, tmp2.cvt32());
code.mov(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(1));
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(qword[tmp], vaddr);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (fastmem_marker) {
Xbyak::Label end;
const auto src_ptr = r13 + vaddr;
const auto location = code.getCurr();
EmitReadMemoryMov<bitsize>(code, value.getIdx(), src_ptr);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_exclusive_fastmem_failure,
});
code.L(end);
} else {
code.call(wrapped_fn);
}
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
EmitWriteMemoryMov<bitsize>(code, tmp, value.getIdx());
EmitExclusiveUnlock(code, conf, tmp, tmp2.cvt32());
ctx.reg_alloc.DefineValue(inst, value);
}
template<std::size_t bitsize, auto callback>
void A32EmitX64::ExclusiveWriteMemoryInline(A32EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
ExclusiveWriteMemory<bitsize, callback>(ctx, inst);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
const Xbyak::Reg64 value = ctx.reg_alloc.UseGpr(args[1]);
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const Xbyak::Reg32 status = ctx.reg_alloc.ScratchGpr().cvt32();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
const auto fallback_fn = exclusive_write_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value.getIdx())];
EmitExclusiveLock(code, conf, tmp, eax);
Xbyak::Label end;
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(status, u32(1));
code.cmp(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(0));
code.je(end, code.T_NEAR);
code.cmp(qword[tmp], vaddr);
code.jne(end, code.T_NEAR);
EmitExclusiveTestAndClear(code, conf, vaddr, tmp, rax);
code.mov(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(0));
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
EmitReadMemoryMov<bitsize>(code, rax.getIdx(), tmp);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (fastmem_marker) {
const auto dest_ptr = r13 + vaddr;
const auto location = code.getCurr();
switch (bitsize) {
case 8:
code.lock();
code.cmpxchg(code.byte[dest_ptr], value.cvt8());
break;
case 16:
code.lock();
code.cmpxchg(word[dest_ptr], value.cvt16());
break;
case 32:
code.lock();
code.cmpxchg(dword[dest_ptr], value.cvt32());
break;
case 64:
code.lock();
code.cmpxchg(qword[dest_ptr], value.cvt64());
break;
default:
UNREACHABLE();
}
code.setnz(status.cvt8());
code.SwitchToFarCode();
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(fallback_fn),
*fastmem_marker,
conf.recompile_on_exclusive_fastmem_failure,
});
code.cmp(al, 0);
code.setz(status.cvt8());
code.movzx(status.cvt32(), status.cvt8());
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
} else {
code.call(fallback_fn);
code.cmp(al, 0);
code.setz(status.cvt8());
code.movzx(status.cvt32(), status.cvt8());
}
code.L(end);
EmitExclusiveUnlock(code, conf, tmp, eax);
ctx.reg_alloc.DefineValue(inst, status);
}
void A32EmitX64::EmitA32ClearExclusive(A32EmitContext&, IR::Inst*) { void A32EmitX64::EmitA32ClearExclusive(A32EmitContext&, IR::Inst*) {
code.mov(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(0)); code.mov(code.byte[r15 + offsetof(A32JitState, exclusive_state)], u8(0));
} }
void A32EmitX64::EmitA32ExclusiveReadMemory8(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveReadMemory8(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveReadMemoryInline<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst); EmitExclusiveReadMemoryInline<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst);
} else { } else {
ExclusiveReadMemory<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst); EmitExclusiveReadMemory<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveReadMemory16(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveReadMemory16(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveReadMemoryInline<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst); EmitExclusiveReadMemoryInline<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst);
} else { } else {
ExclusiveReadMemory<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst); EmitExclusiveReadMemory<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveReadMemory32(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveReadMemory32(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveReadMemoryInline<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst); EmitExclusiveReadMemoryInline<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst);
} else { } else {
ExclusiveReadMemory<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst); EmitExclusiveReadMemory<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveReadMemory64(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveReadMemory64(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveReadMemoryInline<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst); EmitExclusiveReadMemoryInline<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst);
} else { } else {
ExclusiveReadMemory<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst); EmitExclusiveReadMemory<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveWriteMemory8(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveWriteMemory8(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveWriteMemoryInline<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst); EmitExclusiveWriteMemoryInline<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
} else { } else {
ExclusiveWriteMemory<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst); EmitExclusiveWriteMemory<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveWriteMemory16(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveWriteMemory16(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveWriteMemoryInline<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst); EmitExclusiveWriteMemoryInline<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
} else { } else {
ExclusiveWriteMemory<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst); EmitExclusiveWriteMemory<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveWriteMemory32(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveWriteMemory32(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveWriteMemoryInline<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst); EmitExclusiveWriteMemoryInline<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
} else { } else {
ExclusiveWriteMemory<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst); EmitExclusiveWriteMemory<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
} }
} }
void A32EmitX64::EmitA32ExclusiveWriteMemory64(A32EmitContext& ctx, IR::Inst* inst) { void A32EmitX64::EmitA32ExclusiveWriteMemory64(A32EmitContext& ctx, IR::Inst* inst) {
if (conf.fastmem_exclusive_access) { if (conf.fastmem_exclusive_access) {
ExclusiveWriteMemoryInline<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst); EmitExclusiveWriteMemoryInline<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
} else { } else {
ExclusiveWriteMemory<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst); EmitExclusiveWriteMemory<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
} }
} }

6
externals/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h vendored
View file

@ -71,9 +71,9 @@ protected:
void (*memory_exclusive_write_128)(); void (*memory_exclusive_write_128)();
void GenMemory128Accessors(); void GenMemory128Accessors();
std::map<std::tuple<size_t, int, int>, void (*)()> read_fallbacks; std::map<std::tuple<bool, size_t, int, int>, void (*)()> read_fallbacks;
std::map<std::tuple<size_t, int, int>, void (*)()> write_fallbacks; std::map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks;
std::map<std::tuple<size_t, int, int>, void (*)()> exclusive_write_fallbacks; std::map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks;
void GenFastmemFallbacks(); void GenFastmemFallbacks();
const void* terminal_handler_pop_rsb_hint; const void* terminal_handler_pop_rsb_hint;

677
externals/dynarmic/src/dynarmic/backend/x64/a64_emit_x64_memory.cpp vendored
View file

@ -131,301 +131,157 @@ void A64EmitX64::GenFastmemFallbacks() {
{64, Devirtualize<&A64::UserCallbacks::MemoryWriteExclusive64>(conf.callbacks)}, {64, Devirtualize<&A64::UserCallbacks::MemoryWriteExclusive64>(conf.callbacks)},
}}; }};
for (int vaddr_idx : idxes) { for (bool ordered : {false, true}) {
if (vaddr_idx == 4 || vaddr_idx == 15) { for (int vaddr_idx : idxes) {
continue; if (vaddr_idx == 4 || vaddr_idx == 15) {
}
for (int value_idx : idxes) {
code.align();
read_fallbacks[std::make_tuple(128, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(value_idx));
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
code.call(memory_read_128);
if (value_idx != 1) {
code.movaps(Xbyak::Xmm{value_idx}, xmm1);
}
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(value_idx));
code.ret();
PerfMapRegister(read_fallbacks[std::make_tuple(128, vaddr_idx, value_idx)], code.getCurr(), "a64_read_fallback_128");
code.align();
write_fallbacks[std::make_tuple(128, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStack(code);
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
if (value_idx != 1) {
code.movaps(xmm1, Xbyak::Xmm{value_idx});
}
code.call(memory_write_128);
ABI_PopCallerSaveRegistersAndAdjustStack(code);
code.ret();
PerfMapRegister(write_fallbacks[std::make_tuple(128, vaddr_idx, value_idx)], code.getCurr(), "a64_write_fallback_128");
code.align();
exclusive_write_fallbacks[std::make_tuple(128, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
if (value_idx != 1) {
code.movaps(xmm1, Xbyak::Xmm{value_idx});
}
if (code.HasHostFeature(HostFeature::SSE41)) {
code.movq(xmm2, rax);
code.pinsrq(xmm2, rdx, 1);
} else {
code.movq(xmm2, rax);
code.movq(xmm0, rdx);
code.punpcklqdq(xmm2, xmm0);
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
code.call(memory_exclusive_write_128);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
code.ret();
PerfMapRegister(exclusive_write_fallbacks[std::make_tuple(128, vaddr_idx, value_idx)], code.getCurr(), "a64_write_fallback_128");
if (value_idx == 4 || value_idx == 15) {
continue; continue;
} }
for (const auto& [bitsize, callback] : read_callbacks) { for (int value_idx : idxes) {
code.align(); code.align();
read_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>(); read_fallbacks[std::make_tuple(ordered, 128, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx)); ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(value_idx));
if (vaddr_idx != code.ABI_PARAM2.getIdx()) { if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx}); code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
} }
callback.EmitCall(code); if (ordered) {
if (value_idx != code.ABI_RETURN.getIdx()) { code.mfence();
code.mov(Xbyak::Reg64{value_idx}, code.ABI_RETURN);
} }
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx)); code.call(memory_read_128);
code.ZeroExtendFrom(bitsize, Xbyak::Reg64{value_idx}); if (value_idx != 1) {
code.movaps(Xbyak::Xmm{value_idx}, xmm1);
}
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(value_idx));
code.ret(); code.ret();
PerfMapRegister(read_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a64_read_fallback_{}", bitsize)); PerfMapRegister(read_fallbacks[std::make_tuple(ordered, 128, vaddr_idx, value_idx)], code.getCurr(), "a64_read_fallback_128");
}
for (const auto& [bitsize, callback] : write_callbacks) {
code.align(); code.align();
write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>(); write_fallbacks[std::make_tuple(ordered, 128, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStack(code); ABI_PushCallerSaveRegistersAndAdjustStack(code);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) { if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3);
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx}); code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
} }
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3); if (value_idx != 1) {
callback.EmitCall(code); code.movaps(xmm1, Xbyak::Xmm{value_idx});
}
code.call(memory_write_128);
if (ordered) {
code.mfence();
}
ABI_PopCallerSaveRegistersAndAdjustStack(code); ABI_PopCallerSaveRegistersAndAdjustStack(code);
code.ret(); code.ret();
PerfMapRegister(write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a64_write_fallback_{}", bitsize)); PerfMapRegister(write_fallbacks[std::make_tuple(ordered, 128, vaddr_idx, value_idx)], code.getCurr(), "a64_write_fallback_128");
}
for (const auto& [bitsize, callback] : exclusive_write_callbacks) {
code.align(); code.align();
exclusive_write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>(); exclusive_write_fallbacks[std::make_tuple(ordered, 128, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX); ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) { if (value_idx != 1) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3); code.movaps(xmm1, Xbyak::Xmm{value_idx});
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) { }
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx}); if (code.HasHostFeature(HostFeature::SSE41)) {
if (value_idx != code.ABI_PARAM3.getIdx()) { code.movq(xmm2, rax);
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx}); code.pinsrq(xmm2, rdx, 1);
}
} else { } else {
if (value_idx != code.ABI_PARAM3.getIdx()) { code.movq(xmm2, rax);
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx}); code.movq(xmm0, rdx);
} code.punpcklqdq(xmm2, xmm0);
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
code.call(memory_exclusive_write_128);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
code.ret();
PerfMapRegister(exclusive_write_fallbacks[std::make_tuple(ordered, 128, vaddr_idx, value_idx)], code.getCurr(), "a64_exclusive_write_fallback_128");
if (value_idx == 4 || value_idx == 15) {
continue;
}
for (const auto& [bitsize, callback] : read_callbacks) {
code.align();
read_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx));
if (vaddr_idx != code.ABI_PARAM2.getIdx()) { if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx}); code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
} }
if (ordered) {
code.mfence();
}
callback.EmitCall(code);
if (value_idx != code.ABI_RETURN.getIdx()) {
code.mov(Xbyak::Reg64{value_idx}, code.ABI_RETURN);
}
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocRegIdx(value_idx));
code.ZeroExtendFrom(bitsize, Xbyak::Reg64{value_idx});
code.ret();
PerfMapRegister(read_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a64_read_fallback_{}", bitsize));
}
for (const auto& [bitsize, callback] : write_callbacks) {
code.align();
write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStack(code);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3);
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
}
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
callback.EmitCall(code);
if (ordered) {
code.mfence();
}
ABI_PopCallerSaveRegistersAndAdjustStack(code);
code.ret();
PerfMapRegister(write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a64_write_fallback_{}", bitsize));
}
for (const auto& [bitsize, callback] : exclusive_write_callbacks) {
code.align();
exclusive_write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)] = code.getCurr<void (*)()>();
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
if (vaddr_idx == code.ABI_PARAM3.getIdx() && value_idx == code.ABI_PARAM2.getIdx()) {
code.xchg(code.ABI_PARAM2, code.ABI_PARAM3);
} else if (vaddr_idx == code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
} else {
if (value_idx != code.ABI_PARAM3.getIdx()) {
code.mov(code.ABI_PARAM3, Xbyak::Reg64{value_idx});
}
if (vaddr_idx != code.ABI_PARAM2.getIdx()) {
code.mov(code.ABI_PARAM2, Xbyak::Reg64{vaddr_idx});
}
}
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
code.mov(code.ABI_PARAM4, rax);
code.ZeroExtendFrom(bitsize, code.ABI_PARAM4);
callback.EmitCall(code);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
code.ret();
PerfMapRegister(exclusive_write_fallbacks[std::make_tuple(ordered, bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a64_exclusive_write_fallback_{}", bitsize));
} }
code.ZeroExtendFrom(bitsize, code.ABI_PARAM3);
code.mov(code.ABI_PARAM4, rax);
code.ZeroExtendFrom(bitsize, code.ABI_PARAM4);
callback.EmitCall(code);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLoc::RAX);
code.ret();
PerfMapRegister(exclusive_write_fallbacks[std::make_tuple(bitsize, vaddr_idx, value_idx)], code.getCurr(), fmt::format("a64_exclusive_write_fallback_{}", bitsize));
} }
} }
} }
} }
std::optional<A64EmitX64::DoNotFastmemMarker> A64EmitX64::ShouldFastmem(A64EmitContext& ctx, IR::Inst* inst) const { #define Axx A64
if (!conf.fastmem_pointer || !exception_handler.SupportsFastmem()) { #include "emit_x64_memory.cpp.inc"
return std::nullopt; #undef Axx
}
const auto marker = std::make_tuple(ctx.Location(), ctx.GetInstOffset(inst));
if (do_not_fastmem.count(marker) > 0) {
return std::nullopt;
}
return marker;
}
FakeCall A64EmitX64::FastmemCallback(u64 rip_) {
const auto iter = fastmem_patch_info.find(rip_);
if (iter == fastmem_patch_info.end()) {
fmt::print("dynarmic: Segfault happened within JITted code at rip = {:016x}\n", rip_);
fmt::print("Segfault wasn't at a fastmem patch location!\n");
fmt::print("Now dumping code.......\n\n");
Common::DumpDisassembledX64((void*)(rip_ & ~u64(0xFFF)), 0x1000);
ASSERT_FALSE("iter != fastmem_patch_info.end()");
}
if (iter->second.recompile) {
const auto marker = iter->second.marker;
do_not_fastmem.emplace(marker);
InvalidateBasicBlocks({std::get<0>(marker)});
}
return FakeCall{
.call_rip = iter->second.callback,
.ret_rip = iter->second.resume_rip,
};
}
template<std::size_t bitsize, auto callback>
void A64EmitX64::EmitMemoryRead(A64EmitContext& ctx, IR::Inst* inst) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (!conf.page_table && !fastmem_marker) {
// Neither fastmem nor page table: Use callbacks
if constexpr (bitsize == 128) {
ctx.reg_alloc.HostCall(nullptr, {}, args[0]);
code.CallFunction(memory_read_128);
ctx.reg_alloc.DefineValue(inst, xmm1);
} else {
ctx.reg_alloc.HostCall(inst, {}, args[0]);
Devirtualize<callback>(conf.callbacks).EmitCall(code);
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
}
return;
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx();
const auto wrapped_fn = read_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value_idx)];
Xbyak::Label abort, end;
bool require_abort_handling = false;
if (fastmem_marker) {
// Use fastmem
const auto src_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling);
const auto location = code.getCurr();
EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_fastmem_failure,
});
} else {
// Use page table
ASSERT(conf.page_table);
const auto src_ptr = EmitVAddrLookup(code, ctx, bitsize, abort, vaddr);
require_abort_handling = true;
EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr);
}
code.L(end);
if (require_abort_handling) {
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
if constexpr (bitsize == 128) {
ctx.reg_alloc.DefineValue(inst, Xbyak::Xmm{value_idx});
} else {
ctx.reg_alloc.DefineValue(inst, Xbyak::Reg64{value_idx});
}
}
template<std::size_t bitsize, auto callback>
void A64EmitX64::EmitMemoryWrite(A64EmitContext& ctx, IR::Inst* inst) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (!conf.page_table && !fastmem_marker) {
// Neither fastmem nor page table: Use callbacks
if constexpr (bitsize == 128) {
ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.Use(args[1], HostLoc::XMM1);
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(nullptr);
code.CallFunction(memory_write_128);
} else {
ctx.reg_alloc.HostCall(nullptr, {}, args[0], args[1]);
Devirtualize<callback>(conf.callbacks).EmitCall(code);
}
return;
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.UseXmm(args[1]).getIdx() : ctx.reg_alloc.UseGpr(args[1]).getIdx();
const auto wrapped_fn = write_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value_idx)];
Xbyak::Label abort, end;
bool require_abort_handling = false;
if (fastmem_marker) {
// Use fastmem
const auto dest_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling);
const auto location = code.getCurr();
EmitWriteMemoryMov<bitsize>(code, dest_ptr, value_idx);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_fastmem_failure,
});
} else {
// Use page table
ASSERT(conf.page_table);
const auto dest_ptr = EmitVAddrLookup(code, ctx, bitsize, abort, vaddr);
require_abort_handling = true;
EmitWriteMemoryMov<bitsize>(code, dest_ptr, value_idx);
}
code.L(end);
if (require_abort_handling) {
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
}
void A64EmitX64::EmitA64ReadMemory8(A64EmitContext& ctx, IR::Inst* inst) { void A64EmitX64::EmitA64ReadMemory8(A64EmitContext& ctx, IR::Inst* inst) {
EmitMemoryRead<8, &A64::UserCallbacks::MemoryRead8>(ctx, inst); EmitMemoryRead<8, &A64::UserCallbacks::MemoryRead8>(ctx, inst);
@ -467,295 +323,6 @@ void A64EmitX64::EmitA64WriteMemory128(A64EmitContext& ctx, IR::Inst* inst) {
EmitMemoryWrite<128, &A64::UserCallbacks::MemoryWrite64>(ctx, inst); EmitMemoryWrite<128, &A64::UserCallbacks::MemoryWrite64>(ctx, inst);
} }
template<std::size_t bitsize, auto callback>
void A64EmitX64::EmitExclusiveReadMemory(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
if constexpr (bitsize != 128) {
using T = mp::unsigned_integer_of_size<bitsize>;
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr) -> T {
return conf.global_monitor->ReadAndMark<T>(conf.processor_id, vaddr, [&]() -> T {
return (conf.callbacks->*callback)(vaddr);
});
});
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
} else {
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(nullptr);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
ctx.reg_alloc.AllocStackSpace(16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, A64::Vector& ret) {
ret = conf.global_monitor->ReadAndMark<A64::Vector>(conf.processor_id, vaddr, [&]() -> A64::Vector {
return (conf.callbacks->*callback)(vaddr);
});
});
code.movups(result, xword[rsp + ABI_SHADOW_SPACE]);
ctx.reg_alloc.ReleaseStackSpace(16 + ABI_SHADOW_SPACE);
ctx.reg_alloc.DefineValue(inst, result);
}
}
template<std::size_t bitsize, auto callback>
void A64EmitX64::EmitExclusiveWriteMemory(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
if constexpr (bitsize != 128) {
ctx.reg_alloc.HostCall(inst, {}, args[0], args[1]);
} else {
ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.Use(args[1], HostLoc::XMM1);
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(inst);
}
Xbyak::Label end;
code.mov(code.ABI_RETURN, u32(1));
code.cmp(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.je(end);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
if constexpr (bitsize != 128) {
using T = mp::unsigned_integer_of_size<bitsize>;
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, T value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr,
[&](T expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
})
? 0
: 1;
});
} else {
ctx.reg_alloc.AllocStackSpace(16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
code.movaps(xword[code.ABI_PARAM3], xmm1);
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, A64::Vector& value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<A64::Vector>(conf.processor_id, vaddr,
[&](A64::Vector expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
})
? 0
: 1;
});
ctx.reg_alloc.ReleaseStackSpace(16 + ABI_SHADOW_SPACE);
}
code.L(end);
}
template<std::size_t bitsize, auto callback>
void A64EmitX64::EmitExclusiveReadMemoryInline(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
EmitExclusiveReadMemory<bitsize, callback>(ctx, inst);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
const Xbyak::Reg64 tmp2 = ctx.reg_alloc.ScratchGpr();
const auto wrapped_fn = read_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value_idx)];
EmitExclusiveLock(code, conf, tmp, tmp2.cvt32());
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(qword[tmp], vaddr);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (fastmem_marker) {
Xbyak::Label abort, end;
bool require_abort_handling = false;
const auto src_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling);
const auto location = code.getCurr();
EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_exclusive_fastmem_failure,
});
code.L(end);
if (require_abort_handling) {
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
} else {
code.call(wrapped_fn);
}
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
EmitWriteMemoryMov<bitsize>(code, tmp, value_idx);
EmitExclusiveUnlock(code, conf, tmp, tmp2.cvt32());
if constexpr (bitsize == 128) {
ctx.reg_alloc.DefineValue(inst, Xbyak::Xmm{value_idx});
} else {
ctx.reg_alloc.DefineValue(inst, Xbyak::Reg64{value_idx});
}
}
template<std::size_t bitsize, auto callback>
void A64EmitX64::EmitExclusiveWriteMemoryInline(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
EmitExclusiveWriteMemory<bitsize, callback>(ctx, inst);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const auto value = [&] {
if constexpr (bitsize == 128) {
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
ctx.reg_alloc.ScratchGpr(HostLoc::RBX);
ctx.reg_alloc.ScratchGpr(HostLoc::RCX);
ctx.reg_alloc.ScratchGpr(HostLoc::RDX);
return ctx.reg_alloc.UseXmm(args[1]);
} else {
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
return ctx.reg_alloc.UseGpr(args[1]);
}
}();
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const Xbyak::Reg32 status = ctx.reg_alloc.ScratchGpr().cvt32();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
const auto fallback_fn = exclusive_write_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value.getIdx())];
EmitExclusiveLock(code, conf, tmp, eax);
Xbyak::Label end;
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(status, u32(1));
code.cmp(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.je(end, code.T_NEAR);
code.cmp(qword[tmp], vaddr);
code.jne(end, code.T_NEAR);
EmitExclusiveTestAndClear(code, conf, vaddr, tmp, rax);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
if constexpr (bitsize == 128) {
code.mov(rax, qword[tmp + 0]);
code.mov(rdx, qword[tmp + 8]);
if (code.HasHostFeature(HostFeature::SSE41)) {
code.movq(rbx, value);
code.pextrq(rcx, value, 1);
} else {
code.movaps(xmm0, value);
code.movq(rbx, xmm0);
code.punpckhqdq(xmm0, xmm0);
code.movq(rcx, xmm0);
}
} else {
EmitReadMemoryMov<bitsize>(code, rax.getIdx(), tmp);
}
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (fastmem_marker) {
Xbyak::Label abort;
bool require_abort_handling = false;
const auto dest_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling, tmp);
const auto location = code.getCurr();
if constexpr (bitsize == 128) {
code.lock();
code.cmpxchg16b(ptr[dest_ptr]);
} else {
switch (bitsize) {
case 8:
code.lock();
code.cmpxchg(code.byte[dest_ptr], value.cvt8());
break;
case 16:
code.lock();
code.cmpxchg(word[dest_ptr], value.cvt16());
break;
case 32:
code.lock();
code.cmpxchg(dword[dest_ptr], value.cvt32());
break;
case 64:
code.lock();
code.cmpxchg(qword[dest_ptr], value.cvt64());
break;
default:
UNREACHABLE();
}
}
code.setnz(status.cvt8());
code.SwitchToFarCode();
code.L(abort);
code.call(fallback_fn);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(fallback_fn),
*fastmem_marker,
conf.recompile_on_exclusive_fastmem_failure,
});
code.cmp(al, 0);
code.setz(status.cvt8());
code.movzx(status.cvt32(), status.cvt8());
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
} else {
code.call(fallback_fn);
code.cmp(al, 0);
code.setz(status.cvt8());
code.movzx(status.cvt32(), status.cvt8());
}
code.L(end);
EmitExclusiveUnlock(code, conf, tmp, eax);
ctx.reg_alloc.DefineValue(inst, status);
}
void A64EmitX64::EmitA64ClearExclusive(A64EmitContext&, IR::Inst*) { void A64EmitX64::EmitA64ClearExclusive(A64EmitContext&, IR::Inst*) {
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0)); code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
} }

497
externals/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc vendored Executable file
View file

@ -0,0 +1,497 @@
/* This file is part of the dynarmic project.
* Copyright (c) 2022 MerryMage
* SPDX-License-Identifier: 0BSD
*/
#include "dynarmic/common/macro_util.h"
#define AxxEmitX64 CONCATENATE_TOKENS(Axx, EmitX64)
#define AxxEmitContext CONCATENATE_TOKENS(Axx, EmitContext)
#define AxxJitState CONCATENATE_TOKENS(Axx, JitState)
#define AxxUserConfig Axx::UserConfig
namespace {
using Vector = std::array<u64, 2>;
}
std::optional<AxxEmitX64::DoNotFastmemMarker> AxxEmitX64::ShouldFastmem(AxxEmitContext& ctx, IR::Inst* inst) const {
if (!conf.fastmem_pointer || !exception_handler.SupportsFastmem()) {
return std::nullopt;
}
const auto marker = std::make_tuple(ctx.Location(), ctx.GetInstOffset(inst));
if (do_not_fastmem.count(marker) > 0) {
return std::nullopt;
}
return marker;
}
FakeCall AxxEmitX64::FastmemCallback(u64 rip_) {
const auto iter = fastmem_patch_info.find(rip_);
if (iter == fastmem_patch_info.end()) {
fmt::print("dynarmic: Segfault happened within JITted code at rip = {:016x}\n", rip_);
fmt::print("Segfault wasn't at a fastmem patch location!\n");
fmt::print("Now dumping code.......\n\n");
Common::DumpDisassembledX64((void*)(rip_ & ~u64(0xFFF)), 0x1000);
ASSERT_FALSE("iter != fastmem_patch_info.end()");
}
if (iter->second.recompile) {
const auto marker = iter->second.marker;
do_not_fastmem.emplace(marker);
InvalidateBasicBlocks({std::get<0>(marker)});
}
return FakeCall{
.call_rip = iter->second.callback,
.ret_rip = iter->second.resume_rip,
};
}
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = IsOrdered(args[1].GetImmediateAccType());
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (!conf.page_table && !fastmem_marker) {
// Neither fastmem nor page table: Use callbacks
if constexpr (bitsize == 128) {
ctx.reg_alloc.HostCall(nullptr, {}, args[0]);
if (ordered) {
code.mfence();
}
code.CallFunction(memory_read_128);
ctx.reg_alloc.DefineValue(inst, xmm1);
} else {
ctx.reg_alloc.HostCall(inst, {}, args[0]);
if (ordered) {
code.mfence();
}
Devirtualize<callback>(conf.callbacks).EmitCall(code);
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
}
return;
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx();
const auto wrapped_fn = read_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value_idx)];
Xbyak::Label abort, end;
bool require_abort_handling = false;
if (fastmem_marker) {
// Use fastmem
const auto src_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling);
const auto location = EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr, ordered);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_fastmem_failure,
});
} else {
// Use page table
ASSERT(conf.page_table);
const auto src_ptr = EmitVAddrLookup(code, ctx, bitsize, abort, vaddr);
require_abort_handling = true;
EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr, ordered);
}
code.L(end);
if (require_abort_handling) {
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
if constexpr (bitsize == 128) {
ctx.reg_alloc.DefineValue(inst, Xbyak::Xmm{value_idx});
} else {
ctx.reg_alloc.DefineValue(inst, Xbyak::Reg64{value_idx});
}
}
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = IsOrdered(args[2].GetImmediateAccType());
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (!conf.page_table && !fastmem_marker) {
// Neither fastmem nor page table: Use callbacks
if constexpr (bitsize == 128) {
ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.Use(args[1], HostLoc::XMM1);
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(nullptr);
code.CallFunction(memory_write_128);
} else {
ctx.reg_alloc.HostCall(nullptr, {}, args[0], args[1]);
Devirtualize<callback>(conf.callbacks).EmitCall(code);
}
if (ordered) {
code.mfence();
}
return;
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128
? ctx.reg_alloc.UseXmm(args[1]).getIdx()
: (ordered ? ctx.reg_alloc.UseScratchGpr(args[1]).getIdx() : ctx.reg_alloc.UseGpr(args[1]).getIdx());
const auto wrapped_fn = write_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value_idx)];
Xbyak::Label abort, end;
bool require_abort_handling = false;
if (fastmem_marker) {
// Use fastmem
const auto dest_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling);
const auto location = EmitWriteMemoryMov<bitsize>(code, dest_ptr, value_idx, ordered);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_fastmem_failure,
});
} else {
// Use page table
ASSERT(conf.page_table);
const auto dest_ptr = EmitVAddrLookup(code, ctx, bitsize, abort, vaddr);
require_abort_handling = true;
EmitWriteMemoryMov<bitsize>(code, dest_ptr, value_idx, ordered);
}
code.L(end);
if (require_abort_handling) {
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
}
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitExclusiveReadMemory(AxxEmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = IsOrdered(args[1].GetImmediateAccType());
if constexpr (bitsize != 128) {
using T = mp::unsigned_integer_of_size<bitsize>;
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
if (ordered) {
code.mfence();
}
code.CallLambda(
[](AxxUserConfig& conf, Axx::VAddr vaddr) -> T {
return conf.global_monitor->ReadAndMark<T>(conf.processor_id, vaddr, [&]() -> T {
return (conf.callbacks->*callback)(vaddr);
});
});
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
} else {
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(nullptr);
code.mov(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
ctx.reg_alloc.AllocStackSpace(16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
if (ordered) {
code.mfence();
}
code.CallLambda(
[](AxxUserConfig& conf, Axx::VAddr vaddr, Vector& ret) {
ret = conf.global_monitor->ReadAndMark<Vector>(conf.processor_id, vaddr, [&]() -> Vector {
return (conf.callbacks->*callback)(vaddr);
});
});
code.movups(result, xword[rsp + ABI_SHADOW_SPACE]);
ctx.reg_alloc.ReleaseStackSpace(16 + ABI_SHADOW_SPACE);
ctx.reg_alloc.DefineValue(inst, result);
}
}
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitExclusiveWriteMemory(AxxEmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = IsOrdered(args[2].GetImmediateAccType());
if constexpr (bitsize != 128) {
ctx.reg_alloc.HostCall(inst, {}, args[0], args[1]);
} else {
ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.Use(args[1], HostLoc::XMM1);
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(inst);
}
Xbyak::Label end;
code.mov(code.ABI_RETURN, u32(1));
code.cmp(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(0));
code.je(end);
code.mov(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(0));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
if constexpr (bitsize != 128) {
using T = mp::unsigned_integer_of_size<bitsize>;
code.CallLambda(
[](AxxUserConfig& conf, Axx::VAddr vaddr, T value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr,
[&](T expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
})
? 0
: 1;
});
if (ordered) {
code.mfence();
}
} else {
ctx.reg_alloc.AllocStackSpace(16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
code.movaps(xword[code.ABI_PARAM3], xmm1);
code.CallLambda(
[](AxxUserConfig& conf, Axx::VAddr vaddr, Vector& value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<Vector>(conf.processor_id, vaddr,
[&](Vector expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
})
? 0
: 1;
});
if (ordered) {
code.mfence();
}
ctx.reg_alloc.ReleaseStackSpace(16 + ABI_SHADOW_SPACE);
}
code.L(end);
}
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
EmitExclusiveReadMemory<bitsize, callback>(ctx, inst);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = IsOrdered(args[1].GetImmediateAccType());
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
const Xbyak::Reg64 tmp2 = ctx.reg_alloc.ScratchGpr();
const auto wrapped_fn = read_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value_idx)];
EmitExclusiveLock(code, conf, tmp, tmp2.cvt32());
code.mov(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(1));
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(qword[tmp], vaddr);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (fastmem_marker) {
Xbyak::Label abort, end;
bool require_abort_handling = false;
const auto src_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling);
const auto location = EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr, ordered);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_exclusive_fastmem_failure,
});
code.L(end);
if (require_abort_handling) {
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
}
} else {
code.call(wrapped_fn);
}
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
EmitWriteMemoryMov<bitsize>(code, tmp, value_idx, false);
EmitExclusiveUnlock(code, conf, tmp, tmp2.cvt32());
if constexpr (bitsize == 128) {
ctx.reg_alloc.DefineValue(inst, Xbyak::Xmm{value_idx});
} else {
ctx.reg_alloc.DefineValue(inst, Xbyak::Reg64{value_idx});
}
}
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
EmitExclusiveWriteMemory<bitsize, callback>(ctx, inst);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = IsOrdered(args[2].GetImmediateAccType());
const auto value = [&] {
if constexpr (bitsize == 128) {
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
ctx.reg_alloc.ScratchGpr(HostLoc::RBX);
ctx.reg_alloc.ScratchGpr(HostLoc::RCX);
ctx.reg_alloc.ScratchGpr(HostLoc::RDX);
return ctx.reg_alloc.UseXmm(args[1]);
} else {
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
return ctx.reg_alloc.UseGpr(args[1]);
}
}();
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const Xbyak::Reg32 status = ctx.reg_alloc.ScratchGpr().cvt32();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
const auto wrapped_fn = exclusive_write_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value.getIdx())];
EmitExclusiveLock(code, conf, tmp, eax);
Xbyak::Label end;
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(status, u32(1));
code.cmp(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(0));
code.je(end, code.T_NEAR);
code.cmp(qword[tmp], vaddr);
code.jne(end, code.T_NEAR);
EmitExclusiveTestAndClear(code, conf, vaddr, tmp, rax);
code.mov(code.byte[r15 + offsetof(AxxJitState, exclusive_state)], u8(0));
code.mov(tmp, Common::BitCast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
if constexpr (bitsize == 128) {
code.mov(rax, qword[tmp + 0]);
code.mov(rdx, qword[tmp + 8]);
if (code.HasHostFeature(HostFeature::SSE41)) {
code.movq(rbx, value);
code.pextrq(rcx, value, 1);
} else {
code.movaps(xmm0, value);
code.movq(rbx, xmm0);
code.punpckhqdq(xmm0, xmm0);
code.movq(rcx, xmm0);
}
} else {
EmitReadMemoryMov<bitsize>(code, rax.getIdx(), tmp, false);
}
const auto fastmem_marker = ShouldFastmem(ctx, inst);
if (fastmem_marker) {
Xbyak::Label abort;
bool require_abort_handling = false;
const auto dest_ptr = EmitFastmemVAddr(code, ctx, abort, vaddr, require_abort_handling, tmp);
const auto location = code.getCurr();
if constexpr (bitsize == 128) {
code.lock();
code.cmpxchg16b(ptr[dest_ptr]);
} else {
switch (bitsize) {
case 8:
code.lock();
code.cmpxchg(code.byte[dest_ptr], value.cvt8());
break;
case 16:
code.lock();
code.cmpxchg(word[dest_ptr], value.cvt16());
break;
case 32:
code.lock();
code.cmpxchg(dword[dest_ptr], value.cvt32());
break;
case 64:
code.lock();
code.cmpxchg(qword[dest_ptr], value.cvt64());
break;
default:
UNREACHABLE();
}
}
code.setnz(status.cvt8());
code.SwitchToFarCode();
code.L(abort);
code.call(wrapped_fn);
fastmem_patch_info.emplace(
Common::BitCast<u64>(location),
FastmemPatchInfo{
Common::BitCast<u64>(code.getCurr()),
Common::BitCast<u64>(wrapped_fn),
*fastmem_marker,
conf.recompile_on_exclusive_fastmem_failure,
});
code.cmp(al, 0);
code.setz(status.cvt8());
code.movzx(status.cvt32(), status.cvt8());
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
} else {
code.call(wrapped_fn);
code.cmp(al, 0);
code.setz(status.cvt8());
code.movzx(status.cvt32(), status.cvt8());
}
code.L(end);
EmitExclusiveUnlock(code, conf, tmp, eax);
ctx.reg_alloc.DefineValue(inst, status);
}
#undef AxxEmitX64
#undef AxxEmitContext
#undef AxxJitState
#undef AxxUserConfig

93
externals/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h vendored
View file

@ -10,6 +10,7 @@
#include "dynarmic/backend/x64/exclusive_monitor_friend.h" #include "dynarmic/backend/x64/exclusive_monitor_friend.h"
#include "dynarmic/common/spin_lock_x64.h" #include "dynarmic/common/spin_lock_x64.h"
#include "dynarmic/interface/exclusive_monitor.h" #include "dynarmic/interface/exclusive_monitor.h"
#include "dynarmic/ir/acc_type.h"
namespace Dynarmic::Backend::X64 { namespace Dynarmic::Backend::X64 {
@ -198,49 +199,113 @@ template<>
} }
template<std::size_t bitsize> template<std::size_t bitsize>
void EmitReadMemoryMov(BlockOfCode& code, int value_idx, const Xbyak::RegExp& addr) { const void* EmitReadMemoryMov(BlockOfCode& code, int value_idx, const Xbyak::RegExp& addr, bool ordered) {
if (ordered) {
if constexpr (bitsize == 128) {
code.mfence();
} else {
code.xor_(Xbyak::Reg32{value_idx}, Xbyak::Reg32{value_idx});
}
const void* fastmem_location = code.getCurr();
switch (bitsize) {
case 8:
code.lock();
code.xadd(code.byte[addr], Xbyak::Reg32{value_idx}.cvt8());
break;
case 16:
code.lock();
code.xadd(word[addr], Xbyak::Reg32{value_idx});
break;
case 32:
code.lock();
code.xadd(dword[addr], Xbyak::Reg32{value_idx});
break;
case 64:
code.lock();
code.xadd(qword[addr], Xbyak::Reg64{value_idx});
break;
case 128:
// TODO (HACK): Detect CPUs where this load is not atomic
code.movaps(Xbyak::Xmm{value_idx}, xword[addr]);
break;
default:
ASSERT_FALSE("Invalid bitsize");
}
return fastmem_location;
}
const void* fastmem_location = code.getCurr();
switch (bitsize) { switch (bitsize) {
case 8: case 8:
code.movzx(Xbyak::Reg32{value_idx}, code.byte[addr]); code.movzx(Xbyak::Reg32{value_idx}, code.byte[addr]);
return; break;
case 16: case 16:
code.movzx(Xbyak::Reg32{value_idx}, word[addr]); code.movzx(Xbyak::Reg32{value_idx}, word[addr]);
return; break;
case 32: case 32:
code.mov(Xbyak::Reg32{value_idx}, dword[addr]); code.mov(Xbyak::Reg32{value_idx}, dword[addr]);
return; break;
case 64: case 64:
code.mov(Xbyak::Reg64{value_idx}, qword[addr]); code.mov(Xbyak::Reg64{value_idx}, qword[addr]);
return; break;
case 128: case 128:
code.movups(Xbyak::Xmm{value_idx}, xword[addr]); code.movups(Xbyak::Xmm{value_idx}, xword[addr]);
return; break;
default: default:
ASSERT_FALSE("Invalid bitsize"); ASSERT_FALSE("Invalid bitsize");
} }
return fastmem_location;
} }
template<std::size_t bitsize> template<std::size_t bitsize>
void EmitWriteMemoryMov(BlockOfCode& code, const Xbyak::RegExp& addr, int value_idx) { const void* EmitWriteMemoryMov(BlockOfCode& code, const Xbyak::RegExp& addr, int value_idx, bool ordered) {
if (ordered) {
const void* fastmem_location = code.getCurr();
switch (bitsize) {
case 8:
code.xchg(code.byte[addr], Xbyak::Reg64{value_idx}.cvt8());
break;
case 16:
code.xchg(word[addr], Xbyak::Reg16{value_idx});
break;
case 32:
code.xchg(dword[addr], Xbyak::Reg32{value_idx});
break;
case 64:
code.xchg(qword[addr], Xbyak::Reg64{value_idx});
break;
case 128:
code.movaps(xword[addr], Xbyak::Xmm{value_idx});
code.mfence();
break;
default:
ASSERT_FALSE("Invalid bitsize");
}
return fastmem_location;
}
const void* fastmem_location = code.getCurr();
switch (bitsize) { switch (bitsize) {
case 8: case 8:
code.mov(code.byte[addr], Xbyak::Reg64{value_idx}.cvt8()); code.mov(code.byte[addr], Xbyak::Reg64{value_idx}.cvt8());
return; break;
case 16: case 16:
code.mov(word[addr], Xbyak::Reg16{value_idx}); code.mov(word[addr], Xbyak::Reg16{value_idx});
return; break;
case 32: case 32:
code.mov(dword[addr], Xbyak::Reg32{value_idx}); code.mov(dword[addr], Xbyak::Reg32{value_idx});
return; break;
case 64: case 64:
code.mov(qword[addr], Xbyak::Reg64{value_idx}); code.mov(qword[addr], Xbyak::Reg64{value_idx});
return; break;
case 128: case 128:
code.movups(xword[addr], Xbyak::Xmm{value_idx}); code.movups(xword[addr], Xbyak::Xmm{value_idx});
return; break;
default: default:
ASSERT_FALSE("Invalid bitsize"); ASSERT_FALSE("Invalid bitsize");
} }
return fastmem_location;
} }
template<typename UserConfig> template<typename UserConfig>
@ -284,6 +349,10 @@ void EmitExclusiveTestAndClear(BlockOfCode& code, const UserConfig& conf, Xbyak:
} }
} }
inline bool IsOrdered(IR::AccType acctype) {
return acctype == IR::AccType::ORDERED || acctype == IR::AccType::ORDEREDRW || acctype == IR::AccType::LIMITEDORDERED;
}
} // namespace } // namespace
} // namespace Dynarmic::Backend::X64 } // namespace Dynarmic::Backend::X64

5
externals/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp vendored
View file

@ -208,6 +208,11 @@ IR::Cond Argument::GetImmediateCond() const {
return value.GetCond(); return value.GetCond();
} }
IR::AccType Argument::GetImmediateAccType() const {
ASSERT(IsImmediate() && GetType() == IR::Type::AccType);
return value.GetAccType();
}
bool Argument::IsInGpr() const { bool Argument::IsInGpr() const {
if (IsImmediate()) if (IsImmediate())
return false; return false;

5
externals/dynarmic/src/dynarmic/backend/x64/reg_alloc.h vendored
View file

@ -21,6 +21,10 @@
#include "dynarmic/ir/microinstruction.h" #include "dynarmic/ir/microinstruction.h"
#include "dynarmic/ir/value.h" #include "dynarmic/ir/value.h"
namespace Dynarmic::IR {
enum class AccType;
} // namespace Dynarmic::IR
namespace Dynarmic::Backend::X64 { namespace Dynarmic::Backend::X64 {
class RegAlloc; class RegAlloc;
@ -75,6 +79,7 @@ public:
u64 GetImmediateS32() const; u64 GetImmediateS32() const;
u64 GetImmediateU64() const; u64 GetImmediateU64() const;
IR::Cond GetImmediateCond() const; IR::Cond GetImmediateCond() const;
IR::AccType GetImmediateAccType() const;
/// Is this value currently in a GPR? /// Is this value currently in a GPR?
bool IsInGpr() const; bool IsInGpr() const;