drashna/ryujinx
1
0
Fork 0
Code Activity Actions

Implement inline memory load/store exclusive and ordered (#1413)

Browse source 
* Implement inline memory load/store exclusive

* Fix missing REX prefix on 8-bits CMPXCHG

* Increment PTC version due to bugfix

* Remove redundant memory checks

* Address PR feedback

* Increment PPTC version
This commit is contained in:
gdkchan 2020-07-30 11:29:28 -03:00 • committed by GitHub
parent 57bb0abda3
commit 9878fc2d3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 385 additions and 376 deletions
Download patch file Download diff file Expand all files Collapse all files

187
ARMeilleure/Instructions/InstEmitMemoryExHelper.cs
View file

@ -1,87 +1,180 @@
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.State;
using ARMeilleure.Translation;
using System.Reflection;
using static ARMeilleure.Instructions.InstEmitHelper;
using static ARMeilleure.IntermediateRepresentation.OperandHelper;
namespace ARMeilleure.Instructions
{
static class InstEmitMemoryExHelper
{
public static Operand EmitLoadExclusive(
ArmEmitterContext context,
Operand address,
bool exclusive,
int size)
{
MethodInfo info = null;
private const int ErgSizeLog2 = 4;
public static Operand EmitLoadExclusive(ArmEmitterContext context, Operand address, bool exclusive, int size)
{
if (exclusive)
{
switch (size)
Operand value;
if (size == 4)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByteExclusive)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16Exclusive)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32Exclusive)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64Exclusive)); break;
case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128Exclusive)); break;
Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
context.MarkLabel(lblFastPath);
// Only 128-bit CAS is guaranteed to have a atomic load.
Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false);
Operand zero = context.VectorZero();
value = context.CompareAndSwap(physAddr, zero, zero);
}
else
{
value = InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
}
Operand arg0 = context.LoadArgument(OperandType.I64, 0);
Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
context.Store(exAddrPtr, context.BitwiseAnd(address, Const(address.Type, GetExclusiveAddressMask())));
context.Store(exValuePtr, value);
return value;
}
else
{
switch (size)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByte)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64)); break;
case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128)); break;
}
return InstEmitMemoryHelper.EmitReadIntAligned(context, address, size);
}
return context.Call(info, address);
}
public static Operand EmitStoreExclusive(
public static void EmitStoreExclusive(
ArmEmitterContext context,
Operand address,
Operand value,
bool exclusive,
int size)
int size,
int rs,
bool a32)
{
if (size < 3)
{
value = context.ConvertI64ToI32(value);
}
MethodInfo info = null;
if (exclusive)
{
switch (size)
void SetRs(Operand value)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByteExclusive)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16Exclusive)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32Exclusive)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64Exclusive)); break;
case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128Exclusive)); break;
if (a32)
{
SetIntA32(context, rs, value);
}
else
{
SetIntOrZR(context, rs, value);
}
}
return context.Call(info, address, value);
Operand arg0 = context.LoadArgument(OperandType.I64, 0);
Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
Operand exAddr = context.Load(address.Type, exAddrPtr);
// STEP 1: Check if we have exclusive access to this memory region. If not, fail and skip store.
Operand maskedAddress = context.BitwiseAnd(address, Const(GetExclusiveAddressMask()));
Operand exFailed = context.ICompareNotEqual(exAddr, maskedAddress);
Operand lblExit = Label();
SetRs(exFailed);
context.BranchIfTrue(lblExit, exFailed);
// STEP 2: We have exclusive access, make sure that the address is valid.
Operand isUnalignedAddr = InstEmitMemoryHelper.EmitAddressCheck(context, address, size);
Operand lblFastPath = Label();
context.BranchIfFalse(lblFastPath, isUnalignedAddr);
// The call is not expected to return (it should throw).
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ThrowInvalidMemoryAccess)), address);
// STEP 3: We have exclusive access and the address is valid, attempt the store using CAS.
context.MarkLabel(lblFastPath);
Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true);
Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
Operand exValue = size switch
{
0 => context.Load8(exValuePtr),
1 => context.Load16(exValuePtr),
2 => context.Load(OperandType.I32, exValuePtr),
3 => context.Load(OperandType.I64, exValuePtr),
_ => context.Load(OperandType.V128, exValuePtr)
};
Operand currValue = size switch
{
0 => context.CompareAndSwap8(physAddr, exValue, value),
1 => context.CompareAndSwap16(physAddr, exValue, value),
_ => context.CompareAndSwap(physAddr, exValue, value)
};
// STEP 4: Check if we succeeded by comparing expected and in-memory values.
Operand storeFailed;
if (size == 4)
{
Operand currValueLow = context.VectorExtract(OperandType.I64, currValue, 0);
Operand currValueHigh = context.VectorExtract(OperandType.I64, currValue, 1);
Operand exValueLow = context.VectorExtract(OperandType.I64, exValue, 0);
Operand exValueHigh = context.VectorExtract(OperandType.I64, exValue, 1);
storeFailed = context.BitwiseOr(
context.ICompareNotEqual(currValueLow, exValueLow),
context.ICompareNotEqual(currValueHigh, exValueHigh));
}
else
{
storeFailed = context.ICompareNotEqual(currValue, exValue);
}
SetRs(storeFailed);
context.MarkLabel(lblExit);
}
else
{
switch (size)
{
case 0: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte)); break;
case 1: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16)); break;
case 2: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32)); break;
case 3: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64)); break;
case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128)); break;
}
context.Call(info, address, value);
return null;
InstEmitMemoryHelper.EmitWriteIntAligned(context, address, value, size);
}
}
public static void EmitClearExclusive(ArmEmitterContext context)
{
Operand arg0 = context.LoadArgument(OperandType.I64, 0);
Operand exAddrPtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveAddressOffset()));
// We store ULONG max to force any exclusive address checks to fail,
// since this value is not aligned to the ERG mask.
context.Store(exAddrPtr, Const(ulong.MaxValue));
}
private static long GetExclusiveAddressMask() => ~((4L << ErgSizeLog2) - 1);
}
}