kernel: Implement thread pinning support (#2840)
* kernel: Implement Thread pinning support This commit adds support for 8.x thread pinning changes and implement SynchronizePreemptionState syscall. Based on kernel 13.x reverse. * Address gdkchan's comment * kernel: fix missing critical section leave in SetActivity Fix Unity games * Implement missing bits on the interrupt handler and inline update pinning function as it cannot be generic * Fix some bugs in SetActivity and SetCoreAndAffinityMask * Address gdkchan's comments
This commit is contained in:
Mary
GitHub
parent
8544b1445b
commit
e96ef6d532
9 changed files with 436 additions and 95 deletions
|
|
@ -11,6 +11,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
{
|
||||
class KThread : KSynchronizationObject, IKFutureSchedulerObject
|
||||
{
|
||||
private const int TlsUserDisableCountOffset = 0x100;
|
||||
private const int TlsUserInterruptFlagOffset = 0x102;
|
||||
|
||||
public const int MaxWaitSyncObjects = 64;
|
||||
|
||||
private ManualResetEvent _schedulerWaitEvent;
|
||||
|
|
@ -43,6 +46,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
public bool IsSchedulable => _customThreadStart == null && !_forcedUnschedulable;
|
||||
|
||||
public ulong MutexAddress { get; set; }
|
||||
public int KernelWaitersCount { get; private set; }
|
||||
|
||||
public KProcess Owner { get; private set; }
|
||||
|
||||
|
|
@ -65,11 +69,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
private LinkedList<KThread> _mutexWaiters;
|
||||
private LinkedListNode<KThread> _mutexWaiterNode;
|
||||
|
||||
private LinkedList<KThread> _pinnedWaiters;
|
||||
|
||||
public KThread MutexOwner { get; private set; }
|
||||
|
||||
public int ThreadHandleForUserMutex { get; set; }
|
||||
|
||||
private ThreadSchedState _forcePauseFlags;
|
||||
private ThreadSchedState _forcePausePermissionFlags;
|
||||
|
||||
public KernelResult ObjSyncResult { get; set; }
|
||||
|
||||
|
|
@ -79,11 +86,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
public int CurrentCore { get; set; }
|
||||
public int ActiveCore { get; set; }
|
||||
|
||||
private long _affinityMaskOverride;
|
||||
private int _preferredCoreOverride;
|
||||
#pragma warning disable CS0649
|
||||
private int _affinityOverrideCount;
|
||||
#pragma warning restore CS0649
|
||||
public bool IsPinned { get; private set; }
|
||||
|
||||
private long _originalAffinityMask;
|
||||
private int _originalPreferredCore;
|
||||
private int _originalBasePriority;
|
||||
private int _coreMigrationDisableCount;
|
||||
|
||||
public ThreadSchedState SchedFlags { get; private set; }
|
||||
|
||||
|
|
@ -108,6 +116,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
|
||||
public long LastPc { get; set; }
|
||||
|
||||
private object ActivityOperationLock = new object();
|
||||
|
||||
public KThread(KernelContext context) : base(context)
|
||||
{
|
||||
WaitSyncObjects = new KSynchronizationObject[MaxWaitSyncObjects];
|
||||
|
|
@ -116,6 +126,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
|
||||
|
||||
_mutexWaiters = new LinkedList<KThread>();
|
||||
_pinnedWaiters = new LinkedList<KThread>();
|
||||
}
|
||||
|
||||
public KernelResult Initialize(
|
||||
|
|
@ -147,6 +158,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
DynamicPriority = priority;
|
||||
BasePriority = priority;
|
||||
CurrentCore = cpuCore;
|
||||
IsPinned = false;
|
||||
|
||||
_entrypoint = entrypoint;
|
||||
_customThreadStart = customThreadStart;
|
||||
|
|
@ -204,6 +216,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
|
||||
_hasBeenInitialized = true;
|
||||
|
||||
_forcePausePermissionFlags = ThreadSchedState.ForcePauseMask;
|
||||
|
||||
if (owner != null)
|
||||
{
|
||||
owner.SubscribeThreadEventHandlers(Context);
|
||||
|
|
@ -301,6 +315,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
{
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
if (Owner != null && Owner.PinnedThreads[KernelStatic.GetCurrentThread().CurrentCore] == this)
|
||||
{
|
||||
Owner.UnpinThread(this);
|
||||
}
|
||||
|
||||
ThreadSchedState result;
|
||||
|
||||
if (Interlocked.CompareExchange(ref _shallBeTerminated, 1, 0) == 0)
|
||||
|
|
@ -405,6 +424,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
_forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
|
||||
_forcePausePermissionFlags = 0;
|
||||
|
||||
bool decRef = ExitImpl();
|
||||
|
||||
|
|
@ -433,6 +453,19 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
return decRef;
|
||||
}
|
||||
|
||||
private int GetEffectiveRunningCore()
|
||||
{
|
||||
for (int coreNumber = 0; coreNumber < KScheduler.CpuCoresCount; coreNumber++)
|
||||
{
|
||||
if (KernelContext.Schedulers[coreNumber].CurrentThread == this)
|
||||
{
|
||||
return coreNumber;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
public KernelResult Sleep(long timeout)
|
||||
{
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
|
@ -465,7 +498,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
{
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
BasePriority = priority;
|
||||
if (IsPinned)
|
||||
{
|
||||
_originalBasePriority = priority;
|
||||
}
|
||||
else
|
||||
{
|
||||
BasePriority = priority;
|
||||
}
|
||||
|
||||
UpdatePriorityInheritance();
|
||||
|
||||
|
|
@ -497,53 +537,96 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
|
||||
public KernelResult SetActivity(bool pause)
|
||||
{
|
||||
KernelResult result = KernelResult.Success;
|
||||
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
|
||||
|
||||
if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
|
||||
lock (ActivityOperationLock)
|
||||
{
|
||||
KernelResult result = KernelResult.Success;
|
||||
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
|
||||
|
||||
if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
|
||||
{
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
return KernelResult.InvalidState;
|
||||
}
|
||||
|
||||
if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
|
||||
{
|
||||
if (pause)
|
||||
{
|
||||
// Pause, the force pause flag should be clear (thread is NOT paused).
|
||||
if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
|
||||
{
|
||||
Suspend(ThreadSchedState.ThreadPauseFlag);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = KernelResult.InvalidState;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Unpause, the force pause flag should be set (thread is paused).
|
||||
if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
|
||||
{
|
||||
Resume(ThreadSchedState.ThreadPauseFlag);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = KernelResult.InvalidState;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
return KernelResult.InvalidState;
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
|
||||
{
|
||||
if (pause)
|
||||
if (result == KernelResult.Success && pause)
|
||||
{
|
||||
// Pause, the force pause flag should be clear (thread is NOT paused).
|
||||
if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
|
||||
bool isThreadRunning = true;
|
||||
|
||||
while (isThreadRunning)
|
||||
{
|
||||
Suspend(ThreadSchedState.ThreadPauseFlag);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = KernelResult.InvalidState;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Unpause, the force pause flag should be set (thread is paused).
|
||||
if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
|
||||
{
|
||||
Resume(ThreadSchedState.ThreadPauseFlag);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = KernelResult.InvalidState;
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
if (TerminationRequested)
|
||||
{
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
isThreadRunning = false;
|
||||
|
||||
if (IsPinned)
|
||||
{
|
||||
KThread currentThread = KernelStatic.GetCurrentThread();
|
||||
|
||||
if (currentThread.TerminationRequested)
|
||||
{
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
result = KernelResult.ThreadTerminating;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
_pinnedWaiters.AddLast(currentThread);
|
||||
|
||||
currentThread.Reschedule(ThreadSchedState.Paused);
|
||||
}
|
||||
else
|
||||
{
|
||||
isThreadRunning = GetEffectiveRunningCore() >= 0;
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Leave();
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Leave();
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public void CancelSynchronization()
|
||||
|
|
@ -579,58 +662,105 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
|
||||
public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
|
||||
{
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
bool useOverride = _affinityOverrideCount != 0;
|
||||
|
||||
// The value -3 is "do not change the preferred core".
|
||||
if (newCore == -3)
|
||||
lock (ActivityOperationLock)
|
||||
{
|
||||
newCore = useOverride ? _preferredCoreOverride : PreferredCore;
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
if ((newAffinityMask & (1 << newCore)) == 0)
|
||||
bool isCoreMigrationDisabled = _coreMigrationDisableCount != 0;
|
||||
|
||||
// The value -3 is "do not change the preferred core".
|
||||
if (newCore == -3)
|
||||
{
|
||||
KernelContext.CriticalSection.Leave();
|
||||
newCore = isCoreMigrationDisabled ? _originalPreferredCore : PreferredCore;
|
||||
|
||||
return KernelResult.InvalidCombination;
|
||||
}
|
||||
}
|
||||
|
||||
if (useOverride)
|
||||
{
|
||||
_preferredCoreOverride = newCore;
|
||||
_affinityMaskOverride = newAffinityMask;
|
||||
}
|
||||
else
|
||||
{
|
||||
long oldAffinityMask = AffinityMask;
|
||||
|
||||
PreferredCore = newCore;
|
||||
AffinityMask = newAffinityMask;
|
||||
|
||||
if (oldAffinityMask != newAffinityMask)
|
||||
{
|
||||
int oldCore = ActiveCore;
|
||||
|
||||
if (oldCore >= 0 && ((AffinityMask >> oldCore) & 1) == 0)
|
||||
if ((newAffinityMask & (1 << newCore)) == 0)
|
||||
{
|
||||
if (PreferredCore < 0)
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
return KernelResult.InvalidCombination;
|
||||
}
|
||||
}
|
||||
|
||||
if (isCoreMigrationDisabled)
|
||||
{
|
||||
_originalPreferredCore = newCore;
|
||||
_originalAffinityMask = newAffinityMask;
|
||||
}
|
||||
else
|
||||
{
|
||||
long oldAffinityMask = AffinityMask;
|
||||
|
||||
PreferredCore = newCore;
|
||||
AffinityMask = newAffinityMask;
|
||||
|
||||
if (oldAffinityMask != newAffinityMask)
|
||||
{
|
||||
int oldCore = ActiveCore;
|
||||
|
||||
if (oldCore >= 0 && ((AffinityMask >> oldCore) & 1) == 0)
|
||||
{
|
||||
ActiveCore = sizeof(ulong) * 8 - 1 - BitOperations.LeadingZeroCount((ulong)AffinityMask);
|
||||
if (PreferredCore < 0)
|
||||
{
|
||||
ActiveCore = sizeof(ulong) * 8 - 1 - BitOperations.LeadingZeroCount((ulong)AffinityMask);
|
||||
}
|
||||
else
|
||||
{
|
||||
ActiveCore = PreferredCore;
|
||||
}
|
||||
}
|
||||
|
||||
AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
|
||||
}
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
bool targetThreadPinned = true;
|
||||
|
||||
while (targetThreadPinned)
|
||||
{
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
if (TerminationRequested)
|
||||
{
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
targetThreadPinned = false;
|
||||
|
||||
int coreNumber = GetEffectiveRunningCore();
|
||||
bool isPinnedThreadCurrentlyRunning = coreNumber >= 0;
|
||||
|
||||
if (isPinnedThreadCurrentlyRunning && ((1 << coreNumber) & AffinityMask) == 0)
|
||||
{
|
||||
if (IsPinned)
|
||||
{
|
||||
KThread currentThread = KernelStatic.GetCurrentThread();
|
||||
|
||||
if (currentThread.TerminationRequested)
|
||||
{
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
return KernelResult.ThreadTerminating;
|
||||
}
|
||||
|
||||
_pinnedWaiters.AddLast(currentThread);
|
||||
|
||||
currentThread.Reschedule(ThreadSchedState.Paused);
|
||||
}
|
||||
else
|
||||
{
|
||||
ActiveCore = PreferredCore;
|
||||
targetThreadPinned = true;
|
||||
}
|
||||
}
|
||||
|
||||
AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
|
||||
KernelContext.CriticalSection.Leave();
|
||||
}
|
||||
|
||||
return KernelResult.Success;
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Leave();
|
||||
|
||||
return KernelResult.Success;
|
||||
}
|
||||
|
||||
private void CombineForcePauseFlags()
|
||||
|
|
@ -638,7 +768,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
ThreadSchedState oldFlags = SchedFlags;
|
||||
ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
|
||||
|
||||
SchedFlags = lowNibble | _forcePauseFlags;
|
||||
SchedFlags = lowNibble | (_forcePauseFlags & _forcePausePermissionFlags);
|
||||
|
||||
AdjustScheduling(oldFlags);
|
||||
}
|
||||
|
|
@ -1106,7 +1236,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
foreach (KThread thread in _mutexWaiters)
|
||||
{
|
||||
thread.MutexOwner = null;
|
||||
thread._preferredCoreOverride = 0;
|
||||
thread._originalPreferredCore = 0;
|
||||
thread.ObjSyncResult = KernelResult.InvalidState;
|
||||
|
||||
thread.ReleaseAndResume();
|
||||
|
|
@ -1116,5 +1246,113 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
|
|||
|
||||
Owner?.DecrementThreadCountAndTerminateIfZero();
|
||||
}
|
||||
|
||||
public void Pin()
|
||||
{
|
||||
IsPinned = true;
|
||||
_coreMigrationDisableCount++;
|
||||
|
||||
int activeCore = ActiveCore;
|
||||
|
||||
_originalPreferredCore = PreferredCore;
|
||||
_originalAffinityMask = AffinityMask;
|
||||
|
||||
ActiveCore = CurrentCore;
|
||||
PreferredCore = CurrentCore;
|
||||
AffinityMask = 1 << CurrentCore;
|
||||
|
||||
if (activeCore != CurrentCore || _originalAffinityMask != AffinityMask)
|
||||
{
|
||||
AdjustSchedulingForNewAffinity(_originalAffinityMask, activeCore);
|
||||
}
|
||||
|
||||
_originalBasePriority = BasePriority;
|
||||
BasePriority = Math.Min(_originalBasePriority, BitOperations.TrailingZeroCount(Owner.Capabilities.AllowedThreadPriosMask) - 1);
|
||||
UpdatePriorityInheritance();
|
||||
|
||||
// Disallows thread pausing
|
||||
_forcePausePermissionFlags &= ~ThreadSchedState.ThreadPauseFlag;
|
||||
CombineForcePauseFlags();
|
||||
|
||||
// TODO: Assign reduced SVC permissions
|
||||
}
|
||||
|
||||
public void Unpin()
|
||||
{
|
||||
IsPinned = false;
|
||||
_coreMigrationDisableCount--;
|
||||
|
||||
long affinityMask = AffinityMask;
|
||||
int activeCore = ActiveCore;
|
||||
|
||||
PreferredCore = _originalPreferredCore;
|
||||
AffinityMask = _originalAffinityMask;
|
||||
|
||||
if (AffinityMask != affinityMask)
|
||||
{
|
||||
if ((AffinityMask & 1 << ActiveCore) != 0)
|
||||
{
|
||||
if (PreferredCore >= 0)
|
||||
{
|
||||
ActiveCore = PreferredCore;
|
||||
}
|
||||
else
|
||||
{
|
||||
ActiveCore = sizeof(ulong) * 8 - 1 - BitOperations.LeadingZeroCount((ulong)AffinityMask);
|
||||
}
|
||||
|
||||
AdjustSchedulingForNewAffinity(affinityMask, activeCore);
|
||||
}
|
||||
}
|
||||
|
||||
BasePriority = _originalBasePriority;
|
||||
UpdatePriorityInheritance();
|
||||
|
||||
if (!TerminationRequested)
|
||||
{
|
||||
// Allows thread pausing
|
||||
_forcePausePermissionFlags |= ThreadSchedState.ThreadPauseFlag;
|
||||
CombineForcePauseFlags();
|
||||
|
||||
// TODO: Restore SVC permissions
|
||||
}
|
||||
|
||||
// Wake up waiters
|
||||
foreach (KThread waiter in _pinnedWaiters)
|
||||
{
|
||||
waiter.ReleaseAndResume();
|
||||
}
|
||||
|
||||
_pinnedWaiters.Clear();
|
||||
}
|
||||
|
||||
public void SynchronizePreemptionState()
|
||||
{
|
||||
KernelContext.CriticalSection.Enter();
|
||||
|
||||
if (Owner != null && Owner.PinnedThreads[CurrentCore] == this)
|
||||
{
|
||||
ClearUserInterruptFlag();
|
||||
|
||||
Owner.UnpinThread(this);
|
||||
}
|
||||
|
||||
KernelContext.CriticalSection.Leave();
|
||||
}
|
||||
|
||||
public ushort GetUserDisableCount()
|
||||
{
|
||||
return Owner.CpuMemory.Read<ushort>(_tlsAddress + TlsUserDisableCountOffset);
|
||||
}
|
||||
|
||||
public void SetUserInterruptFlag()
|
||||
{
|
||||
Owner.CpuMemory.Write<ushort>(_tlsAddress + TlsUserInterruptFlagOffset, 1);
|
||||
}
|
||||
|
||||
public void ClearUserInterruptFlag()
|
||||
{
|
||||
Owner.CpuMemory.Write<ushort>(_tlsAddress + TlsUserInterruptFlagOffset, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue