Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PPTC Profiles #370

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

PPTC Profiles #370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2af9a33
Add cacheselector and allow PPTC with exefs mods
LotP1 Dec 5, 2024
91518ac
Fix incorrect hash logic
LotP1 Dec 10, 2024
2874c40
Implement blacklist functionality
LotP1 Dec 10, 2024
2c53242
Remove outdated comment
LotP1 Dec 14, 2024
b560431
Add nuke PPTC option to cache management
LotP1 Dec 14, 2024
e0acefe
default value of dishCacheSelector should be null
LotP1 Dec 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 50 additions & 4 deletions src/ARMeilleure/Translation/PTC/Ptc.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using ARMeilleure.CodeGen.Unwinding;
using ARMeilleure.Common;
using ARMeilleure.Memory;
using ARMeilleure.State;
using Ryujinx.Common;
using Ryujinx.Common.Configuration;
using Ryujinx.Common.Logging;
Expand Down Expand Up @@ -30,7 +31,7 @@ class Ptc : IPtcLoadState
private const string OuterHeaderMagicString = "PTCohd\0\0";
private const string InnerHeaderMagicString = "PTCihd\0\0";

private const uint InternalVersion = 6997; //! To be incremented manually for each change to the ARMeilleure project.
private const uint InternalVersion = 7007; //! To be incremented manually for each change to the ARMeilleure project.

private const string ActualDir = "0";
private const string BackupDir = "1";
Expand Down Expand Up @@ -183,6 +184,36 @@ private void ResetCarriersIfNeeded()
InitializeCarriers();
}

private bool ContainsBlacklistedFunctions()
{
List<ulong> blacklist = Profiler.GetBlacklistedFunctions();
bool containsBlacklistedFunctions = false;
_infosStream.Seek(0L, SeekOrigin.Begin);
bool foundBadFunction = false;

for (int index = 0; index < GetEntriesCount(); index++)
{
InfoEntry infoEntry = DeserializeStructure<InfoEntry>(_infosStream);
foreach (ulong address in blacklist)
{
if (infoEntry.Address == address)
{
containsBlacklistedFunctions = true;
Logger.Warning?.Print(LogClass.Ptc, "PPTC cache invalidated: Found blacklisted functions in PPTC cache");
foundBadFunction = true;
break;
}
}

if (foundBadFunction)
{
break;
}
}

return containsBlacklistedFunctions;
}

private void PreLoad()
{
string fileNameActual = $"{CachePathActual}.cache";
Expand Down Expand Up @@ -531,7 +562,7 @@ private unsafe void Save(string fileName)

public void LoadTranslations(Translator translator)
{
if (AreCarriersEmpty())
if (AreCarriersEmpty() || ContainsBlacklistedFunctions())
{
return;
}
Expand Down Expand Up @@ -834,10 +865,18 @@ void TranslateFuncs()
while (profiledFuncsToTranslate.TryDequeue(out var item))
{
ulong address = item.address;
ExecutionMode executionMode = item.funcProfile.Mode;
bool highCq = item.funcProfile.HighCq;

Debug.Assert(Profiler.IsAddressInStaticCodeRange(address));

TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq);
TranslatedFunction func = translator.Translate(address, executionMode, highCq);

if (func == null)
{
Profiler.UpdateEntry(address, executionMode, true, true);
continue;
}

bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func);

Expand Down Expand Up @@ -884,7 +923,14 @@ void TranslateFuncs()

PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount);

Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
if (_translateCount == _translateTotalCount)
{
Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
}
else
{
Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | {_translateTotalCount - _translateCount} function{(_translateTotalCount - _translateCount != 1 ? "s" : "")} blacklisted | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
}

Thread preSaveThread = new(PreSave)
{
Expand Down
86 changes: 76 additions & 10 deletions src/ARMeilleure/Translation/PTC/PtcProfiler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,11 @@ class PtcProfiler
{
private const string OuterHeaderMagicString = "Pohd\0\0\0\0";

private const uint InternalVersion = 5518; //! Not to be incremented manually for each change to the ARMeilleure project.
private const uint InternalVersion = 7007; //! Not to be incremented manually for each change to the ARMeilleure project.

private static readonly uint[] _migrateInternalVersions = {
1866,
5518,
};

private const int SaveInterval = 30; // Seconds.
Expand Down Expand Up @@ -72,20 +73,30 @@ public PtcProfiler(Ptc ptc)
Enabled = false;
}

public void AddEntry(ulong address, ExecutionMode mode, bool highCq)
public void AddEntry(ulong address, ExecutionMode mode, bool highCq, bool blacklist = false)
{
if (IsAddressInStaticCodeRange(address))
{
Debug.Assert(!highCq);

lock (_lock)
if (blacklist)
{
ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false));
lock (_lock)
{
ProfiledFuncs[address] = new FuncProfile(mode, highCq: false, true);
}
}
else
{
lock (_lock)
{
ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false, false));
}
}
}
}

public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq)
public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq, bool? blacklist = null)
{
if (IsAddressInStaticCodeRange(address))
{
Expand All @@ -95,7 +106,7 @@ public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq)
{
Debug.Assert(ProfiledFuncs.ContainsKey(address));

ProfiledFuncs[address] = new FuncProfile(mode, highCq: true);
ProfiledFuncs[address] = new FuncProfile(mode, highCq: true, blacklist ?? ProfiledFuncs[address].Blacklist);
}
}
}
Expand All @@ -111,7 +122,7 @@ public bool IsAddressInStaticCodeRange(ulong address)

foreach (var profiledFunc in ProfiledFuncs)
{
if (!funcs.ContainsKey(profiledFunc.Key))
if (!funcs.ContainsKey(profiledFunc.Key) && !profiledFunc.Value.Blacklist)
{
profiledFuncsToTranslate.Enqueue((profiledFunc.Key, profiledFunc.Value));
}
Expand All @@ -126,6 +137,24 @@ public void ClearEntries()
ProfiledFuncs.TrimExcess();
}

public List<ulong> GetBlacklistedFunctions()
{
List<ulong> funcs = new List<ulong>();

foreach (var profiledFunc in ProfiledFuncs)
{
if (profiledFunc.Value.Blacklist)
{
if (!funcs.Contains(profiledFunc.Key))
{
funcs.Add(profiledFunc.Key);
}
}
}

return funcs;
}

public void PreLoad()
{
_lastHash = default;
Expand Down Expand Up @@ -216,13 +245,18 @@ private bool Load(string fileName, bool isBackup)
return false;
}

Func<ulong, FuncProfile, (ulong, FuncProfile)> migrateEntryFunc = null;

switch (outerHeader.InfoFileVersion)
{
case InternalVersion:
ProfiledFuncs = Deserialize(stream);
break;
case 1866:
ProfiledFuncs = Deserialize(stream, (address, profile) => (address + 0x500000UL, profile));
migrateEntryFunc = (address, profile) => (address + 0x500000UL, profile);
goto case 5518;
case 5518:
ProfiledFuncs = DeserializeAddBlacklist(stream, migrateEntryFunc);
break;
default:
Logger.Error?.Print(LogClass.Ptc, $"No migration path for {nameof(outerHeader.InfoFileVersion)} '{outerHeader.InfoFileVersion}'. Discarding cache.");
Expand Down Expand Up @@ -252,6 +286,16 @@ private static Dictionary<ulong, FuncProfile> Deserialize(Stream stream, Func<ul
return DeserializeDictionary<ulong, FuncProfile>(stream, DeserializeStructure<FuncProfile>);
}

private static Dictionary<ulong, FuncProfile> DeserializeAddBlacklist(Stream stream, Func<ulong, FuncProfile, (ulong, FuncProfile)> migrateEntryFunc = null)
{
if (migrateEntryFunc != null)
{
return DeserializeAndUpdateDictionary(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure<FuncProfilePreBlacklist>(stream)); }, migrateEntryFunc);
}

return DeserializeDictionary<ulong, FuncProfile>(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure<FuncProfilePreBlacklist>(stream)); });
}

private static ReadOnlySpan<byte> GetReadOnlySpan(MemoryStream memoryStream)
{
return new(memoryStream.GetBuffer(), (int)memoryStream.Position, (int)memoryStream.Length - (int)memoryStream.Position);
Expand Down Expand Up @@ -383,13 +427,35 @@ public bool IsHeaderValid()
}
}

[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)]
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 6*/)]
public struct FuncProfile
{
public ExecutionMode Mode;
public bool HighCq;
public bool Blacklist;

public FuncProfile(ExecutionMode mode, bool highCq, bool blacklist)
{
Mode = mode;
HighCq = highCq;
Blacklist = blacklist;
}

public FuncProfile(FuncProfilePreBlacklist fp)
{
Mode = fp.Mode;
HighCq = fp.HighCq;
Blacklist = false;
}
}

[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)]
public struct FuncProfilePreBlacklist
{
public ExecutionMode Mode;
public bool HighCq;

public FuncProfile(ExecutionMode mode, bool highCq)
public FuncProfilePreBlacklist(ExecutionMode mode, bool highCq)
{
Mode = mode;
HighCq = highCq;
Expand Down
10 changes: 10 additions & 0 deletions src/ARMeilleure/Translation/Translator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,11 @@ internal TranslatedFunction Translate(ulong address, ExecutionMode mode, bool hi

ControlFlowGraph cfg = EmitAndGetCFG(context, blocks, out Range funcRange, out Counter<uint> counter);

if (cfg == null)
{
return null;
}

ulong funcSize = funcRange.End - funcRange.Start;

Logger.EndPass(PassName.Translation, cfg);
Expand Down Expand Up @@ -407,6 +412,11 @@ private static ControlFlowGraph EmitAndGetCFG(
if (opCode.Instruction.Emitter != null)
{
opCode.Instruction.Emitter(context);
if (opCode.Instruction.Name == InstName.Und && blkIndex == 0)
{
range = new Range(rangeStart, rangeEnd);
return null;
}
}
else
{
Expand Down
5 changes: 4 additions & 1 deletion src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class ArmProcessContextFactory : IProcessContextFactory
private readonly string _titleIdText;
private readonly string _displayVersion;
private readonly bool _diskCacheEnabled;
private readonly string _diskCacheSelector;
private readonly ulong _codeAddress;
private readonly ulong _codeSize;

Expand All @@ -31,6 +32,7 @@ public ArmProcessContextFactory(
string titleIdText,
string displayVersion,
bool diskCacheEnabled,
string diskCacheSelector,
ulong codeAddress,
ulong codeSize)
{
Expand All @@ -39,6 +41,7 @@ public ArmProcessContextFactory(
_titleIdText = titleIdText;
_displayVersion = displayVersion;
_diskCacheEnabled = diskCacheEnabled;
_diskCacheSelector = diskCacheSelector;
_codeAddress = codeAddress;
_codeSize = codeSize;
}
Expand Down Expand Up @@ -114,7 +117,7 @@ public IProcessContext Create(KernelContext context, ulong pid, ulong addressSpa
}
}

DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, "default"); //Ready for exefs profiles
DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector ?? "default");

return processContext;
}
Expand Down
26 changes: 24 additions & 2 deletions src/Ryujinx.HLE/HOS/ModLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using LibHac.Loader;
using LibHac.Tools.FsSystem;
using LibHac.Tools.FsSystem.RomFs;
using LibHac.Util;
using Ryujinx.Common.Configuration;
using Ryujinx.Common.Logging;
using Ryujinx.Common.Utilities;
Expand All @@ -18,6 +19,7 @@
using System.Globalization;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using LazyFile = Ryujinx.HLE.HOS.Services.Fs.FileSystemProxy.LazyFile;
using Path = System.IO.Path;

Expand Down Expand Up @@ -583,6 +585,7 @@ public struct ModLoadResult
public BitVector32 Stubs;
public BitVector32 Replaces;
public MetaLoader Npdm;
public string Hash;

public bool Modified => (Stubs.Data | Replaces.Data) != 0;
}
Expand All @@ -593,8 +596,11 @@ internal ModLoadResult ApplyExefsMods(ulong applicationId, NsoExecutable[] nsos)
{
Stubs = new BitVector32(),
Replaces = new BitVector32(),
Hash = null,
};

string tempHash = string.Empty;

if (!_appMods.TryGetValue(applicationId, out ModCache mods) || mods.ExefsDirs.Count == 0)
{
return modLoadResult;
Expand Down Expand Up @@ -630,8 +636,16 @@ internal ModLoadResult ApplyExefsMods(ulong applicationId, NsoExecutable[] nsos)

modLoadResult.Replaces[1 << i] = true;

nsos[i] = new NsoExecutable(nsoFile.OpenRead().AsStorage(), nsoName);
Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced");
using (FileStream stream = nsoFile.OpenRead())
{
nsos[i] = new NsoExecutable(stream.AsStorage(), nsoName);
Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced");
using (MD5 md5 = MD5.Create())
{
stream.Seek(0, SeekOrigin.Begin);
tempHash += BitConverter.ToString(md5.ComputeHash(stream)).Replace("-", "").ToLowerInvariant();
}
}
}

modLoadResult.Stubs[1 << i] |= File.Exists(Path.Combine(mod.Path.FullName, nsoName + StubExtension));
Expand Down Expand Up @@ -663,6 +677,14 @@ internal ModLoadResult ApplyExefsMods(ulong applicationId, NsoExecutable[] nsos)
}
}

if (!string.IsNullOrEmpty(tempHash))
{
using (MD5 md5 = MD5.Create())
{
modLoadResult.Hash += BitConverter.ToString(md5.ComputeHash(tempHash.ToBytes())).Replace("-", "").ToLowerInvariant();
}
}

return modLoadResult;
}

Expand Down
Loading
Loading