From 8c919085e8389f07c32e4ba63a9ef361b62bfad1 Mon Sep 17 00:00:00 2001 From: LotP1 Date: Thu, 5 Dec 2024 19:20:06 +0100 Subject: [PATCH 1/3] Add cacheselector and allow PPTC with exefs mods this is currently broken with Exlaunch mods that use hooks --- .../HOS/ArmProcessContextFactory.cs | 5 +++- src/Ryujinx.HLE/HOS/ModLoader.cs | 25 +++++++++++++++++-- .../Extensions/FileSystemExtensions.cs | 10 ++------ .../Loaders/Processes/ProcessLoader.cs | 1 + .../Loaders/Processes/ProcessLoaderHelper.cs | 3 +++ 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs index 14775fb1d..d03873ffe 100644 --- a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs +++ b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs @@ -20,6 +20,7 @@ namespace Ryujinx.HLE.HOS private readonly string _titleIdText; private readonly string _displayVersion; private readonly bool _diskCacheEnabled; + private readonly string _diskCacheSelector; private readonly ulong _codeAddress; private readonly ulong _codeSize; @@ -31,6 +32,7 @@ namespace Ryujinx.HLE.HOS string titleIdText, string displayVersion, bool diskCacheEnabled, + string diskCacheSelector, ulong codeAddress, ulong codeSize) { @@ -39,6 +41,7 @@ namespace Ryujinx.HLE.HOS _titleIdText = titleIdText; _displayVersion = displayVersion; _diskCacheEnabled = diskCacheEnabled; + _diskCacheSelector = diskCacheSelector; _codeAddress = codeAddress; _codeSize = codeSize; } @@ -114,7 +117,7 @@ namespace Ryujinx.HLE.HOS } } - DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, "default"); //Ready for exefs profiles + DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector ?? "default"); //Ready for exefs profiles return processContext; } diff --git a/src/Ryujinx.HLE/HOS/ModLoader.cs b/src/Ryujinx.HLE/HOS/ModLoader.cs index 7cbe1afca..c866fb063 100644 --- a/src/Ryujinx.HLE/HOS/ModLoader.cs +++ b/src/Ryujinx.HLE/HOS/ModLoader.cs @@ -5,6 +5,7 @@ using LibHac.FsSystem; using LibHac.Loader; using LibHac.Tools.FsSystem; using LibHac.Tools.FsSystem.RomFs; +using LibHac.Util; using Ryujinx.Common.Configuration; using Ryujinx.Common.Logging; using Ryujinx.Common.Utilities; @@ -18,6 +19,7 @@ using System.Collections.Specialized; using System.Globalization; using System.IO; using System.Linq; +using System.Security.Cryptography; using LazyFile = Ryujinx.HLE.HOS.Services.Fs.FileSystemProxy.LazyFile; using Path = System.IO.Path; @@ -583,6 +585,7 @@ namespace Ryujinx.HLE.HOS public BitVector32 Stubs; public BitVector32 Replaces; public MetaLoader Npdm; + public string Hash; public bool Modified => (Stubs.Data | Replaces.Data) != 0; } @@ -593,8 +596,11 @@ namespace Ryujinx.HLE.HOS { Stubs = new BitVector32(), Replaces = new BitVector32(), + Hash = null, }; + string tempHash = string.Empty; + if (!_appMods.TryGetValue(applicationId, out ModCache mods) || mods.ExefsDirs.Count == 0) { return modLoadResult; @@ -630,8 +636,15 @@ namespace Ryujinx.HLE.HOS modLoadResult.Replaces[1 << i] = true; - nsos[i] = new NsoExecutable(nsoFile.OpenRead().AsStorage(), nsoName); - Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced"); + using (FileStream stream = nsoFile.OpenRead()) + { + nsos[i] = new NsoExecutable(stream.AsStorage(), nsoName); + Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced"); + using (MD5 md5 = MD5.Create()) + { + tempHash += BitConverter.ToString(md5.ComputeHash(stream)).Replace("-", "").ToLowerInvariant(); + } + } } modLoadResult.Stubs[1 << i] |= File.Exists(Path.Combine(mod.Path.FullName, nsoName + StubExtension)); @@ -663,6 +676,14 @@ namespace Ryujinx.HLE.HOS } } + if (!string.IsNullOrEmpty(tempHash)) + { + using (MD5 md5 = MD5.Create()) + { + modLoadResult.Hash += BitConverter.ToString(md5.ComputeHash(tempHash.ToBytes())).Replace("-", "").ToLowerInvariant(); + } + } + return modLoadResult; } diff --git a/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs b/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs index cd215781f..690f44241 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs @@ -82,13 +82,6 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions // Apply Nsos patches. device.Configuration.VirtualFileSystem.ModLoader.ApplyNsoPatches(programId, nsoExecutables); - // Don't use PTC if ExeFS files have been replaced. - bool enablePtc = device.System.EnablePtc && !modLoadResult.Modified; - if (!enablePtc) - { - Logger.Warning?.Print(LogClass.Ptc, "Detected unsupported ExeFs modifications. PTC disabled."); - } - string programName = string.Empty; if (!isHomebrew && programId > 0x010000000000FFFF) @@ -115,7 +108,8 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions device.System.KernelContext, metaLoader, nacpData, - enablePtc, + device.System.EnablePtc, + modLoadResult.Hash, true, programName, metaLoader.GetProgramId(), diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs index a0e7e0fa1..33d87b60a 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs @@ -212,6 +212,7 @@ namespace Ryujinx.HLE.Loaders.Processes dummyExeFs.GetNpdm(), nacpData, diskCacheEnabled: false, + diskCacheSelector: string.Empty, allowCodeMemoryForJit: true, programName, programId, diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs index 33aee1c4c..744dd7658 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs @@ -186,6 +186,7 @@ namespace Ryujinx.HLE.Loaders.Processes string.Empty, string.Empty, false, + string.Empty, codeAddress, codeSize); @@ -226,6 +227,7 @@ namespace Ryujinx.HLE.Loaders.Processes MetaLoader metaLoader, BlitStruct applicationControlProperties, bool diskCacheEnabled, + string diskCacheSelector, bool allowCodeMemoryForJit, string name, ulong programId, @@ -374,6 +376,7 @@ namespace Ryujinx.HLE.Loaders.Processes $"{programId:x16}", displayVersion, diskCacheEnabled, + diskCacheSelector, codeStart, codeSize); From 8c358880cfc0bee0b99d36c2ee0bf5d66626d481 Mon Sep 17 00:00:00 2001 From: LotP1 Date: Tue, 10 Dec 2024 22:42:39 +0100 Subject: [PATCH 2/3] Fix incorrect hash logic The stream hadn't been reset causing all hashes to be the same in most cases --- src/Ryujinx.HLE/HOS/ModLoader.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Ryujinx.HLE/HOS/ModLoader.cs b/src/Ryujinx.HLE/HOS/ModLoader.cs index c866fb063..0a00b8c9b 100644 --- a/src/Ryujinx.HLE/HOS/ModLoader.cs +++ b/src/Ryujinx.HLE/HOS/ModLoader.cs @@ -642,6 +642,7 @@ namespace Ryujinx.HLE.HOS Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced"); using (MD5 md5 = MD5.Create()) { + stream.Seek(0, SeekOrigin.Begin); tempHash += BitConverter.ToString(md5.ComputeHash(stream)).Replace("-", "").ToLowerInvariant(); } } From c61e05e4f44af2de323908a6da6dfe8d2366f268 Mon Sep 17 00:00:00 2001 From: LotP1 Date: Tue, 10 Dec 2024 22:48:47 +0100 Subject: [PATCH 3/3] Implement blacklist functionality - Added blacklist status to FunctionProfile - Added PPTC Info file updater from v5518 and updated old updater logic to allow multiple update passes - Added blacklist check to PPTC Cache loading - Added marking functions as blacklisted if they do not yet exist at PPTC translation time - Logger now shows how many functions were blacklisted when translating new functions to PPTC cache --- src/ARMeilleure/Translation/PTC/Ptc.cs | 54 +++++++++++- .../Translation/PTC/PtcProfiler.cs | 86 ++++++++++++++++--- src/ARMeilleure/Translation/Translator.cs | 10 +++ 3 files changed, 136 insertions(+), 14 deletions(-) diff --git a/src/ARMeilleure/Translation/PTC/Ptc.cs b/src/ARMeilleure/Translation/PTC/Ptc.cs index 841e5fefa..0d99a2e87 100644 --- a/src/ARMeilleure/Translation/PTC/Ptc.cs +++ b/src/ARMeilleure/Translation/PTC/Ptc.cs @@ -3,6 +3,7 @@ using ARMeilleure.CodeGen.Linking; using ARMeilleure.CodeGen.Unwinding; using ARMeilleure.Common; using ARMeilleure.Memory; +using ARMeilleure.State; using Ryujinx.Common; using Ryujinx.Common.Configuration; using Ryujinx.Common.Logging; @@ -30,7 +31,7 @@ namespace ARMeilleure.Translation.PTC private const string OuterHeaderMagicString = "PTCohd\0\0"; private const string InnerHeaderMagicString = "PTCihd\0\0"; - private const uint InternalVersion = 6997; //! To be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 7007; //! To be incremented manually for each change to the ARMeilleure project. private const string ActualDir = "0"; private const string BackupDir = "1"; @@ -185,6 +186,36 @@ namespace ARMeilleure.Translation.PTC InitializeCarriers(); } + private bool ContainsBlacklistedFunctions() + { + List blacklist = Profiler.GetBlacklistedFunctions(); + bool containsBlacklistedFunctions = false; + _infosStream.Seek(0L, SeekOrigin.Begin); + bool foundBadFunction = false; + + for (int index = 0; index < GetEntriesCount(); index++) + { + InfoEntry infoEntry = DeserializeStructure(_infosStream); + foreach (ulong address in blacklist) + { + if (infoEntry.Address == address) + { + containsBlacklistedFunctions = true; + Logger.Warning?.Print(LogClass.Ptc, "PPTC cache invalidated: Found blacklisted functions in PPTC cache"); + foundBadFunction = true; + break; + } + } + + if (foundBadFunction) + { + break; + } + } + + return containsBlacklistedFunctions; + } + private void PreLoad() { string fileNameActual = $"{CachePathActual}.cache"; @@ -533,7 +564,7 @@ namespace ARMeilleure.Translation.PTC public void LoadTranslations(Translator translator) { - if (AreCarriersEmpty()) + if (AreCarriersEmpty() || ContainsBlacklistedFunctions()) { return; } @@ -836,10 +867,18 @@ namespace ARMeilleure.Translation.PTC while (profiledFuncsToTranslate.TryDequeue(out var item)) { ulong address = item.address; + ExecutionMode executionMode = item.funcProfile.Mode; + bool highCq = item.funcProfile.HighCq; Debug.Assert(Profiler.IsAddressInStaticCodeRange(address)); - TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq); + TranslatedFunction func = translator.Translate(address, executionMode, highCq); + + if (func == null) + { + Profiler.UpdateEntry(address, executionMode, true, true); + continue; + } bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func); @@ -886,7 +925,14 @@ namespace ARMeilleure.Translation.PTC PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount); - Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); + if (_translateCount == _translateTotalCount) + { + Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); + } + else + { + Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | {_translateTotalCount - _translateCount} function{(_translateTotalCount - _translateCount != 1 ? "s" : "")} blacklisted | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); + } Thread preSaveThread = new(PreSave) { diff --git a/src/ARMeilleure/Translation/PTC/PtcProfiler.cs b/src/ARMeilleure/Translation/PTC/PtcProfiler.cs index 8e95c5e4b..8f0ca8371 100644 --- a/src/ARMeilleure/Translation/PTC/PtcProfiler.cs +++ b/src/ARMeilleure/Translation/PTC/PtcProfiler.cs @@ -23,10 +23,11 @@ namespace ARMeilleure.Translation.PTC { private const string OuterHeaderMagicString = "Pohd\0\0\0\0"; - private const uint InternalVersion = 5518; //! Not to be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 7007; //! Not to be incremented manually for each change to the ARMeilleure project. private static readonly uint[] _migrateInternalVersions = { 1866, + 5518, }; private const int SaveInterval = 30; // Seconds. @@ -74,20 +75,30 @@ namespace ARMeilleure.Translation.PTC Enabled = false; } - public void AddEntry(ulong address, ExecutionMode mode, bool highCq) + public void AddEntry(ulong address, ExecutionMode mode, bool highCq, bool blacklist = false) { if (IsAddressInStaticCodeRange(address)) { Debug.Assert(!highCq); - lock (_lock) + if (blacklist) { - ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false)); + lock (_lock) + { + ProfiledFuncs[address] = new FuncProfile(mode, highCq: false, true); + } + } + else + { + lock (_lock) + { + ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false, false)); + } } } } - public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq) + public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq, bool? blacklist = null) { if (IsAddressInStaticCodeRange(address)) { @@ -97,7 +108,7 @@ namespace ARMeilleure.Translation.PTC { Debug.Assert(ProfiledFuncs.ContainsKey(address)); - ProfiledFuncs[address] = new FuncProfile(mode, highCq: true); + ProfiledFuncs[address] = new FuncProfile(mode, highCq: true, blacklist ?? ProfiledFuncs[address].Blacklist); } } } @@ -113,7 +124,7 @@ namespace ARMeilleure.Translation.PTC foreach (var profiledFunc in ProfiledFuncs) { - if (!funcs.ContainsKey(profiledFunc.Key)) + if (!funcs.ContainsKey(profiledFunc.Key) && !profiledFunc.Value.Blacklist) { profiledFuncsToTranslate.Enqueue((profiledFunc.Key, profiledFunc.Value)); } @@ -128,6 +139,24 @@ namespace ARMeilleure.Translation.PTC ProfiledFuncs.TrimExcess(); } + public List GetBlacklistedFunctions() + { + List funcs = new List(); + + foreach (var profiledFunc in ProfiledFuncs) + { + if (profiledFunc.Value.Blacklist) + { + if (!funcs.Contains(profiledFunc.Key)) + { + funcs.Add(profiledFunc.Key); + } + } + } + + return funcs; + } + public void PreLoad() { _lastHash = default; @@ -218,13 +247,18 @@ namespace ARMeilleure.Translation.PTC return false; } + Func migrateEntryFunc = null; + switch (outerHeader.InfoFileVersion) { case InternalVersion: ProfiledFuncs = Deserialize(stream); break; case 1866: - ProfiledFuncs = Deserialize(stream, (address, profile) => (address + 0x500000UL, profile)); + migrateEntryFunc = (address, profile) => (address + 0x500000UL, profile); + goto case 5518; + case 5518: + ProfiledFuncs = DeserializeAddBlacklist(stream, migrateEntryFunc); break; default: Logger.Error?.Print(LogClass.Ptc, $"No migration path for {nameof(outerHeader.InfoFileVersion)} '{outerHeader.InfoFileVersion}'. Discarding cache."); @@ -254,6 +288,16 @@ namespace ARMeilleure.Translation.PTC return DeserializeDictionary(stream, DeserializeStructure); } + private static Dictionary DeserializeAddBlacklist(Stream stream, Func migrateEntryFunc = null) + { + if (migrateEntryFunc != null) + { + return DeserializeAndUpdateDictionary(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure(stream)); }, migrateEntryFunc); + } + + return DeserializeDictionary(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure(stream)); }); + } + private static ReadOnlySpan GetReadOnlySpan(MemoryStream memoryStream) { return new(memoryStream.GetBuffer(), (int)memoryStream.Position, (int)memoryStream.Length - (int)memoryStream.Position); @@ -385,13 +429,35 @@ namespace ARMeilleure.Translation.PTC } } - [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)] + [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 6*/)] public struct FuncProfile { public ExecutionMode Mode; public bool HighCq; + public bool Blacklist; - public FuncProfile(ExecutionMode mode, bool highCq) + public FuncProfile(ExecutionMode mode, bool highCq, bool blacklist) + { + Mode = mode; + HighCq = highCq; + Blacklist = blacklist; + } + + public FuncProfile(FuncProfilePreBlacklist fp) + { + Mode = fp.Mode; + HighCq = fp.HighCq; + Blacklist = false; + } + } + + [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)] + public struct FuncProfilePreBlacklist + { + public ExecutionMode Mode; + public bool HighCq; + + public FuncProfilePreBlacklist(ExecutionMode mode, bool highCq) { Mode = mode; HighCq = highCq; diff --git a/src/ARMeilleure/Translation/Translator.cs b/src/ARMeilleure/Translation/Translator.cs index 162368782..8e07837ee 100644 --- a/src/ARMeilleure/Translation/Translator.cs +++ b/src/ARMeilleure/Translation/Translator.cs @@ -249,6 +249,11 @@ namespace ARMeilleure.Translation ControlFlowGraph cfg = EmitAndGetCFG(context, blocks, out Range funcRange, out Counter counter); + if (cfg == null) + { + return null; + } + ulong funcSize = funcRange.End - funcRange.Start; Logger.EndPass(PassName.Translation, cfg); @@ -407,6 +412,11 @@ namespace ARMeilleure.Translation if (opCode.Instruction.Emitter != null) { opCode.Instruction.Emitter(context); + if (opCode.Instruction.Name == InstName.Und && blkIndex == 0) + { + range = new Range(rangeStart, rangeEnd); + return null; + } } else {